diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7e4edc2dc36..427cf21600b 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -3,12 +3,21 @@ pool: vmImage: 'ubuntu-20.04' +trigger: + branches: + include: + - development pr: autoCancel: true drafts: false + paths: + exclude: + - Docs + - '**/*.rst' jobs: - job: + # FIXME remove unused variables variables: BLASPP_HOME: '/usr/local' CEI_SUDO: 'sudo' @@ -16,29 +25,28 @@ jobs: CMAKE_GENERATOR: 'Ninja' FFTW_HOME: '/usr' LAPACKPP_HOME: '/usr/local' - OMP_NUM_THREADS: 1 WARPX_CI_CCACHE: 'TRUE' - WARPX_CI_NUM_MAKE_JOBS: 2 - WARPX_CI_OPENPMD: 'TRUE' - WARPX_CI_TMP: '/tmp/ci' + #WARPX_OPENPMD: 'TRUE' strategy: matrix: - cartesian1d: - WARPX_CI_REGULAR_CARTESIAN_1D: 'TRUE' - WARPX_CI_PSATD: 'FALSE' - cartesian2d: - WARPX_CI_REGULAR_CARTESIAN_2D: 'TRUE' - cartesian3d: - WARPX_CI_REGULAR_CARTESIAN_3D: 'TRUE' - single_precision: - WARPX_CI_SINGLE_PRECISION: 'TRUE' - rz_or_nompi: - WARPX_CI_RZ_OR_NOMPI: 'TRUE' - qed: - WARPX_CI_QED: 'TRUE' - embedded_boundary: - WARPX_CI_EB: 'TRUE' + # Cartesian 1D + cartesian_1d: + WARPX_CMAKE_FLAGS: -DWarpX_DIMS=1 -DWarpX_FFT=ON -DWarpX_PYTHON=ON + # Cartesian 2D + cartesian_2d: + WARPX_CMAKE_FLAGS: -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_PYTHON=ON + # Cartesian 3D + cartesian_3d: + WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON + # Cylindrical RZ + cylindrical_rz: + WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON + WARPX_RZ_FFT: 'TRUE' + # single precision + #single_precision: + # WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE + # WARPX_RZ_FFT: 'TRUE' # default: 60; maximum: 360 timeoutInMinutes: 240 @@ -50,28 +58,16 @@ jobs: - task: Cache@2 continueOnError: true inputs: - key: 'Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake | run_test.sh' + key: 'Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake' restoreKeys: | - Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake | run_test.sh Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake Ccache | "$(System.JobName)" | .azure-pipelines.yml path: /home/vsts/.ccache cacheHitVar: CCACHE_CACHE_RESTORED displayName: Cache Ccache Objects - - task: Cache@2 - continueOnError: true - inputs: - key: 'Python3 | "$(System.JobName)" | .azure-pipelines.yml | run_test.sh' - restoreKeys: | - Python3 | "$(System.JobName)" | .azure-pipelines.yml | run_test.sh - Python3 | "$(System.JobName)" | .azure-pipelines.yml - path: /home/vsts/.local/lib/python3.8 - cacheHitVar: PYTHON38_CACHE_RESTORED - displayName: Cache Python Libraries - - bash: | - set -eu -o pipefail + set -o nounset errexit pipefail cat /proc/cpuinfo | grep "model name" | sort -u df -h echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries @@ -82,6 +78,8 @@ jobs: python3 python3-pandas python3-pip python3-venv python3-setuptools libblas-dev liblapack-dev ccache --set-config=max_size=10.0G python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build + python3 -m pip install --upgrade packaging python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade wheel python3 -m pip install --upgrade virtualenv @@ -91,32 +89,86 @@ jobs: export PATH="$HOME/.local/bin:$PATH" sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall sudo chmod a+x /usr/local/bin/cmake-easyinstall - if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then - cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.14.3 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF - python3 -m pip install --upgrade openpmd-api - fi - if [[ "${WARPX_CI_RZ_OR_NOMPI:-FALSE}" == "TRUE" ]]; then - cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/blaspp.git \ + #if [ "${WARPX_OPENPMD:-FALSE}" == "TRUE" ]; then + # cmake-easyinstall --prefix=/usr/local \ + # git+https://github.com/openPMD/openPMD-api.git@0.14.3 \ + # -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + # -DCMAKE_VERBOSE_MAKEFILE=ON \ + # -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF + # #python3 -m pip install --upgrade openpmd-api + #fi + if [ "${WARPX_RZ_FFT:-FALSE}" == "TRUE" ]; then + # BLAS++ + cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/blaspp.git \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_CXX_STANDARD=17 \ -Duse_openmp=OFF -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON - cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/lapackpp.git \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 \ + # LAPACK++ + cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/lapackpp.git \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi + # Python modules required for test analysis + python3 -m pip install --upgrade -r Regression/requirements.txt + python3 -m pip cache purge + # external repositories required for test analysis + cd .. + git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git + # TODO select only specific datasets? + git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git + cd - rm -rf ${CEI_TMP} df -h displayName: 'Install dependencies' - bash: | - set -eu -o pipefail + # set options + set -o nounset errexit pipefail + # display disk space usage df -h - ./run_test.sh - rm -rf ${WARPX_CI_TMP} + # configure + export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON" + export WARPX_TEST_FLAGS="-DWarpX_TEST_CLEANUP=ON -DWarpX_TEST_FPETRAP=ON -DWarpX_BACKTRACE_INFO=ON" + cmake -S . -B build \ + ${AMReX_CMAKE_FLAGS} \ + ${WARPX_CMAKE_FLAGS} \ + ${WARPX_TEST_FLAGS} + # build + cmake --build build -j 2 + # display disk space usage df -h - displayName: 'Build & test' + displayName: 'Build' + + - bash: | + # set options + set -o nounset errexit pipefail + # determine if the build was triggered by a push to the development branch + if [[ "$(Build.SourceBranch)" == "refs/heads/development" ]]; then + # run tests (exclude pytest.AMReX when running Python tests) + # and submit results to CDash as Experimental + ctest --test-dir build --output-on-failure -E AMReX \ + -D ExperimentalTest -D ExperimentalSubmit + else + # run tests (exclude pytest.AMReX when running Python tests) + ctest --test-dir build --output-on-failure -E AMReX + fi + displayName: 'Test' + + - bash: | + # set options + set -o nounset errexit pipefail + # find and print backtrace + find build/bin/ -type f -name "Backtrace*" \ + -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ + -exec cat {} \; + displayName: 'Logs' + condition: always() + + - bash: | + # clean out so the Post-job Cache "tar" command has more disk space available + rm -rf build + displayName: 'Clean Build Directory' + condition: always() diff --git a/.clang-tidy b/.clang-tidy index 04d1419c5c7..8111fc2fc25 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -19,6 +19,7 @@ Checks: ' -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, + -cppcoreguidelines-missing-std-forward, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, -cppcoreguidelines-owning-memory, @@ -29,6 +30,7 @@ Checks: ' misc-*, -misc-no-recursion, -misc-non-private-member-variables-in-classes, + -misc-include-cleaner, modernize-*, -modernize-avoid-c-arrays, -modernize-return-braced-init-list, @@ -55,6 +57,9 @@ CheckOptions: value: "H," - key: modernize-pass-by-value.ValuesOnly value: "true" - +- key: misc-use-anonymous-namespace.HeaderFileExtensions + value: "H," +- key: performance-move-const-arg.CheckTriviallyCopyableMove + value: "false" HeaderFilterRegex: 'Source[a-z_A-Z0-9\/]+\.H$' diff --git a/.github/ISSUE_TEMPLATE/blank_issue.md b/.github/ISSUE_TEMPLATE/blank_issue.md new file mode 100644 index 00000000000..2d5216c8fc8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/blank_issue.md @@ -0,0 +1,14 @@ +--- +name: Blank issue +about: Ask us a question +labels: [question] +--- + +Are you here because you have something to report that is neither a bug, a new feature, nor an installation problem? + +Before opening this issue, consider opening a [discussion](https://github.com/ECP-WarpX/WarpX/discussions) instead! + +Issues are used to report bugs, installation problems or to request new features. +Discussions are used to ask more open-ended questions, brainstorm, ask our feedback, etc. + +You can find more details on how to use issues and discussions [here](https://github.com/ECP-WarpX/WarpX/blob/development/CONTRIBUTING.rst). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..a5a64487646 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,58 @@ +--- +name: Bug report +about: Report a bug or unexpected behavior. +labels: [bug] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Description +A clear and concise description of the bug. + +## Expected behavior +What did you expect to happen when you encountered the issue? + +## How to reproduce +Please provide (if available): +- WarpX inputs files +- PICMI Python files +- Python post-processing scripts + +If you are unable to provide certain files or scripts, please describe the steps you took to encounter the issue. + +Please minimize your inputs/scripts to be concise and focused on the issue. +For instance, make the simulation scripts as small and fast to run as possible. + +## System information +Please check all relevant boxes and provide details. + +- Operating system (name and version): + - [ ] Linux: e.g., Ubuntu 22.04 LTS + - [ ] macOS: e.g., macOS Monterey 12.4 + - [ ] Windows: e.g., Windows 11 Pro +- Version of WarpX: e.g., latest, 24.10, etc. +- Installation method: + - [ ] Conda + - [ ] Spack + - [ ] PyPI + - [ ] Brew + - [ ] From source with CMake + - [ ] Module system on an HPC cluster +- Other dependencies: yes/no, describe +- Computational resources: + - [ ] MPI: e.g., 2 MPI processes + - [ ] OpenMP: e.g., 2 OpenMP threads + - [ ] CPU: e.g., 2 CPUs + - [ ] GPU: e.g., 2 GPUs (NVIDIA, AMD, etc.) + +If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported. + +## Steps taken so far +What troubleshooting steps have you taken so far, and what were the results? + +Have you tried debugging the code, following the instructions in our [debugging documentation](https://warpx.readthedocs.io/en/latest/usage/workflows/debugging.html)? + +## Additional information +If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..3ba13e0cec6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..8e4630cc098 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,27 @@ +--- +name: Feature request +about: Suggest a new feature or enhancement. +labels: [enhancement] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Context and motivation +Please provide a clear and concise description of the context that is prompting you to request a new feature. What problem are you trying to solve, and how will this feature help you achieve your goals? + +## Proposed feature +Describe the feature you would like to add to WarpX in detail. Please include: +- A clear and concise description of the feature +- Any relevant technical requirements or specifications +- How you envision the feature being used + +## Alternative solutions +Have you considered any alternative solutions or features that could achieve the same goal? If so, please describe them and explain why you believe the proposed feature is the best solution. + +## Additional information +If applicable, please provide any additional information that may be relevant to the feature request, such as: +- Links to existing codes or implementations +- References to relevant publications or research +- Any specific use cases or scenarios where the feature would be particularly useful diff --git a/.github/ISSUE_TEMPLATE/installation-issue.md b/.github/ISSUE_TEMPLATE/installation-issue.md new file mode 100644 index 00000000000..7cc937d91c0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/installation-issue.md @@ -0,0 +1,49 @@ +--- +name: Installation issue +about: Report an issue with installing or setting up WarpX. +labels: [install] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Description +A clear and concise description of the issue. + +## System information +- Operating system (name and version): + - [ ] Linux: e.g., Ubuntu 22.04 LTS + - [ ] macOS: e.g., macOS Monterey 12.4 + - [ ] Windows: e.g., Windows 11 Pro +- Version of WarpX: e.g., latest, 24.10, etc. +- Installation method: + - [ ] Conda + - [ ] Spack + - [ ] PyPI + - [ ] Brew + - [ ] From source with CMake + - [ ] Module system on an HPC cluster +- Other dependencies: yes/no, describe +- Computational resources: + - [ ] CPU + - [ ] GPU: e.g., NVIDIA, AMD, etc. + +If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported. + +If you encountered the issue installing from source with CMake, please provide the output of the following steps: +1. buildsystem generation: output of `cmake --fresh -S . -B build` (include your specific build options, e.g., `-DWarpX_DIMS=3`) +2. project build: output of `cmake --build build` (include your specific build options, e.g., `-j 4`) + +If applicable, please add any additional information about your software environment: +- [ ] CMake: e.g., 3.24.0 +- [ ] C++ compiler: e.g., GNU 11.3 with NVCC 12.0.76 +- [ ] Python: e.g., CPython 3.12 +- [ ] MPI: e.g., OpenMPI 4.1.1 +- [ ] FFTW: e.g., 3.3.10 +- [ ] HDF5: e.g., 1.14.0 +- [ ] ADIOS2: e.g., 2.10.0 +- Other dependencies: yes/no, describe + +## Additional information +If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details. diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml new file mode 100644 index 00000000000..e0947916d3f --- /dev/null +++ b/.github/workflows/clang_sanitizers.yml @@ -0,0 +1,160 @@ +name: 🧴 clang sanitizers + +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers + cancel-in-progress: true + +jobs: + build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-24.04 + if: github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang.sh 17 + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with UB sanitizer + run: | + + export OMP_NUM_THREADS=2 + + #MPI implementations often leak memory + export "ASAN_OPTIONS=detect_leaks=0" + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d + + build_thread_sanitizer: + name: Clang thread sanitizer + runs-on: ubuntu-24.04 + # TODO Fix data race conditions and re-enable job + if: 0 #github.event.pull_request.draft == false + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang.sh 17 + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=thread" + + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=OFF \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build -j 4 + + cmake -S . -B build_EB \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="2" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=ON \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build_EB -j 4 + + ccache -s + du -hs ~/.cache/ccache + + - name: run with thread sanitizer + run: | + export PMIX_MCA_gds=hash + export TSAN_OPTIONS='ignore_noninstrumented_modules=1' + export ARCHER_OPTIONS="verbose=1" + + export OMP_NUM_THREADS=2 + + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d warpx.serialize_initial_conditions = 0 + + git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data + cd Examples/Tests/embedded_circle + + ulimit -c unlimited + + mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_test_2d_embedded_circle warpx.serialize_initial_conditions = 0 diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index af8632ed698..49f2a5b6e25 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -1,6 +1,13 @@ name: 🧹 clang-tidy -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-clangtidy @@ -13,13 +20,13 @@ jobs: dim: [1, 2, RZ, 3] name: clang-tidy-${{ matrix.dim }}D runs-on: ubuntu-22.04 - timeout-minutes: 120 + timeout-minutes: 250 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - name: install dependencies run: | - .github/workflows/dependencies/clang15.sh + .github/workflows/dependencies/clang.sh 17 - name: set up cache uses: actions/cache@v4 with: @@ -36,8 +43,8 @@ jobs: export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt ccache -z - export CXX=$(which clang++-15) - export CC=$(which clang-15) + export CXX=$(which clang++-17) + export CC=$(which clang-17) cmake -S . -B build_clang_tidy \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -55,23 +62,8 @@ jobs: ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt make -j4 --keep-going -f clang-tidy-ccache-misses.mak \ - CLANG_TIDY=clang-tidy-15 \ + CLANG_TIDY=clang-tidy-17 \ CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 9a2ffb0f61a..5e9a70cd5a4 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -23,8 +23,11 @@ jobs: REPO=${{ github.repository }} - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + # For debugging cat ${GITHUB_EVENT_PATH} to see the payload. + + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number') + echo "Post-PR cache cleanup for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge # Setting this to not fail the workflow while deleting cache keys. diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index bd1a518acf4..c71a48cdad8 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -2,7 +2,7 @@ name: CleanUpCache on: workflow_run: - workflows: [🧹 clang-tidy, 🔍 CodeQL, 🐧 CUDA, 🐧 HIP, 🐧 Intel, 🍏 macOS, 🐧 OpenMP] + workflows: [🧴 clang sanitizers, 🧹 clang-tidy, 🔍 CodeQL, 🐧 CUDA, 🐧 HIP, 🐧 Intel, 🍏 macOS, 🐧 OpenMP] types: - completed @@ -29,9 +29,12 @@ jobs: # Triggering workflow run name (e.g., LinuxClang) WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload. + if [[ $EVENT == "pull_request" ]]; then - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number') + echo "Clean up cache for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge else BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} @@ -54,6 +57,7 @@ jobs: IFS=$'\n' for j in $cached_jobs do + # Delete all entries except the last used one old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bc0bee545cc..e3549ae340a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,6 +31,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: '3.x' + - name: Install Packages (C++) if: ${{ matrix.language == 'cpp' }} run: | @@ -38,9 +43,10 @@ jobs: sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev ccache python -m pip install --upgrade pip + python -m pip install --upgrade pipx python -m pip install --upgrade wheel python -m pip install --upgrade cmake - export CMAKE="$HOME/.local/bin/cmake" && echo "CMAKE=$CMAKE" >> $GITHUB_ENV + python -m pipx install cmake - name: Set Up Cache if: ${{ matrix.language == 'cpp' }} @@ -54,7 +60,7 @@ jobs: - name: Configure (C++) if: ${{ matrix.language == 'cpp' }} run: | - $CMAKE -S . -B build -DWarpX_OPENPMD=ON + cmake -S . -B build -DWarpX_OPENPMD=ON - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -75,7 +81,7 @@ jobs: export CCACHE_MAXSIZE=100M ccache -z - $CMAKE --build build -j 4 + cmake --build build -j 4 ccache -s du -hs ~/.cache/ccache @@ -83,7 +89,7 @@ jobs: # Make sure CodeQL has something to do touch Source/Utils/WarpXVersion.cpp export CCACHE_DISABLE=1 - $CMAKE --build build -j 4 + cmake --build build -j 4 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 @@ -110,18 +116,3 @@ jobs: uses: github/codeql-action/upload-sarif@v3 with: sarif_file: sarif-results/${{ matrix.language }}.sarif - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 8f5e59ccaa4..029d1e4db89 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,6 +1,13 @@ name: 🐧 CUDA -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-cuda @@ -43,8 +50,13 @@ jobs: export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} + which nvcc || echo "nvcc not in PATH!" + cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ + git+https://github.com/openPMD/openPMD-api.git@0.16.1 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ @@ -115,7 +127,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - + cd ../amrex && git checkout --detach b364becad939a490bca4e7f8b23f7392c558a311 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s @@ -187,18 +199,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/dependencies/clang.sh b/.github/workflows/dependencies/clang.sh new file mode 100755 index 00000000000..3ffe6dbc675 --- /dev/null +++ b/.github/workflows/dependencies/clang.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Copyright 2025 The WarpX Community +# +# License: BSD-3-Clause-LBNL + +set -eu -o pipefail + +# `man apt.conf`: number of retries to perform (if non-zero, +# APT will retry failed files the given number of times). +echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries + +sudo apt-get update +sudo apt-get install -y \ + cmake \ + libblas-dev \ + libboost-math-dev \ + libfftw3-dev \ + libfftw3-mpi-dev \ + libhdf5-openmpi-dev \ + liblapack-dev \ + libopenmpi-dev \ + ninja-build + +# parse clang version number from command line +version_number=${1} + +# add LLVM repository and install clang tools +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh ${version_number} + +# install clang, clang-tidy, and +# LLVM implementations of C++ standard library and OpenMP +sudo apt-get update +sudo apt-get install -y \ + clang-${version_number} \ + clang-tidy-${version_number} \ + libc++-${version_number}-dev \ + libomp-${version_number}-dev + +# export compiler flags +export CXX=$(which clang++-${version_number}) +export CC=$(which clang-${version_number}) + +# ccache +$(dirname "$0")/ccache.sh + +# cmake-easyinstall +# +sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall +sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# BLAS++ & LAPACK++ +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/blaspp.git \ + -Duse_openmp=OFF \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON + +cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/lapackpp.git \ + -Duse_cmake_find_lapack=ON \ + -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/dependencies/clang15.sh b/.github/workflows/dependencies/clang15.sh deleted file mode 100755 index 63d5d70956f..00000000000 --- a/.github/workflows/dependencies/clang15.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2023 The WarpX Community -# -# License: BSD-3-Clause-LBNL -# Authors: Luca Fedeli - -set -eu -o pipefail - -# `man apt.conf`: -# Number of retries to perform. If this is non-zero APT will retry -# failed files the given number of times. -echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries - -sudo apt-get -qqq update -sudo apt-get install -y \ - cmake \ - clang-15 \ - clang-tidy-15 \ - libblas-dev \ - libc++-15-dev \ - libboost-math-dev \ - libfftw3-dev \ - libfftw3-mpi-dev \ - libhdf5-openmpi-dev \ - liblapack-dev \ - libopenmpi-dev \ - libomp-15-dev \ - ninja-build - -# ccache -$(dirname "$0")/ccache.sh - -# cmake-easyinstall -# -sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall -sudo chmod a+x /usr/local/bin/cmake-easyinstall -export CEI_SUDO="sudo" -export CEI_TMP="/tmp/cei" - -# BLAS++ & LAPACK++ -cmake-easyinstall \ - --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/blaspp.git \ - -Duse_openmp=OFF \ - -Dbuild_tests=OFF \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_VERBOSE_MAKEFILE=ON - -cmake-easyinstall \ - --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/lapackpp.git \ - -Duse_cmake_find_lapack=ON \ - -Dbuild_tests=OFF \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 3b146405b4b..2ca89e03d3f 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -29,13 +29,16 @@ df -h # https://github.com/ECP-WarpX/WarpX/pull/1566#issuecomment-790934878 # try apt install up to five times, to avoid connection splits +# FIXME install latest version of IntelLLVM, Intel MKL +# after conflicts with openPMD are resolved status=1 for itry in {1..5} do sudo apt-get install -y --no-install-recommends \ build-essential \ cmake \ - intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ + intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \ + intel-oneapi-mkl-devel=2024.2.1-103 \ g++ gfortran \ libopenmpi-dev \ openmpi-bin \ diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index ae897786acd..bf15c2f7101 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -53,6 +53,7 @@ sudo apt-get install -y --no-install-recommends \ rocm-dev \ rocfft-dev \ rocprim-dev \ + rocsparse-dev \ rocrand-dev \ hiprand-dev @@ -65,6 +66,13 @@ source /etc/profile.d/rocm.sh hipcc --version which clang which clang++ +export CXX=$(which clang++) +export CC=$(which clang) + +# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3 +# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786 +# https://github.com/open-mpi/ompi/issues/9317 +export LDFLAGS="-lopen-pal" # cmake-easyinstall # diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index fae6e22d45a..63763421d31 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -58,7 +58,7 @@ export CEI_TMP="/tmp/cei" CXX=$(which icpc) CC=$(which icc) \ cmake-easyinstall \ --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.15.2 \ + git+https://github.com/openPMD/openPMD-api.git@0.16.1 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ diff --git a/.github/workflows/dependencies/nvcc11-3.sh b/.github/workflows/dependencies/nvcc11-3.sh index 92e2717e425..050b58b5947 100755 --- a/.github/workflows/dependencies/nvcc11-3.sh +++ b/.github/workflows/dependencies/nvcc11-3.sh @@ -41,7 +41,8 @@ sudo apt-get install -y \ cuda-nvml-dev-11-3 \ cuda-nvtx-11-3 \ libcufft-dev-11-3 \ - libcurand-dev-11-3 + libcurand-dev-11-3 \ + libcusparse-dev-11-3 sudo ln -s cuda-11.3 /usr/local/cuda # if we run out of temporary storage in CI: diff --git a/.github/workflows/dependencies/nvcc11-8.sh b/.github/workflows/dependencies/nvcc11-8.sh index 6089360392b..608f6c7a817 100755 --- a/.github/workflows/dependencies/nvcc11-8.sh +++ b/.github/workflows/dependencies/nvcc11-8.sh @@ -41,7 +41,8 @@ sudo apt-get install -y \ cuda-nvml-dev-11-8 \ cuda-nvtx-11-8 \ libcufft-dev-11-8 \ - libcurand-dev-11-8 + libcurand-dev-11-8 \ + libcusparse-dev-11-8 sudo ln -s cuda-11.8 /usr/local/cuda # if we run out of temporary storage in CI: diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0f6710b1405..f61c8fe1313 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -1,6 +1,13 @@ name: 🐧 HIP -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-hip @@ -125,18 +132,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index 42923d3df8e..3d3942174a7 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -1,6 +1,13 @@ name: 🐧 In Situ Vis -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-insituvis @@ -17,10 +24,13 @@ jobs: CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code -Wno-error=pass-failed" CMAKE_GENERATOR: Ninja CMAKE_PREFIX_PATH: /root/install/sensei/v4.0.0/lib64/cmake + OMP_NUM_THREADS: 1 container: image: senseiinsitu/ci:fedora35-amrex-20220613 steps: - uses: actions/checkout@v4 + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v2 - name: Configure run: | cmake -S . -B build \ @@ -38,6 +48,7 @@ jobs: CXX: g++ CC: gcc CMAKE_PREFIX_PATH: /ascent/install/lib/cmake/ + OMP_NUM_THREADS: 1 container: image: alpinedav/ascent:0.9.2 steps: @@ -54,10 +65,10 @@ jobs: cmake --build build -j 4 - name: Test run: | - cp Examples/Physics_applications/laser_acceleration/inputs_3d . + cp Examples/Physics_applications/laser_acceleration/inputs_base_3d . cp Examples/Physics_applications/laser_acceleration/3d_ascent_actions.yaml ascent_actions.yaml mpiexec -n 2 ./build/bin/warpx.3d \ - inputs_3d \ + inputs_base_3d \ max_step = 40 \ diag1.intervals = 30:40:10 \ diag1.format = ascent @@ -68,3 +79,58 @@ jobs: *.png conduit_* if-no-files-found: error + + catalyst: + name: Catalyst + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CXX: g++ + CC: gcc + CMAKE_PREFIX_PATH: "/opt/conduit:/opt/catalyst" + CATALYST_DEBUG: 1 + CATALYST_IMPLEMENTATION_PATHS: /opt/paraview/lib/catalyst + OMP_NUM_THREADS: 1 + + # Container build scripts: + # https://gitlab.kitware.com/christos.tsolakis/catalyst-amrex-docker-images + container: + image: kitware/paraview:ci-catalyst-amrex-warpx-20240828 + steps: + - uses: actions/checkout@v4 + - name: Configure + run: | + cmake -S . -B build \ + -DWarpX_DIMS="2;3" \ + -DWarpX_CATALYST=ON + - name: Build + run: | + cmake --build build -j 10 + - name: 2D Test + run: | + cp Examples/Tests/field_ionization/inputs_test_2d_ionization_lab . + cp Examples/Tests/field_ionization/catalyst_pipeline.py . + mpiexec -n 2 ./build/bin/warpx.2d \ + inputs_test_2d_ionization_lab \ + catalyst.script_paths = catalyst_pipeline.py\ + catalyst.implementation = paraview\ + diag1.intervals = 16\ + diag1.species = "electrons ions"\ + diag1.format = catalyst + - name: 3D Test + run: | + cp Examples/Tests/electrostatic_sphere/inputs_base_3d . + cp Examples/Tests/electrostatic_sphere/catalyst_pipeline.py . + mpiexec -n 2 ./build/bin/warpx.3d \ + inputs_base_3d \ + catalyst.script_paths = catalyst_pipeline.py \ + catalyst.implementation = paraview \ + diagnostics.diags_names = diag1 \ + diag1.format = catalyst\ + diag1.intervals = 3 + - uses: actions/upload-artifact@v4 + with: + name: catalyst-test-artifacts + path: | + datasets/*.png + if-no-files-found: error diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 3b1d6b546a4..25819e188e3 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -1,6 +1,13 @@ name: 🐧 Intel -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-intel @@ -106,6 +113,7 @@ jobs: set +e source /opt/intel/oneapi/setvars.sh set -e + export PATH=$PATH:/opt/intel/oneapi/compiler/2024.2/bin # FIXME export CXX=$(which icpx) export CC=$(which icx) @@ -115,7 +123,7 @@ jobs: cmake -S . -B build_sp \ -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_EB=ON \ + -DWarpX_EB=OFF \ -DWarpX_PYTHON=ON \ -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ @@ -132,7 +140,7 @@ jobs: source /opt/intel/oneapi/setvars.sh set -e export OMP_NUM_THREADS=2 - Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py build_dpcc: name: oneAPI DPC++ SP @@ -170,6 +178,7 @@ jobs: set +e source /opt/intel/oneapi/setvars.sh set -e + export PATH=$PATH:/opt/intel/oneapi/compiler/2024.2/bin # FIXME export CXX=$(which icpx) export CC=$(which icx) export CXXFLAGS="-fsycl ${CXXFLAGS}" @@ -180,6 +189,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=SYCL \ -DWarpX_EB=ON \ + -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ @@ -194,18 +204,3 @@ jobs: # python3 -m pip install --upgrade build packaging setuptools wheel # PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . # python3 -m pip install *.whl - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 00ac8f06b5d..87482cc6166 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,6 +1,13 @@ name: 🍏 macOS -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-macos @@ -12,19 +19,21 @@ jobs: runs-on: macos-latest if: github.event.pull_request.draft == false env: - CXXFLAGS: "-Werror -Wno-error=pass-failed" HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v4 - - name: install dependencies + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: '3.x' + - name: install brew dependencies run: | set +e brew unlink gcc brew update brew upgrade || true - brew install --overwrite python brew install ccache brew install fftw brew install libomp @@ -35,12 +44,12 @@ jobs: set -e brew tap openpmd/openpmd brew install openpmd-api - - python3 -m venv py-venv - source py-venv/bin/activate + - name: install pip dependencies + run: | python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade mpi4py + python3 -m pip install --upgrade -r Regression/requirements.txt - name: CCache Cache uses: actions/cache@v4 with: @@ -56,19 +65,18 @@ jobs: export CCACHE_SLOPPINESS=time_macros ccache -z - source py-venv/bin/activate + export CXXFLAGS="-Werror -Wno-error=pass-failed" cmake -S . -B build_dp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_EB=ON \ + -DWarpX_EB=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF cmake --build build_dp -j 3 cmake -S . -B build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DPython_EXECUTABLE=$(which python3) \ - -DWarpX_EB=ON \ + -DWarpX_EB=OFF \ -DWarpX_PYTHON=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ @@ -81,22 +89,6 @@ jobs: - name: run pywarpx run: | - source py-venv/bin/activate export OMP_NUM_THREADS=1 - mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + mpirun -n 2 Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef376cc..5f0b1534970 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -4,17 +4,13 @@ on: types: - closed +# This workflow does not have the permission to clean up cache for PRs +# originated from a fork. The purpose here is to trigger a workflow_run +# cleanup-cache-postpr.yml that has the right permission. + jobs: - cleanup: + noop: runs-on: ubuntu-latest steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + - name: No OP + run: echo "This workflow is going to trigger CleanUpCachePostPR." diff --git a/.github/workflows/scripts/checkQEDTableGenerator.sh b/.github/workflows/scripts/checkQEDTableGenerator.sh index e14a7a2d6f2..a773015c6c7 100755 --- a/.github/workflows/scripts/checkQEDTableGenerator.sh +++ b/.github/workflows/scripts/checkQEDTableGenerator.sh @@ -29,7 +29,7 @@ export OMP_NUM_THREADS=2 # Generate QED lookup tables using WarpX # ./build/bin/warpx.2d \ - ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + ./Examples/Tests/qed/inputs_test_2d_qed_quantum_sync \ qed_bw.lookup_table_mode = "generate" \ qed_bw.tab_dndt_chi_min = 0.01 \ qed_bw.tab_dndt_chi_max = 100.0 \ @@ -70,7 +70,7 @@ diff qs_table_dndt qs_table_tool_dndt # Run a WarpX simulation using the lookup tables generated by the external tool # ./build/bin/warpx.2d \ - ./Examples/Tests/qed/quantum_synchrotron/inputs_2d \ + ./Examples/Tests/qed/inputs_test_2d_qed_quantum_sync \ qed_bw.lookup_table_mode = "load" \ qed_bw.load_table_from = "bw_table_tool" \ qed_qs.lookup_table_mode = "load" \ diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index a1c29416b3e..b97afe016c0 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -6,7 +6,11 @@ name: 📜 Source -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-source @@ -25,12 +29,8 @@ jobs: run: .github/workflows/source/hasTabs - name: End-of-Line whitespaces run: .github/workflows/source/hasEOLwhiteSpace - - name: Proper file names in Examples - run: .github/workflows/source/wrongFileNameInExamples - - name: Examples are tested - run: .github/workflows/source/inputsNotTested - - name: Check that the test matrix for CI includes all tests - run: .github/workflows/source/test_ci_matrix.sh + - name: Check test input files + run: .github/workflows/source/check_inputs.py - name: Doxygen run: | sudo apt-get install -y --no-install-recommends doxygen diff --git a/.github/workflows/source/check_inputs.py b/.github/workflows/source/check_inputs.py new file mode 100755 index 00000000000..84ea70cba60 --- /dev/null +++ b/.github/workflows/source/check_inputs.py @@ -0,0 +1,109 @@ +#! /usr/bin/env python3 + +import os +import re +import sys + +# mandatory prefixes for test names +testname_prefix = ["test_1d_", "test_2d_", "test_3d_", "test_rz_"] + +# collect all test names and test input filenames from CMakeLists.txt files +tests = [] +# walk through all files under Examples/, including subdirectories +for dirpath, dirnames, filenames in os.walk(top="./Examples"): + # loop over CMakeLists.txt files + for name in [filename for filename in filenames if filename == "CMakeLists.txt"]: + filepath = os.path.join(dirpath, name) + # open CMakeLists.txt file + with open(filepath) as f: + # loop over lines of CMakeLists.txt + for line in f: + # strip leading whitespaces + line = line.lstrip() + # find lines where 'add_warpx_test' is called + if re.match("add_warpx_test", line): + # strip leading whitespaces, remove end-of-line comments + testname = next(f).lstrip().split(" ")[0] + # skip lines related to other function arguments + # NOTE: update range call to reflect changes + # in the interface of 'add_warpx_test' + for _ in range(2): # skip over: dims, nprocs + next(f) + # strip leading whitespaces, remove end-of-line comments + testinput = next(f).lstrip().split(" ")[0] + # some Python input scripts are quoted + # to account for command-line arguments: + # strip initial quotation mark from string + if testinput.startswith('"'): + testinput = re.sub('"', "", testinput) + # extract filename from path + testinput = os.path.split(testinput)[1] + tests.append( + {"name": testname, "input": testinput, "path": filepath} + ) + +# check consistency of test names and test input filenames +print("\nCheck that test names and input names are correct...") +wrong_testname = False +wrong_testinput = False +for test in tests: + testname = test["name"].rstrip() + testinput = test["input"].rstrip() + testpath = test["path"].rstrip() + if not testname.startswith(tuple(testname_prefix)): + print(f"Wrong test name: {testname}") + print(f"(from {testpath})") + wrong_testname = True + # PICMI tests + if "picmi" in testname: + if not testname.endswith("_picmi") and not testname.endswith("_picmi_restart"): + print(f"Wrong test name: {testname}") + print(f"(from {testpath})") + wrong_testname = True + # restart tests + if "restart" in testname: + if not testname.endswith("_restart"): + print(f"Wrong test name: {testname}") + print(f"(from {testpath})") + wrong_testname = True + # test input file names + if ( + not testinput == f"inputs_{testname}" + and not testinput == f"inputs_{testname}.py" + ): + # we may be running a base input file/script or a restart PICMI test + if not testinput.startswith("inputs_base") and not testinput.endswith( + "_picmi.py" + ): + print(f"Wrong input name: {testinput}") + print(f"(from {testpath})") + wrong_testinput = True + +if wrong_testname: + print(f"NOTE: Test names must start with one of {testname_prefix}.") + print(" Test names must end with '_restart' for restart tests.") + print(" Test names must end with '_picmi' for PICMI tests.") +if wrong_testinput: + print("NOTE: Test input names must start with 'inputs_' followed by the test name.") + print(" Test input names must end with '.py' for PICMI tests.") + +# check that all input files in Examples/ are tested +print("\nCheck that all test input files are tested...") +missing_input = False +# walk through all files under Examples/, including subdirectories +for dirpath, dirnames, filenames in os.walk(top="./Examples"): + # loop over files starting with "inputs_test_" + for name in [ + filename for filename in filenames if filename.startswith("inputs_test_") + ]: + if name not in [test["input"] for test in tests]: + print(f"Input not tested: {os.path.join(dirpath, name)}") + missing_input = True + +if missing_input: + print("NOTE: All test input files must be tested.\n") +else: + print() + +if wrong_testname or wrong_testinput or missing_input: + sys.exit("FAILED\n") diff --git a/.github/workflows/source/ci_matrix.py b/.github/workflows/source/ci_matrix.py deleted file mode 100644 index 79f4d878c5d..00000000000 --- a/.github/workflows/source/ci_matrix.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python - -# Concatenation of tests in each of the 6 elements in CI matrix -f = open('./ci_matrix_elements.txt') ; matrix_elements = f.readlines() ; f.close() -# All tests read by prepare_file_ci.py -f = open('./ci_all_tests.txt') ; all_tests = f.readlines() ; f.close() - -# Now let's make sure these two are equal - -# Remove these elements from both lists, as they are are not test names -elements_to_remove = ['[main]\n', '[AMReX]\n', '[source]\n', '[extra-PICSAR]\n'] -for element in elements_to_remove: - for x in range(matrix_elements.count(element)): - matrix_elements.remove(element) - for x in range(all_tests.count(element)): - all_tests.remove(element) - -# Sort lists, and make sure they are equal -matrix_elements.sort() -all_tests.sort() -print("Tests in matrix, but not in initial list (typically if a test is done twice):") -print(list(set(matrix_elements) - set(all_tests))) -print("Tests in initial list but not in the matrix:") -print(list(set(all_tests) - set(matrix_elements))) - -assert( matrix_elements == all_tests ) diff --git a/.github/workflows/source/inputsNotTested b/.github/workflows/source/inputsNotTested deleted file mode 100755 index 497d322a610..00000000000 --- a/.github/workflows/source/inputsNotTested +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Search input files in Examples/ and verify if all input files are tested - -set -eu -o pipefail - -ok=0 - -for file in $(find Examples -type f) -do - # Name of file without path - filename=$(basename $file) - # If file is an input file - if [[ ${filename:0:6 } =~ inputs ]] || - [[ ${filename:0:12} =~ PICMI_inputs ]] - then - cr=$'$' - file_cr="$file$cr" - # Search file name in test list - string_match=$(grep -m1 "$file_cr" Regression/WarpX-tests.ini || echo "") - # If match is empty, inputs examples is not tested - if [[ -z $string_match ]] - then - echo "$file is not tested!" - ok=1 - fi - fi -done - -if [ $ok -ne 0 ] -then - echo "" - echo "All files in Examples that start with one of" - echo " - inputs" - echo " - PICMI_inputs" - echo "must have an automated test." - echo "Please add a test in Regression/WarpX-tests.ini" - echo "for all files listed above." -fi - -exit $ok diff --git a/.github/workflows/source/makeMakefileForClangTidy.py b/.github/workflows/source/makeMakefileForClangTidy.py index 07809187dbd..13460b9e548 100755 --- a/.github/workflows/source/makeMakefileForClangTidy.py +++ b/.github/workflows/source/makeMakefileForClangTidy.py @@ -14,40 +14,49 @@ def makeMakefileForClangTidy(argv): parser = argparse.ArgumentParser() - parser.add_argument("--input", - help="Ccache log file", - default="ccache.log.txt") - parser.add_argument("--identifier", - help="Unique identifier for finding compilation line in the log file", - default="WarpX/Source") + parser.add_argument("--input", help="Ccache log file", default="ccache.log.txt") + parser.add_argument( + "--identifier", + help="Unique identifier for finding compilation line in the log file", + default="WarpX/Source", + ) # We assume WarpX/Source can be used as an identifier to distinguish # WarpX code from amrex, openMPD, and cmake's temporary files like # build/CMakeFiles/CMakeScratch/TryCompile-hw3x4m/test_mpi.cpp - parser.add_argument("--output", - help="Make file for clang-tidy", - default="clang-tidy-ccache-misses.mak") + parser.add_argument( + "--output", + help="Make file for clang-tidy", + default="clang-tidy-ccache-misses.mak", + ) args = parser.parse_args() fin = open(args.input, "r") fout = open(args.output, "w") fout.write("CLANG_TIDY ?= clang-tidy\n") - fout.write("override CLANG_TIDY_ARGS += --extra-arg=-Wno-unknown-warning-option --extra-arg-before=--driver-mode=g++\n") + fout.write( + "override CLANG_TIDY_ARGS += --extra-arg=-Wno-unknown-warning-option --extra-arg-before=--driver-mode=g++\n" + ) fout.write("\n") fout.write(".SECONDEXPANSION:\n") fout.write("clang-tidy: $$(all_targets)\n") fout.write("\t@echo SUCCESS\n\n") - exe_re = re.compile(r" Executing .*? (-.*{}.*) -c .* -o .* (\S*)".format(args.identifier)) + exe_re = re.compile( + r" Executing .*? (-.*{}.*) -c .* -o .* (\S*)".format(args.identifier) + ) count = 0 for line in fin.readlines(): ret_exe_re = exe_re.search(line) - if (ret_exe_re): + if ret_exe_re: fout.write("target_{}: {}\n".format(count, ret_exe_re.group(2))) - fout.write("\t$(CLANG_TIDY) $(CLANG_TIDY_ARGS) $< -- {}\n".format - (ret_exe_re.group(1))) + fout.write( + "\t$(CLANG_TIDY) $(CLANG_TIDY_ARGS) $< -- {}\n".format( + ret_exe_re.group(1) + ) + ) fout.write("\ttouch target_{}\n\n".format(count)) count = count + 1 @@ -61,5 +70,6 @@ def makeMakefileForClangTidy(argv): fout.close() fin.close() + if __name__ == "__main__": makeMakefileForClangTidy(sys.argv) diff --git a/.github/workflows/source/test_ci_matrix.sh b/.github/workflows/source/test_ci_matrix.sh deleted file mode 100755 index 62903b66e45..00000000000 --- a/.github/workflows/source/test_ci_matrix.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail - -cp .github/workflows/source/ci_matrix.py Regression/ -cd Regression/ - -# Put the name of all CI tests into a text file -python prepare_file_ci.py -grep "^\[" ci-tests.ini > ci_all_tests.txt - - -export WARPX_CI_PSATD=TRUE - -# Concatenate the names of all elements in CI matrix into another test file -WARPX_CI_REGULAR_CARTESIAN_1D=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini > ci_matrix_elements.txt -WARPX_CI_REGULAR_CARTESIAN_2D=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt -WARPX_CI_REGULAR_CARTESIAN_3D=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt -WARPX_CI_SINGLE_PRECISION=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt -WARPX_CI_RZ_OR_NOMPI=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt -WARPX_CI_QED=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt -WARPX_CI_EB=TRUE python prepare_file_ci.py -grep "^\[" ci-tests.ini >> ci_matrix_elements.txt - -# Check that the resulting lists are equal -{ - python ci_matrix.py && - rm ci_all_tests.txt ci_matrix_elements.txt ci_matrix.py && - echo "test passed" && - exit 0 -} || { - rm ci_all_tests.txt ci_matrix_elements.txt ci_matrix.py && - echo "tests failed" && - exit 1 -} diff --git a/.github/workflows/source/wrongFileNameInExamples b/.github/workflows/source/wrongFileNameInExamples deleted file mode 100755 index 0de69d69c9c..00000000000 --- a/.github/workflows/source/wrongFileNameInExamples +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# -# Search inside Examples/ and check that file names start with -# inputs -# PICMI_inputs -# analysis -# README - -set -eu -o pipefail - -ok=0 -files=() - -for pathtofile in $(find Examples -type f) -do - file=$(basename $pathtofile) - if [[ ${file:0:6 } != inputs ]] && - [[ ${file:0:12} != PICMI_inputs ]] && - [[ ${file:0:8 } != analysis ]] && - [[ ${file: -4} != yaml ]] && - [[ ${file:0:4 } != plot ]] && - [[ ${file:0:6 } != README ]] - then - files+=($file) - echo "$pathtofile does not have a proper name!" - ok=1 - fi -done - -if [ $ok -ne 0 ] -then - echo "" - echo "Files in Examples/ must start with one of:" - echo " - inputs : for WarpX input files" - echo " - PICMI_inputs : for PICMI-compliant input scripts" - echo " - analysis : for scripts testing the accuracy of a test" - echo " - *.yaml : for third-party input, e.g. Ascent in situ visualization" - echo " - README : for readme files" - echo "" - echo "Please rename the file(s) to comply, or move to another folder" -fi - -exit $ok diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9f4953b7a98..d657daf5793 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,6 +1,13 @@ name: 🐧 OpenMP -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-ubuntu @@ -39,8 +46,8 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_QED=OFF cmake --build build -j 4 - ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d - ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d + ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz ccache -s du -hs ~/.cache/ccache @@ -82,8 +89,8 @@ jobs: -DWarpX_QED_TOOLS=ON cmake --build build -j 4 - ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d - ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d + ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d ccache -s du -hs ~/.cache/ccache @@ -133,8 +140,8 @@ jobs: -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 4 - ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d - ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d + ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz ccache -s du -hs ~/.cache/ccache @@ -210,7 +217,6 @@ jobs: cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_APP=OFF \ -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON @@ -222,74 +228,4 @@ jobs: - name: run pywarpx run: | export OMP_NUM_THREADS=1 - mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - - build_UB_sanitizer: - name: Clang UB sanitizer - runs-on: ubuntu-22.04 - if: github.event.pull_request.draft == false - env: - CC: clang - CXX: clang++ - # On CI for this test, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - .github/workflows/dependencies/clang15.sh - - name: CCache Cache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: build WarpX - run: | - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=10 - export CCACHE_MAXSIZE=100M - ccache -z - - export CXX=$(which clang++-15) - export CC=$(which clang-15) - export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all" - - cmake -S . -B build \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE \ - -DWarpX_PARTICLE_PRECISION=SINGLE - cmake --build build -j 4 - - ccache -s - du -hs ~/.cache/ccache - - - name: run with UB sanitizer - run: | - export OMP_NUM_THREADS=2 - mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz - mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d - mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d - mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + mpirun -n 2 Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2ef74cdb7f9..7f964239a02 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,6 +1,13 @@ name: 🪟 Windows -on: [push, pull_request] +on: + push: + branches: + - "development" + pull_request: + paths-ignore: + - "Docs/**" + - "**.rst" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-windows @@ -10,7 +17,9 @@ jobs: build_win_msvc: name: MSVC C++17 w/o MPI runs-on: windows-latest - if: github.event.pull_request.draft == false + # disabled due to issues in #5230 + if: 0 + #if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -54,7 +63,7 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - python3 Examples\Tests\gaussian_beam\PICMI_inputs_gaussian_beam.py + python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py # JSON writes are currently very slow (50min) with MSVC # --diagformat=openpmd @@ -91,7 +100,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DWarpX_COMPUTE=OMP ^ - -DWarpX_EB=ON ^ + -DWarpX_EB=OFF ^ -DWarpX_PYTHON=ON ^ -DWarpX_MPI=OFF ^ -DWarpX_OPENPMD=ON @@ -118,5 +127,5 @@ jobs: call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64 set "PATH=C:/Program Files (x86)/WarpX/bin/;%PATH%" - python3 Examples\Tests\gaussian_beam\PICMI_inputs_gaussian_beam.py --diagformat=openpmd + python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py --diagformat=openpmd if errorlevel 1 exit 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cef24aed2cc..e113fa4c8e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -66,44 +66,16 @@ repos: # C++ formatting # clang-format -# Autoremoves unused Python imports -- repo: https://github.com/hadialqattan/pycln - rev: v2.4.0 +# Python: Ruff linter & formatter +# https://docs.astral.sh/ruff/ +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.6 hooks: - - id: pycln - name: pycln (python) - -# Sorts Python imports according to PEP8 -# https://www.python.org/dev/peps/pep-0008/#imports -- repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort (python) - args: ['--profile black'] - -# Python: Flake8 (checks only, does this support auto-fixes?) -#- repo: https://github.com/PyCQA/flake8 -# rev: 4.0.1 -# hooks: -# - id: flake8 -# additional_dependencies: &flake8_dependencies -# - flake8-bugbear -# - pep8-naming -# exclude: ^(docs/.*|tools/.*)$ -# Alternatively: use autopep8? - -# Python Formatting -#- repo: https://github.com/psf/black -# rev: 21.10b0 # Keep in sync with blacken-docs -# hooks: -# - id: black -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.11.0 -# hooks: -# - id: blacken-docs -# additional_dependencies: -# - black==21.10b0 # keep in sync with black hook + # Run the linter + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + # Run the formatter + - id: ruff-format # Jupyter Notebooks: clean up all cell outputs - repo: https://github.com/roy-ht/pre-commit-jupyter @@ -117,7 +89,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.49" + rev: "0.50" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed diff --git a/.readthedocs.yml b/.readthedocs.yml index 3da9bc77140..95f86fe4ff2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,14 +9,17 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "mambaforge-latest" + # python: "3.11" sphinx: - configuration: Docs/source/conf.py + configuration: Docs/source/conf.py -python: - install: - - requirements: Docs/requirements.txt +conda: + environment: Docs/conda.yml +# python: +# install: +# - requirements: Docs/requirements.txt formats: - htmlzip diff --git a/.zenodo.json b/.zenodo.json index a2c9b443f99..d753e33d017 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -39,7 +39,7 @@ "name": "Blelly, Aurore" }, { - "affiliation": "Lawrence Livermore National Laboratory", + "affiliation": "Helion Energy, Inc.", "name": "Clark, Stephen Eric", "orcid": "0000-0002-0117-776X" }, diff --git a/CMakeLists.txt b/CMakeLists.txt index 66332e9ff50..aec21a308bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # -cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.07) +cmake_minimum_required(VERSION 3.24.0) +project(WarpX VERSION 25.02) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -29,13 +29,6 @@ if(POLICY CMP0104) cmake_policy(SET CMP0104 OLD) endif() -# We use simple syntax in cmake_dependent_option, so we are compatible with the -# extended syntax in CMake 3.22+ -# https://cmake.org/cmake/help/v3.22/policy/CMP0127.html -if(POLICY CMP0127) - cmake_policy(SET CMP0127 NEW) -endif() - # C++ Standard in Superbuilds ################################################# # @@ -74,12 +67,12 @@ set_default_install_dirs() include(CMakeDependentOption) option(WarpX_APP "Build the WarpX executable application" ON) option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) -option(WarpX_EB "Embedded boundary support" OFF) +option(WarpX_CATALYST "Catalyst in situ diagnostics" OFF) +option(WarpX_EB "Embedded boundary support" ON) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_FFT "FFT-based solvers" OFF) -option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -88,6 +81,22 @@ option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" option(WarpX_QED_TOOLS "Build external tool to generate QED lookup tables (requires PICSAR and Boost)" OFF) +# Advanced option to run tests +option(WarpX_TEST_CLEANUP "Clean up automated test directories" OFF) +option(WarpX_TEST_DEBUGGER "Run automated tests without AMReX signal handling (to attach debuggers)" OFF) +option(WarpX_TEST_FPETRAP "Run automated tests with FPE-trapping runtime parameters" OFF) +mark_as_advanced(WarpX_TEST_CLEANUP) +mark_as_advanced(WarpX_TEST_DEBUGGER) +mark_as_advanced(WarpX_TEST_FPETRAP) + +# Advanced option to compile with the -g1 option for minimal debug symbols +# (useful to see, e.g., line numbers in backtraces) +option(WarpX_BACKTRACE_INFO "Compile with -g1 for minimal debug symbols (currently used in CI tests)" OFF) +mark_as_advanced(WarpX_BACKTRACE_INFO) +if(WarpX_BACKTRACE_INFO) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g1") +endif() + set(WarpX_DIMS_VALUES 1 2 3 RZ) set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality <1;2;3;RZ>") list(REMOVE_DUPLICATES WarpX_DIMS) @@ -137,10 +146,6 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) -if(WarpX_HEFFTE AND NOT WarpX_MPI) - message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") -endif() - # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") @@ -188,13 +193,10 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) if(WarpX_FFT) set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() -option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) -if(WarpX_HEFFTE) - set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) -endif() -# this defined the variable BUILD_TESTING which is ON by default -#include(CTest) +# Define the variable BUILD_TESTING (ON by default), +# include CDash dashboard testing module +include(CTest) # Dependencies ################################################################ @@ -233,26 +235,9 @@ if(WarpX_FFT) endif() endif() -# multi-node FFT -if(WarpX_HEFFTE) - if(WarpX_COMPUTE STREQUAL CUDA) - set(_heFFTe_COMPS CUDA) - elseif(WarpX_COMPUTE STREQUAL HIP) - set(_heFFTe_COMPS ROCM) - elseif(WarpX_COMPUTE STREQUAL SYCL) - set(_heFFTe_COMPS ONEAPI) - else() # NOACC, OMP - set(_heFFTe_COMPS FFTW) # or MKL - endif() - # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be - # disabled at runtime - - find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) -endif() - # Python if(WarpX_PYTHON) - find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) + find_package(Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED) # default installation directories: Python warpx_set_default_install_dirs_python() @@ -493,10 +478,6 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(ABLASTR_HEFFTE) - target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) - endif() - if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) @@ -504,7 +485,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_enable_IPO(pyWarpX_${SD}) else() # conditionally defined target in pybind11 - # https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403 + # https://github.com/pybind/pybind11/blob/v2.13.0/tools/pybind11Common.cmake#L407-L413 target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) endif() endif() @@ -587,13 +568,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() - if(WarpX_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) - endif() - if(ABLASTR_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) - endif() - if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE @@ -714,8 +688,10 @@ endforeach() # pip helpers for the pywarpx package ######################################### # if(WarpX_PYTHON) - set(PYINSTALLOPTIONS "" CACHE STRING - "Additional parameters to pass to `pip install`") + set(PY_PIP_OPTIONS "-v" CACHE STRING + "Additional parameters to pass to `pip` as ; separated list") + set(PY_PIP_INSTALL_OPTIONS "" CACHE STRING + "Additional parameters to pass to `pip install` as ; separated list") # ensure all targets are built before we package them in a wheel set(pyWarpX_INSTALL_TARGET_NAMES) @@ -738,7 +714,8 @@ if(WarpX_PYTHON) ${CMAKE_COMMAND} -E rm -f -r warpx-whl COMMAND ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$ - ${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl "${WarpX_SOURCE_DIR}" + COMMAND_EXPAND_LISTS VERBATIM WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -752,7 +729,8 @@ if(WarpX_PYTHON) set(pyWarpX_REQUIREMENT_FILE "requirements.txt") endif() add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements - ${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install ${PY_PIP_INSTALL_OPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" + COMMAND_EXPAND_LISTS VERBATIM WORKING_DIRECTORY ${WarpX_BINARY_DIR} ) @@ -769,7 +747,8 @@ if(WarpX_PYTHON) # because otherwise pip would also force reinstall all dependencies. add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx + COMMAND_EXPAND_LISTS VERBATIM WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -782,7 +761,8 @@ if(WarpX_PYTHON) # this is for package managers only add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx + COMMAND_EXPAND_LISTS VERBATIM WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -794,12 +774,12 @@ endif() # Tests ####################################################################### # - -#if(BUILD_TESTING) -# enable_testing() -# -# add_test(...) -#endif() +if(BUILD_TESTING) + enable_testing() + if(WarpX_APP OR WarpX_PYTHON) + add_subdirectory(Examples) + endif() +endif() # Status Summary for Build Options ############################################ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 77b8200b0d5..3affc9f4aaa 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -12,6 +12,36 @@ Git workflow The WarpX project uses `git `_ for version control. If you are new to git, you can follow `this tutorial `__. + +What to do when +^^^^^^^^^^^^^^^ + +Issues +"""""" + +`Issues `__ are used to track tasks that the contributors and/or maintainers can work on. +Use issues for reporting bugs or installation problems and for requesting new features. + +If you've found a bug and wish to report it, first search the open issues and `pull requests `__ to see if someone else has already reported the same thing. +If it's something new, open an issue using a template. +We'll use the issue to address the problem you've encountered. + +Discussions +""""""""""" + +`Discussions `__ are for open-ended conversations, general questions, brainstorming ideas. +Please, use discussions if you want to ask us something that is not technically a bug or a feature. +Feel free to ping us there! + +Pull Requests (PRs) +""""""""""""""""""" + +Open a `pull request `__ if you want to add a new feature yourself. +Follow the guide below for more details. + + +Thank you for contributing! 🥰 + Configure your GitHub Account & Development Machine ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 00000000000..8af4cb36ac1 --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,20 @@ +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## +## # The following are required to submit to the CDash dashboard: +## ENABLE_TESTING() +## INCLUDE(CTest) + +set(CTEST_PROJECT_NAME WarpX) +set(CTEST_NIGHTLY_START_TIME 08:00:00 UTC) + +set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=WarpX) + +set(CTEST_DROP_SITE_CDASH TRUE) + +# Set site and build names +# - CTest script variables: CTEST_SITE, CTEST_BUILD_NAME +# - CTest module variables: SITE, BUILDNAME +set(SITE "Azure-Pipelines") +set(BUILDNAME "CI-Development") diff --git a/Docs/Doxyfile b/Docs/Doxyfile index 5fbb7651b18..f7740bc0328 100644 --- a/Docs/Doxyfile +++ b/Docs/Doxyfile @@ -2245,7 +2245,7 @@ ENABLE_PREPROCESSING = YES # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -MACRO_EXPANSION = NO +MACRO_EXPANSION = YES # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then # the macro expansion is limited to the macros specified with the PREDEFINED and @@ -2253,7 +2253,7 @@ MACRO_EXPANSION = NO # The default value is: NO. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -EXPAND_ONLY_PREDEF = NO +EXPAND_ONLY_PREDEF = YES # If the SEARCH_INCLUDES tag is set to YES, the include files in the # INCLUDE_PATH will be searched if a #include is found. @@ -2305,6 +2305,8 @@ PREDEFINED = AMREX_Linux=1 \ WARPX_QED=1 \ WARPX_QED_TABLE_GEN=1 +PREDEFINED += "AMREX_ENUM(CLASS,...)=\"enum class CLASS : int { __VA_ARGS__ };\"" + # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The # macro definition that is found in the sources will be used. Use the PREDEFINED @@ -2312,7 +2314,7 @@ PREDEFINED = AMREX_Linux=1 \ # definition found in the source code. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -EXPAND_AS_DEFINED = +EXPAND_AS_DEFINED = AMREX_ENUM # If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will # remove all references to function-like macros that are alone on a line, have diff --git a/Docs/README.md b/Docs/README.md index e6fac921b04..6d3903ab327 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -9,12 +9,13 @@ More information can be found in Docs/source/developers/documentation.rst. Install the Python requirements for compiling the documentation: ``` -python3 -m pip install -r Docs/requirements.txt +cd Docs/ +python3 -m pip install -r requirements.txt ``` ### Compiling the documentation -`cd` into the `Docs/` directory and type +Still in the `Docs/` directory, type ``` make html ``` diff --git a/Docs/conda.yml b/Docs/conda.yml new file mode 100644 index 00000000000..1e23c203b2b --- /dev/null +++ b/Docs/conda.yml @@ -0,0 +1,12 @@ +name: readthedocs + +channels: + - conda-forge + - nodefaults + +dependencies: + - python + - doxygen + - pip + - pip: + - -r requirements.txt diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 1ce1ee6c1a0..14d07e29f6e 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -5,7 +5,7 @@ # License: BSD-3-Clause-LBNL # WarpX PICMI bindings w/o C++ component (used for autoclass docs) --e Python +-e ../Python breathe docutils>=0.17.1 @@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.29.0 +picmistandard==0.33.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python @@ -27,5 +27,6 @@ sphinx-copybutton sphinx-design sphinx_rtd_theme>=1.1.1 sphinxcontrib-bibtex +sphinxcontrib-googleanalytics sphinxcontrib-napoleon yt # for checksumAPI diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index b01ebf00a39..8c9b8dcf15c 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -53,6 +53,11 @@ Prior WarpX references If your project uses a specific algorithm or component, please consider citing the respective publications in addition. +- Shapoval O, Zoni E, Lehe R, Thévenet M, and Vay J-L. + **Pseudospectral particle-in-cell formulation with arbitrary charge and current-density time dependencies for the modeling of relativistic plasmas**. + Physical Review E **110**, 025206, 2024. + `DOI:10.1103/PhysRevE.110.025206 `__ + - Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 7997b1e338c..a5fed3a4614 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -31,10 +31,12 @@ import urllib.request import pybtex.plugin -import sphinx_rtd_theme +import sphinx_rtd_theme # noqa from pybtex.style.formatting.unsrt import Style as UnsrtStyle -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) +module_path = os.path.dirname(os.path.abspath(__file__)) +checksum_path = os.path.join(module_path, "../../Regression/Checksum") +sys.path.insert(0, checksum_path) # -- General configuration ------------------------------------------------ @@ -46,21 +48,27 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.mathjax', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_copybutton', - 'sphinx_design', - 'breathe', - 'sphinxcontrib.bibtex' - ] + "sphinx.ext.autodoc", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_copybutton", + "sphinx_design", + "breathe", + "sphinxcontrib.bibtex", + "sphinxcontrib.googleanalytics", +] + +# Google Analytics +googleanalytics_id = "G-QZGY5060MZ" +googleanalytics_enabled = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # Relative path to bibliography file, bibliography style -bibtex_bibfiles = ['latex_theory/allbibs.bib', 'refs.bib'] +bibtex_bibfiles = ["latex_theory/allbibs.bib", "refs.bib"] + # An brief introduction to custom BibTex formatting can be found in the Sphinx documentation: # https://sphinxcontrib-bibtex.readthedocs.io/en/latest/usage.html#bibtex-custom-formatting @@ -77,42 +85,43 @@ def __init__(self, *args, **kwargs): # This option makes the given names of an author abbreviated to just initials. # Example: "Jean-Luc" becomes "J.-L." # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer - kwargs['abbreviate_names'] = True + kwargs["abbreviate_names"] = True super().__init__(*args, **kwargs) -pybtex.plugin.register_plugin('pybtex.style.formatting', 'warpxbibstyle', WarpXBibStyle) -bibtex_default_style = 'warpxbibstyle' +pybtex.plugin.register_plugin("pybtex.style.formatting", "warpxbibstyle", WarpXBibStyle) + +bibtex_default_style = "warpxbibstyle" # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'WarpX' -copyright = '2017-2021, WarpX collaboration' -author = 'WarpX collaboration' +project = "WarpX" +copyright = "2017-2021, WarpX collaboration" +author = "WarpX collaboration" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'24.07' +version = "25.02" # The full version, including alpha/beta/rc tags. -release = u'24.07' +release = "25.02" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -120,7 +129,7 @@ def __init__(self, *args, **kwargs): exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -131,14 +140,16 @@ def __init__(self, *args, **kwargs): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" numfig = True math_eqref_format = "{number}" -numfig_format = {'figure': 'Fig. %s', - 'table': 'Table %s', - 'code-block': 'Listing %s', - 'section': 'Section %s'} +numfig_format = { + "figure": "Fig. %s", + "table": "Table %s", + "code-block": "Listing %s", + "section": "Section %s", +} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -149,16 +160,16 @@ def __init__(self, *args, **kwargs): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = [ - 'custom.css', + "custom.css", ] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'WarpXdoc' +htmlhelp_basename = "WarpXdoc" # -- Options for LaTeX output --------------------------------------------- @@ -167,15 +178,12 @@ def __init__(self, *args, **kwargs): # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -185,8 +193,7 @@ def __init__(self, *args, **kwargs): # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'WarpX.tex', 'WarpX Documentation', - 'WarpX collaboration', 'manual'), + (master_doc, "WarpX.tex", "WarpX Documentation", "WarpX collaboration", "manual"), ] @@ -194,10 +201,7 @@ def __init__(self, *args, **kwargs): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'warpx', 'WarpX Documentation', - [author], 1) -] +man_pages = [(master_doc, "warpx", "WarpX Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -206,40 +210,44 @@ def __init__(self, *args, **kwargs): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'WarpX', 'WarpX Documentation', - author, 'WarpX', 'WarpX is an advanced electromagnetic Particle-In-Cell code.', - 'Miscellaneous'), + ( + master_doc, + "WarpX", + "WarpX Documentation", + author, + "WarpX", + "WarpX is an advanced electromagnetic Particle-In-Cell code.", + "Miscellaneous", + ), ] - - # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://amrex-codes.github.io/': None} +intersphinx_mapping = {"https://amrex-codes.github.io/": None} # Setup the breathe extension -breathe_projects = { - "WarpX": "../doxyxml/" -} +breathe_projects = {"WarpX": "../doxyxml/"} breathe_default_project = "WarpX" # Tell sphinx what the primary language being documented is. -primary_domain = 'cpp' +primary_domain = "cpp" # Tell sphinx what the pygments highlight language should be. -highlight_language = 'cpp' +highlight_language = "cpp" # Download AMReX & openPMD-api Doxygen Tagfile to interlink Doxygen docs -url = 'https://amrex-codes.github.io/amrex/docs_xml/doxygen/amrex-doxygen-web.tag.xml' -urllib.request.urlretrieve(url, '../amrex-doxygen-web.tag.xml') +url = "https://amrex-codes.github.io/amrex/docs_xml/doxygen/amrex-doxygen-web.tag.xml" +urllib.request.urlretrieve(url, "../amrex-doxygen-web.tag.xml") -url = 'https://openpmd-api.readthedocs.io/en/latest/_static/doxyhtml/openpmd-api-doxygen-web.tag.xml' -urllib.request.urlretrieve(url, '../openpmd-api-doxygen-web.tag.xml') +url = "https://openpmd-api.readthedocs.io/en/latest/_static/doxyhtml/openpmd-api-doxygen-web.tag.xml" +urllib.request.urlretrieve(url, "../openpmd-api-doxygen-web.tag.xml") # Build Doxygen -subprocess.call('cd ../; doxygen;' - 'mkdir -p source/_static;' - 'cp -r doxyhtml source/_static/;' - 'cp warpx-doxygen-web.tag.xml source/_static/doxyhtml/', - shell=True) +subprocess.call( + "cd ../; doxygen;" + "mkdir -p source/_static;" + "cp -r doxyhtml source/_static/;" + "cp warpx-doxygen-web.tag.xml source/_static/doxyhtml/", + shell=True, +) suppress_warnings = ["bibtex.duplicate_label"] diff --git a/Docs/source/dataanalysis/catalyst.rst b/Docs/source/dataanalysis/catalyst.rst new file mode 100644 index 00000000000..939b6b134bd --- /dev/null +++ b/Docs/source/dataanalysis/catalyst.rst @@ -0,0 +1,135 @@ +.. _visualization-catalyst: + +In situ Visualization with Catalyst 2 +===================================== +Catalyst 2 (further referred to as just Catalyst) is a lightweight in-situ visualization and analysis framework API developed for simulations and other scientific data producers. It has a lightweight implementation +(or **stub**) and an SDK to develop custom implementations of Catalyst. ParaView comes with its own implementation (known as **ParaView Catalyst**) for leveraging ParaView's +visualization and analysis capabilities, which is what this document will focus on. + + +Enabling Catalyst +----------------- +In order to use Catalyst with WarpX, we need to ensure that we will be using the same version of +conduit across all libraries i.e Catalyst, AMReX and ParaView. One way to achieve this is to +build conduit externally and use it for compiling all the above packages. +This ensures compatibility when passing conduit nodes between WarpX and ParaView. + +First, we build +`Conduit `_ and then +build `Catalyst 2 `_ +using the conduit library created in the previous step. +The latter can be achieved by adding the installation path of conduit to the environmental +variable ``CMAKE_PREFIX_PATH`` and setting ``CATALYST_WITH_EXTERNAL_CONDUIT=ON`` during the configuration step of Catalyst. + +Then we build ParaView master (on a commit after 2024.07.01, tested on ``4ef351a54ff747ef7169e2e52e77d9703a9dfa77``) following the developer instructions provided +`here `__ . +A representative set of options for a headless ParaView installation is provided +`here `__ +Afterward, WarpX must be built with ``WarpX_CATALYST=ON``. +Also, make sure to provide the installed paths of Conduit and Catalyst via +``CMAKE_PREFIX_PATH`` before configuring WarpX. + +Inputs File Configuration +------------------------- +Once WarpX has been compiled with Catalyst support, it will need to be enabled and configured at runtime. +This is done using our usual inputs file (read with ``amrex::ParmParse``). +The supported parameters are part of the :ref:`FullDiagnostics ` with ``.format`` parameter set to ``catalyst``. + +In addition to configuring the diagnostics, the following parameters must be included: + * ``catalyst.script_paths``: The locations of the pipeline scripts, separated by either a colon or semicolon (e.g. ``/path/to/script1.py;/path/to/script2.py``). + * ``catalyst.implementation`` (default ``paraview``): The name of the implementation being used (case sensitive). + * ``catalyst.implementation_search_paths``: The locations to search for the given implementation. The specific file being searched for will be ``catalyst_{implementation}.so``. + +The latter two can also be given via the environmental variables +``CATALYST_IMPLEMENTATION_NAME`` and ``CATALYST_IMPLEMENTATION_PATHS`` +respectively. + +Because the scripts and implementations are global, Catalyst does not benefit from nor differentiate between multiple diagnostics. + + +Visualization/Analysis Pipeline Configuration +--------------------------------------------- +Catalyst uses the files specified in ``catalyst.script_paths`` to run all analysis. + +The following script, :code:`simple_catalyst_pipeline.py`, automatically detects the type of data for both the mesh and particles, then creates an extractor for them. In most +cases, these will be saved as ``.VTPC`` files which can be read with the ``XML Partitioned Dataset Collection Reader``. + +.. literalinclude:: catalyst/catalyst_simple_pipeline.py + :language: python + :caption: You can copy this file from ``Docs/source/dataanalysis/catalyst/catalyst_simple_pipeline.py``. + + + +For the case of ParaView Catalyst, pipelines are run with ParaView's included ``pvbatch`` executable and use the ``paraview`` library to modify the data. While pipeline scripts +could be written manually, this is not advised for anything beyond the script above. It is much more practical to use ParaView's built in ``Save Catalyst State`` button. + +The process for creating a pipeline is as follows: + 1. Run at least one step of simulation and save the data in a ParaView compatible format, then open it in ParaView. + 2. Set up the desired scene, including filters, camera and views, and extractors. + 3. Press ``Save Catalyst State``, or the multicolored flask icon in the top left corner, and save it to a desired location. + 4. Open the script and replace the used producer with ``PVTrivialProducer``, setting the ``registrationName`` to either ``mesh`` or ``particles`` based on what data is used. + +As an example for step four, here are a few lines from a script directly exported from ParaView: + +.. code-block:: python + + # create a new 'XML Image Data Reader' + meshdatavti = XMLImageDataReader(registrationName='meshdata.vti', FileName=['/path/to/meshdata.vti']) + meshdatavti.CellArrayStatus = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez'] + meshdatavti.TimeArray = 'None' + + # Calculator sample filter + calculator1 = Calculator(registrationName='Calculator1', Input=meshdatavti) + calculator1.AttributeType = 'Cell Data' + calculator1.ResultArrayName = 'BField' + calculator1.Function = 'sqrt(Bx^2 + By^2 + Bz^2)' + +In order to use it with the mesh data coming from the simulation, the above code would be changed to: + +.. code-block:: python + + # create the producer + meshdata = PVTrivialProducer(registrationName='mesh') + meshdata.CellArrayStatus = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez'] + meshdata.TimeArray = 'None' + + # Calculator sample filter + calculator1 = Calculator(registrationName='Calculator1', Input=meshdata) + calculator1.AttributeType = 'Cell Data' + calculator1.ResultArrayName = 'BField' + calculator1.Function = 'sqrt(Bx^2 + By^2 + Bz^2)' + +Steps one is advised so that proper scaling and framing can be done, however in certain cases it may not be possible. If this is the case, a dummy object can be used instead +(such as a wavelet or geometric shape scaled appropriately) and the rest of the steps can be followed as usual. + +Replay +------ +Catalyst 2.0 supports generating binary data dumps for the conduit nodes passed to each ``catalyst_`` call at each iteration. This allows to debug/adapt catalyst scripts without having to rerun the simulation each time. + +To generate the data dumps one must first set the environmental variable ``CATALYST_DATA_DUMP_DIRECTORY`` to the path where the dumps should be saved. Then, run the simulation as normal but replace ``catalyst.implementation=stub`` either in the calling script of WarpX or as an additional argument. + +This will run the simulation and write the conduit nodes under ``CATALYST_DATA_DUMP_DIRECTORY``. + +Afterward, one can replay the generated nodes by setting up the ``CATALYST_IMPLEMENTATION_*`` variables for the ``catalyst_replay`` executable (which can be found in the catalyst build directory) appropriately. For example: + +.. code-block:: bash + + # dump conduit nodes + export CATALYST_DATA_DUMP_DIRECTORY=./raw_data + mpiexec -n N /bin/warpx.2d ./inputs_2d catalyst.script_paths=catalyst_pipeline.py catalyst.implementation="stub" + # validate that files have been written + ls ./raw_data/ + ... many files of the format XXXX.conduit_bin.Y.Z + + # replay them + export CATALYST_IMPLEMENTATION_NAME=paraview + export CATALYST_IMPLEMENTATION_PATHS=/lib/catalyst + export CATALYST_IMPLEMENTATION_PREFER_ENV=YES + export CATALYST_DEBUG=1 # optional but helps to make sure the right paths are used + export PYTHONPATH=${PYTHONPATH}/$(pwd) # or the path containing catalyst_pipeline.py in general + # N needs to be the same as when we generated the dump + mpiexec -n N /bin/catalyst_replay ./raw_data + # check extractor output e.g + ls ./datasets/ + +For more information see the documentation for catalyst replay `here `__ . diff --git a/Docs/source/dataanalysis/catalyst/catalyst_simple_pipeline.py b/Docs/source/dataanalysis/catalyst/catalyst_simple_pipeline.py new file mode 100644 index 00000000000..c74b5d3206d --- /dev/null +++ b/Docs/source/dataanalysis/catalyst/catalyst_simple_pipeline.py @@ -0,0 +1,101 @@ +from paraview import catalyst +from paraview.simple import * # noqa: F403 + + +# Helper function +def create_data_extractor(data_node, filename="Dataset"): + """Creates a data extractor that saves `data_node` to a datafile named `filename`. + The filetype is chosen based on the type of `data_note`. + + Note: no rendering is performed by such an extractor. The data are + written directly to a file via VTK. + """ + VTK_TYPES = [ + "vtkImageData", + "vtkRectilinearGrid", + "vtkStructuredGrid", + "vtkPolyData", + "vtkUnstructuredGrid", + "vtkUniformGridAMR", + "vtkMultiBlockDataSet", + "vtkPartitionedDataSet", + "vtkPartitionedDataSetCollection", + "vtkHyperTreeGrid", + ] + FILE_ASSOCIATIONS = [ + "VTI", + "VTR", + "VTS", + "VTP", + "VTU", + "VTH", + "VTM", + "VTPD", + "VTPC", + "HTG", + ] + clientside_data = data_node.GetClientSideObject().GetOutputDataObject( + 0 + ) # Gets the dataobject from the default output port + + # Loop is required because .IsA() detects valid classes that inherit from the VTK_TYPES + for i, vtk_type in enumerate(VTK_TYPES): + if clientside_data.IsA(vtk_type): + filetype = FILE_ASSOCIATIONS[i] + extractor = CreateExtractor( + filetype, data_node, registrationName=f"_{filetype}" + ) + extractor.Writer.FileName = filename + "_{timestep:}" + f".{filetype}" + return extractor + + raise RuntimeError(f"Unsupported data type: {clientside_data.GetClassName()}") + + +# Camera settings +paraview.simple._DisableFirstRenderCameraReset() # Prevents the camera from being shown + +# Options +options = catalyst.Options() + +options.CatalystLiveTrigger = "TimeStep" # "Python", "TimeStep", "TimeValue" +options.EnableCatalystLive = 0 # 0 (disabled), 1 (enabled) +if options.EnableCatalystLive == 1: + options.CatalystLiveURL = "localhost:22222" # localhost:22222 is default + +options.ExtractsOutputDirectory = "datasets" # Base for where all files are saved +options.GenerateCinemaSpecification = 0 # 0 (disabled), 1 (enabled), generates additional descriptor files for cinema exports +options.GlobalTrigger = "TimeStep" # "Python", "TimeStep", "TimeValue" + +meshSource = PVTrivialProducer( + registrationName="mesh" +) # "mesh" is the node where the mesh data is stored +create_extractor(meshSource, filename="meshdata") +particleSource = PVTrivialProducer( + registrationName="particles" +) # "particles" is the node where particle data is stored +create_extractor(particleSource, filename="particledata") + + +# Called on catalyst initialize (after Cxx side initialize) +def catalyst_initialize(): + return + + +# Called on catalyst execute (after Cxx side update) +def catalyst_execute(info): + print(f"Time: {info.time}, Timestep: {info.timestep}, Cycle: {info.cycle}") + return + + +# Callback if global trigger is set to "Python" +def is_activated(controller): + return True + + +# Called on catalyst finalize (after Cxx side finalize) +def catalyst_finalize(): + return + + +if __name__ == "__main__": + paraview.simple.SaveExtractsUsingCatalystOptions(options) diff --git a/Docs/source/dataanalysis/formats.rst b/Docs/source/dataanalysis/formats.rst index a7836d41fef..12b85cadfbb 100644 --- a/Docs/source/dataanalysis/formats.rst +++ b/Docs/source/dataanalysis/formats.rst @@ -41,8 +41,9 @@ files to disk). .. toctree:: :maxdepth: 1 - sensei ascent + catalyst + sensei If you like the 3D rendering of laser wakefield acceleration on the WarpX documentation front page (which is diff --git a/Docs/source/dataanalysis/paraview.rst b/Docs/source/dataanalysis/paraview.rst index 4c90f7179b3..939a23008a8 100644 --- a/Docs/source/dataanalysis/paraview.rst +++ b/Docs/source/dataanalysis/paraview.rst @@ -54,3 +54,9 @@ Plotfiles (AMReX) ParaView also supports visualizing AMReX plotfiles. Please see `the AMReX documentation `__ for more details. + + +In Situ Analysis with Catalyst 2 +-------------------------------- + +Continue reading :ref:`here `. diff --git a/Docs/source/dataanalysis/workflows.rst b/Docs/source/dataanalysis/workflows.rst index dc611b00e53..ab4ca88ddab 100644 --- a/Docs/source/dataanalysis/workflows.rst +++ b/Docs/source/dataanalysis/workflows.rst @@ -1,7 +1,8 @@ +.. _dataanalysis-how-to: .. _dataanalysis-workflows: -Workflows -========= +How-To Guides +============= This section collects typical user workflows and best practices for data analysis with WarpX. diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 81d16809d50..1e71ee3ddae 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -1,32 +1,31 @@ .. _developers-checksum: -Checksum regression tests -========================= +Checksums on Tests +================== -WarpX has checksum regression tests: as part of CI testing, when running a given test, the checksum module computes one aggregated number per field (``Ex_checksum = np.sum(np.abs(Ex))``) and compares it to a reference (benchmark). This should be sensitive enough to make the test fail if your PR causes a significant difference, print meaningful error messages, and give you a chance to fix a bug or reset the benchmark if needed. +When running an automated test, we often compare the data of final time step of the test with expected values to catch accidental changes. +Instead of relying on reference files that we would have to store in their full size, we calculate an aggregate checksum. -The checksum module is located in ``Regression/Checksum/``, and the benchmarks are stored as human-readable `JSON `__ files in ``Regression/Checksum/benchmarks_json/``, with one file per benchmark (for instance, test ``Langmuir_2d`` has a corresponding benchmark ``Regression/Checksum/benchmarks_json/Langmuir_2d.json``). +For this purpose, the checksum Python module computes one aggregated number per field (e.g., the sum of the absolute values of the array elements) and compares it to a reference value (benchmark). +This should be sensitive enough to make the test fail if your PR causes a significant difference, print meaningful error messages, and give you a chance to fix a bug or reset the benchmark if needed. -For more details on the implementation, the Python files in ``Regression/Checksum/`` should be well documented. +The checksum module is located in ``Regression/Checksum/``, and the benchmarks are stored as human-readable `JSON `__ files in ``Regression/Checksum/benchmarks_json/``, with one file per benchmark (for example, the test ``test_2d_langmuir_multi`` has a corresponding benchmark ``Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json``). -From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipes for the main tasks related to checksum regression tests in WarpX CI. +For more details on the implementation, please refer to the Python implementation in ``Regression/Checksum/``. -Include a checksum regression test in an analysis Python script ---------------------------------------------------------------- +From a user point of view, you should only need to use ``checksumAPI.py``, which contains Python functions that can be imported and used from an analysis Python script or can also be executed directly as a Python script. + +How to compare checksums in your analysis script +------------------------------------------------ This relies on the function ``evaluate_checksum``: .. autofunction:: checksumAPI.evaluate_checksum -For an example, see - -.. literalinclude:: ../../../Examples/analysis_default_regression.py - :language: python - -This can also be included in an existing analysis script. Note that the plotfile must be ``_plt?????``, as is generated by the CI framework. +This can also be included as part of an existing analysis script. -Evaluate a checksum regression test from a bash terminal --------------------------------------------------------- +How to evaluate checksums from the command line +----------------------------------------------- You can execute ``checksumAPI.py`` as a Python script for that, and pass the plotfile that you want to evaluate, as well as the test name (so the script knows which benchmark to compare it to). @@ -41,11 +40,8 @@ See additional options * ``--rtol`` relative tolerance for the comparison * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``) -Create/Reset a benchmark with new values that you know are correct ------------------------------------------------------------------- - -Create/Reset a benchmark from a plotfile generated locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How to create or reset checksums with local benchmark values +------------------------------------------------------------ This is using ``checksumAPI.py`` as a Python script. @@ -65,8 +61,23 @@ Since this will automatically change the JSON file stored on the repo, make a se git add .json git commit -m "reset benchmark for because ..." --author="Tools " -Reset a benchmark from the Azure pipeline output on Github -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How to reset checksums for a list of tests with local benchmark values +---------------------------------------------------------------------- + +If you set the environment variable ``export CHECKSUM_RESET=ON`` before running tests that are compared against existing benchmarks, the test analysis will reset the benchmarks to the new values, skipping the comparison. + +With `CTest `__ (coming soon), select the test(s) to reset by `name `__ or `label `__. + +.. code-block:: bash + + # regex filter: matched names + CHECKSUM_RESET=ON ctest --test-dir build -R "Langmuir_multi|LaserAcceleration" + + # ... check and commit changes ... + + +How to reset checksums for a list of tests with benchmark values from the Azure pipeline output +----------------------------------------------------------------------------------------------- Alternatively, the benchmarks can be reset using the output of the Azure continuous intergration (CI) tests on Github. The output can be accessed by following the steps below: diff --git a/Docs/source/developers/developers.rst b/Docs/source/developers/developers.rst index aa2e6196377..222f665b563 100644 --- a/Docs/source/developers/developers.rst +++ b/Docs/source/developers/developers.rst @@ -15,7 +15,6 @@ Implementation Details initialization diagnostics moving_window - qed portability warning_logger python diff --git a/Docs/source/developers/documentation.rst b/Docs/source/developers/documentation.rst deleted file mode 100644 index a5013299336..00000000000 --- a/Docs/source/developers/documentation.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _developers-docs: - -Documentation -============= - -Doxygen documentation ---------------------- - -WarpX uses a `Doxygen documentation `__. -Whenever you create a new class, please document it where it is declared (typically in the header file): - -.. code-block:: cpp - - /** \brief A brief title - * - * few-line description explaining the purpose of MyClass. - * - * If you are kind enough, also quickly explain how things in MyClass work. - * (typically a few more lines) - */ - class MyClass - { ... } - -Doxygen reads this docstring, so please be accurate with the syntax! See `Doxygen manual `__ for more information. Similarly, please document functions when you declare them (typically in a header file) like: - -.. code-block:: cpp - - /** \brief A brief title - * - * few-line description explaining the purpose of my_function. - * - * \param[in,out] my_int a pointer to an integer variable on which - * my_function will operate. - * \return what is the meaning and value range of the returned value - */ - int MyClass::my_function (int* my_int); - -An online version of this documentation is :ref:`linked here `. - -Breathe documentation ---------------------- - -Your Doxygen documentation is not only useful for people looking into the code, it is also part of the `WarpX online documentation `_ based on `Sphinx `_! -This is done using the Python module `Breathe `_, that allows you to write Doxygen documentation directly in the source and have it included it in your Sphinx documentation, by calling Breathe functions. -For instance, the following line will get the Doxygen documentation for ``WarpXParticleContainer`` in ``Source/Particles/WarpXParticleContainer.H`` and include it to the html page generated by Sphinx: - -.. code-block:: rst - - .. doxygenclass:: WarpXParticleContainer - -Building the documentation --------------------------- - -To build the documentation on your local computer, you will need to install Doxygen as well as the Python module `breathe`. -First, make sure you are in the root directory of WarpX's source and install the Python requirements: - -.. code-block:: sh - - python3 -m pip install -r Docs/requirements.txt - -You will also need Doxygen (macOS: ``brew install doxygen``; Ubuntu: ``sudo apt install doxygen``). - -Then, to compile the documentation, use - -.. code-block:: sh - - cd Docs/ - - make html - # This will first compile the Doxygen documentation (execute doxygen) - # and then build html pages from rst files using sphinx and breathe. - -Open the created ``build/html/index.html`` file with your favorite browser. -Rebuild and refresh as needed. diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index d0af160afef..ee782570bad 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -37,6 +37,13 @@ The ``MultiFab`` constructor (for, e.g., ``Ex`` on level ``lev``) is called in ` By default, the ``MultiFab`` are set to ``0`` at initialization. They can be assigned a different value in ``WarpX::InitLevelData``. +Field Names +----------- + +The commonly used WarpX field names are defined in: + +.. doxygenenum:: warpx::fields::FieldType + Field solver ------------ @@ -112,9 +119,9 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) +.. doxygenfunction:: WarpX::ApplyFilterMF(const ablastr::fields::MultiLevelVectorField &mfvec, int lev, int idim) -.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) +.. doxygenfunction:: WarpX::SumBoundaryJ(const ablastr::fields::MultiLevelVectorField ¤t, int lev, int idim, const amrex::Periodicity &period) Godfrey's anti-NCI filter for FDTD simulations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Docs/source/developers/gnumake/openpmd.rst b/Docs/source/developers/gnumake/openpmd.rst index 3215c9461a1..0497ecf044b 100644 --- a/Docs/source/developers/gnumake/openpmd.rst +++ b/Docs/source/developers/gnumake/openpmd.rst @@ -9,7 +9,7 @@ therefore we recommend to use `spack `__ in order to facilitate the installation. More specifically, we recommend that you try installing the -`openPMD-api library 0.15.1 or newer `__ +`openPMD-api library 0.16.1 or newer `__ using spack (first section below). If this fails, a back-up solution is to install parallel HDF5 with spack, and then install the openPMD-api library from source. diff --git a/Docs/source/developers/gnumake/python.rst b/Docs/source/developers/gnumake/python.rst index 543b80d5ddd..06dbd5ac737 100644 --- a/Docs/source/developers/gnumake/python.rst +++ b/Docs/source/developers/gnumake/python.rst @@ -3,7 +3,7 @@ Installing WarpX as a Python package ==================================== -A full Python installation of WarpX can be done, which includes a build of all of the C++ code, or a pure Python version can be made which only installs the Python scripts. WarpX requires Python version 3.8 or newer. +A full Python installation of WarpX can be done, which includes a build of all of the C++ code, or a pure Python version can be made which only installs the Python scripts. WarpX requires Python version 3.9 or newer. For a full Python installation of WarpX --------------------------------------- diff --git a/Docs/source/developers/how_to_compile_locally.rst b/Docs/source/developers/how_to_compile_locally.rst new file mode 100644 index 00000000000..9fc1b397b78 --- /dev/null +++ b/Docs/source/developers/how_to_compile_locally.rst @@ -0,0 +1,138 @@ +.. _developers-local-compile: + +How to compile locally and fast +=============================== + +For simplicity, WarpX :ref:`compilation with CMake ` by default downloads, configures and compiles compatible versions of :ref:`central dependencies ` such as: + +* `AMReX `__ +* `PICSAR `__ +* `openPMD-api `__ +* `pyAMReX `__ +* `pybind11 `__ + +on-the-fly, which is called a *superbuild*. + +In some scenarios, e.g., when compiling without internet, with slow internet access, or when working on WarpX and its dependencies, modifications to the superbuild strategy might be preferable. +In the below workflows, you as the developer need to make sure to use compatible versions of the dependencies you provide. + + +.. _developers-local-compile-src: + +Compiling From Local Sources +---------------------------- + +This workflow is best for developers that make changes to WarpX, AMReX, PICSAR, openPMD-api and/or pyAMReX at the same time. +For instance, use this if you add a feature in AMReX and want to try it in WarpX before it is proposed as a pull request for inclusion in AMReX. + +Instead of downloading the source code of the above dependencies, one can also use an already cloned source copy. +For instance, clone these dependencies to ``$HOME/src``: + +.. code-block:: bash + + cd $HOME/src + + git clone https://github.com/ECP-WarpX/WarpX.git warpx + git clone https://github.com/AMReX-Codes/amrex.git + git clone https://github.com/openPMD/openPMD-api.git + git clone https://github.com/ECP-WarpX/picsar.git + git clone https://github.com/AMReX-Codes/pyamrex.git + git clone https://github.com/pybind/pybind11.git + +Now modify the dependencies as needed in their source locations, update sources if you cloned them earlier, etc. +When building WarpX, :ref:`the following CMake flags ` will use the respective local sources: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_src=$HOME/src/amrex \ + -DWarpX_openpmd_src=$HOME/src/openPMD-api \ + -DWarpX_picsar_src=$HOME/src/picsar \ + -DWarpX_pyamrex_src=$HOME/src/pyamrex \ + -DWarpX_pybind11_src=$HOME/src/pybind11 + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + + +.. _developers-local-compile-findpackage: + +Compiling With Pre-Compiled Dependencies +---------------------------------------- + +This workflow is the best and fastest to compile WarpX, when you just want to change code in WarpX and have the above central dependencies already made available *in the right configurations* (e.g., w/ or w/o MPI or GPU support) from a :ref:`module system ` or :ref:`package manager `. + +Instead of downloading the source code of the above central dependencies, or using a local copy of their source, we can compile and install those dependencies once. +By setting the `CMAKE_PREFIX_PATH `__ environment variable to the respective dependency install location prefixes, we can instruct CMake to `find their install locations and configurations `__. + +WarpX supports this with :ref:`the following CMake flags `: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_internal=OFF \ + -DWarpX_openpmd_internal=OFF \ + -DWarpX_picsar_internal=OFF \ + -DWarpX_pyamrex_internal=OFF \ + -DWarpX_pybind11_internal=OFF + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + +As a background, this is also the workflow how WarpX is built in :ref:`package managers such as Spack and Conda-Forge `. + + +.. _developers-local-compile-pylto: + +Faster Python Builds +-------------------- + +The Python bindings of WarpX and AMReX (pyAMReX) use `pybind11 `__. +Since pybind11 relies heavily on `C++ metaprogramming `__, speeding up the generated binding code requires that we perform a `link-time optimization (LTO) `__ step, also known as `interprocedural optimization (IPO) `__. + +For fast local development cycles, one can skip LTO/IPO with the following flags: + +.. code-block:: bash + + cd src/warpx + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_PYTHON_IPO=OFF \ + -DpyAMReX_IPO=OFF + + cmake --build build -j 8 --target pip_install + +.. note:: + + We might transition to `nanobind `__ in the future, which `does not rely on LTO/IPO `__ for optimal binaries. + You can contribute to `this pyAMReX pull request `__ to help exploring this library (and if it works for the HPC/GPU compilers that we need to support). + +For robustness, our ``pip_install`` target performs a regular ``wheel`` build and then installs it with ``pip``. +This step will check every time of WarpX dependencies are properly installed, to avoid broken installations. +When developing without internet or after the first ``pip_install`` succeeded in repeated installations in rapid development cycles, this check of ``pip`` can be skipped by using the ``pip_install_nodeps`` target instead: + +.. code-block:: bash + + cmake --build build -j 8 --target pip_install_nodeps + + +.. _developers-local-compile-ccache: + +CCache +------ + +WarpX builds will automatically search for `CCache `__ to speed up subsequent compilations in development cycles. +Make sure a :ref:`recent CCache version ` is installed to make use of this feature. + +For power developers that switch a lot between fundamentally different WarpX configurations (e.g., 1D to 3D, GPU and CPU builds, many branches with different bases, developing AMReX and WarpX at the same time), also consider increasing the `CCache cache size `__ and changing the `cache directory `__ if needed, e.g., due to storage quota constraints or to choose a fast(er) filesystem for the cache files. diff --git a/Docs/source/developers/how_to_guides.rst b/Docs/source/developers/how_to_guides.rst new file mode 100644 index 00000000000..25ed4f2d30d --- /dev/null +++ b/Docs/source/developers/how_to_guides.rst @@ -0,0 +1,13 @@ +.. _development-howtoguides: + +How-To Guides +============= + +.. toctree:: + :maxdepth: 1 + + how_to_profile + how_to_test + how_to_run_clang_tidy + how_to_compile_locally + how_to_write_the_docs diff --git a/Docs/source/developers/how_to_profile.rst b/Docs/source/developers/how_to_profile.rst new file mode 100644 index 00000000000..e756d8362c2 --- /dev/null +++ b/Docs/source/developers/how_to_profile.rst @@ -0,0 +1,280 @@ +.. _developers-profiling: + +How to profile the code +======================= + +Profiling allows us to find the bottle-necks of the code as it is currently implemented. +Bottle-necks are the parts of the code that may delay the simulation, making it more computationally expensive. +Once found, we can update the related code sections and improve its efficiency. +Profiling tools can also be used to check how load balanced the simulation is, i.e. if the work is well distributed across all MPI ranks used. +Load balancing can be activated in WarpX by setting input parameters, see the :ref:`parallelization input parameter section `. + +.. _developers-profiling-tiny-profiler: + +AMReX's Tiny Profiler +--------------------- + +By default, WarpX uses the AMReX baseline tool, the TINYPROFILER, to evaluate the time information for different parts of the code (functions) between the different MPI ranks. +The results, timers, are stored into four tables in the standard output, stdout, that are located below the simulation steps information and above the warnings regarding unused input file parameters (if there were any). + +The timers are displayed in tables for which the columns correspond to: + +* name of the function +* number of times it is called in total +* minimum of time spent exclusively/inclusively in it, between all ranks +* average of time, between all ranks +* maximum time, between all ranks +* maximum percentage of time spent, across all ranks + +If the simulation is well load balanced the minimum, average and maximum times should be identical. + +The top two tables refer to the complete simulation information. +The bottom two are related to the Evolve() section of the code (where each time step is computed). + +Each set of two timers show the exclusive, top, and inclusive, bottom, information depending on whether the time spent in nested sections of the codes are included. + +.. note:: + + When creating performance-related issues on the WarpX GitHub repo, please include Tiny Profiler tables (besides the usual issue description, input file and submission script), or (even better) the whole standard output. + +For more detailed information please visit the `AMReX profiling documentation `__. +There is a script located `here `__ that parses the Tiny Profiler output and generates a JSON file that can be used with `Hatchet `__ in order to analyze performance. + +AMReX's Full Profiler +--------------------- + +The Tiny Profiler provides a summary across all MPI ranks. However, when analyzing +load-balancing, it can be useful to have more detailed information about the +behavior of *each* individual MPI rank. The workflow for doing so is the following: + +- Compile WarpX with full profiler support: + + .. code-block:: bash + + cmake -S . -B build -DAMReX_BASE_PROFILE=YES -DAMReX_TRACE_PROFILE=YES -DAMReX_COMM_PROFILE=YES -DAMReX_TINY_PROFILE=OFF + cmake --build build -j 4 + + .. warning:: + + Please note that the `AMReX build options `__ for ``AMReX_TINY_PROFILE`` (our default: ``ON``) and full profiling traces via ``AMReX_BASE_PROFILE`` are mutually exclusive. + Further tracing options are sub-options of ``AMReX_BASE_PROFILE``. + + To turn on the tiny profiler again, remove the ``build`` directory or turn off ``AMReX_BASE_PROFILE`` again: + + .. code-block:: bash + + cmake -S . -B build -DAMReX_BASE_PROFILE=OFF -DAMReX_TINY_PROFILE=ON + +- Run the simulation to be profiled. Note that the WarpX executable will create + a new folder `bl_prof`, which contains the profiling data. + + .. note:: + + When using the full profiler, it is usually useful to profile only + a few PIC iterations (e.g. 10-20 PIC iterations), in order to improve + readability. If the interesting PIC iterations occur only late in a + simulation, you can run the first part of the simulation without + profiling, the create a checkpoint, and then restart the simulation for + 10-20 steps with the full profiler on. + +.. note:: + + The next steps can be done on a local computer (even if + the simulation itself ran on an HPC cluster). In this + case, simply copy the folder `bl_prof` to your local computer. + +- In order, to visualize the profiling data, install `amrvis` using `spack`: + + .. code-block:: bash + + spack install amrvis dims=2 +profiling + +- Then create timeline database from the `bl_prof` data and open it: + + .. code-block:: bash + + -timelinepf bl_prof/ + pltTimeline/ + + In the above, `` should be replaced by the actual of your + `amrvis` executable, which can be found starting to type `amrvis` and then + using Tab completion, in a Terminal. + +- This will pop-up a window with the timeline. Here are few guidelines to navigate it: + - Use the horizontal scroller to find the area where the 10-20 PIC steps occur. + - In order to zoom on an area, you can drag and drop with the mouse, and the hit `Ctrl-S` on a keyboard. + - You can directly click on the timeline to see which actual MPI call is being perform. (Note that the colorbar can be misleading.) + +.. _developers-profiling-nsight-systems: + +Nvidia Nsight-Systems +--------------------- + +`Vendor homepage `__ and `product manual `__. + +Nsight-Systems provides system level profiling data, including CPU and GPU +interactions. It runs quickly, and provides a convenient visualization of +profiling results including NVTX timers. + + +Perlmutter Example +"""""""""""""""""" + +Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). +You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. + +.. code-block:: bash + + # GPU-aware MPI + export MPICH_GPU_SUPPORT_ENABLED=1 + # 1 OpenMP thread + export OMP_NUM_THREADS=1 + + export TMPDIR="$PWD/tmp" + rm -rf ${TMPDIR} profiling* + mkdir -p ${TMPDIR} + + # record + srun --ntasks=4 --gpus=4 --cpu-bind=cores \ + nsys profile -f true \ + -o profiling_%q{SLURM_TASK_PID} \ + -t mpi,cuda,nvtx,osrt,openmp \ + --mpi-impl=mpich \ + ./warpx.3d.MPI.CUDA.DP.QED \ + inputs_3d \ + warpx.numprocs=1 1 4 amr.n_cell=512 512 2048 max_step=10 + +.. note:: + + If everything went well, you will obtain as many output files named ``profiling_.nsys-rep`` as active MPI ranks. + Each MPI rank's performance trace can be analyzed with the Nsight System graphical user interface (GUI). + In WarpX, every MPI rank is associated with one GPU, which each creates one trace file. + +.. warning:: + + The last line of the sbatch file has to match the data of your input files. + +Summit Example +"""""""""""""" + + Example on how to create traces on a multi-GPU system that uses the ``jsrun`` scheduler (e.g., `OLCF's Summit system `__): + +.. code-block:: bash + + # nsys: remove old traces + rm -rf profiling* tmp-traces + # nsys: a location where we can write temporary nsys files to + export TMPDIR=$PWD/tmp-traces + mkdir -p $TMPDIR + # WarpX: one OpenMP thread per MPI rank + export OMP_NUM_THREADS=1 + + # record + jsrun -n 4 -a 1 -g 1 -c 7 --bind=packed:$OMP_NUM_THREADS \ + nsys profile -f true \ + -o profiling_%p \ + -t mpi,cuda,nvtx,osrt,openmp \ + --mpi-impl=openmpi \ + ./warpx.3d.MPI.CUDA.DP.QED inputs_3d \ + warpx.numprocs=1 1 4 amr.n_cell=512 512 2048 max_step=10 + +.. warning:: + + Sep 10th, 2021 (OLCFHELP-3580): + The Nsight-Compute (``nsys``) version installed on Summit does not record details of GPU kernels. + This is reported to Nvidia and OLCF. + +Details +""""""" + +In these examples, the individual lines for recording a trace profile are: + +* ``srun``: execute multi-GPU runs with ``srun`` (Slurm's ``mpiexec`` wrapper), here for four GPUs +* ``-f true`` overwrite previously written trace profiles +* ``-o``: record one profile file per MPI rank (per GPU); if you run ``mpiexec``/``mpirun`` with OpenMPI directly, replace ``SLURM_TASK_PID`` with ``OMPI_COMM_WORLD_RANK`` +* ``-t``: select a couple of APIs to trace +* ``--mpi--impl``: optional, hint the MPI flavor +* ``./warpx...``: select the WarpX executable and a good inputs file +* ``warpx.numprocs=...``: make the run short, reasonably small, and run only a few steps + +Now open the created trace files (per rank) in the Nsight-Systems GUI. +This can be done on another system than the one that recorded the traces. +For example, if you record on a cluster and open the analysis GUI on your laptop, it is recommended to make sure that versions of Nsight-Systems match on the remote and local system. + +Nvidia Nsight-Compute +--------------------- + +`Vendor homepage `__ and `product manual `__. + +Nsight-Compute captures fine grained information at the kernel level +concerning resource utilization. By default, it collects a lot of data and runs +slowly (can be a few minutes per step), but provides detailed information about +occupancy, and memory bandwidth for a kernel. + + +Example +""""""" + +Example of how to create traces on a single-GPU system. A jobscript for +Perlmutter is shown, but the `SBATCH` headers are not strictly necessary as the +command only profiles a single process. This can also be run on an interactive +node, or without a workload management system. + +.. code-block:: bash + + #!/bin/bash -l + #SBATCH -t 00:30:00 + #SBATCH -N 1 + #SBATCH -J ncuProfiling + #SBATCH -A + #SBATCH -q regular + #SBATCH -C gpu + #SBATCH --ntasks-per-node=1 + #SBATCH --gpus-per-task=1 + #SBATCH --gpu-bind=map_gpu:0 + #SBATCH --mail-user= + #SBATCH --mail-type=ALL + + # record + dcgmi profile --pause + ncu -f -o out \ + --target-processes all \ + --set detailed \ + --nvtx --nvtx-include="WarpXParticleContainer::DepositCurrent::CurrentDeposition/" \ + ./warpx input max_step=1 \ + &> warpxOut.txt + +.. note:: + + To collect full statistics, Nsight-Compute reruns kernels, + temporarily saving device memory in host memory. This makes it + slower than Nsight-Systems, so the provided script profiles only a single + step of a single process. This is generally enough to extract relevant + information. + +Details +""""""" +In the example above, the individual lines for recording a trace profile are: + +* ``dcgmi profile --pause`` other profiling tools can't be collecting data, + `see this Q&A `__. +* ``-f`` overwrite previously written trace profiles. +* ``-o``: output file for profiling. +* ``--target-processes all``: required for multiprocess code. +* ``--set detailed``: controls what profiling data is collected. If only + interested in a few things, this can improve profiling speed. + ``detailed`` gets pretty much everything. +* ``--nvtx``: collects NVTX data. See note. +* ``--nvtx-include``: tells the profiler to only profile the given sections. + You can also use ``-k`` to profile only a given kernel. +* ``./warpx...``: select the WarpX executable and a good inputs file. + +Now open the created trace file in the Nsight-Compute GUI. As with +Nsight-Systems, +this can be done on another system than the one that recorded the traces. +For example, if you record on a cluster and open the analysis GUI on your laptop, it is recommended to make sure that versions of Nsight-Compute match on the remote and local system. + +.. note:: + + nvtx-include syntax is very particular. The trailing / in the example is + significant. For full information, see the Nvidia's documentation on `NVTX filtering `__ . diff --git a/Docs/source/developers/how_to_run_clang_tidy.rst b/Docs/source/developers/how_to_run_clang_tidy.rst new file mode 100644 index 00000000000..bbcd7b80130 --- /dev/null +++ b/Docs/source/developers/how_to_run_clang_tidy.rst @@ -0,0 +1,51 @@ +.. _developers-run_clang_tidy_locally: + +How to run the clang-tidy linter +================================ + +WarpX's CI tests include several checks performed with the `clang-tidy `__ linter. +The complete list of checks performed is defined in the ``.clang-tidy`` configuration file. + +.. dropdown:: clang-tidy configuration file + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../.clang-tidy + :language: yaml + +Under `Tools/Linter `__, the script ``runClangTidy.sh`` can be used to run the clang-tidy linter locally. + +.. dropdown:: clang-tidy local run script + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../Tools/Linter/runClangTidy.sh + :language: bash + +It is a prerequisite that WarpX is compiled following the instructions that you find in our :ref:`Users ` or :ref:`Developers ` sections. + +The script generates a wrapper to ensure that clang-tidy is only applied to WarpX source files and compiles WarpX in 1D, 2D, 3D, and RZ geometry, using such wrapper. + +By default WarpX is compiled in single precision with PSATD solver, QED module, QED table generator and embedded boundary in order to ensure broader coverage with the clang-tidy tool. + +Few optional environment variables can be set to tune the behavior of the script: + +* ``WARPX_TOOLS_LINTER_PARALLEL``: set the number of cores used for compilation; + +* ``CLANG``, ``CLANGXX``, and ``CLANGTIDY``: set the version of the compiler and the linter. + +For continuous integration we currently use clang version 15.0.0 and it is recommended to use this version locally as well. +A newer version may find issues not currently covered by CI tests (checks are opt-in), while older versions may not find all the issues. + +Here's an example of how to run the script after setting the appropriate environment variables: + +.. code-block:: bash + + export WARPX_TOOLS_LINTER_PARALLEL=12 + export CLANG=clang-15 + export CLANGXX=clang++-15 + export CLANGTIDY=clang-tidy-15 + + ./Tools/Linter/runClangTidy.sh diff --git a/Docs/source/developers/how_to_test.rst b/Docs/source/developers/how_to_test.rst new file mode 100644 index 00000000000..12dc653ac61 --- /dev/null +++ b/Docs/source/developers/how_to_test.rst @@ -0,0 +1,254 @@ +.. _developers-testing: + +How to test the code +==================== + +When you propose code changes, you want to make sure that + +* the code changes do not break the behavior of the rest of the code; +* the code changes give correct results (numerics, physics, etc.). + +Following the continuous integration (CI) software development practice, WarpX runs automated builds and tests after a commit is pushed to an open PR as well as after a PR is merged into the main branch. + +How to run pre-commit tests locally +----------------------------------- + +First, WarpX uses `pre-commit `__ to perform automated style and correctness checks. + +Here is how to install ``pre-commit`` locally: + +#. Install ``pre-commit``: + + .. code-block:: sh + + python -m pip install -U pre-commit + +#. Install the git hook scripts: + + .. code-block:: sh + + pre-commit install + +If you install ``pre-commit`` locally, the style and correctness checks will run automatically on your computer, after you commit the code changes and before you push them to the remote repository. + +If you do not install ``pre-commit`` locally, these checks will run automatically as part of our CI workflows and a commit containing style and correctness changes might be added automatically to your branch after you have pushed your own commit. +In that case, you will need to pull that automated commit before pushing further commits. + +The configuration options for ``pre-commit`` are set in the `pre-commit-config.yaml `__ file. + +How to configure the automated tests +------------------------------------ + +Our regression tests are run with `CTest `__, an executable that comes with CMake. + +The test suite is ready to run once you have configured and built WarpX with CMake, following the instructions that you find in our :ref:`Users ` or :ref:`Developers ` sections. + +A test that requires a build option that was not configured and built will be skipped automatically. For example, if you configure and build WarpX in 1D only, any test of dimensionality other than 1D, which would require WarpX to be configured and built in the corresponding dimensionality, will be skipped automatically. + +How to run automated tests locally +---------------------------------- + +Once your code changes are ready, there are ways to check that they do not break the rest of the code. +WarpX has automated tests running every time a commit is pushed to an open pull request. +The input files and scripts used by the automated tests can be found in the `Examples `__ directory, either under `Physics_applications `__ or `Tests `__. + +For easier debugging, it can be convenient to run the tests on your local computer by executing CTest as illustrated in the examples below (where we assume that WarpX was configured and built in the directory ``build``): + +* List tests available for the current build options: + + .. code-block:: sh + + ctest --test-dir build -N + +* Run tests available for the current build options: + + .. code-block:: sh + + ctest --test-dir build + +* Run tests available for the current build options in parallel (while preserving existing dependencies between tests): + + .. code-block:: sh + + ctest --test-dir build -j 2 + +* Run tests available for the current build options and output anything outputted by the test program if the test should fail: + + .. code-block:: sh + + ctest --test-dir build --output-on-failure + +* Run tests available for the current build options with verbose output: + + .. code-block:: sh + + ctest --test-dir build --verbose + +* Run tests matching the regular expression ``laser_acceleration``: + + .. code-block:: sh + + ctest --test-dir build -R laser_acceleration + +* Run tests except those matching the regular expression ``laser_acceleration``: + + .. code-block:: sh + + ctest --test-dir build -E laser_acceleration + +* Sometimes two or more tests share a large number of input parameters and differ by a small set of options. + Such tests typically also share a base string in their names. + For example, you can find three different tests named ``test_3d_langmuir_multi``, ``test_3d_langmuir_multi_nodal`` and ``test_3d_langmuir_multi_picmi``. + In such a case, if you wish to run the test ``test_3d_langmuir_multi`` only, this can be done again with the ``-R`` regular `expression filter `__ via + + .. code-block:: sh + + ctest --test-dir build -R "test_3d_langmuir_multi\..*" + + Note that filtering with ``-R "test_3d_langmuir_multi"`` would include the additional tests that have the same substring in their name and would not be sufficient to isolate a single test. + Note also that the escaping ``\.`` in the regular expression is necessary in order to take into account the fact that each test is automatically appended with the strings ``.run``, ``.analysis``, ``.checksum`` and possibly ``.cleanup``. + +* Run only tests not labeled with the ``slow`` label: + + .. code-block:: sh + + ctest --test-dir build -LE slow + +Once the execution of CTest is completed, you can find all files associated with each test in its corresponding directory under ``build/bin/``. +For example, if you run the single test ``test_3d_laser_acceleration``, you can find all files associated with this test in the directory ``build/bin/test_3d_laser_acceleration/``. + +If you modify the code base locally and want to assess the effects of your code changes on the automated tests, you need to first rebuild WarpX including your code changes and then rerun CTest. + +How to add automated tests +-------------------------- + +An automated test typically consists of the following components: + +* input file or PICMI input script; + +* analysis script; + +* checksum file. + +As mentioned above, the input files and scripts used by the automated tests can be found in the `Examples `__ directory, under either `Physics_applications `__ or `Tests `__. + +Each test directory must contain a file named ``CMakeLists.txt`` where all tests associated with the input files and scripts in that directory must be listed. + +A checksum file is a file that contains reference values obtained by computing a chosen checksum for a set of fields. +More precisely, we compute the sums of the absolute values of the arrays corresponding to each field from the results produced by the automated test and compare these checksums with the reference ones stored in the checksum file of that test, with respect to specific tolerances. +This is expected to be sensitive enough to make the automated test fail if the code changes cause significant differences in the final results, thus catching possible bugs. + +A new test can be added by calling the function ``add_warpx_test`` in ``CMakeLists.txt``. The function has the following signature: + +.. code-block:: cmake + + function(add_warpx_test + name # unique test name: + # test_1d_example, test_2d_example_picmi, etc. + dims # dimensionality: 1, 2, 3, RZ + nprocs # number of processes: 1, 2 + inputs # inputs file or PICMI script: + # inputs_test_1d_example, inputs_test_2d_example_picmi.py, "inputs_test_2d_example_picmi.py arg1 arg2", etc. + analysis # custom test analysis command: + # OFF, "analysis.py", "analysis.py arg1 arg2", etc. + checksum # default regression analysis command: + # OFF, "analysis_default_regression.py --path diags/diag1", etc. + dependency # name of base test that must run first (must match name exactly): + # OFF, test_1d_example_prepare, etc. + ) + +Here's how to add an automated test: + +#. Choose the test directory, either an existing one or a new one. + +#. Add an input file or PICMI input script. + The name must follow the naming conventions described in the section :ref:`developers-testing-naming` below. + +#. Add a Python analysis script to analyze the results of the test. + +#. Add the test to the ``CMakeLists.txt`` file (add such file if you are adding the test in a new test directory) using the function ``add_warpx_test`` mentioned above. + +#. If the test directory is new, add the directory with the command ``add_subdirectory`` in `Physics_applications/CMakeLists.txt `__ or `Tests/CMakeLists.txt `__, depending on where the test directory is located. + +#. If the test directory is new, make a symbolic link to the default regression analysis script ``analysis_default_regression.py`` from `Examples/analysis_default_regression.py `__, by running ``ln -s ../../analysis_default_regression.py analysis_default_regression.py`` from the test directory. + +#. Run the test locally with ``ctest``, after setting the environment variable ``CHECKSUM_RESET=ON``, in order to generate automatically the checksum file. + +Once you have added the test, run the test locally again, after resetting ``CHECKSUM_RESET=OFF``, to check that everything works as expected. + +The ``analysis`` and ``checksum`` commands passed as arguments to ``add_warpx_test`` can be set to ``OFF`` if the intention is to skip the respective analysis for a given test. + +If you need a new Python package dependency for testing, please add it in `Regression/requirements.txt `__. + +Sometimes two or more tests share a large number of input parameters. +The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``. + +Here is the help message of the default regression analysis script, including usage and list of available options and arguments: + + .. code-block:: bash + + usage: analysis_default_regression.py [-h] [--path PATH] [--rtol RTOL] [--skip-fields] [--skip-particles] + options: + -h, --help show this help message and exit + --path PATH path to output file(s) + --rtol RTOL relative tolerance to compare checksums + --skip-fields skip fields when comparing checksums + --skip-particles skip particles when comparing checksums + +How to reset checksums locally +------------------------------ + +It is possible to reset a checksum file locally by running the corresponding test with ``ctest`` with the environment variable ``CHECKSUM_RESET=ON``. For example: + + .. code-block:: bash + + CHECKSUM_RESET=ON ctest --test-dir build -R laser_acceleration + +Alternatively, it is also possible to reset multiple checksum files using the output of our Azure pipelines, which can be useful for code changes that result in resetting a large numbers of checksum files. +Here's how to do so: + +#. On the GitHub page of the pull request, find (one of) the pipeline(s) failing due to checksum regressions and click on "Details" (highlighted in blue). + + .. figure:: https://gist.github.com/user-attachments/assets/09db91b9-5711-4250-8b36-c52a6049e38e + +#. In the new page that opens up, click on "View more details on Azure pipelines" (highlighted in blue). + + .. figure:: https://gist.github.com/user-attachments/assets/ab0c9a24-5518-4da7-890f-d79fa1c8de4c + +#. In the new page that opens up, select the group of tests for which you want to reset the checksum files (e.g., ``cartesian_3d``) and click on "View raw log". + + .. figure:: https://gist.github.com/user-attachments/assets/06c1fe27-2c13-4bd3-b6b8-8b8941b37889 + +#. Save the raw log as a text file on your computer. + +#. Go to the directory `Tools/DevUtils `__ and run the Python script `update_benchmarks_from_azure_output.py `__ passing the path of the raw log text file as a command line argument: + + .. code:: bash + + python update_benchmarks_from_azure_output.py path/to/raw_log.txt + + This will update the checksum files for all the tests in the raw log that did not pass the checksum analysis. + +.. _developers-testing-naming: + +Naming conventions for automated tests +-------------------------------------- + +Note that we currently obey the following snake\_case naming conventions for test names and test input files (which make automation tasks easier, e.g., parsing visually, parsing through code, sorting alphabetically, filtering tests in CTest via ``-R``, etc.): + +#. **Regular test names** start with the string ``test_1d_``, ``test_2d_``, ``test_3d_`` or ``test_rz_``, followed by a string that is descriptive of the test. For example, ``test_3d_laser_acceleration``. + +#. **PICMI test names** start with the string ``test_1d_``, ``test_2d_``, ``test_3d_`` or ``test_rz_``, followed by a string that is descriptive of the test, and end with the string ``_picmi``. For example, ``test_3d_laser_acceleration_picmi``. + +#. **Restart test names** end with the string ``_restart``. For example, ``test_3d_laser_acceleration_restart``. + +#. **Test input files** start with the string ``inputs_`` followed by the test name. For example, ``inputs_test_3d_laser_acceleration`` or ``inputs_test_3d_laser_acceleration_picmi.py`` or ``inputs_test_3d_laser_acceleration_restart``. + +#. **Base input files** (that is, files collecting input parameters shared between two or more tests) are typically named ``inputs_base_1d``, ``inputs_base_2d``, ``inputs_base_3d`` or ``inputs_base_rz``, possibly followed by additional strings if need be. + +Other resources +--------------- + +With regard to testing the code more generally, not necessarily in the context of continuous integration, AMReX provides a number of useful post-processing tools for plotfiles. +The complete list of tools can be found `here `__. +One tool that traditionally stood out as especially useful for core developers and maintainers is `fcompare `__. diff --git a/Docs/source/developers/how_to_write_the_docs.rst b/Docs/source/developers/how_to_write_the_docs.rst new file mode 100644 index 00000000000..827b5950d80 --- /dev/null +++ b/Docs/source/developers/how_to_write_the_docs.rst @@ -0,0 +1,73 @@ +.. _developers-docs: + +How to write documentation +========================== + +Doxygen documentation +--------------------- + +WarpX uses a `Doxygen documentation `__. +Whenever you create a new class, please document it where it is declared (typically in the header file): + +.. code-block:: cpp + + /** \brief A brief title + * + * few-line description explaining the purpose of MyClass. + * + * If you are kind enough, also quickly explain how things in MyClass work. + * (typically a few more lines) + */ + class MyClass + { ... } + +Doxygen reads this docstring, so please be accurate with the syntax! See `Doxygen manual `__ for more information. Similarly, please document functions when you declare them (typically in a header file) like: + +.. code-block:: cpp + + /** \brief A brief title + * + * few-line description explaining the purpose of my_function. + * + * \param[in,out] my_int a pointer to an integer variable on which + * my_function will operate. + * \return what is the meaning and value range of the returned value + */ + int MyClass::my_function (int* my_int); + +An online version of this documentation is :ref:`linked here `. + +Breathe documentation +--------------------- + +Your Doxygen documentation is not only useful for people looking into the code, it is also part of the `WarpX online documentation `_ based on `Sphinx `_! +This is done using the Python module `Breathe `_, that allows you to write Doxygen documentation directly in the source and have it included it in your Sphinx documentation, by calling Breathe functions. +For instance, the following line will get the Doxygen documentation for ``WarpXParticleContainer`` in ``Source/Particles/WarpXParticleContainer.H`` and include it to the html page generated by Sphinx: + +.. code-block:: rst + + .. doxygenclass:: WarpXParticleContainer + +Building the documentation +-------------------------- + +To build the documentation on your local computer, you will need to install Doxygen as well as the Python module `breathe`. +First, make sure you are in the root directory of WarpX's source and install the Python requirements: + +.. code-block:: sh + + cd Docs/ + python3 -m pip install -r requirements.txt + +You will also need Doxygen (macOS: ``brew install doxygen``; Ubuntu: ``sudo apt install doxygen``). + +Still in the ``Docs/`` directory, compile the documentation via + +.. code-block:: sh + + make html + # This will first compile the Doxygen documentation (execute doxygen) + # and then build html pages from rst files using sphinx and breathe. + +Open the created ``build/html/index.html`` file with your favorite browser. +Rebuild and refresh as needed. diff --git a/Docs/source/developers/local_compile.rst b/Docs/source/developers/local_compile.rst deleted file mode 100644 index 8bfa033a92d..00000000000 --- a/Docs/source/developers/local_compile.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. _developers-local-compile: - -Fast, Local Compilation -======================= - -For simplicity, WarpX :ref:`compilation with CMake ` by default downloads, configures and compiles compatible versions of :ref:`central dependencies ` such as: - -* `AMReX `__ -* `PICSAR `__ -* `openPMD-api `__ -* `pyAMReX `__ -* `pybind11 `__ - -on-the-fly, which is called a *superbuild*. - -In some scenarios, e.g., when compiling without internet, with slow internet access, or when working on WarpX and its dependencies, modifications to the superbuild strategy might be preferable. -In the below workflows, you as the developer need to make sure to use compatible versions of the dependencies you provide. - - -.. _developers-local-compile-src: - -Compiling From Local Sources ----------------------------- - -This workflow is best for developers that make changes to WarpX, AMReX, PICSAR, openPMD-api and/or pyAMReX at the same time. -For instance, use this if you add a feature in AMReX and want to try it in WarpX before it is proposed as a pull request for inclusion in AMReX. - -Instead of downloading the source code of the above dependencies, one can also use an already cloned source copy. -For instance, clone these dependencies to ``$HOME/src``: - -.. code-block:: bash - - cd $HOME/src - - git clone https://github.com/ECP-WarpX/WarpX.git warpx - git clone https://github.com/AMReX-Codes/amrex.git - git clone https://github.com/openPMD/openPMD-api.git - git clone https://github.com/ECP-WarpX/picsar.git - git clone https://github.com/AMReX-Codes/pyamrex.git - git clone https://github.com/pybind/pybind11.git - -Now modify the dependencies as needed in their source locations, update sources if you cloned them earlier, etc. -When building WarpX, :ref:`the following CMake flags ` will use the respective local sources: - -.. code-block:: bash - - cd src/warpx - - rm -rf build - - cmake -S . -B build \ - -DWarpX_PYTHON=ON \ - -DWarpX_amrex_src=$HOME/src/amrex \ - -DWarpX_openpmd_src=$HOME/src/openPMD-api \ - -DWarpX_picsar_src=$HOME/src/picsar \ - -DWarpX_pyamrex_src=$HOME/src/pyamrex \ - -DWarpX_pybind11_src=$HOME/src/pybind11 - - cmake --build build -j 8 - cmake --build build -j 8 --target pip_install - - -.. _developers-local-compile-findpackage: - -Compiling With Pre-Compiled Dependencies ----------------------------------------- - -This workflow is the best and fastest to compile WarpX, when you just want to change code in WarpX and have the above central dependencies already made available *in the right configurations* (e.g., w/ or w/o MPI or GPU support) from a :ref:`module system ` or :ref:`package manager `. - -Instead of downloading the source code of the above central dependencies, or using a local copy of their source, we can compile and install those dependencies once. -By setting the `CMAKE_PREFIX_PATH `__ environment variable to the respective dependency install location prefixes, we can instruct CMake to `find their install locations and configurations `__. - -WarpX supports this with :ref:`the following CMake flags `: - -.. code-block:: bash - - cd src/warpx - - rm -rf build - - cmake -S . -B build \ - -DWarpX_PYTHON=ON \ - -DWarpX_amrex_internal=OFF \ - -DWarpX_openpmd_internal=OFF \ - -DWarpX_picsar_internal=OFF \ - -DWarpX_pyamrex_internal=OFF \ - -DWarpX_pybind11_internal=OFF - - cmake --build build -j 8 - cmake --build build -j 8 --target pip_install - -As a background, this is also the workflow how WarpX is built in :ref:`package managers such as Spack and Conda-Forge `. - - -.. _developers-local-compile-pylto: - -Faster Python Builds --------------------- - -The Python bindings of WarpX and AMReX (pyAMReX) use `pybind11 `__. -Since pybind11 relies heavily on `C++ metaprogramming `__, speeding up the generated binding code requires that we perform a `link-time optimization (LTO) `__ step, also known as `interprocedural optimization (IPO) `__. - -For fast local development cycles, one can skip LTO/IPO with the following flags: - -.. code-block:: bash - - cd src/warpx - - cmake -S . -B build \ - -DWarpX_PYTHON=ON \ - -DWarpX_PYTHON_IPO=OFF \ - -DpyAMReX_IPO=OFF - - cmake --build build -j 8 --target pip_install - -.. note:: - - We might transition to `nanobind `__ in the future, which `does not rely on LTO/IPO `__ for optimal binaries. - You can contribute to `this pyAMReX pull request `__ to help exploring this library (and if it works for the HPC/GPU compilers that we need to support). - -For robustness, our ``pip_install`` target performs a regular ``wheel`` build and then installs it with ``pip``. -This step will check every time of WarpX dependencies are properly installed, to avoid broken installations. -When developing without internet or after the first ``pip_install`` succeeded in repeated installations in rapid development cycles, this check of ``pip`` can be skipped by using the ``pip_install_nodeps`` target instead: - -.. code-block:: bash - - cmake --build build -j 8 --target pip_install_nodeps - - -.. _developers-local-compile-ccache: - -CCache ------- - -WarpX builds will automatically search for `CCache `__ to speed up subsequent compilations in development cycles. -Make sure a :ref:`recent CCache version ` is installed to make use of this feature. - -For power developers that switch a lot between fundamentally different WarpX configurations (e.g., 1D to 3D, GPU and CPU builds, many branches with different bases, developing AMReX and WarpX at the same time), also consider increasing the `CCache cache size `__ and changing the `cache directory `__ if needed, e.g., due to storage quota constraints or to choose a fast(er) filesystem for the cache files. diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index 53a0090b5c9..9f199bdbb91 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -83,7 +83,7 @@ Main functions .. doxygenfunction:: PhysicalParticleContainer::PushPX -.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, amrex::Real dt, amrex::Real relative_time) +.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(ablastr::fields::MultiLevelVectorField const &J, amrex::Real dt, amrex::Real relative_time) .. note:: The current deposition is used both by ``PhysicalParticleContainer`` and ``LaserParticleContainer``, so it is in the parent class ``WarpXParticleContainer``. @@ -160,7 +160,7 @@ Attribute name ``int``/``real`` Description Default when they were created. ================== ================ ================================= ============== -A Python example that adds runtime options can be found in :download:`Examples/Tests/particle_data_python <../../../Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py>` +A Python example that adds runtime options can be found in :download:`Examples/Tests/particle_data_python <../../../Examples/Tests/particle_data_python/inputs_test_2d_prev_positions_picmi.py>` .. note:: diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst deleted file mode 100644 index 5acea786920..00000000000 --- a/Docs/source/developers/profiling.rst +++ /dev/null @@ -1,280 +0,0 @@ -.. _developers-profiling: - -Profiling the Code -================== - -Profiling allows us to find the bottle-necks of the code as it is currently implemented. -Bottle-necks are the parts of the code that may delay the simulation, making it more computationally expensive. -Once found, we can update the related code sections and improve its efficiency. -Profiling tools can also be used to check how load balanced the simulation is, i.e. if the work is well distributed across all MPI ranks used. -Load balancing can be activated in WarpX by setting input parameters, see the :ref:`parallelization input parameter section `. - -.. _developers-profiling-tiny-profiler: - -AMReX's Tiny Profiler ---------------------- - -By default, WarpX uses the AMReX baseline tool, the TINYPROFILER, to evaluate the time information for different parts of the code (functions) between the different MPI ranks. -The results, timers, are stored into four tables in the standard output, stdout, that are located below the simulation steps information and above the warnings regarding unused input file parameters (if there were any). - -The timers are displayed in tables for which the columns correspond to: - -* name of the function -* number of times it is called in total -* minimum of time spent exclusively/inclusively in it, between all ranks -* average of time, between all ranks -* maximum time, between all ranks -* maximum percentage of time spent, across all ranks - -If the simulation is well load balanced the minimum, average and maximum times should be identical. - -The top two tables refer to the complete simulation information. -The bottom two are related to the Evolve() section of the code (where each time step is computed). - -Each set of two timers show the exclusive, top, and inclusive, bottom, information depending on whether the time spent in nested sections of the codes are included. - -.. note:: - - When creating performance-related issues on the WarpX GitHub repo, please include Tiny Profiler tables (besides the usual issue description, input file and submission script), or (even better) the whole standard output. - -For more detailed information please visit the `AMReX profiling documentation `__. -There is a script located `here `__ that parses the Tiny Profiler output and generates a JSON file that can be used with `Hatchet `__ in order to analyze performance. - -AMReX's Full Profiler ---------------------- - -The Tiny Profiler provides a summary across all MPI ranks. However, when analyzing -load-balancing, it can be useful to have more detailed information about the -behavior of *each* individual MPI rank. The workflow for doing so is the following: - -- Compile WarpX with full profiler support: - - .. code-block:: bash - - cmake -S . -B build -DAMReX_BASE_PROFILE=YES -DAMReX_TRACE_PROFILE=YES -DAMReX_COMM_PROFILE=YES -DAMReX_TINY_PROFILE=OFF - cmake --build build -j 4 - - .. warning:: - - Please note that the `AMReX build options `__ for ``AMReX_TINY_PROFILE`` (our default: ``ON``) and full profiling traces via ``AMReX_BASE_PROFILE`` are mutually exclusive. - Further tracing options are sub-options of ``AMReX_BASE_PROFILE``. - - To turn on the tiny profiler again, remove the ``build`` directory or turn off ``AMReX_BASE_PROFILE`` again: - - .. code-block:: bash - - cmake -S . -B build -DAMReX_BASE_PROFILE=OFF -DAMReX_TINY_PROFILE=ON - -- Run the simulation to be profiled. Note that the WarpX executable will create - a new folder `bl_prof`, which contains the profiling data. - - .. note:: - - When using the full profiler, it is usually useful to profile only - a few PIC iterations (e.g. 10-20 PIC iterations), in order to improve - readability. If the interesting PIC iterations occur only late in a - simulation, you can run the first part of the simulation without - profiling, the create a checkpoint, and then restart the simulation for - 10-20 steps with the full profiler on. - -.. note:: - - The next steps can be done on a local computer (even if - the simulation itself ran on an HPC cluster). In this - case, simply copy the folder `bl_prof` to your local computer. - -- In order, to visualize the profiling data, install `amrvis` using `spack`: - - .. code-block:: bash - - spack install amrvis dims=2 +profiling - -- Then create timeline database from the `bl_prof` data and open it: - - .. code-block:: bash - - -timelinepf bl_prof/ - pltTimeline/ - - In the above, `` should be replaced by the actual of your - `amrvis` executable, which can be found starting to type `amrvis` and then - using Tab completion, in a Terminal. - -- This will pop-up a window with the timeline. Here are few guidelines to navigate it: - - Use the horizontal scroller to find the area where the 10-20 PIC steps occur. - - In order to zoom on an area, you can drag and drop with the mouse, and the hit `Ctrl-S` on a keyboard. - - You can directly click on the timeline to see which actual MPI call is being perform. (Note that the colorbar can be misleading.) - -.. _developers-profiling-nsight-systems: - -Nvidia Nsight-Systems ---------------------- - -`Vendor homepage `__ and `product manual `__. - -Nsight-Systems provides system level profiling data, including CPU and GPU -interactions. It runs quickly, and provides a convenient visualization of -profiling results including NVTX timers. - - -Perlmutter Example -"""""""""""""""""" - -Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). -You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. - -.. code-block:: bash - - # GPU-aware MPI - export MPICH_GPU_SUPPORT_ENABLED=1 - # 1 OpenMP thread - export OMP_NUM_THREADS=1 - - export TMPDIR="$PWD/tmp" - rm -rf ${TMPDIR} profiling* - mkdir -p ${TMPDIR} - - # record - srun --ntasks=4 --gpus=4 --cpu-bind=cores \ - nsys profile -f true \ - -o profiling_%q{SLURM_TASK_PID} \ - -t mpi,cuda,nvtx,osrt,openmp \ - --mpi-impl=mpich \ - ./warpx.3d.MPI.CUDA.DP.QED \ - inputs_3d \ - warpx.numprocs=1 1 4 amr.n_cell=512 512 2048 max_step=10 - -.. note:: - - If everything went well, you will obtain as many output files named ``profiling_.nsys-rep`` as active MPI ranks. - Each MPI rank's performance trace can be analyzed with the Nsight System graphical user interface (GUI). - In WarpX, every MPI rank is associated with one GPU, which each creates one trace file. - -.. warning:: - - The last line of the sbatch file has to match the data of your input files. - -Summit Example -"""""""""""""" - - Example on how to create traces on a multi-GPU system that uses the ``jsrun`` scheduler (e.g., `OLCF's Summit system `__): - -.. code-block:: bash - - # nsys: remove old traces - rm -rf profiling* tmp-traces - # nsys: a location where we can write temporary nsys files to - export TMPDIR=$PWD/tmp-traces - mkdir -p $TMPDIR - # WarpX: one OpenMP thread per MPI rank - export OMP_NUM_THREADS=1 - - # record - jsrun -n 4 -a 1 -g 1 -c 7 --bind=packed:$OMP_NUM_THREADS \ - nsys profile -f true \ - -o profiling_%p \ - -t mpi,cuda,nvtx,osrt,openmp \ - --mpi-impl=openmpi \ - ./warpx.3d.MPI.CUDA.DP.QED inputs_3d \ - warpx.numprocs=1 1 4 amr.n_cell=512 512 2048 max_step=10 - -.. warning:: - - Sep 10th, 2021 (OLCFHELP-3580): - The Nsight-Compute (``nsys``) version installed on Summit does not record details of GPU kernels. - This is reported to Nvidia and OLCF. - -Details -""""""" - -In these examples, the individual lines for recording a trace profile are: - -* ``srun``: execute multi-GPU runs with ``srun`` (Slurm's ``mpiexec`` wrapper), here for four GPUs -* ``-f true`` overwrite previously written trace profiles -* ``-o``: record one profile file per MPI rank (per GPU); if you run ``mpiexec``/``mpirun`` with OpenMPI directly, replace ``SLURM_TASK_PID`` with ``OMPI_COMM_WORLD_RANK`` -* ``-t``: select a couple of APIs to trace -* ``--mpi--impl``: optional, hint the MPI flavor -* ``./warpx...``: select the WarpX executable and a good inputs file -* ``warpx.numprocs=...``: make the run short, reasonably small, and run only a few steps - -Now open the created trace files (per rank) in the Nsight-Systems GUI. -This can be done on another system than the one that recorded the traces. -For example, if you record on a cluster and open the analysis GUI on your laptop, it is recommended to make sure that versions of Nsight-Systems match on the remote and local system. - -Nvidia Nsight-Compute ---------------------- - -`Vendor homepage `__ and `product manual `__. - -Nsight-Compute captures fine grained information at the kernel level -concerning resource utilization. By default, it collects a lot of data and runs -slowly (can be a few minutes per step), but provides detailed information about -occupancy, and memory bandwidth for a kernel. - - -Example -""""""" - -Example of how to create traces on a single-GPU system. A jobscript for -Perlmutter is shown, but the `SBATCH` headers are not strictly necessary as the -command only profiles a single process. This can also be run on an interactive -node, or without a workload management system. - -.. code-block:: bash - - #!/bin/bash -l - #SBATCH -t 00:30:00 - #SBATCH -N 1 - #SBATCH -J ncuProfiling - #SBATCH -A - #SBATCH -q regular - #SBATCH -C gpu - #SBATCH --ntasks-per-node=1 - #SBATCH --gpus-per-task=1 - #SBATCH --gpu-bind=map_gpu:0 - #SBATCH --mail-user= - #SBATCH --mail-type=ALL - - # record - dcgmi profile --pause - ncu -f -o out \ - --target-processes all \ - --set detailed \ - --nvtx --nvtx-include="WarpXParticleContainer::DepositCurrent::CurrentDeposition/" \ - ./warpx input max_step=1 \ - &> warpxOut.txt - -.. note:: - - To collect full statistics, Nsight-Compute reruns kernels, - temporarily saving device memory in host memory. This makes it - slower than Nsight-Systems, so the provided script profiles only a single - step of a single process. This is generally enough to extract relevant - information. - -Details -""""""" -In the example above, the individual lines for recording a trace profile are: - -* ``dcgmi profile --pause`` other profiling tools can't be collecting data, - `see this Q&A `__. -* ``-f`` overwrite previously written trace profiles. -* ``-o``: output file for profiling. -* ``--target-processes all``: required for multiprocess code. -* ``--set detailed``: controls what profiling data is collected. If only - interested in a few things, this can improve profiling speed. - ``detailed`` gets pretty much everything. -* ``--nvtx``: collects NVTX data. See note. -* ``--nvtx-include``: tells the profiler to only profile the given sections. - You can also use ``-k`` to profile only a given kernel. -* ``./warpx...``: select the WarpX executable and a good inputs file. - -Now open the created trace file in the Nsight-Compute GUI. As with -Nsight-Systems, -this can be done on another system than the one that recorded the traces. -For example, if you record on a cluster and open the analysis GUI on your laptop, it is recommended to make sure that versions of Nsight-Compute match on the remote and local system. - -.. note:: - - nvtx-include syntax is very particular. The trailing / in the example is - significant. For full information, see the Nvidia's documentation on `NVTX filtering `__ . diff --git a/Docs/source/developers/run_clang_tidy_locally.rst b/Docs/source/developers/run_clang_tidy_locally.rst deleted file mode 100644 index 3f600019fe7..00000000000 --- a/Docs/source/developers/run_clang_tidy_locally.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _developers-run_clang_tidy_locally: - -The clang-tidy linter -===================== - -Clang-tidy CI test ------------------- - -WarpX's CI tests include several checks performed with the -`clang-tidy `__ linter -(currently the version 15 of this tool). The complete list of checks -enforced in CI tests can be found in the ``.clang-tidy`` configuration file. - -.. dropdown:: clang-tidy configuration file - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../.clang-tidy - :language: yaml - -Run clang-tidy linter locally ------------------------------ - -We provide a script to run clang-tidy locally. The script can be run as follows, -provided that all the requirements to compile WarpX are met (see `building from source `). -The script generates a simple wrapper to ensure that `clang-tidy` is only applied to WarpX source files -and compiles WarpX in 1D,2D,3D, and RZ using such wrapper. By default WarpX is compiled in single precision -with PSATD solver, QED module, QED table generator and Embedded boundary in order to find more -potential issues with the `clang-tidy` tool. - -Few optional environment variables can be set to tune the behavior of the script: - -* ``WARPX_TOOLS_LINTER_PARALLEL``: sets the number of cores to be used for the compilation -* ``CLANG``, ``CLANGXX``, and ``CLANGTIDY`` : set the version of the compiler and of the linter - -Note: clang v15 is currently used in CI tests. It is therefore recommended to use this version. -Otherwise, a newer version may find issues not currently covered by CI tests (checks are opt-in) -while older versions may not find all the issues. - -.. code-block:: bash - - export WARPX_TOOLS_LINTER_PARALLEL=12 - export CLANG=clang-15 - export CLANGXX=clang++-15 - export CLANGTIDY=clang-tidy-15 - ./Tools/Linter/runClangTidy.sh - -.. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../Tools/Linter/runClangTidy.sh - :language: bash diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst deleted file mode 100644 index c6a09d970df..00000000000 --- a/Docs/source/developers/testing.rst +++ /dev/null @@ -1,178 +0,0 @@ -.. _developers-testing: - -Testing the code -================ - -When adding a new feature, you want to make sure that (i) you did not break the existing code and (ii) your contribution gives correct results. While existing capabilities are tested regularly remotely (when commits are pushed to an open PR on CI, and every night on local clusters), it can also be useful to run tests on your custom input file. This section details how to use both automated and custom tests. - -Continuous Integration in WarpX -------------------------------- - -Configuration -^^^^^^^^^^^^^ - -Our regression tests are using the suite published and documented at `AMReX-Codes/regression_testing `__. - -Most of the configuration of our regression tests happens in ``Regression/Warpx-tests.ini``. -We slightly modify this file in ``Regression/prepare_file_ci.py``. - -For example, if you like to change the compiler to compilation to build on Nvidia GPUs, modify this block to add ``-DWarpX_COMPUTE=CUDA``: - -.. code-block:: ini - - [source] - dir = /home/regtester/AMReX_RegTesting/warpx - branch = development - cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_COMPUTE=CUDA - -We also support changing compilation options via the usual :ref:`build environment variables `. -For instance, compiling with ``clang++ -Werror`` would be: - -.. code-block:: sh - - export CXX=$(which clang++) - export CXXFLAGS="-Werror" - - -Run Pre-Commit Tests Locally ----------------------------- - -When proposing code changes to Warpx, we perform a couple of automated stylistic and correctness checks on the code change. -You can run those locally before you push to save some time, install them once like this: - -.. code-block:: sh - - python -m pip install -U pre-commit - pre-commit install - -See `pre-commit.com `__ and our ``.pre-commit-config.yaml`` file in the repository for more details. - - -Run the test suite locally --------------------------- - -Once your new feature is ready, there are ways to check that you did not break anything. -WarpX has automated tests running every time a commit is added to an open pull request. -The list of automated tests is defined in `./Regression/WarpX-tests.ini `__. - -For easier debugging, it can be convenient to run the tests on your local machine by executing the script -`./run_test.sh `__ from WarpX's root folder, as illustrated in the examples below: - -.. code-block:: sh - - # Example: - # run all tests defined in ./Regression/WarpX-tests.ini - ./run_test.sh - - # Example: - # run only the test named 'pml_x_yee' - ./run_test.sh pml_x_yee - - # Example: - # run only the tests named 'pml_x_yee', 'pml_x_ckc' and 'pml_x_psatd' - ./run_test.sh pml_x_yee pml_x_ckc pml_x_psatd - -Note that the script `./run_test.sh `__ runs the tests with the exact same compile-time options and runtime options used to run the tests remotely. - -Moreover, the script `./run_test.sh `__ compiles all the executables that are necessary in order to run the chosen tests. -The default number of threads allotted for compiling is set with ``numMakeJobs = 8`` in `./Regression/WarpX-tests.ini `__. -However, when running the tests on a local machine, it is usually possible and convenient to allot more threads for compiling, in order to speed up the builds. -This can be accomplished by setting the environment variable ``WARPX_CI_NUM_MAKE_JOBS``, with the preferred number of threads that fits your local machine, e.g. ``export WARPX_CI_NUM_MAKE_JOBS=16`` (or less if your machine is smaller). -On public CI, we overwrite the value to ``WARPX_CI_NUM_MAKE_JOBS=2``, in order to avoid overloading the available remote resources. -Note that this will not change the number of threads used to run each test, but only the number of threads used to compile each executable necessary to run the tests. - -Once the execution of `./run_test.sh `__ is completed, you can find all the relevant files associated with each test in one single directory. -For example, if you run the single test ``pml_x_yee``, as shown above, on 04/30/2021, you can find all relevant files in the directory ``./test_dir/rt-WarpX/WarpX-tests/2021-04-30/pml_x_yee/``. -The content of this directory will look like the following (possibly including backtraces if the test crashed at runtime): - -.. code-block:: sh - - $ ls ./test_dir/rt-WarpX/WarpX-tests/2021-04-30/pml_x_yee/ - analysis_pml_yee.py # Python analysis script - inputs_2d # input file - main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.ex # executable - pml_x_yee.analysis.out # Python analysis output - pml_x_yee.err.out # error output - pml_x_yee.make.out # build output - pml_x_yee_plt00000/ # data output (initialization) - pml_x_yee_plt00300/ # data output (last time step) - pml_x_yee.run.out # test output - - -Add a test to the suite ------------------------ - -There are three steps to follow to add a new automated test (illustrated here for PML boundary conditions): - -* An input file for your test, in folder `Example/Tests/...`. For the PML test, the input file is at ``Examples/Tests/pml/inputs_2d``. You can also re-use an existing input file (even better!) and pass specific parameters at runtime (see below). -* A Python script that reads simulation output and tests correctness versus theory or calibrated results. For the PML test, see ``Examples/Tests/pml/analysis_pml_yee.py``. It typically ends with Python statement ``assert( error<0.01 )``. -* If you need a new Python package dependency for testing, add it in ``Regression/requirements.txt`` -* Add an entry to ``Regression/WarpX-tests.ini``, so that a WarpX simulation runs your test in the continuous integration process, and the Python script is executed to assess the correctness. For the PML test, the entry is - -.. code-block:: - - [pml_x_yee] - buildDir = . - inputFile = Examples/Tests/pml/inputs2d - runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee - dim = 2 - addToCompileString = - cmakeSetupOpts = -DWarpX_DIMS=2 - restartTest = 0 - useMPI = 1 - numprocs = 2 - useOMP = 1 - numthreads = 1 - compileTest = 0 - doVis = 0 - analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py - -If you re-use an existing input file, you can add arguments to ``runtime_params``, like ``runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=100 plasma_e.zmin=-200.e-6``. - -.. note:: - - If you added ``analysisRoutine = Examples/analysis_default_regression.py``, then run the new test case locally and add the :ref:`checksum ` file for the expected output. - -.. note:: - - We run those tests on our continuous integration services, which at the moment only have 2 virtual CPU cores. - Thus, make sure that the product of ``numprocs`` and ``numthreads`` for a test is ``<=2``. - - -Useful tool for plotfile comparison: ``fcompare`` -------------------------------------------------- - -AMReX provides ``fcompare``, an executable that takes two ``plotfiles`` as input and returns the absolute and relative difference for each field between these two plotfiles. For some changes in the code, it is very convenient to run the same input file with an old and your current version, and ``fcompare`` the plotfiles at the same iteration. To use it: - -.. code-block:: sh - - # Compile the executable - cd /Tools/Plotfile/ # This may change - make -j 8 - # Run the executable to compare old and new versions - /Tools/Plotfile/fcompare.gnu.ex old/plt00200 new/plt00200 - -which should return something like - -.. code-block:: sh - - variable name absolute error relative error - (||A - B||) (||A - B||/||A||) - ---------------------------------------------------------------------------- - level = 0 - jx 1.044455105e+11 1.021651316 - jy 4.08631977e+16 7.734299273 - jz 1.877301764e+14 1.073458933 - Ex 4.196315448e+10 1.253551615 - Ey 3.330698083e+12 6.436470137 - Ez 2.598167798e+10 0.6804387128 - Bx 273.8687473 2.340209782 - By 152.3911863 1.10952567 - Bz 37.43212767 2.1977289 - part_per_cell 15 0.9375 - Ex_fp 4.196315448e+10 1.253551615 - Ey_fp 3.330698083e+12 6.436470137 - Ez_fp 2.598167798e+10 0.6804387128 - Bx_fp 273.8687473 2.340209782 - By_fp 152.3911863 1.10952567 - Bz_fp 37.43212767 2.1977289 diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst deleted file mode 100644 index 00279018e9d..00000000000 --- a/Docs/source/developers/workflows.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _development-workflows: - -Workflows -========= - -.. toctree:: - :maxdepth: 1 - - profiling - testing - documentation - checksum - local_compile - run_clang_tidy_locally diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 108f685a551..81cc53c3eab 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,7 +14,12 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. -#. Shrock JE, Rockafellow E, Miao B, Le M, Hollinger RC, Wang S, Gonsalves AJ, Picksley A, Rocca JJ, and Milchberg HM +#. Ma M, Zeng M, Wang J, Lu G, Yan W, Chen L, and Li D. + **Particle-in-cell simulation of laser wakefield accelerators with oblique lasers in quasicylindrical geometry**. + Phys. Rev. Accel. Beams **28**, 021301, 2025 + `DOI:10.1103/PhysRevAccelBeams.28.021301 `__ + +#. Shrock JE, Rockafellow E, Miao B, Le M, Hollinger RC, Wang S, Gonsalves AJ, Picksley A, Rocca JJ, and Milchberg HM. **Guided Mode Evolution and Ionization Injection in Meter-Scale Multi-GeV Laser Wakefield Accelerators**. Phys. Rev. Lett. **133**, 045002, 2024 `DOI:10.1103/PhysRevLett.133.045002 `__ @@ -85,6 +90,16 @@ Laser-Plasma Interaction Scientific works in laser-ion acceleration and laser-matter interaction. +#. Garten M, Bulanov S S, Hakimi S, Obst-Huebl L, Mitchell C E, Schroeder C B, Esarey E, Geddes C G R, Vay J-L, Huebl A. + **Laser-plasma ion beam booster based on hollow-channel magnetic vortex acceleration**. + Physical Review Research **6**, 033148, 2024. + `DOI:10.1103/PhysRevResearch.6.033148 `__ + +#. Zaïm N, Sainte-Marie A, Fedeli L, Bartoli P, Huebl A, Leblanc A, Vay J-L, Vincenti H. + **Light-matter interaction near the Schwinger limit using tightly focused doppler-boosted lasers**. + Physical Review Letters **132**, 175002, 2024. + `DOI:10.1103/PhysRevLett.132.175002 `__ + #. Knight B, Gautam C, Stoner C, Egner B, Smith J, Orban C, Manfredi J, Frische K, Dexter M, Chowdhury E, Patnaik A (2023). **Detailed Characterization of a kHz-rate Laser-Driven Fusion at a Thin Liquid Sheet with a Neutron Detection Suite**. High Power Laser Science and Engineering, 1-13, 2023. @@ -105,6 +120,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction. Phys. Rev. Accel. Beams **25**, 093402, 2022. `DOI:10.1103/PhysRevAccelBeams.25.093402 `__ +#. Fedeli L, Sainte-Marie A, Zaïm N, Thévenet M, Vay J-L, Myers A, Quéré F, Vincenti H. + **Probing strong-field QED with Doppler-boosted PetaWatt-class lasers**. + Physical Review Letters **127**, 114801, 2021. + `DOI:10.1103/PhysRevLett.127.114801 `__ + Particle Accelerator & Beam Physics *********************************** @@ -139,10 +159,25 @@ High Energy Astrophysical Plasma Physics Scientific works in astrophysical plasma modeling. +#. Sam A, Kumar P, Fletcher AC, Crabtree C, Lee N, Elschot S. + **Nonlinear evolution, propagation, electron-trapping, and damping effects of ion-acoustic solitons using fully kinetic PIC simulations**. + Phys. Plasmas **32** 022103, 2025 + `DOI:10.1063/5.0249525 `__ + +#. Jambunathan R, Jones H, Corrales L, Klion H, Roward ME, Myers A, Zhang W, Vay J-L. + **Application of mesh refinement to relativistic magnetic reconnection**. + Physics of Plasmas ***32*** 1, 2025 + `DOI:10.1063/5.0233583 `__ + +#. Ghosh S, Bhat P. + **Magnetic Reconnection: An Alternative Explanation of Radio Emission in Galaxy Clusters**. + The Astrophysical Journal Letters **979** 1, 2025. + `DOI:10.3847/2041-8213/ad9f2d `__ + #. Klion H, Jambunathan R, Rowan ME, Yang E, Willcox D, Vay J-L, Lehe R, Myers A, Huebl A, Zhang W. **Particle-in-Cell simulations of relativistic magnetic reconnection with advanced Maxwell solver algorithms**. - arXiv pre-print, 2023. - `DOI:10.48550/arXiv.2304.10566 `__ + The Astrophysical Journal **952** 8, 2023. + `DOI:10.3847/1538-4357/acd75b `__ Microelectronics @@ -173,8 +208,31 @@ Scientific works in High-Performance Computing, applied mathematics and numerics Please see :ref:`this section `. -Nuclear Fusion - Magnetically Confined Plasmas -********************************************** +Related works using WarpX: + +#. Yan Y., Du F., Tang J., Yu D and Zhao Y., + **Numerical study on wave attenuation via 1D fully kinetic electromagnetic particle-in-cell simulations**. + Plasma Sources Sci. Technol. **33** 115013, 2024 + `DOI:10.1088/1361-6595/ad8c7c `__ + + +Nuclear Fusion and Plasma Confinement +************************************* + +#. Tyushev M., Papahn Zadeh M., Chopra N. S., Raitses Y., Romadanov I., Likhanskii A., Fubiani G., Garrigues L., Groenewald R. and Smolyakov A. + **Mode transitions and spoke structures in E×B Penning discharge**. + Physics of Plasmas **32**, 013511, 2025. + `DOI:10.1063/5.0238577 `__ + +#. Scheffel J. and Jäderberg J. and Bendtz K. and Holmberg R. and Lindvall K., + **Axial Confinement in the Novatron Mirror Machine**. + arXiv 2410.20134 + `DOI:10.48550/arXiv.2410.20134 `__ + +#. Affolter M., Thompson R., Hepner S., Hayes E. C., Podolsky V., Borghei M., Carlsson J., Gargone A., Merthe D., McKee E., Langtry R., + **The Orbitron: A crossed-field device for co-confinement of high energy ions and electrons**. + AIP Advances **14**, 085025, 2024. + `DOI:10.1063/5.0201470 `__ #. Nicks B. S., Putvinski S. and Tajima T. **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**. @@ -185,3 +243,16 @@ Nuclear Fusion - Magnetically Confined Plasmas **Accelerated kinetic model for global macro stability studies of high-beta fusion reactors**. Physics of Plasmas **30**, 122508, 2023. `DOI:10.1063/5.0178288 `__ + +Plasma Thrusters and Electric Propulsion +**************************************** + +#. Xie L., Luo X., Zhou Z. and Zhao Y., + **Effect of plasma initialization on 3D PIC simulation of Hall thruster azimuthal instability**. + Physica Scripta, **99**, 095602, 2024. + `DOI:10.1088/1402-4896/ad69e5 `__ + +#. Marks T. A. and Gorodetsky A. A., + **Hall thruster simulations in WarpX**. + 38th International Electric Propulsion Conference, Toulouse, France, 2024. + `DOI:10.7302/234915 `__ diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 89bc2e417cb..dfdeb3b9530 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -3,7 +3,7 @@ WarpX ----- -WarpX is an advanced, time-based, **electromagnetic & electrostatic Particle-In-Cell** code. +WarpX is an advanced **electromagnetic & electrostatic Particle-In-Cell** code. It supports many features including: @@ -111,8 +111,7 @@ Theory theory/amr theory/boundary_conditions theory/boosted_frame - theory/input_output - theory/collisions + theory/multiphysics_extensions theory/kinetic_fluid_hybrid_model theory/cold_fluid_model @@ -127,7 +126,7 @@ Development developers/developers developers/doxygen developers/gnumake - developers/workflows + developers/how_to_guides developers/faq .. good to have in the future: .. developers/repostructure @@ -140,7 +139,6 @@ Maintenance :hidden: maintenance/release - maintenance/performance_tests Epilogue -------- diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index fde927c10e1..fbdc6809853 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -77,18 +77,18 @@ For example, this builds WarpX in all geometries, enables Python bindings and Nv Build Options ------------- -============================= ============================================ ========================================================= +============================= ============================================ =========================================================== CMake Option Default & Values Description -============================= ============================================ ========================================================= +============================= ============================================ =========================================================== ``CMAKE_BUILD_TYPE`` RelWithDebInfo/**Release**/Debug `Type of build, symbols & optimizations `__ ``CMAKE_INSTALL_PREFIX`` system-dependent path `Install path prefix `__ ``CMAKE_VERBOSE_MAKEFILE`` ON/**OFF** `Print all compiler commands to the terminal during build `__ -``PYINSTALLOPTIONS`` Additional options for ``pip install``, e.g., ``-v --user`` ``WarpX_APP`` **ON**/OFF Build the WarpX executable application ``WarpX_ASCENT`` ON/**OFF** Ascent in situ visualization +``WarpX_CATALYST`` ON/**OFF** Catalyst in situ visualization ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WarpX_DIMS`` **3**/2/1/RZ Simulation dimensionality. Use ``"1;2;RZ;3"`` for all. -``WarpX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) +``WarpX_EB`` **ON**/OFF Embedded boundary support (not supported in RZ yet) ``WarpX_IPO`` ON/**OFF** Compile WarpX with interprocedural optimization (aka LTO) ``WarpX_LIB`` ON/**OFF** Build WarpX as a library, e.g., for PICMI Python ``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) @@ -97,14 +97,16 @@ CMake Option Default & Values Descr ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WarpX_FFT`` ON/**OFF** FFT-based solvers -``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) ``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost) ``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation ``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization -============================= ============================================ ========================================================= +``Python_EXECUTABLE`` (newest found) Path to Python executable +``PY_PIP_OPTIONS`` ``-v`` Additional options for ``pip``, e.g., ``-vvv;-q`` +``PY_PIP_INSTALL_OPTIONS`` Additional options for ``pip install``, e.g., ``--user;-q`` +============================= ============================================ =========================================================== WarpX can be configured in further detail with options from AMReX, which are documented in the AMReX manual: @@ -126,7 +128,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.15.2`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``0.16.1`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from @@ -141,6 +143,10 @@ CMake Option Default & Values Des ``WarpX_pybind11_repo`` ``https://github.com/pybind/pybind11.git`` Repository URI to pull and build pybind11 from ``WarpX_pybind11_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_pybind11_repo`` ``WarpX_pybind11_internal`` **ON**/OFF Needs a pre-installed pybind11 library if set to ``OFF`` +``WarpX_TEST_CLEANUP`` ON/**OFF** Clean up automated test directories +``WarpX_TEST_DEBUGGER`` ON/**OFF** Run automated tests without AMReX signal handling (to attach debuggers) +``WarpX_TEST_FPETRAP`` ON/**OFF** Run automated tests with FPE-trapping runtime parameters +``WarpX_BACKTRACE_INFO`` ON/**OFF** Compile with -g1 for minimal debug symbols (currently used in CI tests) ============================= ============================================== =========================================================== For example, one can also build against a local AMReX copy. @@ -239,7 +245,7 @@ Developers could now change the WarpX source code and then call the build line a .. tip:: If you do *not* develop with :ref:`a user-level package manager `, e.g., because you rely on a HPC system's environment modules, then consider to set up a virtual environment via `Python venv `__. - Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPYINSTALLOPTIONS="--user"``. + Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPY_PIP_INSTALL_OPTIONS="--user"``. .. _building-pip-python: @@ -266,13 +272,12 @@ Environment Variable Default & Values Descr ============================= ============================================ ================================================================ ``WARPX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WARPX_DIMS`` ``"1;2;3;RZ"`` Simulation dimensionalities (semicolon-separated list) -``WARPX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) +``WARPX_EB`` **ON**/OFF Embedded boundary support (not supported in RZ yet) ``WARPX_MPI`` ON/**OFF** Multi-node support (message-passing) ``WARPX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WARPX_FFT`` ON/**OFF** FFT-based solvers -``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 3bab32b7502..facaa3a5614 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -7,7 +7,7 @@ WarpX depends on the following popular third party software. Please see installation instructions below. - a mature `C++17 `__ compiler, e.g., GCC 8.4+, Clang 7, NVCC 11.0, MSVC 19.15 or newer -- `CMake 3.20.0+ `__ +- `CMake 3.24.0+ `__ - `Git 2.18+ `__ - `AMReX `__: we automatically download and compile a copy of AMReX - `PICSAR `__: we automatically download and compile a copy of PICSAR @@ -28,17 +28,16 @@ Optional dependencies include: - `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `heFFTe 2.4.0+ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support -- `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support +- `openPMD-api 0.16.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support - see `optional I/O backends `__, i.e., ADIOS2 and/or HDF5 - `Ascent 0.8.0+ `__: for in situ 3D visualization - `SENSEI 4.0.0+ `__: for in situ analysis and visualization - `CCache `__: to speed up rebuilds (For CUDA support, needs version 3.7.9+ and 4.2+ is recommended) - `Ninja `__: for faster parallel compiles -- `Python 3.8+ `__ +- `Python 3.9+ `__ - `mpi4py `__ - `numpy `__ @@ -53,27 +52,26 @@ For all other systems, we recommend to use a **package dependency manager**: Pick *one* of the installation methods below to install all dependencies for WarpX development in a consistent manner. -Conda (Linux/macOS/Windows) ---------------------------- +Conda-Forge (Linux/macOS/Windows) +--------------------------------- -`Conda `__/`Mamba `__ are cross-compatible, user-level package managers. +`Conda-Forge `__ is a repository for cross-compatible, user-level packages. .. tip:: - We recommend to configure your conda to use the faster ``libmamba`` `dependency solver `__. + We recommend to deactivate that conda self-activates its ``base`` environment. + This `avoids interference with the system and other package managers `__. .. code-block:: bash - conda update -y -n base conda - conda install -y -n base conda-libmamba-solver - conda config --set solver libmamba + conda config --set auto_activate_base false - We recommend to deactivate that conda self-activates its ``base`` environment. - This `avoids interference with the system and other package managers `__. + In order to make sure that the conda configuration uses ``conda-forge`` as the only channel, which will help avoid issues with blocked ``defaults`` or ``anaconda`` repositories, please set the following configurations: .. code-block:: bash - conda config --set auto_activate_base false + conda config --add channels conda-forge + conda config --set channel_priority strict .. tab-set:: @@ -81,7 +79,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer packaging pytest python python-build make numpy pandas scipy setuptools yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv wheel conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -91,7 +89,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api openpmd-viewer python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api openpmd-viewer packaging pytest python python-build make numpy pandas scipy setuptools yt fftw pkg-config matplotlib mamba ninja pip virtualenv wheel conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF @@ -105,19 +103,19 @@ For OpenMP support, you will further need: .. code-block:: bash - conda install -c conda-forge libgomp + mamba install -c conda-forge libgomp .. tab-item:: macOS or Windows .. code-block:: bash - conda install -c conda-forge llvm-openmp + mamba install -c conda-forge llvm-openmp For Nvidia CUDA GPU support, you will need to have `a recent CUDA driver installed `__ or you can lower the CUDA version of `the Nvidia cuda package `__ and `conda-forge to match your drivers `__ and then add these packages: .. code-block:: bash - conda install -c nvidia -c conda-forge cuda cupy + mamba install -c nvidia -c conda-forge cuda cuda-nvtx-dev cupy More info for `CUDA-enabled ML packages `__. @@ -228,7 +226,7 @@ The `Advanced Package Tool (APT) ` .. code-block:: bash sudo apt update - sudo apt install build-essential ccache cmake g++ git libfftw3-mpi-dev libfftw3-dev libhdf5-openmpi-dev libopenmpi-dev pkg-config python3 python3-matplotlib python3-mpi4py python3-numpy python3-pandas python3-pip python3-scipy python3-venv + sudo apt install build-essential ccache cmake g++ git libfftw3-mpi-dev libfftw3-dev libhdf5-openmpi-dev libopenmpi-dev pkg-config python3 python3-dev python3-matplotlib python3-mpi4py python3-numpy python3-pandas python3-pip python3-scipy python3-venv # optional: # for CUDA, either install @@ -244,7 +242,7 @@ The `Advanced Package Tool (APT) ` .. code-block:: bash sudo apt update - sudo apt install build-essential ccache cmake g++ git libfftw3-dev libfftw3-dev libhdf5-dev pkg-config python3 python3-matplotlib python3-numpy python3-pandas python3-pip python3-scipy python3-venv + sudo apt install build-essential ccache cmake g++ git libfftw3-dev libfftw3-dev libhdf5-dev pkg-config python3 python3-dev python3-matplotlib python3-numpy python3-pandas python3-pip python3-scipy python3-venv # optional: # for CUDA, either install diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 5b388b9d0b2..61e60359e59 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -43,16 +43,17 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lassen hpc/lawrencium hpc/leonardo + hpc/lonestar6 hpc/lumi hpc/lxplus hpc/ookami hpc/perlmutter hpc/pitzer hpc/polaris - hpc/quartz - hpc/spock + hpc/dane hpc/summit hpc/taurus + hpc/tioga .. tip:: diff --git a/Docs/source/install/hpc/dane.rst b/Docs/source/install/hpc/dane.rst new file mode 100644 index 00000000000..9c3c9077df5 --- /dev/null +++ b/Docs/source/install/hpc/dane.rst @@ -0,0 +1,167 @@ +.. _building-dane: + +Dane (LLNL) +============= + +The `Dane Intel CPU cluster `__ is located at LLNL. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `LLNL user account `__ (login required) +* `Jupyter service `__ (`documentation `__, login required) +* `Production directories `__: + + * ``/p/lustre1/$(whoami)`` and ``/p/lustre2/$(whoami)``: personal directory on the parallel filesystem + * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation. + + +.. _building-dane-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code. +Note that these commands and the shell scripts all assume the bash shell. + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/dane/src/warpx + +We use system software modules, add environment hints and further dependencies via the file ``$HOME/dane_warpx.profile``. +Create it now: + +.. code-block:: bash + + cp /usr/workspace/${USER}/dane/src/warpx/Tools/machines/dane-llnl/dane_warpx.profile.example $HOME/dane_warpx.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/dane-llnl/dane_warpx.profile.example + :language: bash + +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``tps``, then run ``vi $HOME/dane_warpx.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: + +.. code-block:: bash + + export proj="tps" + +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + +.. important:: + + Now, and as the first step on future logins to Dane, activate these environment settings: + + .. code-block:: bash + + source $HOME/dane_warpx.profile + +Finally, since Dane does not yet provide software modules for some of our dependencies, install them once: + +.. code-block:: bash + + bash /usr/workspace/${USER}/dane/src/warpx/Tools/machines/dane-llnl/install_dependencies.sh + source /usr/workspace/${USER}/dane/venvs/warpx-dane/bin/activate + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/dane-llnl/install_dependencies.sh + :language: bash + + +.. _building-dane-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd /usr/workspace/${USER}/dane/src/warpx + rm -rf build_dane + + cmake -S . -B build_dane -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_dane -j 6 + +The WarpX application executables are now in ``/usr/workspace/${USER}/dane/src/warpx/build_dane/bin/``. +Additionally, the following commands will install WarpX as a Python module: + +.. code-block:: bash + + rm -rf build_dane_py + + cmake -S . -B build_dane_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_dane_py -j 6 --target pip_install + +Now, you can :ref:`submit Dane compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Dane jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PROJWORK/$proj/``. + + +.. _building-dane-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd /usr/workspace/${USER}/dane/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the dane_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/dane/src/warpx/build_dane`` and rebuild WarpX. + + +.. _running-cpp-dane: + +Running +------- + +.. _running-cpp-dane-CPUs: + +Intel Sapphire Rapids CPUs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The batch script below can be used to run a WarpX simulation on 2 nodes on the supercomputer Dane at LLNL. +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + +.. literalinclude:: ../../../../Tools/machines/dane-llnl/dane.sbatch + :language: bash + :caption: You can copy this file from ``Tools/machines/dane-llnl/dane.sbatch``. + +To run a simulation, copy the lines above to a file ``dane.sbatch`` and run + +.. code-block:: bash + + sbatch dane.sbatch + +to submit the job. diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst index 2217c5a31ce..f163531a29a 100644 --- a/Docs/source/install/hpc/lawrencium.rst +++ b/Docs/source/install/hpc/lawrencium.rst @@ -69,7 +69,7 @@ And since Lawrencium does not yet provide a module for them, install ADIOS2, BLA cmake -S src/lapackpp -B src/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dgpu_backend=cuda -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/v100/lapackpp-master -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${LAPACK_DIR}/lib/libblas.a -DLAPACK_LIBRARIES=${LAPACK_DIR}/lib/liblapack.a cmake --build src/lapackpp-v100-build --target install --parallel 12 -Optionally, download and install Python packages for :ref:`PICMI ` or dynamic ensemble optimizations (:ref:`libEnsemble `): +Optionally, download and install Python packages for :ref:`PICMI ` or dynamic ensemble optimizations (`libEnsemble `__): .. code-block:: bash diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 568a5612250..bfd584288fe 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -65,7 +65,7 @@ Finally, since Leonardo does not yet provide software modules for some of our de .. code-block:: bash - bash $HOME/src/warpx/Tools/machines/leonardo_cineca/install_gpu_dependencies.sh + bash $HOME/src/warpx/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh source $HOME/sw/venvs/warpx/bin/activate .. dropdown:: Script Details diff --git a/Docs/source/install/hpc/lonestar6.rst b/Docs/source/install/hpc/lonestar6.rst new file mode 100644 index 00000000000..f1512e4a508 --- /dev/null +++ b/Docs/source/install/hpc/lonestar6.rst @@ -0,0 +1,139 @@ +.. _building-lonestar6: + +Lonestar6 (TACC) +================ + +The `Lonestar6 cluster `_ is located at `TACC `__. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `TACC user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ +* `Filesystem directories `__: + + * ``$HOME``: per-user home directory, backed up (10 GB) + * ``$WORK``: per-user production directory, not backed up, not purged, Lustre (1 TB) + * ``$SCRATCH``: per-user production directory, not backed up, purged every 10 days, Lustre (no limits, 8PByte total) + + +Installation +------------ + +Use the following commands to download the WarpX source code and switch to the correct branch: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx + +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lonestar6_warpx_a100.profile``. +Create it now: + +.. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example $HOME/lonestar6_warpx_a100.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example + :language: bash + +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``abcde``, then run ``nano $HOME/lonestar6_warpx_a100.profile`` and edit line 2 to read: + +.. code-block:: bash + + export proj="abcde" + +Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + +.. important:: + + Now, and as the first step on future logins to Lonestar6, activate these environment settings: + + .. code-block:: bash + + source $HOME/lonestar6_warpx_a100.profile + +Finally, since Lonestar6 does not yet provide software modules for some of our dependencies, install them once: + +.. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh + source ${SW_DIR}/venvs/warpx-a100/bin/activate + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/install_a100_dependencies.sh + :language: bash + + +.. _building-lonestar6-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu + + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 16 + +The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: + +.. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu_py + + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 16 --target pip_install + +Now, you can :ref:`submit Lonestar6 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Lonestar6 jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$WORK`` or ``$SCRATCH``. + + +.. _running-cpp-lonestar6: + +Running +------- + +.. _running-cpp-lonestar6-A100-GPUs: + +A100 GPUs (40 GB) +^^^^^^^^^^^^^^^^^ + +`84 GPU nodes, each with 2 A100 GPUs (40 GB) `__. + +The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer lonestar6 at tacc. +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. +Note that we run one MPI rank per GPU. + + +.. literalinclude:: ../../../../Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch + :language: bash + :caption: You can copy this file from ``Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch``. + +To run a simulation, copy the lines above to a file ``lonestar6.sbatch`` and run + +.. code-block:: bash + + sbatch lonestar6_a100.sbatch + +to submit the job. diff --git a/Docs/source/install/hpc/lxplus.rst b/Docs/source/install/hpc/lxplus.rst index 5bd8a3c9795..fc1e5d4286d 100644 --- a/Docs/source/install/hpc/lxplus.rst +++ b/Docs/source/install/hpc/lxplus.rst @@ -44,7 +44,7 @@ The easiest way to install the dependencies is to use the pre-prepared ``warpx.p .. code-block:: bash - cp $WORK/warpx/WarpX/Tools/machines/lxplus-cern/lxplus_warpx.profile.example $WORK/lxplus_warpx.profile + cp $WORK/warpx/Tools/machines/lxplus-cern/lxplus_warpx.profile.example $WORK/lxplus_warpx.profile source $WORK/lxplus_warpx.profile When doing this one can directly skip to the :ref:`Building WarpX ` section. diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index dc5a985e99f..7e2ae31630e 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -76,7 +76,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate + source ${PSCRATCH}/storage/sw/warpx/perlmutter/gpu/venvs/warpx-gpu/bin/activate .. dropdown:: Script Details :color: light @@ -126,7 +126,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate + source ${PSCRATCH}/storage/sw/warpx/perlmutter/cpu/venvs/warpx-cpu/bin/activate .. dropdown:: Script Details :color: light @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst deleted file mode 100644 index a49327e8613..00000000000 --- a/Docs/source/install/hpc/quartz.rst +++ /dev/null @@ -1,168 +0,0 @@ -.. _building-quartz: - -Quartz (LLNL) -============= - -The `Quartz Intel CPU cluster `_ is located at LLNL. - - -Introduction ------------- - -If you are new to this system, **please see the following resources**: - -* `LLNL user account `__ (login required) -* `Quartz user guide `_ -* Batch system: `Slurm `_ -* `Jupyter service `__ (`documentation `__, login required) -* `Production directories `_: - - * ``/p/lustre1/$(whoami)`` and ``/p/lustre2/$(whoami)``: personal directory on the parallel filesystem - * Note that the ``$HOME`` directory and the ``/usr/workspace/$(whoami)`` space are NFS mounted and *not* suitable for production quality data generation. - - -.. _building-quartz-preparation: - -Preparation ------------ - -Use the following commands to download the WarpX source code: - -.. code-block:: bash - - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx - -We use system software modules, add environment hints and further dependencies via the file ``$HOME/quartz_warpx.profile``. -Create it now: - -.. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz_warpx.profile.example $HOME/quartz_warpx.profile - -.. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz_warpx.profile.example - :language: bash - -Edit the 2nd line of this script, which sets the ``export proj=""`` variable. -For example, if you are member of the project ``tps``, then run ``vi $HOME/quartz_warpx.profile``. -Enter the edit mode by typing ``i`` and edit line 2 to read: - -.. code-block:: bash - - export proj="tps" - -Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - -.. important:: - - Now, and as the first step on future logins to Quartz, activate these environment settings: - - .. code-block:: bash - - source $HOME/quartz_warpx.profile - -Finally, since Quartz does not yet provide software modules for some of our dependencies, install them once: - -.. code-block:: bash - - bash $HOME/src/warpx/Tools/machines/quartz-llnl/install_dependencies.sh - source /usr/workspace/${USER}/quartz/venvs/warpx-quartz/bin/activate - -.. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/quartz-llnl/install_dependencies.sh - :language: bash - - -.. _building-quartz-compilation: - -Compilation ------------ - -Use the following :ref:`cmake commands ` to compile the application executable: - -.. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_quartz - - cmake -S . -B build_quartz -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_quartz -j 6 - -The WarpX application executables are now in ``$HOME/src/warpx/build_quartz/bin/``. -Additionally, the following commands will install WarpX as a Python module: - -.. code-block:: bash - - rm -rf build_quartz_py - - cmake -S . -B build_quartz_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_quartz_py -j 6 --target pip_install - -Now, you can :ref:`submit Quartz compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). -Or, you can use the WarpX executables to submit Quartz jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PROJWORK/$proj/``. - - -.. _building-quartz-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the quartz_warpx.profile file `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_quartz`` and rebuild WarpX. - - -.. _running-cpp-quartz: - -Running -------- - -.. _running-cpp-quartz-CPUs: - -Intel Xeon E5-2695 v4 CPUs -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The batch script below can be used to run a WarpX simulation on 2 nodes on the supercomputer Quartz at LLNL. -Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. - -.. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz.sbatch - :language: bash - :caption: You can copy this file from ``Tools/machines/quartz-llnl/quartz.sbatch``. - -To run a simulation, copy the lines above to a file ``quartz.sbatch`` and run - -.. code-block:: bash - - sbatch quartz.sbatch - -to submit the job. diff --git a/Docs/source/install/hpc/spock.rst b/Docs/source/install/hpc/spock.rst deleted file mode 100644 index 144257289f1..00000000000 --- a/Docs/source/install/hpc/spock.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. _building-spock: - -Spock (OLCF) -============= - -The `Spock cluster `_ is located at OLCF. - - -Introduction ------------- - -If you are new to this system, **please see the following resources**: - -* `Spock user guide `_ -* Batch system: `Slurm `_ -* `Production directories `_: - - * ``$PROJWORK/$proj/``: shared with all members of a project (recommended) - * ``$MEMBERWORK/$proj/``: single user (usually smaller quota) - * ``$WORLDWORK/$proj/``: shared with all users - * Note that the ``$HOME`` directory is mounted as read-only on compute nodes. - That means you cannot run in your ``$HOME``. - - -Installation ------------- - -Use the following commands to download the WarpX source code and switch to the correct branch: - -.. code-block:: bash - - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx - -We use the following modules and environments on the system (``$HOME/spock_warpx.profile``). - -.. literalinclude:: ../../../../Tools/machines/spock-olcf/spock_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/spock-olcf/spock_warpx.profile.example``. - -We recommend to store the above lines in a file, such as ``$HOME/spock_warpx.profile``, and load it into your shell after a login: - -.. code-block:: bash - - source $HOME/spock_warpx.profile - - -Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: - -.. code-block:: bash - - cd $HOME/src/warpx - rm -rf build - - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" - cmake --build build -j 10 - -The general :ref:`cmake compile-time options ` apply as usual. - -**That's it!** -A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. -Most people execute the binary directly or copy it out to a location in ``$PROJWORK/$proj/``. - - -.. _running-cpp-spock: - -Running -------- - -.. _running-cpp-spock-MI100-GPUs: - -MI100 GPUs (32 GB) -^^^^^^^^^^^^^^^^^^ - -After requesting an interactive node with the ``getNode`` alias above, run a simulation like this, here using 4 MPI ranks: - -.. code-block:: bash - - srun -n 4 -c 2 --ntasks-per-node=4 ./warpx inputs - -Or in non-interactive runs started with ``sbatch``: - -.. literalinclude:: ../../../../Tools/machines/spock-olcf/spock_mi100.sbatch - :language: bash - :caption: You can copy this file from ``Tools/machines/spock-olcf/spock_mi100.sbatch``. - -We can currently use up to ``4`` nodes with ``4`` GPUs each (maximum: ``-N 4 -n 16``). - - -.. _post-processing-spock: - -Post-Processing ---------------- - -For post-processing, most users use Python via OLCFs's `Jupyter service `__ (`Docs `__). - -Please follow the same guidance as for :ref:`OLCF Summit post-processing `. diff --git a/Docs/source/install/hpc/tioga.rst b/Docs/source/install/hpc/tioga.rst new file mode 100644 index 00000000000..2599d6d8ac4 --- /dev/null +++ b/Docs/source/install/hpc/tioga.rst @@ -0,0 +1,210 @@ +.. _building-tioga: + +Tioga (LLNL) +============ + +The `Tioga AMD GPU cluster `__ is located at LLNL. +It is equipped with two nodes that have each four AMD MI300A APUs. +Tioga is an LLNL El Capitan Early Access System. + +There are also "conventional" MI250X GPUs on Tioga nodes, which we did not yet document. +El Capitan will use MI300A GPUs. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `LLNL user account `__ (login required) +* `Tioga user guide `__ +* Batch system: `Flux with Slurm Wrappers `__ +* `Jupyter service `__ (`documentation `__, login required) +* `Production directories `__: + + * ``/p/lustre1/${USER}``: personal directory on the parallel filesystem (also: ``lustre2``) + * Note that the ``$HOME`` directory and the ``/usr/workspace/${USER}`` space are NFS mounted and *not* suitable for production quality data generation. + + +Login +----- + +.. code-block:: bash + + ssh tioga.llnl.gov + +To use the available MI300A nodes (currently two), request one via + +.. code-block:: bash + + salloc -N 1 -p mi300a -t 30:0 + + +.. _building-tioga-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git /p/lustre1/${USER}/tioga/src/warpx + +We use system software modules, add environment hints and further dependencies via the file ``$HOME/tioga_mi300a_warpx.profile``. +Create it now: + +.. code-block:: bash + + cp /p/lustre1/${USER}/tioga/src/warpx/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example $HOME/tioga_mi300a_warpx.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example + :language: bash + +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +**Currently, this is unused and can be kept empty.** +Once project allocation becomes required, e.g., if you are member of the project ``abcde``, then run ``vi $HOME/tioga_mi300a_warpx.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: + +.. code-block:: bash + + export proj="abcde" + +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + +.. important:: + + Now, and as the first step on future logins to Tioga, activate these environment settings: + + .. code-block:: bash + + source $HOME/tioga_mi300a_warpx.profile + +Finally, since Tioga does not yet provide software modules for some of our dependencies, install them once: + + + .. code-block:: bash + + bash /p/lustre1/${USER}/tioga/src/warpx/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh + source /p/lustre1/${USER}/tioga/warpx/mi300a/gpu/venvs/warpx-trioga-mi300a/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/tioga-llnl/install_mi300a_dependencies.sh + :language: bash + + .. dropdown:: AI/ML Dependencies (Optional) + :animate: fade-in-slide-down + + If you plan to run AI/ML workflows depending on PyTorch et al., run the next step as well. + This will take a while and should be skipped if not needed. + + .. code-block:: bash + + bash /p/lustre1/${USER}/tioga/src/warpx/Tools/machines/tioga-llnl/install_mi300a_ml.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/tioga-llnl/install_mi300a_ml.sh + :language: bash + + +.. _building-tioga-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd /p/lustre1/${USER}/tioga/src/warpx + + cmake --fresh -S . -B build_tioga -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_tioga -j 24 + +The WarpX application executables are now in ``/p/lustre1/${USER}/tioga/src/warpx/build_tioga/bin/``. +Additionally, the following commands will install WarpX as a Python module: + +.. code-block:: bash + + cmake --fresh -S . -B build_tioga_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_tioga_py -j 24 --target pip_install + +Now, you can :ref:`submit tioga compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit tioga jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PROJWORK/$proj/``. + + +.. _building-tioga-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd /p/lustre1/${USER}/tioga/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the tioga_mi300a_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step :ref:`rebuild WarpX `. + + +.. _running-cpp-tioga: + +Running +------- + +.. _running-cpp-tioga-MI300A-APUs: + +MI300A APUs (128GB) +^^^^^^^^^^^^^^^^^^^ + +The batch script below can be used to run a WarpX simulation on 1 node with 4 APUs on the supercomputer Tioga at LLNL. +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. +WarpX runs with one MPI rank per GPU. + +Note that we append these non-default runtime options: + +* ``amrex.use_gpu_aware_mpi=1``: make use of fast APU to APU MPI communications +* ``amrex.the_arena_init_size=1``: avoid overallocating memory that is *shared* on APUs between CPU & GPU + +.. literalinclude:: ../../../../Tools/machines/tioga-llnl/tioga_mi300a.sbatch + :language: bash + :caption: You can copy this file from ``Tools/machines/tioga-llnl/tioga_mi300a.sbatch``. + +To run a simulation, copy the lines above to a file ``tioga_mi300a.sbatch`` and run + +.. code-block:: bash + + sbatch tioga_mi300a.sbatch + +to submit the job. diff --git a/Docs/source/install/users.rst b/Docs/source/install/users.rst index e56d2d8ac43..650cbacd4d0 100644 --- a/Docs/source/install/users.rst +++ b/Docs/source/install/users.rst @@ -38,36 +38,35 @@ If want to use WarpX on a specific high-performance computing (HPC) systems, jum .. image:: conda.svg -Using the Conda Package ------------------------ +Using the Conda-Forge Package +----------------------------- -A package for WarpX is available via the `Conda `_ package manager. +A package for WarpX is available via `Conda-Forge `__. .. tip:: - We recommend to configure your conda to use the faster ``libmamba`` `dependency solver `__. + We recommend to deactivate that conda self-activates its ``base`` environment. + This `avoids interference with the system and other package managers `__. .. code-block:: bash - conda update -y -n base conda - conda install -y -n base conda-libmamba-solver - conda config --set solver libmamba + conda config --set auto_activate_base false - We recommend to deactivate that conda self-activates its ``base`` environment. - This `avoids interference with the system and other package managers `__. + In order to make sure that the conda configuration uses ``conda-forge`` as the only channel, which will help avoid issues with blocked ``defaults`` or ``anaconda`` repositories, please set the following configurations: .. code-block:: bash - conda config --set auto_activate_base false + conda config --add channels conda-forge + conda config --set channel_priority strict .. code-block:: bash - conda create -n warpx -c conda-forge warpx - conda activate warpx + mamba create -n warpx -c conda-forge warpx + mamba activate warpx .. note:: - The ``warpx`` `conda package `__ does not yet provide GPU support. + The ``warpx`` package on conda-forge does not yet provide `GPU support `__. .. _install-spack: @@ -80,7 +79,7 @@ Using the Spack Package ----------------------- Packages for WarpX are available via the `Spack `__ package manager. -The package ``warpx`` installs executables and the package ``py-warpx`` includes Python bindings, i.e. `PICMI `_. +The package ``warpx`` installs executables and the variant ``warpx +python`` also includes Python bindings, i.e. `PICMI `__. .. code-block:: bash @@ -89,11 +88,11 @@ The package ``warpx`` installs executables and the package ``py-warpx`` includes spack buildcache keys --install --trust # see `spack info py-warpx` for build options. - # optional arguments: -mpi ^warpx dims=2 compute=cuda - spack install py-warpx - spack load py-warpx + # optional arguments: -mpi compute=cuda + spack install warpx +python + spack load warpx +python -See ``spack info warpx`` or ``spack info py-warpx`` and `the official Spack tutorial `__ for more information. +See ``spack info warpx`` and `the official Spack tutorial `__ for more information. .. _install-pypi: diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index b3475c5a81b..e44ab5cf112 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -2187,3 +2187,60 @@ @book{godfrey1985iprop title = {{The IPROP Three-Dimensional Beam Propagation Code}}, year = {1985} } + +@article{shapovalPRE2024, +author = {Shapoval, Olga and Zoni, Edoardo and Lehe, Remi and Thevenet, Maxence and Vay, Jean-Luc}, +doi = {10.1103/PhysRevE.110.025206}, +issue = {2}, +journal = {Phys. Rev. E}, +month = {Aug}, +numpages = {19}, +pages = {025206}, +publisher = {American Physical Society}, +title = {{Pseudospectral particle-in-cell formulation with arbitrary charge and current-density time dependencies for the modeling of relativistic plasmas}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.110.025206}, +volume = {110}, +year = {2024} +} + +@article{Ammosov1986, +title = {Tunnel ionization of complex atoms and of atomic ions in an alternating electromagnetic field}, +volume = {64}, +issn = {0044-4510}, +doi = {10.1117/12.938695}, +number = {December 1986}, +journal = {Sov. Phys. JETP}, +author = {Ammosov, M. V. and Delone, N. B. and Krainov, V. P.}, +year = {1986}, +pmid = {22232002}, +note = {ISBN: 0892526998}, +pages = {1191--1194}, +} + + +@article{zhang_empirical_2014, +title = {Empirical formula for over-barrier strong-field ionization}, +volume = {90}, +issn = {1050-2947, 1094-1622}, +doi = {10.1103/PhysRevA.90.043410}, +language = {en}, +number = {4}, +journal = {Physical Review A}, +author = {Zhang, Qingbin and Lan, Pengfei and Lu, Peixiang}, +month = oct, +year = {2014}, +pages = {043410}, +} + + +@book{Mulser2010, +title = {High {Power} {Laser}-{Matter} {Interaction}}, +volume = {238}, +isbn = {978-3-540-50669-0}, +publisher = {Springer Berlin Heidelberg}, +author = {Mulser, Peter and Bauer, Dieter}, +year = {2010}, +pmid = {25246403}, +doi = {10.1007/978-3-540-46065-7}, +note = {Series Title: Springer Tracts in Modern Physics}, +} diff --git a/Docs/source/maintenance/performance_tests.rst b/Docs/source/maintenance/performance_tests.rst deleted file mode 100644 index 8b902297ccb..00000000000 --- a/Docs/source/maintenance/performance_tests.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. _developers-performance_tests: - -Automated performance tests -=========================== - -WarpX has automated performance test scripts, which run weak scalings for various tests on a weekly basis. The results are stored in the `perf_logs repo `_ and plots of the performance history can be found on `this page `_. - -These performance tests run automatically, so they need to do ``git`` operations etc. For this reason, they need a separate clone of the source repos, so they don't conflict with one's usual operations. This is typically in a sub-directory in the ``$HOME``, with variable ``$AUTOMATED_PERF_TESTS`` pointing to it. Similarly, a directory is needed to run the simulations and store the results. By default, it is ``$SCRATCH/performance_warpx``. - -The test runs a weak scaling (1,2,8,64,256,512 nodes) for 6 different tests ``Tools/PerformanceTests/automated_test_{1,2,3,4,5,6}_*``, gathered in 1 batch job per number of nodes to avoid submitting too many jobs. - -Setup on Summit @ OLCF ----------------------- - -Here is an example setup for Summit: - -.. code-block:: sh - - # I put the next three lines in $HOME/my_bashrc.sh - export proj=aph114 # project for job submission - export AUTOMATED_PERF_TESTS=$HOME/AUTOMATED_PERF_TESTS/ - export SCRATCH=/gpfs/alpine/scratch/$(whoami)/$proj/ - - mkdir $HOME/AUTOMATED_PERF_TESTS - cd $AUTOMATED_PERF_TESTS - git clone https://github.com/ECP-WarpX/WarpX.git warpx - git clone https://github.com/ECP-WarpX/picsar.git - git clone https://github.com/AMReX-Codes/amrex.git - git clone https://github.com/ECP-WarpX/perf_logs.git - -Then, in ``$AUTOMATED_PERF_TESTS``, create a file ``run_automated_performance_tests_512.sh`` with the following content: - -.. code-block:: sh - - #!/bin/bash -l - #BSUB -P APH114 - #BSUB -W 00:15 - #BSUB -nnodes 1 - #BSUB -J PERFTEST - #BSUB -e err_automated_tests.txt - #BSUB -o out_automated_tests.txt - - module load nano - module load cmake/3.20.2 - module load gcc/9.3.0 - module load cuda/11.0.3 - module load blaspp/2021.04.01 - module load lapackpp/2021.04.00 - module load boost/1.76.0 - module load adios2/2.7.1 - module load hdf5/1.12.2 - - module unload darshan-runtime - - export AMREX_CUDA_ARCH=7.0 - export CC=$(which gcc) - export CXX=$(which g++) - export FC=$(which gfortran) - export CUDACXX=$(which nvcc) - export CUDAHOSTCXX=$(which g++) - - # Make sure all dependencies are installed and loaded - cd $HOME - module load python/3.8.10 - module load freetype/2.10.4 # matplotlib - module load openblas/0.3.5-omp - export BLAS=$OLCF_OPENBLAS_ROOT/lib/libopenblas.so - export LAPACK=$OLCF_OPENBLAS_ROOT/lib/libopenblas.so - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - python3 -m venv $HOME/sw/venvs/warpx-perftest - source $HOME/sw/venvs/warpx-perftest/bin/activate - # While setting up the performance tests for the first time, - # execute the lines above this comment and then the commented - # lines below this comment once, before submission. - # The commented lines take too long for the job script. - #python3 -m pip install --upgrade pip - #python3 -m pip install --upgrade build packaging setuptools wheel - #python3 -m pip install --upgrade cython - #python3 -m pip install --upgrade numpy - #python3 -m pip install --upgrade markupsafe - #python3 -m pip install --upgrade pandas - #python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself - #python3 -m pip install --upgrade bokeh - #python3 -m pip install --upgrade gitpython - #python3 -m pip install --upgrade tables - - # Run the performance test suite - cd $AUTOMATED_PERF_TESTS/warpx/Tools/PerformanceTests/ - python run_automated.py --n_node_list='1,2,8,64,256,512' --automated - - # submit next week's job - cd $AUTOMATED_PERF_TESTS/ - next_date=`date -d "+7 days" '+%Y:%m:%d:%H:%M'` - bsub -b $next_date ./run_automated_performance_tests_512.sh - -Then, running - -.. code-block:: sh - - bsub run_automated_performance_tests_512.sh - -will submit this job once, and all the following ones. It will: - - - Create directory ``$SCRATCH/performance_warpx`` if doesn't exist. - - Create 1 sub-directory per week per number of nodes (1,2,8,64,256,512). - - Submit one job per number of nodes. It will run 6 different tests, each twice (to detect fluctuations). - - Submit an analysis job, that will read the results ONLY AFTER all runs are finished. This uses the dependency feature of the batch system. - - This job reads the Tiny Profiler output for each run, and stores the results in a pandas file at the hdf5 format. - - Execute ``write_csv.py`` from the ``perf_logs`` repo to append a csv and a hdf5 file with the new results. - - Commit the results (but DO NOT PUSH YET) - -Then, the user periodically has to - -.. code-block:: sh - - cd $AUTOMATED_PERF_TESTS/perf_logs - git pull # to get updates from someone else, or from another supercomputer - git push - -This will update the database but not the online plots. For this, you need to periodically run something like - -.. code-block:: sh - - cd $AUTOMATED_PERF_TESTS/perf_logs - git pull - python generate_index_html.py - git add -u - git commit -m "upload new html page" - git push - -Setup on Cori @ NERSC ---------------------- - -Still to be written! diff --git a/Docs/source/maintenance/release.rst b/Docs/source/maintenance/release.rst index f25b8c313e4..9c6dbcc3f82 100644 --- a/Docs/source/maintenance/release.rst +++ b/Docs/source/maintenance/release.rst @@ -28,7 +28,7 @@ In order to create a GitHub release, you need to: 1. Create a new branch from ``development`` and update the version number in all source files. We usually wait for the AMReX release to be tagged first, then we also point to its tag. - There is a script for updating core dependencies of WarpX and the WarpX version: + There are scripts for updating core dependencies of WarpX and the WarpX version: .. code-block:: sh @@ -42,6 +42,9 @@ In order to create a GitHub release, you need to: Then open a PR, wait for tests to pass and then merge. + The maintainer script ``Tools/Release/releasePR.py`` automates the steps above. + Please read through the instructions in the script before running. + 2. **Local Commit** (Optional): at the moment, ``@ax3l`` is managing releases and signs tags (naming: ``YY.MM``) locally with his GPG key before uploading them to GitHub. **Publish**: On the `GitHub Release page `__, create a new release via ``Draft a new release``. diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 130e0ce4da7..49f4658af4c 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -35,6 +35,17 @@ @ARTICLE{Birdsall1991 year = {1991} } +@misc{Janssen2016 +author = {Janssen, J. F. J. and Pitchford L. C. and Hagelaar G. J. M. and van Dijk J.}, +doi = {10.1088/0963-0252/25/5/055026}, +journal = {Plasma Sources Science and Technology}, +number = {5}, +pages = {055026}, +title = {{Evaluation of angular scattering models for electron-neutral collisions in Monte Carlo simulations}}, +volume = {25}, +year = {2016} +} + @misc{Lim2007, author = {Lim, Chul-Hyun}, issn = {0419-4217}, @@ -444,3 +455,76 @@ @article{Vranic2015 issn = {0010-4655}, doi = {https://doi.org/10.1016/j.cpc.2015.01.020}, } + +@misc{Fallahi2020, + title={MITHRA 2.0: A Full-Wave Simulation Tool for Free Electron Lasers}, + author={Arya Fallahi}, + year={2020}, + eprint={2009.13645}, + archivePrefix={arXiv}, + primaryClass={physics.acc-ph}, + url={https://arxiv.org/abs/2009.13645}, +} + +@article{VayFELA2009, + title = {FULL ELECTROMAGNETIC SIMULATION OF FREE-ELECTRON LASER AMPLIFIER PHYSICS VIA THE LORENTZ-BOOSTED FRAME APPROACH}, + author = {Fawley, William M and Vay, Jean-Luc}, + journal = {}, + abstractNote = {Numerical simulation of some systems containing charged particles with highly relativistic directed motion can by speeded up by orders of magnitude by choice of the proper Lorentz-boosted frame[1]. A particularly good example is that of short wavelength free-electron lasers (FELs) in which a high energy electron beam interacts with a static magnetic undulator. In the optimal boost frame with Lorentz factor gamma_F , the red-shifted FEL radiation and blue shifted undulator have identical wavelengths and the number of required time-steps (presuming the Courant condition applies) decreases by a factor of 2(gamma_F)**2 for fully electromagnetic simulation. We have adapted the WARP code [2]to apply this method to several FEL problems involving coherent spontaneous emission (CSE) from pre-bunched ebeams, including that in a biharmonic undulator.}, + url = {https://www.osti.gov/biblio/964405}, + place = {United States}, + year = {2009}, + month = {4}, +} + +@article{VayFELB2009, + author = {Fawley, W. M. and Vay, J.‐L.}, + title = "{Use of the Lorentz‐Boosted Frame Transformation to Simulate Free‐Electron Laser Amplifier Physics}", + journal = {AIP Conference Proceedings}, + volume = {1086}, + number = {1}, + pages = {346-350}, + year = {2009}, + month = {01}, + abstract = "{Recently [1] it has been pointed out that numerical simulation of some systems containing charged particles with highly relativistic directed motion can by speeded up by orders of magnitude by choice of the proper Lorentz boosted frame. A particularly good example is that of short wavelength free‐electron lasers (FELs) in which a high energy (E0⩾250 MeV) electron beam interacts with a static magnetic undulator. In the optimal boost frame with Lorentz factor γF, the red‐shifted FEL radiation and blue shifted undulator have identical wavelengths and the number of required time‐steps (presuming the Courant condition applies) decreases by a factor of γF2 for fully electromagnetic simulation.We have adapted the WARP code [2] to apply this method to several FEL problems including coherent spontaneous emission (CSE) from pre‐bunched e‐beams, and strong exponential gain in a single pass amplifier configuration. We discuss our results and compare with those from the “standard” FEL simulation approach which adopts the eikonal approximation for propagation of the radiation field.}", + issn = {0094-243X}, + doi = {10.1063/1.3080930}, + url = {https://doi.org/10.1063/1.3080930}, +} + +@article{Barnes2021, + title = {Improved C1 shape functions for simplex meshes}, + author = {D.C. Barnes}, + journal = {Journal of Computational Physics}, + volume = {424}, + pages = {109852}, + year = {2021}, + issn = {0021-9991}, + doi = {https://doi.org/10.1016/j.jcp.2020.109852}, + url = {https://www.sciencedirect.com/science/article/pii/S0021999120306264}, +} + +@article{Rhee1987, + author = {Rhee, M. J. and Schneider, R. F. and Weidman, D. J.}, + title = "{Simple time‐resolving Thomson spectrometer}", + journal = {Review of Scientific Instruments}, + volume = {58}, + number = {2}, + pages = {240-244}, + year = {1987}, + month = {02}, + issn = {0034-6748}, + doi = {10.1063/1.1139314}, + url = {https://doi.org/10.1063/1.1139314}, + eprint = {https://pubs.aip.org/aip/rsi/article-pdf/58/2/240/19154912/240\_1\_online.pdf}, +} + +@misc{holmstrom2013handlingvacuumregionshybrid, + title={Handling vacuum regions in a hybrid plasma solver}, + author={M. Holmstrom}, + year={2013}, + eprint={1301.0272}, + archivePrefix={arXiv}, + primaryClass={physics.space-ph}, + url={https://arxiv.org/abs/1301.0272}, +} diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index ea1f662bd30..04c30bedc98 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -12,7 +12,21 @@ The simulations of plasma accelerators from first principles are extremely compu A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. -Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). +Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. +As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. +This is a result of the properties of Lorentz contraction and dilation of space and time. +In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. +As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity) +The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). + +.. note:: + + For additional reading on inputs and outputs in boosted frame simulations, consider the following pages: + + .. toctree:: + :maxdepth: 1 + + boosted_frame/input_output The modeling of a plasma acceleration stage in a boosted frame involves the fully electromagnetic modeling of a plasma propagating at near the speed of light, for which Numerical Cerenkov diff --git a/Docs/source/theory/Input_output.png b/Docs/source/theory/boosted_frame/Input_output.png similarity index 100% rename from Docs/source/theory/Input_output.png rename to Docs/source/theory/boosted_frame/Input_output.png diff --git a/Docs/source/theory/input_output.rst b/Docs/source/theory/boosted_frame/input_output.rst similarity index 99% rename from Docs/source/theory/input_output.rst rename to Docs/source/theory/boosted_frame/input_output.rst index 21a5f5c8d2c..f47915116df 100644 --- a/Docs/source/theory/input_output.rst +++ b/Docs/source/theory/boosted_frame/input_output.rst @@ -1,4 +1,4 @@ -.. _theory-io: +.. _boosted_frame-io: Inputs and Outputs ================== diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst index 395b072ccbe..d8b3de40c11 100644 --- a/Docs/source/theory/boundary_conditions.rst +++ b/Docs/source/theory/boundary_conditions.rst @@ -301,3 +301,23 @@ the right boundary is reflecting. .. bibliography:: :keyprefix: bc- + +.. _theory-bc-pmc: + +Perfect Magnetic Conductor +---------------------------- + +This boundary can be used to model a symmetric surface, where charges and current are +symmetric across the boundary. +This is equivalent to the Neumann (zero-derivative) boundary condition. +For the electromagnetic solve, at PMC, the tangential magnetic field and the normal electric +field are odd across the boundary and set to 0 on the boundary. +In the guard-cell region, those fields are set equal and +opposite to the respective field component in the mirror location across the PMC boundary. +The other components, the normal magnetic field and tangential electric field, are even +and set equal to the field component in the mirror location in the domain across the PMC boundary. + +The PMC boundary condition also impacts the deposition of charge and current density. +The charge and current densities deposited into the guard cells are reflected back into +the domain, adding them to the mirror cells in the domain. +This represents the charge and current from the virtual symmetric particles in the guard cells. diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index b4f494d8382..f764ce4e02b 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -46,7 +46,7 @@ integrating over velocity), also called the generalized Ohm's law, is given by: .. math:: - en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e + en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e where :math:`\vec{U}_e = \vec{J}_e/(en_e)` is the electron fluid velocity, :math:`{\overleftrightarrow P}_e` is the electron pressure tensor and @@ -64,7 +64,7 @@ Plugging this back into the generalized Ohm' law gives: \left(en_e +\frac{m}{e\mu_0}\nabla\times\nabla\times\right)\vec{E} =& - \frac{m}{e}\left( \frac{\partial\vec{J}_{ext}}{\partial t} + \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t} \right) \\ - &+ \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. + &+ \frac{m}{e}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above equation simplifies to, diff --git a/Docs/source/theory/collisions.rst b/Docs/source/theory/multiphysics/collisions.rst similarity index 90% rename from Docs/source/theory/collisions.rst rename to Docs/source/theory/multiphysics/collisions.rst index 52e36521125..1c7593a0e4e 100644 --- a/Docs/source/theory/collisions.rst +++ b/Docs/source/theory/multiphysics/collisions.rst @@ -1,4 +1,4 @@ -.. _theory-collisions: +.. _multiphysics-collisions: Collisions ========== @@ -8,7 +8,7 @@ including collisions between kinetic particles (Coulomb collisions, DSMC, nuclear fusion) as well as collisions between kinetic particles and a fixed (i.e. non-evolving) background species (MCC, background stopping). -.. _theory-collisions-mcc: +.. _multiphysics-collisions-mcc: Background Monte Carlo Collisions (MCC) --------------------------------------- @@ -45,14 +45,14 @@ where :math:`u` is the speed of the particle as tracked in WarpX (i.e. :math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and :math:`M` are the rest masses of the simulation and background species, respectively. The Lorentz factor is defined in the usual way, -:math:`\gamma \def \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above +:math:`\gamma \equiv \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above expression reduces to the classical equation :math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections for all scattering processes are evaluated at the energy as calculated above. Once a particle is selected for a specific collision process, that process determines how the particle is scattered as outlined below. -.. _theory-collisions-dsmc: +.. _multiphysics-collisions-dsmc: Direct Simulation Monte Carlo (DSMC) ------------------------------------ @@ -121,17 +121,27 @@ The particle velocity in the COM frame is then isotropically scattered using the Back scattering ^^^^^^^^^^^^^^^ -The process is the same as for elastic scattering above expect the scattering angle is fixed at :math:`\pi`, meaning the particle velocity in the COM frame is updated to :math:`-\vec{u}_c`. +The process is the same as for elastic scattering above except the scattering angle is fixed at :math:`\pi`, meaning the particle velocity in the COM frame is updated to :math:`-\vec{u}_c`. Excitation ^^^^^^^^^^ The process is also the same as for elastic scattering except the excitation energy cost is subtracted from the particle energy. This is done by reducing the velocity before a scattering angle is chosen. +Forward scattering +^^^^^^^^^^^^^^^^^^ + +This process operates in two ways: + +1. If an excitation energy cost is provided, the energy cost is subtracted from the particle energy and no scattering is performed. +2. If an excitation energy cost is not provided, the particle is not scattered and the velocity is unchanged (corresponding to a scattering angle of :math:`0` in the elastic scattering process above). + +See :cite:t:`b-Janssen2016` for a recommended use of this process. + Benchmarks ---------- -See the :ref:`MCC example ` for a benchmark of the MCC +See the :ref:`MCC example ` for a benchmark of the MCC implementation against literature results. Particle cooling due to elastic collisions diff --git a/Docs/source/theory/multiphysics/ionization.rst b/Docs/source/theory/multiphysics/ionization.rst new file mode 100644 index 00000000000..5003872b1a1 --- /dev/null +++ b/Docs/source/theory/multiphysics/ionization.rst @@ -0,0 +1,73 @@ +.. _multiphysics-ionization: + +Ionization +========== + +Field Ionization +---------------- + +Under the influence of a sufficiently strong external electric field atoms become ionized. +Particularly the dynamics of interactions between ultra-high intensity laser pulses and matter, e.g., Laser-Plasma Acceleration (LPA) with ionization injection, or Laser-Plasma Interactions with solid density targets (LPI) can depend on field ionization dynamics as well. + +WarpX models field ionization based on a description of the Ammosov-Delone-Krainov model:cite:p:`mpion-Ammosov1986` following :cite:t:`mpion-ChenPRSTAB13`. + +Implementation Details and Assumptions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + The current implementation makes the following assumptions + + * Energy for ionization processes is not removed from the electromagnetic fields + * Only one single-level ionization process can occur per macroparticle and time step + * Ionization happens at the beginning of the PIC loop before the field solve + * Angular momentum quantum number :math:`l = 0` and magnetic quantum number :math:`m = 0` + +The model implements the following equations (assumptions to :math:`l` and :math:`m` have already been applied). + +The electric field amplitude is calculated in the particle's frame of reference. + +.. math:: + + \begin{aligned} + \vec{E}_\mathrm{dc} &= \sqrt{ - \frac{1}{\mathrm{c}^2} \left( \vec{u} \cdot \vec{E} \right)^2 + + \left( \gamma \vec{E} + \vec{u} \times \vec{B} \right)^2 } + \\ + \gamma &= \sqrt{1 + \frac{\vec{u}^2}{\mathrm{c}^2}} + \end{aligned} + +Here, :math:`\vec{u} = (u_x, u_y, u_z)` is the momentum normalized to the particle mass, :math:`u_i = (\beta \gamma)_i \mathrm{c}`. +:math:`E_\mathrm{dc} = |\vec{E}_\mathrm{dc}|` is the DC-field in the frame of the particle. + +.. math:: + + \begin{aligned} + P &= 1 - \mathrm{e}^{-W\mathrm{d}\tau/\gamma} + \\ + W &= \omega_\mathrm{a} \mathcal{C}^2_{n^* l^*} \frac{U_\mathrm{ion}}{2 U_H} + \left[ 2 \frac{E_\mathrm{a}}{E_\mathrm{dc}} \left( \frac{U_\mathrm{ion}}{U_\mathrm{H}} \right)^{3/2} \right]^{2n^*-1} + \times \exp\left[ - \frac{2}{3} \frac{E_\mathrm{a}}{E_\mathrm{dc}} \left( \frac{U_\mathrm{ion}}{U_\mathrm{H}} \right)^{3/2} \right] + \\ + \mathcal{C}^2_{n^* l^*} &= \frac{2^{2n^*}}{n^* \Gamma(n^* + l^* + 1) \Gamma(n^* - l^*)} + \end{aligned} + +where :math:`\mathrm{d}\tau` is the simulation timestep, which is divided by the particle :math:`\gamma` to account for time dilation. The quantities are: :math:`\omega_\mathrm{a}`, the atomic unit frequency, :math:`U_\mathrm{ion}`, the ionization potential, :math:`U_\mathrm{H}`, Hydrogen ground state ionization potential, :math:`E_\mathrm{a}`, the atomic unit electric field, :math:`n^* = Z \sqrt{U_\mathrm{H}/U_\mathrm{ion}}`, the effective principal quantum number (*Attention!* :math:`Z` is the ionization state *after ionization*.) , :math:`l^* = n_0^* - 1`, the effective orbital quantum number. + +Empirical Extension to Over-the-Barrier Regime for Hydrogen +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For hydrogen, WarpX offers the modified empirical ADK extension to the Over-the-Barrier (OTB) published in :cite:t:`mpion-zhang_empirical_2014` Eq. (8) (note there is a typo in the paper and there should not be a minus sign in Eq. 8). + +.. math:: + + W_\mathrm{M} = \exp\left[ a_1 \frac{E^2}{E_\mathrm{b}} + a_2 \frac{E}{E_\mathrm{b}} + a_3 \right] W_\mathrm{ADK} + +The parameters :math:`a_1` through :math:`a_3` are independent of :math:`E` and can be found in the same reference. :math:`E_\mathrm{b}` is the classical Barrier Suppresion Ionization (BSI) field strength :math:`E_\mathrm{b} = U_\mathrm{ion}^2 / (4 Z)` given here in atomic units (AU). For a detailed description of conversion between unit systems consider the book by :cite:t:`mpion-Mulser2010`. + +Testing +^^^^^^^ + +* `Testing the field ionization module <../../../../en/latest/usage/examples/field_ionization/README.html>`_. + +.. bibliography:: + :keyprefix: mpion- diff --git a/Docs/source/developers/qed.rst b/Docs/source/theory/multiphysics/qed.rst similarity index 86% rename from Docs/source/developers/qed.rst rename to Docs/source/theory/multiphysics/qed.rst index f509d6ea386..3e961bb2898 100644 --- a/Docs/source/developers/qed.rst +++ b/Docs/source/theory/multiphysics/qed.rst @@ -1,7 +1,7 @@ -.. _developers-qed: +.. _multiphysics-qed: -QED -==================== +Quantum Electrodynamics (QED) +============================= Quantum synchrotron ------------------- @@ -28,7 +28,8 @@ electron-positron pairs can be created in vacuum in the function ``MultiParticleContainer::doQEDSchwinger`` in turn calls the function ``filterCreateTransformFromFAB``: -.. doxygenfunction:: filterCreateTransformFromFAB(DstTile&, DstTile&, const amrex::Box, const FABs&, const Index, const Index, FilterFunc&&, CreateFunc1&&, CreateFunc2&&, TransFunc&&) +Filter Create Transform Function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``filterCreateTransformFromFAB`` proceeds in three steps. In the filter phase, we loop on every cell and calculate the number of physical pairs created within diff --git a/Docs/source/theory/multiphysics_extensions.rst b/Docs/source/theory/multiphysics_extensions.rst new file mode 100644 index 00000000000..6c9ab040ef2 --- /dev/null +++ b/Docs/source/theory/multiphysics_extensions.rst @@ -0,0 +1,13 @@ +.. _theory-multiphysics: + +Multi-Physics Extensions +======================== + +WarpX includes various extensions to the traditional PIC loop which enable it to model additional physics. + +.. toctree:: + :maxdepth: 1 + + multiphysics/collisions + multiphysics/ionization + multiphysics/qed diff --git a/Docs/source/theory/pic.rst b/Docs/source/theory/pic.rst index 820cdba50e6..8356ba9f0f8 100644 --- a/Docs/source/theory/pic.rst +++ b/Docs/source/theory/pic.rst @@ -434,6 +434,83 @@ of this model can be found in the section .. _current_deposition: +Pseudo Spectral Analytical Time Domain with arbitrary charge and current-density time dependencies (PSATD-JRhom) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In :cite:`pt-shapovalPRE2024` we introduce a formulation of the particle-in-cell (PIC) method for the modeling of relativistic plasmas, which leverages the ability of the pseudo-spectral analytical time-domain solver (PSATD) to handle arbitrary time dependencies of the charge and current densities during one PIC cycle (up to second-order polynomial dependencies here). +The formulation is applied to a modified set of Maxwell's equations, which in Fourier space reads + +.. math:: + + \begin{align} + \frac{\partial\boldsymbol{\widetilde{E}}}{\partial t} & = i\boldsymbol{k}\times\boldsymbol{\widetilde{B}}-\boldsymbol{\widetilde{J}} + i\boldsymbol{k}{\widetilde{F}} \,, \\ + \frac{\partial\boldsymbol{\widetilde{B}}}{\partial t} & = -i\boldsymbol{k}\times\boldsymbol{\widetilde{E}} \,, \\ + \frac{\partial{\widetilde{F}}}{\partial t} & = i\boldsymbol{k}\cdot\boldsymbol{\widetilde{E}} - \widetilde{\rho} \,. + \end{align} + +Here, in addition to the usual Maxwell-Faraday and Ampere-Maxwell equations, the system contains an extra equation for the scalar field :math:`\widetilde{F}`, which propagates deviations to Gauss' law (if Gauss' law is verified in the PIC simulation, :math:`\widetilde{F}=0` and the modified Maxwell’s equations reduce to the standard Maxwell's equations). +These additional terms were introduced in :cite:p:`pt-Vayfed1996,pt-Munzjcp2000` from the potential formulation in the Lorenz gauge and used as a propagative divergence cleaning procedure, as an alternative to the Langdon-Marder :cite:p:`pt-Langdoncpc92,pt-Marderjcp87` diffusive procedures. +The above-mentioned earlier works :cite:p:`pt-Vayfed1996,pt-Munzjcp2000` considered this formulation in the context of the standard PIC method using FDTD discretization, while the PSATD-JRhom method introduced in :cite:`pt-shapovalPRE2024` exploits the PSATD discretization of the modified Maxwell's equations. +In contrast to the standard PSATD algorithm :cite:p:`pt-VayJCP2013`, where :math:`\boldsymbol{\widetilde{J}}` is assumed to be constant in time and :math:`\widetilde{\rho}` is assumed to be linear in time, within a given time step :math:`\Delta t`, the PSATD-JRhom provides more general time dependencies for :math:`\boldsymbol{\widetilde{J}}` and :math:`\widetilde{\rho}` within one timestep, which can be divided into :math:`m` subintervals of equal size :math:`\delta t = \Delta t/m`. +During these subintervals, :math:`\boldsymbol{\widetilde{J}}` and :math:`\widetilde{\rho}` are considered to be either **piecewise constant** (macro-particles deposit their density in the middle of each time subinterval), **piecewise linear** (macro-particles deposit their density at the edge of each time subinterval), or **piecewise quadratic** (macro-particles deposit their density at the edge of each time subinterval) in time. + +.. _fig-psatd_jrhom: + +.. figure:: https://gist.githubusercontent.com/oshapoval/88a73cada764364ad4ffce13563cedf1/raw/697ce1897cde0416bebdde8f1c1e8fcf859cb419/psatd_jrhom.png + :alt: figure not found, caption only + + Diagrams illustrating various time dependencies of the current density :math:`\boldsymbol{\widetilde{J}}` and charge density :math:`\widetilde{\rho}` for constant/linear (CL), both constant (CC), linear (LL) and quadratic (QQ) dependencies with :math:`m` subintervals: (first column) :math:`m=1`, (second) :math:`m=2` and (third) :math:`m=4`. CL1 corresponds to the standard PSATD PIC method. The triangle and circle glyphs represent the times at which the macroparticles deposit :math:`\boldsymbol{\widetilde{J}}` and :math:`\widetilde{\rho}` on the grid, respectively. The dashed and solid lines represent the assumed time dependency of :math:`\boldsymbol{\widetilde{J}}` and :math:`\widetilde{\rho}` within one time step, when integrating the Maxwell equations analytically. + +Using the piecewise definition of :math:`\widetilde{\rho}` and :math:`\boldsymbol{\widetilde{J}}`, the modified Maxwell's equations can be integrated analytically over one time step :math:`\Delta t`, i.e., from :math:`t=n\Delta t` to :math:`t=(n+1)\Delta t`. +In practice, this is done by sequentially integrating these equations over each subinterval :math:`\ell \in [0,m-1]`. +The final discretized equations write as: + +.. math:: + + \begin{align} + \begin{split} + \boldsymbol{\widetilde{E}}^{n+(\ell+1)/m} & = C{\boldsymbol{\widetilde{J}}}^{n+\ell/m}+ic^2\frac{S}{ck}\boldsymbol{k}\times{\boldsymbol{\widetilde{J}}}^{n+\ell/m}+ic^2\frac{S}{ck}\widetilde{F}^{n+\ell/m}\boldsymbol{k} \\ + &\quad + \frac{1}{\varepsilon_0 ck}\left(Y_3\boldsymbol{a_J} + Y_2\boldsymbol{b_J} - S\boldsymbol{c_J}\right) + + \frac{ic^2}{\varepsilon_0 c^2k^2}\left({Y_1}a_{\rho}-Y_{5}b_{\rho}-Y_{4}c_{\rho}\right)\boldsymbol{k}, + \end{split} + \\[4pt] + \begin{split} + \boldsymbol{\widetilde{B}}^{n+(\ell+1)/m} & = C {\boldsymbol{\widetilde{B}}}^{n+\ell/m}-i\frac{S}{ck}\boldsymbol{k}\times{\boldsymbol{\widetilde{E}}}^{n+\ell/m} - \frac{i}{\varepsilon_0 c^2k^2}\boldsymbol{k}\times\left(Y_1\boldsymbol{a_J} -Y_5\boldsymbol{b_J} -Y_4\boldsymbol{c_J} \right), + \end{split} + \\[4pt] + \begin{split} + \widetilde{F}^{n+(\ell+1)/m} & = C \widetilde{F}^{n+\ell/m}+i\frac{S}{ck}\boldsymbol{k} \cdot {\boldsymbol{\widetilde{E}}}^{n+\ell/m}+\frac{i}{\varepsilon_0 c^2k^2}\boldsymbol{k}\cdot\left(Y_1\boldsymbol{a_J}-Y_5\boldsymbol{b_J}-Y_4\boldsymbol{c_J}\right) \\ + &\quad + \frac{1}{\varepsilon_0 ck}\left({Y_3}a_{\rho}+{Y_2}b_{\rho}-Sc_{\rho}\right), + \end{split} + \end{align} + +where + +.. math:: + + \begin{aligned} + C &= \cos(ck\delta t), \ S = \sin(ck\delta t), + \\ + Y_1 & = \frac{(1-C)(8-c^2k^2\delta t^2)-4Sck\delta t}{2 c^2 k^2 \delta t^2}, + \\ + Y_2 & = \frac{2(C-1)+ S ck\delta t }{2 ck\delta t}, + \\ + Y_3 & = \frac{S(8- c^2k^2\delta t^2 ) - 4ck\delta t(1+C)}{2c^2 k^2 \delta t^2}, + \\ + Y_4 &= (1-C), \ Y_5 = \frac{(1+C) ck\delta t - 2S}{2ck \delta t}. + \end{aligned} + +Here, :math:`\boldsymbol{a_J}, \boldsymbol{b_J}, \boldsymbol{c_J}, a_{\rho}, b_{\rho}, c_{\rho}` are polynomial coefficients based on the time dependencies of the current and charge densities, as shown in the following table: + +.. _fig-j_rho_table: + +.. figure:: + https://gist.githubusercontent.com/oshapoval/88a73cada764364ad4ffce13563cedf1/raw/ebc249f8e875a952c65a5319fd523821baccfd5a/j_rho_table.png + :alt: figure not found, caption only + + Polynomial coefficients based on the time dependency of the current and charge densities :math:`{\boldsymbol{\widetilde{J}}}(t)` and :math:`\widetilde{\rho}(t)` over one time subinterval, :math:`\delta t = \Delta t/m`. + +Detailed analysis and tests revealed that, under certain conditions, the formulation can expand the range of numerical parameters under which PIC simulations are stable and accurate when modeling relativistic plasmas, such as, e.g., plasma-based particle accelerators. + Current deposition ------------------ diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 0492372b4e6..4ac80a8bab0 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -43,8 +43,9 @@ Particle Accelerator & Beam Physics :maxdepth: 1 examples/gaussian_beam/README.rst - examples/beam-beam_collision/README.rst - + examples/beam_beam_collision/README.rst + examples/free_electron_laser/README.rst + examples/thomson_parabola_spectrometer/README.rst High Energy Astrophysical Plasma Physics ---------------------------------------- @@ -64,14 +65,6 @@ Microelectronics * `ARTEMIS manual `__ -Nuclear Fusion --------------- - -.. note:: - - TODO - - Fundamental Plasma Physics -------------------------- @@ -101,7 +94,7 @@ examples below were generated at that time. .. toctree:: :maxdepth: 1 - examples/ohm_solver_EM_modes/README.rst + examples/ohm_solver_em_modes/README.rst examples/ohm_solver_ion_beam_instability/README.rst examples/ohm_solver_ion_Landau_damping/README.rst @@ -127,16 +120,21 @@ Manipulating fields via Python An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. -* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` +* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc_picmi.py>` An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. -* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` +* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/inputs_test_2d_python_wrappers_picmi.py>` Many Further Examples, Demos and Tests -------------------------------------- +.. toctree:: + :maxdepth: 1 + + examples/field_ionization/README.rst + WarpX runs over 200 integration tests on a variety of modeling cases, which validate and demonstrate its functionality. Please see the `Examples/Tests/ `__ directory for many more examples. diff --git a/Docs/source/usage/examples/beam-beam_collision b/Docs/source/usage/examples/beam-beam_collision deleted file mode 120000 index 8c6ac6b30b1..00000000000 --- a/Docs/source/usage/examples/beam-beam_collision +++ /dev/null @@ -1 +0,0 @@ -../../../../Examples/Physics_applications/beam-beam_collision \ No newline at end of file diff --git a/Docs/source/usage/examples/beam_beam_collision b/Docs/source/usage/examples/beam_beam_collision new file mode 120000 index 00000000000..2f46224fd8b --- /dev/null +++ b/Docs/source/usage/examples/beam_beam_collision @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/beam_beam_collision/ \ No newline at end of file diff --git a/Docs/source/usage/examples/field_ionization b/Docs/source/usage/examples/field_ionization new file mode 120000 index 00000000000..b1c3e38dab2 --- /dev/null +++ b/Docs/source/usage/examples/field_ionization @@ -0,0 +1 @@ +../../../../Examples/Tests/field_ionization/ \ No newline at end of file diff --git a/Docs/source/usage/examples/free_electron_laser b/Docs/source/usage/examples/free_electron_laser new file mode 120000 index 00000000000..1ce0fedd798 --- /dev/null +++ b/Docs/source/usage/examples/free_electron_laser @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/free_electron_laser \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_EM_modes b/Docs/source/usage/examples/ohm_solver_EM_modes deleted file mode 120000 index 485be7241ae..00000000000 --- a/Docs/source/usage/examples/ohm_solver_EM_modes +++ /dev/null @@ -1 +0,0 @@ -../../../../Examples/Tests/ohm_solver_EM_modes \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_em_modes b/Docs/source/usage/examples/ohm_solver_em_modes new file mode 120000 index 00000000000..03214170a1f --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_em_modes @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_em_modes/ \ No newline at end of file diff --git a/Docs/source/usage/examples/thomson_parabola_spectrometer b/Docs/source/usage/examples/thomson_parabola_spectrometer new file mode 120000 index 00000000000..8e72fba4100 --- /dev/null +++ b/Docs/source/usage/examples/thomson_parabola_spectrometer @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/thomson_parabola_spectrometer \ No newline at end of file diff --git a/Docs/source/usage/faq.rst b/Docs/source/usage/faq.rst index 67cea8d6621..4ed0f8fa6af 100644 --- a/Docs/source/usage/faq.rst +++ b/Docs/source/usage/faq.rst @@ -74,10 +74,10 @@ Several BTD quantities differ slightly from the lab frame domain described in th In the following discussion, we will use a subscript input (e.g. :math:`\Delta z_{\rm input}`) to denote properties of the lab frame domain. -- The first back-transformed diagnostic (BTD) snapshot may not occur at :math:`t=0`. Rather, it occurs at :math:`t_0=\frac{z_{max}}c \beta(1+\beta)\gamma^2`. This is the first time when the boosted frame can complete the snapshot. +- The first back-transformed diagnostic (BTD) snapshot may not occur at :math:`t=0`. Rather, it occurs at :math:`t_0=\frac{z_{max}}c \beta/(1 - \beta \beta_{mw})`, where :math:`\beta_{mw}` represents the speed of the moving window. This is the first time when the boosted frame can complete the snapshot. - The grid spacing of the BTD snapshot is different from the grid spacing indicated in the input script. It is given by :math:`\Delta z_{\rm grid,snapshot}=\frac{c\Delta t_{\rm boost}}{\gamma\beta}`. For a CFL-limited time step, :math:`\Delta z_{\rm grid,snapshot}\approx \frac{1+\beta}{\beta} \Delta z_{\rm input}\approx 2 \Delta z_{\rm input}`. Hence in many common use cases at large boost, it is expected that the BTD snapshot has a grid spacing twice what is expressed in the input script. - The effective length of the BTD snapshot may be longer than anticipated from the input script because the grid spacing is different. Additionally, the number of grid points in the BTD snapshot is a multiple of ``.buffer_size`` whereas the number of grid cells specified in the input deck may not be. -- The code may require longer than anticipated to complete a BTD snapshot. The code starts filling the :math:`i^{th}` snapshot around step :math:`j_{\rm BTD start}={\rm ceil}\left( i\gamma(1-\beta)\frac{\Delta t_{\rm snapshot}}{\Delta t_{\rm boost}}\right)`. The code then saves information for one BTD cell every time step in the boosted frame simulation. The :math:`i^{th}` snapshot is completed and saved :math:`n_{z,{\rm snapshot}}=n_{\rm buffers}\cdot ({\rm buffer\ size})` time steps after it begins, which is when the effective snapshot length is covered by the simulation. +- The code may require longer than anticipated to complete a BTD snapshot. The code starts filling the :math:`i^{th}` snapshot around step :math:`j_{\rm BTD start}={\rm ceil}\left( i\gamma(1-\beta\beta_{mw})\frac{\Delta t_{\rm snapshot}}{\Delta t_{\rm boost}}\right)`. The code then saves information for one BTD cell every time step in the boosted frame simulation. The :math:`i^{th}` snapshot is completed and saved :math:`n_{z,{\rm snapshot}}=n_{\rm buffers}\cdot ({\rm buffer\ size})` time steps after it begins, which is when the effective snapshot length is covered by the simulation. What kinds of RZ output do you support? --------------------------------------- diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4a8c4be0b5f..77f99044448 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -93,6 +93,14 @@ Overall simulation parameters The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + * ``strang_implicit_spectral_em``: Use a fully implicit electromagnetic solver. All of the comments for ``theta_implicit_em`` + above apply here as well (except that theta is fixed to 0.5 and that charge will not be conserved). + In this version, the advance is Strang split, with a half advance of the source free Maxwell's equation (with a spectral solver), a full advance of the particles plus longitudinal E field, and a second half advance of the source free Maxwell's equations. + The advantage of this method is that with the Spectral advance of the fields, it is dispersionless. + Note that exact energy convergence is achieved only with one grid block and ``psatd.periodic_single_box_fft == 1``. Otherwise, + the energy convservation is spoiled because of the inconsistency of the periodic assumption of the spectral solver and the + non-periodic behavior of the individual blocks. + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. @@ -105,16 +113,16 @@ Overall simulation parameters exactly energy conserving, but the solver may perform better. * ``implicit_evolve.nonlinear_solver`` (`string`, default: None) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em`, this sets the nonlinear solver used to advance the field-particle system in time. Options are `picard` or `newton`. * ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. * ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. @@ -228,6 +236,23 @@ Overall simulation parameters \boldsymbol{\nabla}^2 \phi = - \rho/\epsilon_0 \qquad \boldsymbol{E} = - \boldsymbol{\nabla}\phi \\ \boldsymbol{\nabla}^2 \boldsymbol{A} = - \mu_0 \boldsymbol{j} \qquad \boldsymbol{B} = \boldsymbol{\nabla}\times\boldsymbol{A} + * ``labframe-effective-potential``: Poisson's equation is solved with a modified dielectric function + (resulting in an "effective potential") to create a semi-implicit scheme which is robust to the + numerical instability seen in explicit electrostatic PIC when :math:`\Delta t \omega_{pe} > 2`. + If this option is used the additional parameter ``warpx.effective_potential_factor`` can also be + specified to set the value of :math:`C_{EP}` (default 4). The method is stable for :math:`C_{EP} \geq 1` + regardless of :math:`\Delta t`, however, the larger :math:`C_{EP}` is set, the lower the numerical plasma + frequency will be and therefore care must be taken to not set it so high that the plasma mode + hybridizes with other modes of interest. + Details of the method can be found in Appendix A of :cite:t:`param-Barnes2021` (note that in that paper + the method is referred to as "semi-implicit electrostatic" but here it has been renamed to "effective potential" + to avoid confusion with the semi-implicit method of Chen et al.). + In short, the code solves: + + .. math:: + + \boldsymbol{\nabla}\cdot\left(1+\frac{C_{EP}}{4}\sum_{s \in \text{species}}(\omega_{ps}\Delta t)^2 \right)\boldsymbol{\nabla} \phi = - \rho/\epsilon_0 \qquad \boldsymbol{E} = - \boldsymbol{\nabla}\phi + * ``relativistic``: Poisson's equation is solved **for each species** in their respective rest frame. The corresponding field is mapped back to the simulation frame and will produce both E and B @@ -257,7 +282,7 @@ Overall simulation parameters ``warpx.self_fields_absolute_tolerance``). * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). - See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. + See these references for more details :cite:t:`param-QiangPhysRevSTAB2006`, :cite:t:`param-QiangPhysRevSTAB2006err`. It only works in 3D and it requires the compilation flag ``-DWarpX_FFT=ON``. If mesh refinement is enabled, this solver only works on the coarsest level. On the refined patches, the Poisson equation is solved with the multigrid solver. @@ -265,6 +290,21 @@ Overall simulation parameters In electromagnetic mode, this solver can be used to initialize the species' self fields (``.initialize_self_fields=1``) provided that the field BCs are PML (``boundary.field_lo,hi = PML``). + * ``warpx.use_2d_slices_fft_solver`` (`bool`) optional (default: 0): Select the type of Integrated Green Function solver. + If 0, solve Poisson equation in full 3D geometry. + If 1, solve Poisson equation in a quasi 3D geometry, neglecting the :math:`z` derivatives in the Laplacian of the Poisson equation. + In practice, in this case, the code performes many 2D Poisson solves on all :math:`(x,y)` slices, each slice at a given :math:`z`. + This is often a good approximation for ultra-relativistic beams propagating along the :math:`z` direction, with the relativistic solver. + As a consequence, this solver does not need to do an FFT along the :math:`z` direction, + and instead uses only transverse FFTs (along :math:`x` and :math:`y`) at each :math:`z` position (or :math:`z` "slice"). + + * ``ablastr.nprocs_igf_fft`` (`int`) optional (default: number of MPI ranks): Number of MPI ranks used to parallalelize the FFT solver. + This can be less or equal than then number of MPI ranks that are used to run the overall simulation. + It can be useful if the auxiliary simulation boxes fit within a single process, so to avoid extra communications. + The auxiliary boxes are extended boxes in real and spectral space that are used to perform the necessary FFTs. + The extended simulation box size in real space is :math:`2n_x-1, 2n_y-1, 2n_z-1` with the 3D solver, :math:`2n_x-1, 2n_y -1, n_z` with the 2D solver. + The extended simulation box size in spectral space is :math:`n_x, 2n_y-1, 2n_z-1` with the 3D solver, :math:`n_x, 2n_y-1, n_z` with the 2D solver. + * ``warpx.self_fields_required_precision`` (`float`, default: 1.e-11) The relative precision with which the electrostatic space-charge fields should be calculated. More specifically, the space-charge fields are @@ -493,6 +533,39 @@ Domain Boundary Conditions * ``pec``: This option can be used to set a Perfect Electric Conductor at the simulation boundary. Please see the :ref:`PEC theory section ` for more details. Note that PEC boundary is invalid at `r=0` for the RZ solver. Please use ``none`` option. This boundary condition does not work with the spectral solver. + * ``pmc``: This option can be used to set a Perfect Magnetic Conductor at the simulation boundary. Please see the :ref:`PEC theory section ` for more details. This is equivalent to ``Neumann``. This boundary condition does not work with the spectral solver. + + * ``pec_insulator``: This option specifies a mixed perfect electric conductor and insulator boundary, where some part of the + boundary is PEC and some is insulator. In the insulator portion, the normal fields are extrapolated and the tangential fields + are either set to the specified value or extrapolated. The region that is insulator is specified using a spatially dependent expression with the insulator being in the area where the value of the expression is greater than zero. + The expressions are given for the low and high boundary on each axis, as listed below. The tangential fields are specified as + expressions that can depend on the location and time. The tangential fields are in two pairs, the electric fields and the + magnetic fields. In each pair, if one is specified, the other will be set to zero if not also specified. + + * ``insulator.area_x_lo(y,z)``: For the lower x (or r) boundary, expression specifying the insulator location + + * ``insulator.area_x_hi(y,z)``: For the upper x (or r) boundary, expression specifying the insulator location + + * ``insulator.area_y_lo(x,z)``: For the lower y boundary, expression specifying the insulator location + + * ``insulator.area_y_hi(x,z)``: For the upper y boundary, expression specifying the insulator location + + * ``insulator.area_z_lo(x,y)``: For the lower z boundary, expression specifying the insulator location + + * ``insulator.area_z_hi(x,y)``: For the upper z boundary, expression specifying the insulator location + + * ``insulator.Ey_x_lo(y,z,t)``, ``insulator.Ez_x_lo(y,z,t)``, ``insulator.By_x_lo(y,z,t)``, ``insulator.Bz_x_lo(y,z,t)``: expressions of the tangential field values for the lower x (or r) boundary + + * ``insulator.Ey_x_hi(y,z,t)``, ``insulator.Ez_x_hi(y,z,t)``, ``insulator.By_x_hi(y,z,t)``, ``insulator.Bz_x_hi(y,z,t)``: expressions of the tangential field values for the upper x (or r) boundary + + * ``insulator.Ex_y_lo(x,z,t)``, ``insulator.Ez_y_lo(x,z,t)``, ``insulator.Bx_y_lo(x,z,t)``, ``insulator.Bz_y_lo(x,z,t)``: expressions of the tangential field values for the lower y boundary + + * ``insulator.Ex_y_hi(x,z,t)``, ``insulator.Ez_y_hi(x,z,t)``, ``insulator.Bx_y_hi(x,z,t)``, ``insulator.Bz_y_hi(x,z,t)``: expressions of the tangential field values for the upper y boundary + + * ``insulator.Ex_z_lo(x,y,t)``, ``insulator.Ey_z_lo(x,y,t)``, ``insulator.Bx_z_lo(x,y,t)``, ``insulator.By_z_lo(x,y,t)``: expressions of the tangential field values for the lower z boundary + + * ``insulator.Ex_z_hi(x,y,t)``, ``insulator.Ey_z_hi(x,y,t)``, ``insulator.Bx_z_hi(x,y,t)``, ``insulator.By_z_hi(x,y,t)``: expressions of the tangential field values for the upper z boundary + * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. * ``neumann``: For the electrostatic multigrid solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. @@ -590,14 +663,15 @@ In WarpX, the embedded boundary can be defined in either of two ways: A function of `x`, `y`, `z` that defines the surface of the embedded boundary. That surface lies where the function value is 0 ; the physics simulation area is where the function value is negative ; - the interior of the embeddded boundary is where the function value is positive. + the interior of the embedded boundary is where the function value is positive. - **From an STL file:** In that case, you will need to set the following parameters in the input file. * ``eb2.stl_file`` (`string`) - The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``, - in order for the file to be read by WarpX. + The path to an `STL file `__. + In addition, you also need to set ``eb2.geom_type = stl``, in order for the file to be read by WarpX. + `See the AMReX documentation for more details `__. Whether the embedded boundary is defined with an analytical function or an STL file, you can additionally define the electric potential at the embedded boundary with an analytical function: @@ -733,6 +807,14 @@ Distribution across MPI ranks and parallelization * ``warpx.do_dynamic_scheduling`` (`0` or `1`) optional (default `1`) Whether to activate OpenMP dynamic scheduling. +* ``warpx.roundrobin_sfc`` (`0` or `1`) optional (default `0`) + Whether to use AMReX's RRSFS strategy for making DistributionMapping to + override the default space filling curve (SFC) strategy. If this is + enabled, the round robin method is used to distribute Boxes ordered by + SFC. This could potentially mitigate the load imbalance issue during + initialization by avoiding putting neighboring boxes on the same + process. + .. _running-cpp-parameters-parser: Math parser and user-defined constants @@ -812,7 +894,7 @@ Particle initialization * ``particles.rigid_injected_species`` (`strings`, separated by spaces) List of species injected using the rigid injection method. The rigid injection method is useful when injecting a relativistic particle beam in boosted-frame - simulations; see the :ref:`input-output section ` for more details. + simulations; see the :ref:`input-output section ` for more details. For species injected using this method, particles are translated along the `+z` axis with constant velocity as long as their ``z`` coordinate verifies ``zzinject_plane``, @@ -962,16 +1044,21 @@ Particle initialization The ``external_file`` option is currently implemented for 2D, 3D and RZ geometries, with record components in the cartesian coordinates ``(x,y,z)`` for 3D and RZ, and ``(x,z)`` for 2D. For more information on the `openPMD format `__ and how to build WarpX with it, please visit :ref:`the install section `. - * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a planar surface. + * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a surface. The emitting surface can be chosen to be either a plane + defined by the user (using some of the parameters listed below), or the embedded boundary (see :ref:`Embedded Boundary Conditions `). This requires the additional parameters: * ``.flux_profile`` (see the description of this parameter further below) - * ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) + * ``.inject_from_embedded_boundary`` (`0` or `1`, default `0` ; whether to inject from the embedded boundary or from a user-specified plane. + When injecting from the embedded boundary, the momentum distribution specified by the user along ``z`` (see e.g. ``uz_m``, ``uz_th`` below) is interpreted + as the momentum distribution along the local normal to the embedded boundary.) - * ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + * ``.surface_flux_pos`` (only used when injecting from a plane, `double`, location of the injection plane [meter]) - * ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) + * ``.flux_normal_axis`` (only used when injecting from a plane, `x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + + * ``.flux_direction`` (only used when injecting from a plane, `-1` or `+1`, direction of flux relative to the plane) * ``.num_particles_per_cell`` (`double`) @@ -1944,7 +2031,7 @@ Collision models ---------------- WarpX provides several particle collision models, using varying degrees of approximation. -Details about the collision models can be found in the :ref:`theory section `. +Details about the collision models can be found in the :ref:`theory section `. * ``collisions.collision_names`` (`strings`, separated by spaces) The name of each collision type. @@ -1967,10 +2054,10 @@ Details about the collision models can be found in the :ref:`theory section .species_type = 'deuterium'``) - ``dsmc`` for pair-wise, non-Coulomb collisions between kinetic species. This is a "direct simulation Monte Carlo" treatment of collisions between - kinetic species. See :ref:`DSMC section `. + kinetic species. See :ref:`DSMC section `. - ``background_mcc`` for collisions between particles and a neutral background. This is a relativistic Monte Carlo treatment for particles colliding - with a neutral background gas. See :ref:`MCC section `. + with a neutral background gas. See :ref:`MCC section `. - ``background_stopping`` for slowing of ions due to collisions with electrons or ions. This implements the approximate formulae as derived in Introduction to Plasma Physics, from Goldston and Rutherford, section 14.2. @@ -2082,8 +2169,8 @@ Details about the collision models can be found in the :ref:`theory section .scattering_processes`` (`strings` separated by spaces) Only for ``dsmc`` and ``background_mcc``. The scattering processes that should be - included. Available options are ``elastic``, ``back`` & ``charge_exchange`` - for ions and ``elastic``, ``excitationX`` & ``ionization`` for electrons. + included. Available options are ``elastic``, ``excitationX``, ``forward``, ``back``, and ``charge_exchange`` + for ions and ``elastic``, ``excitationX``, ``ionization`` & ``forward`` for electrons. Multiple excitation events can be included for electrons corresponding to excitation to different levels, the ``X`` above can be changed to a unique identifier for each excitation process. For each scattering process specified @@ -2120,21 +2207,34 @@ Time step The ratio between the actual timestep that is used in the simulation and the Courant-Friedrichs-Lewy (CFL) limit. (e.g. for `warpx.cfl=1`, the timestep will be exactly equal to the CFL limit.) - This parameter will only be used with the electromagnetic solver. + For some speed v and grid spacing dx, this limits the timestep to `warpx.cfl * dx / v`. + When used with the electromagnetic solver, `v` is the speed of light. + For the electrostatic solver, `v` is the maximum speed among all particles in the domain. * ``warpx.const_dt`` (`float`) Allows direct specification of the time step size, in units of seconds. - When the electrostatic solver is being used, this must be supplied. + When the electrostatic solver is being used, this must be supplied if not using adaptive timestepping. This can be used with the electromagnetic solver, overriding ``warpx.cfl``, but it is up to the user to ensure that the CFL condition is met. +* ``warpx.dt_update_interval`` (`string`) optional (default `-1`) + How many iterations pass between timestep adaptations when using the electrostatic solver. + Must be greater than `0` to use adaptive timestepping, or else ``warpx.const_dt`` must be specified. + +* ``warpx.max_dt`` (`float`) optional + The maximum timestep permitted for the electrostatic solver, when using adaptive timestepping. + If supplied, also sets the initial timestep for these simulations, before the first timestep update. + Filtering ^^^^^^^^^ -* ``warpx.use_filter`` (`0` or `1`; default: `1`, except for RZ FDTD) - Whether to smooth the charge and currents on the mesh, after depositing them from the macro-particles. +* ``warpx.use_filter`` (`0` or `1`) + Whether to use filtering in the simulation. + With the explicit evolve scheme, the filtering is turned on by default, except for RZ FDTD. + With the implicit evolve schemes, the filtering is turned off by default. + The filtering smoothes the charge and currents on the mesh, after depositing them from the macro-particles. + With implicit schemes, the electric field is also filtered (to maintain consistency for energy conservation). This uses a bilinear filter (see the :ref:`filtering section `). - The default is `1` in all cases, except for simulations in RZ geometry using the FDTD solver. With the RZ PSATD solver, the filtering is done in :math:`k`-space. .. warning:: @@ -2437,6 +2537,27 @@ Maxwell solver: kinetic-fluid hybrid * ``hybrid_pic_model.substeps`` (`int`) optional (default ``10``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. +* ``hybrid_pic_model.holmstrom_vacuum_region`` (`bool`) optional (default ``false``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the vacuum region handling of the generalized Ohm's Law to suppress vacuum fluctuations. :cite:t:`param-holmstrom2013handlingvacuumregionshybrid`. + +* ``hybid_pic_model.add_external_fields`` (`bool`) optional (default ``false``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the hybrid solver to use split external fields defined in external_vector_potential inputs. + +* ``external_vector_potential.fields`` (list of `str`) optional (default ``empty``) + If ``hybid_pic_model.add_external_fields`` is set to ``true``, this adds a list names for external time varying vector potentials to be added to hybrid solver. + +* ``external_vector_potential..read_from_file`` (`bool`) optional (default ``false``) + If ``hybid_pic_model.add_external_fields`` is set to ``true``, this flag determines whether to load an external field or use an implcit function to evaluate the time varying field. + +* ``external_vector_potential..path`` (`str`) optional (default ``""``) + If ``external_vector_potential..read_from_file`` is set to ``true``, sets the path to an OpenPMD file that can be loaded externally in :math:`weber/m`. + +* ``external_vector_potential..A[x,y,z]_external_grid_function(x,y,z)`` (`str`) optional (default ``"0"``) + If ``external_vector_potential..read_from_file`` is set to ``false``, Sets the external vector potential to be populated by an implicit function (on the grid) in :math:`weber/m`. + +* ``external_vector_potential..A_time_external_grid_function(t)`` (`str`) optional (default ``"1"``) + This sets the relative strength of the external vector potential by a dimensionless implicit time function, which can compute the external B fields and E fields based on the value and first time derivative of the function. + .. note:: Based on results from :cite:t:`param-Stanier2020` it is recommended to use @@ -2494,6 +2615,10 @@ Additional parameters to propagate (at the speed of light) to the boundaries of the simulation domain, where it can be absorbed. +* ``warpx.do_divb_cleaning_external`` (`0` or `1` ; default: 0) + Whether to use projection method to scrub B field divergence in externally + loaded fields. This is automatically turned on if external B fields are loaded. + * ``warpx.do_subcycling`` (`0` or `1`; default: 0) Whether or not to use sub-cycling. Different refinement levels have a different cell size, which results in different Courant–Friedrichs–Lewy @@ -2605,11 +2730,12 @@ Diagnostics and output In-situ visualization ^^^^^^^^^^^^^^^^^^^^^ -WarpX has four types of diagnostics: -``FullDiagnostics`` consist in dumps of fields and particles at given iterations, -``BackTransformedDiagnostics`` are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame, -``BoundaryScrapingDiagnostics`` are used to collect the particles that are absorbed at the boundary, throughout the simulation, and -``ReducedDiags`` allow the user to compute some reduced quantity (particle temperature, max of a field) and write a small amount of data to text files. +WarpX has five types of diagnostics: +``Full`` diagnostics consist in dumps of fields and particles at given iterations, +``TimeAveraged`` diagnostics only allow field data, which they output after averaging over a period of time, +``BackTransformed`` diagnostics are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame, +``BoundaryScraping`` diagnostics are used to collect the particles that are absorbed at the boundary, throughout the simulation, and +``ReducedDiags`` enable users to compute specific reduced quantities, such as particle temperature, energy histograms, or maximum field values, and efficiently save this in-situ analyzed data to files. Similar to what is done for physical species, WarpX has a class Diagnostics that allows users to initialize different diagnostics, each of them with different fields, resolution and period. This currently applies to standard diagnostics, but should be extended to back-transformed diagnostics and reduced diagnostics (and others) in a near future. @@ -2677,18 +2803,18 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. * ``.openpmd_encoding`` (optional, ``v`` (variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. - openPMD `file output encoding `__. + openPMD `file output encoding `__. File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). - ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. + ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. Default: ``f`` (full diagnostics) * ``.adios2_operator.type`` (``zfp``, ``blosc``) optional, - `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. * ``.adios2_operator.parameters.*`` optional, - `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. - A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): + A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): .. code-block:: text @@ -2707,11 +2833,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a .adios2_operator.parameters.precision = 3 * ``.adios2_engine.type`` (``bp4``, ``sst``, ``ssc``, ``dataman``) optional, - `ADIOS2 Engine type `__ for `openPMD `_ data dumps. + `ADIOS2 Engine type `__ for `openPMD `_ data dumps. See full list of engines at `ADIOS2 readthedocs `__ * ``.adios2_engine.parameters.*`` optional, - `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. + `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. An example for parameters for the BP engine are setting the number of writers (``NumAggregators``), transparently redirecting data to burst buffers etc. A detailed list of engine-specific parameters are available at the official `ADIOS2 documentation `__ @@ -2854,12 +2980,58 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``warpx.mffile_nstreams`` (`int`) optional (default `4`) Limit the number of concurrent readers per file. + +.. _running-cpp-parameters-diagnostics-timeavg: + +Time-Averaged Diagnostics +^^^^^^^^^^^^^^^^^^^^^^^^^ + +``TimeAveraged`` diagnostics are a special type of ``Full`` diagnostics that allows for the output of time-averaged field data. +This type of diagnostics can be created using ``.diag_type = TimeAveraged``. +We support only field data and related options from the list at `Full Diagnostics`_. + +.. note:: + + As with ``Full`` diagnostics, ``TimeAveraged`` diagnostics output the initial **instantaneous** conditions of the selected fields on step 0 (unless more specific output intervals exclude output for step 0). + +In addition, ``TimeAveraged`` diagnostic options include: + +* ``.time_average_mode`` (`string`, default `none`) + Describes the operating mode for time averaged field output. + + * ``none`` for no averaging (instantaneous fields) + + * ``fixed_start`` for a diagnostic that averages all fields between the current output step and a fixed point in time + + * ``dynamic_start`` for a constant averaging period and output at different points in time (non-overlapping) + + .. note:: + + To enable time-averaged field output with intervals tightly spaced enough for overlapping averaging periods, + please create additional instances of ``TimeAveraged`` diagnostics. + +* ``.average_period_steps`` (`int`) + Configures the number of time steps in an averaging period. + Set this only in the ``dynamic_start`` mode and only if ``average_period_time`` has not already been set. + Will be ignored in the ``fixed_start`` mode (with warning). + +* ``.average_period_time`` (`float`, in seconds) + Configures the time (SI units) in an averaging period. + Set this only in the ``dynamic_start`` mode and only if ``average_period_steps`` has not already been set. + Will be ignored in the ``fixed_start`` mode (with warning). + +* ``.average_start_step`` (`int`) + Configures the time step at which time-averaging begins. + Set this only in the ``fixed_start`` mode. + Will be ignored in the ``dynamic_start`` mode (with warning). + .. _running-cpp-parameters-diagnostics-btd: BackTransformed Diagnostics ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``.diag_type = BackTransformed``. We support the following list of options from `Full Diagnostics`_ + ``.format``, ``.openpmd_backend``, ``.dump_rz_modes``, ``.file_prefix``, ``.diag_lo``, ``.diag_hi``, ``.write_species``, ``.species``. Additional options for this diagnostic include: @@ -2947,15 +3119,14 @@ In addition to their usual attributes, the saved particles have Reduced Diagnostics ^^^^^^^^^^^^^^^^^^^ -``ReducedDiags`` allow the user to compute some reduced quantity (particle temperature, max of a field) and write a small amount of data to text files. +``ReducedDiags`` enable users to compute specific reduced quantities, such as particle temperature, energy histograms, or maximum field values, and efficiently save this in-situ analyzed data to files. +This shifts analysis from post-processing to runtime calculation of reduction operations (average, maximum, ...) and can greatly save disk space when "raw" particle and field outputs from `FullDiagnostics` can be avoided in favor of single values, 1D or 2D data at possibly even higher time resolution. * ``warpx.reduced_diags_names`` (`strings`, separated by spaces) - The names given by the user of simple reduced diagnostics. - Also the names of the output `.txt` files. - This reduced diagnostics aims to produce simple outputs - of the time history of some physical quantities. + A list of user-given names for reduced diagnostics. + By default, these names are also prefixing the names of output files. If ``warpx.reduced_diags_names`` is not provided in the input file, - no reduced diagnostics will be done. + no reduced diagnostics will be activated during the run. This is then used in the rest of the input deck; in this documentation we use ```` as a placeholder. @@ -2963,7 +3134,7 @@ Reduced Diagnostics The type of reduced diagnostics associated with this ````. For example, ``ParticleEnergy``, ``FieldEnergy``, etc. All available types are described below in detail. - For all reduced diagnostics, + For all reduced diagnostics that are writing tabular data into text files, the first and the second columns in the output file are the time step and the corresponding physical time in seconds, respectively. @@ -3034,6 +3205,12 @@ Reduced Diagnostics Note that the fields are averaged on the cell centers before their maximum values are computed. + * ``FieldPoyntingFlux`` + Integrates the normal Poynting flux over each domain boundary surface and also integrates the flux over time. + This provides the power and total energy loss into or out of the simulation domain. + The output columns are the flux for each dimension on the lower boundaries, then the higher boundaries, + then the integrated energy loss for each dimension on the the lower and higher boundaries. + * ``FieldProbe`` This type computes the value of each component of the electric and magnetic fields and of the Poynting vector (a measure of electromagnetic flux) at points in the domain. @@ -3435,22 +3612,111 @@ Reduced Diagnostics For 1D-Z, :math:`x`-related and :math:`y`-related quantities are not outputted. RZ geometry is not supported yet. -* ``.intervals`` (`string`) + * ``DifferentialLuminosity`` + This type computes the differential luminosity between two species, defined as: + + .. math:: + + \frac{d\mathcal{L}}{d\mathcal{E}^*}(\mathcal{E}^*, t) = \int_0^t dt'\int d\boldsymbol{x}\,d\boldsymbol{p}_1 d\boldsymbol{p}_2\; + \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(\mathcal{E}^* - \mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2)) + + where :math:`f_i` is the distribution function of species :math:`i` and + :math:`\mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2) = \sqrt{m_1^2c^4 + m_2^2c^4 + 2 c^2{p_1}^\mu {p_2}_\mu}` + is the energy in the center-of-mass frame, where :math:`p^\mu = (\sqrt{m^2 c^2 + \boldsymbol{p}^2}, \boldsymbol{p})` + represents the 4-momentum. Note that, if :math:`\sigma^*(\mathcal{E}^*)` + is the center-of-mass cross-section of a given collision process, then + :math:`\int d\mathcal{E}^* \frac{d\mathcal{L}}{d\mathcal{E}^*} (\mathcal{E}^*, t)\sigma^*(\mathcal{E}^*)` + gives the total number of collisions of that process (from the beginning of the simulation up until time :math:`t`). + + The differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-1}`. For collider-relevant WarpX simulations + involving two crossing, high-energy beams of particles, the differential luminosity in :math:`\text{s}^{-1}.\text{m}^{-2}.\text{eV}^{-1}` + can be obtained by multiplying the above differential luminosity by the expected repetition rate of the beams. + + In practice, the above expression of the differential luminosity is evaluated over discrete bins in energy :math:`\mathcal{E}^*`, + and by summing over macroparticles. + + * ``.species`` (`list of two strings`) + The names of the two species for which the differential luminosity is computed. + + * ``.bin_number`` (`int` > 0) + The number of bins in energy :math:`\mathcal{E}^*` + + * ``.bin_max`` (`float`, in eV) + The minimum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. + + * ``.bin_min`` (`float`, in eV) + The maximum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. + + * ``DifferentialLuminosity2D`` + This type computes the two-dimensional differential luminosity between two species, defined as: + + .. math:: + + \frac{d^2\mathcal{L}}{dE_1 dE_2}(E_1, E_2, t) = \int_0^t dt'\int d\boldsymbol{x}\, \int d\boldsymbol{p}_1 \int d\boldsymbol{p}_2\; + \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ + f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(E_1 - E_1(\boldsymbol{p}_1)) \delta(E_2 - E_2(\boldsymbol{p}_2)) + + where :math:`f_i` is the distribution function of species :math:`i` + (normalized such that :math:`\int \int f(\boldsymbol{x} \boldsymbol{p}, t )d\boldsymbol{x} d\boldsymbol{p} = N` + is the number of particles in species :math:`i` at time :math:`t`), + :math:`\boldsymbol{p}_i` and :math:`E_i (\boldsymbol{p}_i) = \sqrt{m_1^2c^4 + c^2 |\boldsymbol{p}_i|^2}` + are, respectively, the momentum and the energy of a particle of the :math:`i`-th species. + The 2D differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-2}`. + + * ``.species`` (`list of two strings`) + The names of the two species for which the differential luminosity is computed. + + * ``.bin_number_1`` (`int` > 0) + The number of bins in energy :math:`E_1` + + * ``.bin_max_1`` (`float`, in eV) + The minimum value of :math:`E_1` for which the 2D differential luminosity is computed. + + * ``.bin_min_1`` (`float`, in eV) + The maximum value of :math:`E_2` for which the 2D differential luminosity is compute + + * ``.bin_number_2`` (`int` > 0) + The number of bins in energy :math:`E_2` + + * ``.bin_max_2`` (`float`, in eV) + The minimum value of :math:`E_2` for which the 2D differential luminosity is computed. + + * ``.bin_min_2`` (`float`, in eV) + The minimum value of :math:`E_2` for which the 2D differential luminosity is computed. + + * ``.file_min_digits`` (`int`) optional (default `6`) + The minimum number of digits used for the iteration number appended to the diagnostic file names. + + The output is a ```` folder containing a set of openPMD files. + The values of the diagnostic are stored in a record labeled `d2L_dE1_dE2`. + An example input file and a loading python script of + using the DifferentialLuminosity2D reduced diagnostics + are given in ``Examples/Tests/diff_lumi_diag/``. + + * ``Timestep`` + This type outputs the simulation's physical timestep (in seconds) at each mesh refinement level. + +* ``reduced_diags.intervals`` (`string`) Using the `Intervals Parser`_ syntax, this string defines the timesteps at which reduced - diagnostics are written to file. + diagnostics are written to the file. + This can also be specified for the specific diagnostic by setting ``.intervals``. -* ``.path`` (`string`) optional (default `./diags/reducedfiles/`) - The path that the output file will be stored. +* ``reduced_diags.path`` (`string`) optional (default `./diags/reducedfiles/`) + The path where the output file will be stored. + This can also be specified for the specific diagnostic by setting ``.path``. -* ``.extension`` (`string`) optional (default `txt`) - The extension of the output file. +* ``reduced_diags.extension`` (`string`) optional (default `txt`) + The extension of the output file (the suffix). + This can also be specified for the specific diagnostic by setting ``.extension``. -* ``.separator`` (`string`) optional (default a `whitespace`) +* ``reduced_diags.separator`` (`string`) optional (default a `whitespace`) The separator between row values in the output file. The default separator is a whitespace. + This can also be specified for the specific diagnostic by setting ``.separator``. -* ``.precision`` (`integer`) optional (default `14`) +* ``reduced_diags.precision`` (`integer`) optional (default `14`) The precision used when writing out the data to the text files. + This can also be specified for the specific diagnostic by setting ``.precision``. Lookup tables and other settings for QED modules ------------------------------------------------ diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index 5119184089c..1d3481b5589 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -5,7 +5,7 @@ In-Depth: PWFA As described in the :doc:`../theory/intro`, one of the key applications of the WarpX exascale computing platform is in modelling future, compact and economic plasma-based accelerators. In this section we describe the simulation setup of a realistic *electron beam driven plasma wakefield accelerator* (PWFA) configuration. -For illustration purposes the setup can be explored with **WarpX** using the example input file :download:`PWFA <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d_boost>`. +For illustration purposes the setup can be explored with **WarpX** using the example input file :download:`PWFA <../../../Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_boosted>`. The simulation setup consists of 4 particle species: drive beam (driver), witness beam (beam), plasma electrons (plasma_e), and plasma ions (plasma_p). The species physical parameters are summarized in the following table. diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 38b0a31d7f3..1af884b40e7 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -114,6 +114,8 @@ Diagnostics .. autoclass:: pywarpx.picmi.FieldDiagnostic +.. autoclass:: pywarpx.picmi.TimeAveragedFieldDiagnostic + .. autoclass:: pywarpx.picmi.ElectrostaticFieldDiagnostic .. autoclass:: pywarpx.picmi.Checkpoint @@ -144,6 +146,10 @@ Particle distributions can be used for to initialize particles in a particle spe .. autoclass:: pywarpx.picmi.AnalyticDistribution +.. autoclass:: pywarpx.picmi.UniformFluxDistribution + +.. autoclass:: pywarpx.picmi.AnalyticFluxDistribution + .. autoclass:: pywarpx.picmi.ParticleListDistribution Particle layouts determine how to microscopically place macro particles in a grid cell. diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 5c5329e18b8..fa2f73b35d0 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -1,7 +1,8 @@ +.. _usage-how-to: .. _usage-workflows: -Workflows -========= +How-To Guides +============= This section collects typical user workflows and best practices for WarpX. diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py index ecd182c8802..aefae201617 100644 --- a/Docs/source/usage/workflows/ml_materials/create_dataset.py +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -19,28 +19,31 @@ c = 2.998e8 ############### + def sanitize_dir_strings(*dir_strings): - """append '/' to a string for concatenation in building up file tree descriptions - """ + """append '/' to a string for concatenation in building up file tree descriptions""" dir_strings = list(dir_strings) for ii, dir_string in enumerate(dir_strings): - if dir_string[-1] != '/': - dir_strings[ii] = dir_string + '/' + if dir_string[-1] != "/": + dir_strings[ii] = dir_string + "/" return dir_strings + def download_and_unzip(url, data_dir): request.urlretrieve(url, data_dir) - with zipfile.ZipFile(data_dir, 'r') as zip_dataset: + with zipfile.ZipFile(data_dir, "r") as zip_dataset: zip_dataset.extractall() -def create_source_target_data(data_dir, - species, - source_index=0, - target_index=-1, - survivor_select_index=-1, - particle_selection=None - ): + +def create_source_target_data( + data_dir, + species, + source_index=0, + target_index=-1, + survivor_select_index=-1, + particle_selection=None, +): """Create dataset from openPMD files Parameters @@ -60,35 +63,38 @@ def create_source_target_data(data_dir, target_stds: 6 element array of source particle coordinate standard deviations relevant times: 2 element array of source and target times """ - data_dir, = sanitize_dir_strings(data_dir) + (data_dir,) = sanitize_dir_strings(data_dir) data_path = data_dir - print('loading openPMD data from', data_path) + print("loading openPMD data from", data_path) ts = OpenPMDTimeSeries(data_path) relevant_times = [ts.t[source_index], ts.t[target_index]] # Manual: Particle tracking START iteration = ts.iterations[survivor_select_index] - pt = ParticleTracker( ts, - species=species, - iteration=iteration, - select=particle_selection) + pt = ParticleTracker( + ts, species=species, iteration=iteration, select=particle_selection + ) # Manual: Particle tracking END #### create normalized source, target data sets #### - print('creating data sets') + print("creating data sets") # Manual: Load openPMD START iteration = ts.iterations[source_index] - source_data = ts.get_particle(species=species, - iteration=iteration, - var_list=['x','y','z','ux','uy','uz'], - select=pt) + source_data = ts.get_particle( + species=species, + iteration=iteration, + var_list=["x", "y", "z", "ux", "uy", "uz"], + select=pt, + ) iteration = ts.iterations[target_index] - target_data = ts.get_particle(species=species, - iteration=iteration, - var_list=['x','y','z','ux','uy','uz'], - select=pt) + target_data = ts.get_particle( + species=species, + iteration=iteration, + var_list=["x", "y", "z", "ux", "uy", "uz"], + select=pt, + ) # Manual: Load openPMD END # Manual: Normalization START @@ -114,51 +120,75 @@ def create_source_target_data(data_dir, target_data = torch.tensor(np.column_stack(target_data)) # Manual: Format data END - return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times + return ( + source_data, + source_means, + source_stds, + target_data, + target_means, + target_stds, + relevant_times, + ) -def save_warpx_surrogate_data(dataset_fullpath_filename, - diag_dir, - species, - training_frac, - batch_size, - source_index, - target_index, - survivor_select_index, - particle_selection=None - ): +def save_warpx_surrogate_data( + dataset_fullpath_filename, + diag_dir, + species, + training_frac, + batch_size, + source_index, + target_index, + survivor_select_index, + particle_selection=None, +): source_target_data = create_source_target_data( data_dir=diag_dir, species=species, source_index=source_index, target_index=target_index, survivor_select_index=survivor_select_index, - particle_selection=particle_selection + particle_selection=particle_selection, ) - source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data + ( + source_data, + source_means, + source_stds, + target_data, + target_means, + target_stds, + times, + ) = source_target_data # Manual: Save dataset START - full_dataset = torch.utils.data.TensorDataset(source_data.float(), target_data.float()) + full_dataset = torch.utils.data.TensorDataset( + source_data.float(), target_data.float() + ) n_samples = full_dataset.tensors[0].size(0) - n_train = int(training_frac*n_samples) + n_train = int(training_frac * n_samples) n_test = n_samples - n_train - train_data, test_data = torch.utils.data.random_split(full_dataset, [n_train, n_test]) - - torch.save({'dataset':full_dataset, - 'train_indices':train_data.indices, - 'test_indices':test_data.indices, - 'source_means':source_means, - 'source_stds':source_stds, - 'target_means':target_means, - 'target_stds':target_stds, - 'times':times, - }, - dataset_fullpath_filename - ) + train_data, test_data = torch.utils.data.random_split( + full_dataset, [n_train, n_test] + ) + + torch.save( + { + "dataset": full_dataset, + "train_indices": train_data.indices, + "test_indices": test_data.indices, + "source_means": source_means, + "source_stds": source_stds, + "target_means": target_means, + "target_stds": target_stds, + "times": times, + }, + dataset_fullpath_filename, + ) # Manual: Save dataset END + ######## end utility functions ############# ######## start dataset creation ############ @@ -171,38 +201,40 @@ def save_warpx_surrogate_data(dataset_fullpath_filename, source_index = 0 target_index = 1 survivor_select_index = 1 -batch_size=1200 +batch_size = 1200 training_frac = 0.7 -os.makedirs('datasets', exist_ok=True) +os.makedirs("datasets", exist_ok=True) # improve stage 0 dataset stage_i = 0 -select = {'z':[0.280025, None]} -species = f'beam_stage_{stage_i}' -dataset_filename = f'dataset_{species}.pt' -dataset_file = 'datasets/' + dataset_filename -save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, - diag_dir=data_dir, - species=species, - training_frac=training_frac, - batch_size=batch_size, - source_index=source_index, - target_index=target_index, - survivor_select_index=survivor_select_index, - particle_selection=select - ) - -for stage_i in range(1,15): - species = f'beam_stage_{stage_i}' - dataset_filename = f'dataset_{species}.pt' - dataset_file = 'datasets/' + dataset_filename - save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, - diag_dir=data_dir, - species=species, - training_frac=training_frac, - batch_size=batch_size, - source_index=source_index, - target_index=target_index, - survivor_select_index=survivor_select_index - ) +select = {"z": [0.280025, None]} +species = f"beam_stage_{stage_i}" +dataset_filename = f"dataset_{species}.pt" +dataset_file = "datasets/" + dataset_filename +save_warpx_surrogate_data( + dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=select, +) + +for stage_i in range(1, 15): + species = f"beam_stage_{stage_i}" + dataset_filename = f"dataset_{species}.pt" + dataset_file = "datasets/" + dataset_filename + save_warpx_surrogate_data( + dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + ) diff --git a/Docs/source/usage/workflows/ml_materials/neural_network_classes.py b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py index 58b51a1d364..91090ffae3d 100644 --- a/Docs/source/usage/workflows/ml_materials/neural_network_classes.py +++ b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py @@ -16,11 +16,13 @@ class ActivationType(Enum): """ Activation class provides an enumeration type for the supported activation layers """ + ReLU = 1 Tanh = 2 PReLU = 3 Sigmoid = 4 + def get_enum_type(type_to_test, EnumClass): """ Returns the enumeration type associated to type_to_test in EnumClass @@ -42,28 +44,25 @@ def get_enum_type(type_to_test, EnumClass): raise Exception("unsupported type entered") - class ConnectedNN(nn.Module): """ ConnectedNN is a class of fully connected neural networks """ + def __init__(self, layers): super().__init__() self.stack = nn.Sequential(*layers) + def forward(self, x): return self.stack(x) + class OneActNN(ConnectedNN): """ OneActNN is class of fully connected neural networks admitting only one activation function """ - def __init__(self, - n_in, - n_out, - n_hidden_nodes, - n_hidden_layers, - act): + def __init__(self, n_in, n_out, n_hidden_nodes, n_hidden_layers, act): self.n_in = n_in self.n_out = n_out self.n_hidden_layers = n_hidden_layers @@ -84,7 +83,7 @@ def __init__(self, layers += [nn.Sigmoid()] if ii < self.n_hidden_layers - 1: - layers += [nn.Linear(self.n_hidden_nodes,self.n_hidden_nodes)] + layers += [nn.Linear(self.n_hidden_nodes, self.n_hidden_nodes)] layers += [nn.Linear(self.n_hidden_nodes, self.n_out)] diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 054c1b4cfc0..9a246de1cc2 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -13,66 +13,68 @@ ep0 = picmi.constants.ep0 # Number of cells -dim = '3' +dim = "3" nx = ny = 128 -nz = 35328 #17664 #8832 -if dim == 'rz': - nr = nx//2 +nz = 35328 # 17664 #8832 +if dim == "rz": + nr = nx // 2 # Computational domain -rmin = 0. -rmax = 128e-6 +rmin = 0.0 +rmax = 128e-6 zmin = -180e-6 -zmax = 0. +zmax = 0.0 # Number of processes for static load balancing # Check with your submit script -num_procs = [1, 1, 64*4] -if dim == 'rz': +num_procs = [1, 1, 64 * 4] +if dim == "rz": num_procs = [1, 64] # Number of time steps -gamma_boost = 60. -beta_boost = np.sqrt(1.-gamma_boost**-2) +gamma_boost = 60.0 +beta_boost = np.sqrt(1.0 - gamma_boost**-2) # Create grid -if dim == 'rz': +if dim == "rz": grid = picmi.CylindricalGrid( number_of_cells=[nr, nz], guard_cells=[32, 32], n_azimuthal_modes=2, lower_bound=[rmin, zmin], upper_bound=[rmax, zmax], - lower_boundary_conditions=['none', 'damped'], - upper_boundary_conditions=['none', 'damped'], - lower_boundary_conditions_particles=['absorbing', 'absorbing'], - upper_boundary_conditions_particles=['absorbing', 'absorbing'], - moving_window_velocity=[0., c], + lower_boundary_conditions=["none", "damped"], + upper_boundary_conditions=["none", "damped"], + lower_boundary_conditions_particles=["absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing"], + moving_window_velocity=[0.0, c], warpx_max_grid_size=256, - warpx_blocking_factor=64) + warpx_blocking_factor=64, + ) else: grid = picmi.Cartesian3DGrid( number_of_cells=[nx, ny, nz], guard_cells=[11, 11, 12], lower_bound=[-rmax, -rmax, zmin], upper_bound=[rmax, rmax, zmax], - lower_boundary_conditions=['periodic', 'periodic', 'damped'], - upper_boundary_conditions=['periodic', 'periodic', 'damped'], - lower_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], - upper_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], - moving_window_velocity=[0., 0., c], + lower_boundary_conditions=["periodic", "periodic", "damped"], + upper_boundary_conditions=["periodic", "periodic", "damped"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + moving_window_velocity=[0.0, 0.0, c], warpx_max_grid_size=256, - warpx_blocking_factor=32) + warpx_blocking_factor=32, + ) # plasma region -plasma_rlim = 100.e-6 +plasma_rlim = 100.0e-6 N_stage = 15 L_plasma_bulk = 0.28 -L_ramp = 1.e-9 +L_ramp = 1.0e-9 L_ramp_up = L_ramp L_ramp_down = L_ramp -L_stage = L_plasma_bulk + 2*L_ramp +L_stage = L_plasma_bulk + 2 * L_ramp # focusing # lens external fields @@ -80,124 +82,144 @@ lens_focal_length = 0.015 lens_width = 0.003 -stage_spacing = L_plasma_bulk + 2*lens_focal_length - -def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, - stage_xmin=-plasma_rlim, stage_xmax=plasma_rlim, - stage_ymin=-plasma_rlim, stage_ymax=plasma_rlim, - Lplus = L_ramp_up, Lp = L_plasma_bulk, - Lminus = L_ramp_down): +stage_spacing = L_plasma_bulk + 2 * lens_focal_length + + +def get_species_of_accelerator_stage( + stage_idx, + stage_zmin, + stage_zmax, + stage_xmin=-plasma_rlim, + stage_xmax=plasma_rlim, + stage_ymin=-plasma_rlim, + stage_ymax=plasma_rlim, + Lplus=L_ramp_up, + Lp=L_plasma_bulk, + Lminus=L_ramp_down, +): # Parabolic density profile n0 = 1.7e23 - Rc = 40.e-6 + Rc = 40.0e-6 Lstage = Lplus + Lp + Lminus - if not np.isclose(stage_zmax-stage_zmin, Lstage): - print('Warning: zmax disagrees with stage length') + if not np.isclose(stage_zmax - stage_zmin, Lstage): + print("Warning: zmax disagrees with stage length") parabolic_distribution = picmi.AnalyticDistribution( - density_expression= - f'n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.-cos(pi*(z-{stage_zmin})/Lplus)))*((z-{stage_zmin})=Lplus)*((z-{stage_zmin})<(Lplus+Lp))' \ - + f'+n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.+cos(pi*((z-{stage_zmin})-Lplus-Lp)/Lminus)))*((z-{stage_zmin})>=(Lplus+Lp))*((z-{stage_zmin})<(Lplus+Lp+Lminus))', + density_expression=f"n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.-cos(pi*(z-{stage_zmin})/Lplus)))*((z-{stage_zmin})=Lplus)*((z-{stage_zmin})<(Lplus+Lp))" + + f"+n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.+cos(pi*((z-{stage_zmin})-Lplus-Lp)/Lminus)))*((z-{stage_zmin})>=(Lplus+Lp))*((z-{stage_zmin})<(Lplus+Lp+Lminus))", pi=3.141592653589793, n0=n0, - kp=q_e/c*math.sqrt(n0/(m_e*ep0)), + kp=q_e / c * math.sqrt(n0 / (m_e * ep0)), Rc=Rc, Lplus=Lplus, Lp=Lp, Lminus=Lminus, lower_bound=[stage_xmin, stage_ymin, stage_zmin], upper_bound=[stage_xmax, stage_ymax, stage_zmax], - fill_in=True) + fill_in=True, + ) electrons = picmi.Species( - particle_type='electron', - name=f'electrons{stage_idx}', - initial_distribution=parabolic_distribution) + particle_type="electron", + name=f"electrons{stage_idx}", + initial_distribution=parabolic_distribution, + ) ions = picmi.Species( - particle_type='proton', - name=f'ions{stage_idx}', - initial_distribution=parabolic_distribution) + particle_type="proton", + name=f"ions{stage_idx}", + initial_distribution=parabolic_distribution, + ) return electrons, ions + species_list = [] for i_stage in range(1): # Add plasma zmin_stage = i_stage * stage_spacing zmax_stage = zmin_stage + L_stage - electrons, ions = get_species_of_accelerator_stage(i_stage+1, zmin_stage, zmax_stage) + electrons, ions = get_species_of_accelerator_stage( + i_stage + 1, zmin_stage, zmax_stage + ) species_list.append(electrons) species_list.append(ions) # add beam to species_list -beam_charge = -10.e-15 # in Coulombs +beam_charge = -10.0e-15 # in Coulombs N_beam_particles = int(1e6) -beam_centroid_z = -107.e-6 -beam_rms_z = 2.e-6 +beam_centroid_z = -107.0e-6 +beam_rms_z = 2.0e-6 beam_gammas = [1960 + 13246 * i_stage for i_stage in range(N_stage)] -#beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +# beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run beams = [] for i_stage in range(N_stage): beam_gamma = beam_gammas[i_stage] sigma_gamma = 0.06 * beam_gamma gaussian_distribution = picmi.GaussianBunchDistribution( - n_physical_particles= abs(beam_charge) / q_e, - rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], - rms_velocity=[8*c, 8*c, sigma_gamma*c], - centroid_position=[0., 0., beam_centroid_z], - centroid_velocity=[0., 0., beam_gamma*c], + n_physical_particles=abs(beam_charge) / q_e, + rms_bunch_size=[2.0e-6, 2.0e-6, beam_rms_z], + rms_velocity=[8 * c, 8 * c, sigma_gamma * c], + centroid_position=[0.0, 0.0, beam_centroid_z], + centroid_velocity=[0.0, 0.0, beam_gamma * c], ) beam = picmi.Species( - particle_type='electron', - name=f'beam_stage_{i_stage}', - initial_distribution= gaussian_distribution + particle_type="electron", + name=f"beam_stage_{i_stage}", + initial_distribution=gaussian_distribution, ) beams.append(beam) # Laser antenna_z = -1e-9 profile_t_peak = 1.46764864e-13 + + def get_laser(antenna_z, profile_t_peak, fill_in=True): - profile_focal_distance = 0. + profile_focal_distance = 0.0 laser = picmi.GaussianLaser( wavelength=0.8e-06, waist=36e-06, duration=7.33841e-14, - focal_position=[0., 0., profile_focal_distance + antenna_z], - centroid_position=[0., 0., antenna_z - c*profile_t_peak], - propagation_direction=[0., 0., 1.], - polarization_direction=[0., 1., 0.], + focal_position=[0.0, 0.0, profile_focal_distance + antenna_z], + centroid_position=[0.0, 0.0, antenna_z - c * profile_t_peak], + propagation_direction=[0.0, 0.0, 1.0], + polarization_direction=[0.0, 1.0, 0.0], a0=2.36, - fill_in=fill_in) + fill_in=fill_in, + ) laser_antenna = picmi.LaserAntenna( - position=[0., 0., antenna_z], - normal_vector=[0., 0., 1.]) + position=[0.0, 0.0, antenna_z], normal_vector=[0.0, 0.0, 1.0] + ) return (laser, laser_antenna) + + lasers = [] for i_stage in range(1): fill_in = True if i_stage == 0: fill_in = False lasers.append( - get_laser(antenna_z + i_stage*stage_spacing, - profile_t_peak + i_stage*stage_spacing/c, - fill_in) + get_laser( + antenna_z + i_stage * stage_spacing, + profile_t_peak + i_stage * stage_spacing / c, + fill_in, + ) ) # Electromagnetic solver -psatd_algo = 'multij' -if psatd_algo == 'galilean': - galilean_velocity = [0.,0.] if dim=='3' else [0.] - galilean_velocity += [-c*beta_boost] +psatd_algo = "multij" +if psatd_algo == "galilean": + galilean_velocity = [0.0, 0.0] if dim == "3" else [0.0] + galilean_velocity += [-c * beta_boost] n_pass_z = 1 do_multiJ = None - do_multi_J_n_depositions=None + do_multi_J_n_depositions = None J_in_time = None current_correction = True divE_cleaning = False -elif psatd_algo == 'multij': +elif psatd_algo == "multij": n_pass_z = 4 galilean_velocity = None do_multiJ = True @@ -206,21 +228,23 @@ def get_laser(antenna_z, profile_t_peak, fill_in=True): current_correction = False divE_cleaning = True else: - raise Exception(f'PSATD algorithm \'{psatd_algo}\' is not recognized!\n'\ - 'Valid options are \'multiJ\' or \'galilean\'.') -if dim == 'rz': + raise Exception( + f"PSATD algorithm '{psatd_algo}' is not recognized!\n" + "Valid options are 'multiJ' or 'galilean'." + ) +if dim == "rz": stencil_order = [8, 16] - smoother = picmi.BinomialSmoother(n_pass=[1,n_pass_z]) - grid_type = 'collocated' + smoother = picmi.BinomialSmoother(n_pass=[1, n_pass_z]) + grid_type = "collocated" else: stencil_order = [8, 8, 16] - smoother = picmi.BinomialSmoother(n_pass=[1,1,n_pass_z]) - grid_type = 'hybrid' + smoother = picmi.BinomialSmoother(n_pass=[1, 1, n_pass_z]) + grid_type = "hybrid" solver = picmi.ElectromagneticSolver( grid=grid, - method='PSATD', + method="PSATD", cfl=0.9999, source_smoother=smoother, stencil_order=stencil_order, @@ -228,63 +252,68 @@ def get_laser(antenna_z, profile_t_peak, fill_in=True): warpx_psatd_update_with_rho=True, warpx_current_correction=current_correction, divE_cleaning=divE_cleaning, - warpx_psatd_J_in_time=J_in_time - ) + warpx_psatd_J_in_time=J_in_time, +) # Diagnostics -diag_field_list = ['B', 'E', 'J', 'rho'] -diag_particle_list = ['weighting','position','momentum'] -coarse_btd_end = int((L_plasma_bulk+0.001+stage_spacing*(N_stage-1))*100000) -stage_end_snapshots=[f'{int((L_plasma_bulk+stage_spacing*ii)*100000)}:{int((L_plasma_bulk+stage_spacing*ii)*100000+50)}:5' for ii in range(1)] +diag_field_list = ["B", "E", "J", "rho"] +diag_particle_list = ["weighting", "position", "momentum"] +coarse_btd_end = int((L_plasma_bulk + 0.001 + stage_spacing * (N_stage - 1)) * 100000) +stage_end_snapshots = [ + f"{int((L_plasma_bulk + stage_spacing * ii) * 100000)}:{int((L_plasma_bulk + stage_spacing * ii) * 100000 + 50)}:5" + for ii in range(1) +] btd_particle_diag = picmi.LabFrameParticleDiagnostic( - name='lab_particle_diags', + name="lab_particle_diags", species=beams, grid=grid, - num_snapshots=25*N_stage, - #warpx_intervals=', '.join([f':{coarse_btd_end}:1000']+stage_end_snapshots), - warpx_intervals=', '.join(['0:0']+stage_end_snapshots), - dt_snapshots=0.00001/c, + num_snapshots=25 * N_stage, + # warpx_intervals=', '.join([f':{coarse_btd_end}:1000']+stage_end_snapshots), + warpx_intervals=", ".join(["0:0"] + stage_end_snapshots), + dt_snapshots=0.00001 / c, data_list=diag_particle_list, - write_dir='lab_particle_diags', - warpx_format='openpmd', - warpx_openpmd_backend='bp') + write_dir="lab_particle_diags", + warpx_format="openpmd", + warpx_openpmd_backend="bp", +) btd_field_diag = picmi.LabFrameFieldDiagnostic( - name='lab_field_diags', + name="lab_field_diags", grid=grid, - num_snapshots=25*N_stage, - dt_snapshots=stage_spacing/25/c, + num_snapshots=25 * N_stage, + dt_snapshots=stage_spacing / 25 / c, data_list=diag_field_list, - warpx_lower_bound=[-128.e-6, 0.e-6, -180.e-6], - warpx_upper_bound=[128.e-6, 0.e-6, 0.], - write_dir='lab_field_diags', - warpx_format='openpmd', - warpx_openpmd_backend='bp') + warpx_lower_bound=[-128.0e-6, 0.0e-6, -180.0e-6], + warpx_upper_bound=[128.0e-6, 0.0e-6, 0.0], + write_dir="lab_field_diags", + warpx_format="openpmd", + warpx_openpmd_backend="bp", +) field_diag = picmi.FieldDiagnostic( - name='field_diags', + name="field_diags", data_list=diag_field_list, grid=grid, period=100, - write_dir='field_diags', - lower_bound=[-128.e-6, 0.e-6, -180.e-6], - upper_bound=[128.e-6, 0.e-6, 0.], - warpx_format='openpmd', - warpx_openpmd_backend='h5') + write_dir="field_diags", + lower_bound=[-128.0e-6, 0.0e-6, -180.0e-6], + upper_bound=[128.0e-6, 0.0e-6, 0.0], + warpx_format="openpmd", + warpx_openpmd_backend="h5", +) particle_diag = picmi.ParticleDiagnostic( - name='particle_diags', + name="particle_diags", species=beams, period=100, - write_dir='particle_diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5') + write_dir="particle_diags", + warpx_format="openpmd", + warpx_openpmd_backend="h5", +) beamrel_red_diag = picmi.ReducedDiagnostic( - diag_type='BeamRelevant', - name='beamrel', - species=beam, - period=1) + diag_type="BeamRelevant", name="beamrel", species=beam, period=1 +) # Set up simulation sim = picmi.Simulation( @@ -292,40 +321,42 @@ def get_laser(antenna_z, profile_t_peak, fill_in=True): warpx_numprocs=num_procs, warpx_compute_max_step_from_btd=True, verbose=2, - particle_shape='cubic', + particle_shape="cubic", gamma_boost=gamma_boost, - warpx_charge_deposition_algo='standard', - warpx_current_deposition_algo='direct', - warpx_field_gathering_algo='momentum-conserving', - warpx_particle_pusher_algo='vay', + warpx_charge_deposition_algo="standard", + warpx_current_deposition_algo="direct", + warpx_field_gathering_algo="momentum-conserving", + warpx_particle_pusher_algo="vay", warpx_amrex_the_arena_is_managed=False, warpx_amrex_use_gpu_aware_mpi=True, warpx_do_multi_J=do_multiJ, warpx_do_multi_J_n_depositions=do_multi_J_n_depositions, warpx_grid_type=grid_type, # default: 2 for staggered grids, 8 for hybrid grids - warpx_field_centering_order=[16,16,16], + warpx_field_centering_order=[16, 16, 16], # only for hybrid grids, default: 8 - warpx_current_centering_order=[16,16,16] - ) + warpx_current_centering_order=[16, 16, 16], +) for species in species_list: - if dim=='rz': - n_macroparticle_per_cell=[2,4,2] + if dim == "rz": + n_macroparticle_per_cell = [2, 4, 2] else: - n_macroparticle_per_cell=[2,2,2] + n_macroparticle_per_cell = [2, 2, 2] sim.add_species( species, - layout=picmi.GriddedLayout(grid=grid, - n_macroparticle_per_cell=n_macroparticle_per_cell) + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=n_macroparticle_per_cell + ), ) for i_stage in range(N_stage): sim.add_species_through_plane( species=beams[i_stage], layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=N_beam_particles), - injection_plane_position=0., - injection_plane_normal_vector=[0.,0.,1.]) + injection_plane_position=0.0, + injection_plane_normal_vector=[0.0, 0.0, 1.0], + ) for i_stage in range(1): # Add laser @@ -334,14 +365,14 @@ def get_laser(antenna_z, profile_t_peak, fill_in=True): # Add diagnostics sim.add_diagnostic(btd_particle_diag) -#sim.add_diagnostic(btd_field_diag) -#sim.add_diagnostic(field_diag) -#sim.add_diagnostic(particle_diag) +# sim.add_diagnostic(btd_field_diag) +# sim.add_diagnostic(field_diag) +# sim.add_diagnostic(particle_diag) # Add reduced diagnostic sim.add_diagnostic(beamrel_red_diag) -sim.write_input_file(f'inputs_training_{N_stage}_stages') +sim.write_input_file(f"inputs_training_{N_stage}_stages") # Advance simulation until last time step sim.step() diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py index 4de11b9c99e..35b02c5cd44 100644 --- a/Docs/source/usage/workflows/ml_materials/train.py +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -18,7 +18,7 @@ ############# set model parameters ################# stage_i = 0 -species = f'beam_stage_{stage_i}' +species = f"beam_stage_{stage_i}" source_index = 0 target_index = 1 survivor_select_index = 1 @@ -35,36 +35,52 @@ n_hidden_nodes = 20 n_hidden_layers = 3 -activation_type = 'ReLU' +activation_type = "ReLU" -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -print(f'device={device}') +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"device={device}") #################### load dataset ################ -dataset_filename = f'dataset_{species}.pt' -dataset_file = 'datasets/' + dataset_filename +dataset_filename = f"dataset_{species}.pt" +dataset_file = "datasets/" + dataset_filename print(f"trying to load dataset+test-train split in {dataset_file}") dataset_with_indices = torch.load(dataset_file) -train_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['train_indices']) -test_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['test_indices']) -source_data = dataset_with_indices['dataset'] -source_means = dataset_with_indices['source_means'] -source_stds = dataset_with_indices['source_stds'] -target_means = dataset_with_indices['target_means'] -target_stds = dataset_with_indices['target_stds'] +train_data = torch.utils.data.dataset.Subset( + dataset_with_indices["dataset"], dataset_with_indices["train_indices"] +) +test_data = torch.utils.data.dataset.Subset( + dataset_with_indices["dataset"], dataset_with_indices["test_indices"] +) +source_data = dataset_with_indices["dataset"] +source_means = dataset_with_indices["source_means"] +source_stds = dataset_with_indices["source_stds"] +target_means = dataset_with_indices["target_means"] +target_stds = dataset_with_indices["target_stds"] print("able to load data and test/train split") ###### move data to device (GPU) if available ######## -source_device = train_data.dataset.tensors[0].to(device) # equivalently, test_data.tensors[0].to(device) +source_device = train_data.dataset.tensors[0].to( + device +) # equivalently, test_data.tensors[0].to(device) target_device = train_data.dataset.tensors[1].to(device) -full_dataset_device = torch.utils.data.TensorDataset(source_device.float(), target_device.float()) - -train_data_device = torch.utils.data.dataset.Subset(full_dataset_device, train_data.indices) -test_data_device = torch.utils.data.dataset.Subset(full_dataset_device, test_data.indices) - -train_loader_device = torch.utils.data.DataLoader(train_data_device, batch_size=batch_size, shuffle=True) -test_loader_device = torch.utils.data.DataLoader(test_data_device, batch_size=batch_size, shuffle=True) +full_dataset_device = torch.utils.data.TensorDataset( + source_device.float(), target_device.float() +) + +train_data_device = torch.utils.data.dataset.Subset( + full_dataset_device, train_data.indices +) +test_data_device = torch.utils.data.dataset.Subset( + full_dataset_device, test_data.indices +) + +train_loader_device = torch.utils.data.DataLoader( + train_data_device, batch_size=batch_size, shuffle=True +) +test_loader_device = torch.utils.data.DataLoader( + test_data_device, batch_size=batch_size, shuffle=True +) test_source_device = test_data_device.dataset.tensors[0] test_target_device = test_data_device.dataset.tensors[1] @@ -74,53 +90,61 @@ ###### create model ########### -model = mynn.OneActNN(n_in = n_in, - n_out = n_out, - n_hidden_nodes=n_hidden_nodes, - n_hidden_layers = n_hidden_layers, - act=activation_type - ) +model = mynn.OneActNN( + n_in=n_in, + n_out=n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers=n_hidden_layers, + act=activation_type, +) training_time = 0 train_loss_list = [] test_loss_list = [] -model.to(device=device); +model.to(device=device) + ########## train and test functions #### # Manual: Train function START def train(model, optimizer, train_loader, loss_fun): model.train() - total_loss = 0. + total_loss = 0.0 for batch_idx, (data, target) in enumerate(train_loader): - #evaluate network with data + # evaluate network with data output = model(data) - #compute loss - # sum the differences squared, take mean afterward - loss = loss_fun(output, target,reduction='sum') - #backpropagation: step optimizer and reset gradients + # compute loss + # sum the differences squared, take mean afterward + loss = loss_fun(output, target, reduction="sum") + # backpropagation: step optimizer and reset gradients loss.backward() optimizer.step() optimizer.zero_grad() total_loss += loss.item() return total_loss + + # Manual: Train function END + def test(model, test_loader, loss_fun): model.eval() - total_loss = 0. + total_loss = 0.0 with torch.no_grad(): for batch_idx, (data, target) in enumerate(test_loader): output = model(data) - total_loss += loss_fun(output, target, reduction='sum').item() + total_loss += loss_fun(output, target, reduction="sum").item() return total_loss + # Manual: Test function START def test_dataset(model, test_source, test_target, loss_fun): model.eval() with torch.no_grad(): output = model(test_source) - return loss_fun(output, test_target, reduction='sum').item() + return loss_fun(output, test_target, reduction="sum").item() + + # Manual: Test function END ######## training loop ######## @@ -134,33 +158,47 @@ def test_dataset(model, test_source, test_target, loss_fun): for epoch in range(n_epochs): if do_print: t1 = time.time() - ave_train_loss = train(model, optimizer, train_loader_device, loss_fun) / data_dim / training_set_size - ave_test_loss = test_dataset(model, test_source_device, test_target_device, loss_fun) / data_dim / training_set_size + ave_train_loss = ( + train(model, optimizer, train_loader_device, loss_fun) + / data_dim + / training_set_size + ) + ave_test_loss = ( + test_dataset(model, test_source_device, test_target_device, loss_fun) + / data_dim + / training_set_size + ) train_loss_list.append(ave_train_loss) test_loss_list.append(ave_test_loss) if do_print: t2 = time.time() - print('Train Epoch: {:04d} \tTrain Loss: {:.6f} \tTest Loss: {:.6f}, this epoch: {:.3f} s'.format( - epoch + 1, ave_train_loss, ave_test_loss, t2-t1)) + print( + "Train Epoch: {:04d} \tTrain Loss: {:.6f} \tTest Loss: {:.6f}, this epoch: {:.3f} s".format( + epoch + 1, ave_train_loss, ave_test_loss, t2 - t1 + ) + ) # Manual: Training loop END t4 = time.time() -print(f'total training time: {t4-t3:.3f}s') +print(f"total training time: {t4 - t3:.3f}s") ######### save model ######### -os.makedirs('models', exist_ok=True) +os.makedirs("models", exist_ok=True) # Manual: Save model START -model.to(device='cpu') -torch.save({ - 'n_hidden_layers':n_hidden_layers, - 'n_hidden_nodes':n_hidden_nodes, - 'activation':activation_type, - 'model_state_dict': model.state_dict(), - 'optimizer_state_dict': optimizer.state_dict(), - 'train_loss_list': train_loss_list, - 'test_loss_list': test_loss_list, - 'training_time': training_time, - }, f'models/{species}_model.pt') +model.to(device="cpu") +torch.save( + { + "n_hidden_layers": n_hidden_layers, + "n_hidden_nodes": n_hidden_nodes, + "activation": activation_type, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "train_loss_list": train_loss_list, + "test_loss_list": test_loss_list, + "training_time": training_time, + }, + f"models/{species}_model.pt", +) # Manual: Save model END diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py index 920efe29909..38bce78a91d 100644 --- a/Docs/source/usage/workflows/ml_materials/visualize.py +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -18,67 +18,70 @@ # open model file stage_i = 0 -species = f'beam_stage_{stage_i}' -model_data = torch.load(f'models/{species}_model.pt',map_location=torch.device('cpu')) +species = f"beam_stage_{stage_i}" +model_data = torch.load(f"models/{species}_model.pt", map_location=torch.device("cpu")) data_dim = 6 n_in = data_dim n_out = data_dim -n_hidden_layers = model_data['n_hidden_layers'] -n_hidden_nodes = model_data['n_hidden_nodes'] -activation_type = model_data['activation'] -train_loss_list = model_data['train_loss_list'] -test_loss_list = model_data['test_loss_list'] -training_time = model_data['training_time'] +n_hidden_layers = model_data["n_hidden_layers"] +n_hidden_nodes = model_data["n_hidden_nodes"] +activation_type = model_data["activation"] +train_loss_list = model_data["train_loss_list"] +test_loss_list = model_data["test_loss_list"] +training_time = model_data["training_time"] loss_fun = F.mse_loss n_epochs = len(train_loss_list) -train_counter = np.arange(n_epochs)+1 +train_counter = np.arange(n_epochs) + 1 test_counter = train_counter do_log_plot = False fig, ax = plt.subplots() if do_log_plot: - ax.semilogy(train_counter, train_loss_list, '.-',color='blue',label='training loss') - ax.semilogy(test_counter, test_loss_list, color='green',label='testing loss') + ax.semilogy( + train_counter, train_loss_list, ".-", color="blue", label="training loss" + ) + ax.semilogy(test_counter, test_loss_list, color="green", label="testing loss") else: - ax.plot(train_counter, train_loss_list, '.-',color='blue',label='training loss') - ax.plot(test_counter, test_loss_list, color='green',label='testing loss') -ax.set_xlabel('number of epochs seen') -ax.set_ylabel(' loss') + ax.plot(train_counter, train_loss_list, ".-", color="blue", label="training loss") + ax.plot(test_counter, test_loss_list, color="green", label="testing loss") +ax.set_xlabel("number of epochs seen") +ax.set_ylabel(" loss") ax.legend() -fig_dir = 'figures/' -ax.set_title(f'final test error = {test_loss_list[-1]:.3e} ') +fig_dir = "figures/" +ax.set_title(f"final test error = {test_loss_list[-1]:.3e} ") ax.grid() plt.tight_layout() -plt.savefig(f'{species}_training_testing_error.png') +plt.savefig(f"{species}_training_testing_error.png") ######### plot phase space comparison ####### -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -print(f'device={device}') - -model = mynn.OneActNN(n_in = n_in, - n_out = n_out, - n_hidden_nodes=n_hidden_nodes, - n_hidden_layers = n_hidden_layers, - act = activation_type - ) -model.load_state_dict(model_data['model_state_dict']) -model.to(device=device); +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"device={device}") + +model = mynn.OneActNN( + n_in=n_in, + n_out=n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers=n_hidden_layers, + act=activation_type, +) +model.load_state_dict(model_data["model_state_dict"]) +model.to(device=device) ###### load model data ############### -dataset_filename = f'dataset_{species}.pt' -dataset_dir = 'datasets/' +dataset_filename = f"dataset_{species}.pt" +dataset_dir = "datasets/" model_input_data = torch.load(dataset_dir + dataset_filename) -dataset = model_input_data['dataset'] -train_indices = model_input_data['train_indices'] -test_indices = model_input_data['test_indices'] -source_means = model_input_data['source_means'] -source_stds = model_input_data['source_stds'] -target_means = model_input_data['target_means'] -target_stds = model_input_data['target_stds'] -source_time, target_time = model_input_data['times'] +dataset = model_input_data["dataset"] +train_indices = model_input_data["train_indices"] +test_indices = model_input_data["test_indices"] +source_means = model_input_data["source_means"] +source_stds = model_input_data["source_stds"] +target_means = model_input_data["target_means"] +target_stds = model_input_data["target_stds"] +source_time, target_time = model_input_data["times"] source = dataset.tensors[0] @@ -86,7 +89,7 @@ test_source_device = test_source.to(device) with torch.no_grad(): evaluation_device = model(test_source_device.float()) -eval_cpu = evaluation_device.to('cpu') +eval_cpu = evaluation_device.to("cpu") target = dataset.tensors[1] test_target = target[test_indices] @@ -95,64 +98,89 @@ eval_cpu_si = eval_cpu * target_stds + target_means target_mu = np.copy(target_si) eval_cpu_mu = np.copy(eval_cpu_si) -target_mu[:,2] -= c*target_time -eval_cpu_mu[:,2] -= c*target_time -target_mu[:,:3] *= 1e6 -eval_cpu_mu[:,:3] *= 1e6 +target_mu[:, 2] -= c * target_time +eval_cpu_mu[:, 2] -= c * target_time +target_mu[:, :3] *= 1e6 +eval_cpu_mu[:, :3] *= 1e6 - -loss_tensor = torch.sum(loss_fun(eval_cpu, - test_target, - reduction='none'), - axis=1)/6 +loss_tensor = torch.sum(loss_fun(eval_cpu, test_target, reduction="none"), axis=1) / 6 loss_array = loss_tensor.detach().numpy() tinds = np.nonzero(loss_array > 0.0)[0] skip = 10 plt.figure() -fig, axT = plt.subplots(3,3) -axes_label = {0:r'x [$\mu$m]', 1:r'y [$\mu$m]', 2:r'z - %.2f cm [$\mu$m]'%(c*target_time),3:r'$p_x$',4:r'$p_y$',5:r'$p_z$'} -xy_inds = [(0,1),(2,0),(2,1)] +fig, axT = plt.subplots(3, 3) +axes_label = { + 0: r"x [$\mu$m]", + 1: r"y [$\mu$m]", + 2: r"z - %.2f cm [$\mu$m]" % (c * target_time), + 3: r"$p_x$", + 4: r"$p_y$", + 5: r"$p_z$", +} +xy_inds = [(0, 1), (2, 0), (2, 1)] + + def set_axes(ax, indx, indy): - ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') - ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=0.02, label='surrogate',cmap='YlOrRd') + ax.scatter( + target_mu[::skip, indx], target_mu[::skip, indy], s=8, c="k", label="simulation" + ) + ax.scatter( + eval_cpu_mu[::skip, indx], + eval_cpu_mu[::skip, indy], + marker="*", + c=loss_array[::skip], + s=0.02, + label="surrogate", + cmap="YlOrRd", + ) ax.set_xlabel(axes_label[indx]) ax.set_ylabel(axes_label[indy]) # return for ii in range(3): - ax = axT[0,ii] - indx,indy = xy_inds[ii] - set_axes(ax,indx,indy) + ax = axT[0, ii] + indx, indy = xy_inds[ii] + set_axes(ax, indx, indy) for ii in range(2): - indx,indy = xy_inds[ii] - ax = axT[1,ii] - set_axes(ax,indx+3,indy+3) + indx, indy = xy_inds[ii] + ax = axT[1, ii] + set_axes(ax, indx + 3, indy + 3) for ii in range(3): - ax = axT[2,ii] + ax = axT[2, ii] indx = ii - indy = ii+3 + indy = ii + 3 set_axes(ax, indx, indy) -ax = axT[1,2] +ax = axT[1, 2] indx = 5 indy = 4 -ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') -evalplt = ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=2, label='surrogate',cmap='YlOrRd') +ax.scatter( + target_mu[::skip, indx], target_mu[::skip, indy], s=8, c="k", label="simulation" +) +evalplt = ax.scatter( + eval_cpu_mu[::skip, indx], + eval_cpu_mu[::skip, indy], + marker="*", + c=loss_array[::skip], + s=2, + label="surrogate", + cmap="YlOrRd", +) ax.set_xlabel(axes_label[indx]) ax.set_ylabel(axes_label[indy]) cb = plt.colorbar(evalplt, ax=ax) -cb.set_label('MSE loss') +cb.set_label("MSE loss") -fig.suptitle(f'stage {stage_i} prediction') +fig.suptitle(f"stage {stage_i} prediction") plt.tight_layout() -plt.savefig(f'{species}_model_evaluation.png') +plt.savefig(f"{species}_model_evaluation.png") diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst index 47610e0d7ba..275a4dd134d 100644 --- a/Docs/source/usage/workflows/python_extend.rst +++ b/Docs/source/usage/workflows/python_extend.rst @@ -134,9 +134,12 @@ This example accesses the :math:`E_x(x,y,z)` field at level 0 after every time s warpx = sim.extension.warpx # data access - E_x_mf = warpx.multifab(f"Efield_fp[x][level=0]") + # vector field E, component x, on the fine patch of MR level 0 + E_x_mf = warpx.multifab("Efield_fp", dir=0, level=0) + # scalar field rho, on the fine patch of MR level 0 + rho_mf = warpx.multifab("rho_fp", level=0) - # compute + # compute on E_x_mf # iterate over mesh-refinement levels for lev in range(warpx.finest_level + 1): # grow (aka guard/ghost/halo) regions diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt new file mode 100644 index 00000000000..b77a3790c36 --- /dev/null +++ b/Examples/CMakeLists.txt @@ -0,0 +1,293 @@ +# Configuration ############################################################### +# +if(WarpX_MPI) + # OpenMPI root guard: https://github.com/open-mpi/ompi/issues/4451 + if("$ENV{USER}" STREQUAL "root") + # calling even --help as root will abort and warn on stderr + execute_process( + COMMAND ${MPIEXEC_EXECUTABLE} --help + ERROR_VARIABLE MPIEXEC_HELP_TEXT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(${MPIEXEC_HELP_TEXT} MATCHES "^.*allow-run-as-root.*$") + set(MPI_ALLOW_ROOT --allow-run-as-root) + endif() + endif() +endif() + +# Add a WarpX test set (with sub-tests) +# +# name: unique name of this test +# dims: 1,2,RZ,3 +# nprocs: 1 or 2 (maybe refactor later on to just depend on WarpX_MPI) +# inputs: inputs file or PICMI script, WarpX_MPI decides w/ or w/o MPI +# analysis: custom test analysis command, always run without MPI +# checksum: default regression analysis command (checksum benchmark) +# dependency: name of base test that must run first +# +function(add_warpx_test + name + dims + nprocs + inputs + analysis + checksum + dependency +) + # cannot run MPI tests w/o MPI build + if(nprocs GREATER_EQUAL 2 AND NOT WarpX_MPI) + message(WARNING "${name}: cannot run MPI tests without MPI build") + return() + endif() + + # cannot run tests with unsupported geometry + if(NOT dims IN_LIST WarpX_DIMS) + return() + endif() + + # cannot run tests with unfulfilled dependencies + if(dependency AND NOT TEST ${dependency}.run) + return() + endif() + + # set dimension suffix + warpx_set_suffix_dims(SD ${dims}) + + # make a unique run directory + file(MAKE_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${name}) + set(THIS_WORKING_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${name}) + + # get input file/script and optional command-line arguments + separate_arguments(INPUTS_LIST UNIX_COMMAND "${inputs}") + list(GET INPUTS_LIST 0 INPUTS_FILE) + list(LENGTH INPUTS_LIST INPUTS_LIST_LENGTH) + if(INPUTS_LIST_LENGTH GREATER 1) + list(SUBLIST INPUTS_LIST 1 -1 INPUTS_ARGS) + list(JOIN INPUTS_ARGS " " INPUTS_ARGS) + else() + set(INPUTS_ARGS "") + endif() + + # get analysis script and optional command-line arguments + separate_arguments(ANALYSIS_LIST UNIX_COMMAND "${analysis}") + list(GET ANALYSIS_LIST 0 ANALYSIS_FILE) + cmake_path(SET ANALYSIS_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${ANALYSIS_FILE}") + list(LENGTH ANALYSIS_LIST ANALYSIS_LIST_LENGTH) + if(ANALYSIS_LIST_LENGTH GREATER 1) + list(SUBLIST ANALYSIS_LIST 1 -1 ANALYSIS_ARGS) + list(JOIN ANALYSIS_ARGS " " ANALYSIS_ARGS) + else() + set(ANALYSIS_ARGS "") + endif() + + # get checksum script and optional command-line arguments + separate_arguments(CHECKSUM_LIST UNIX_COMMAND "${checksum}") + list(GET CHECKSUM_LIST 0 CHECKSUM_FILE) + cmake_path(SET CHECKSUM_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${CHECKSUM_FILE}") + list(LENGTH CHECKSUM_LIST CHECKSUM_LIST_LENGTH) + if(CHECKSUM_LIST_LENGTH GREATER 1) + list(SUBLIST CHECKSUM_LIST 1 -1 CHECKSUM_ARGS) + list(JOIN CHECKSUM_ARGS " " CHECKSUM_ARGS) + else() + set(CHECKSUM_ARGS "") + endif() + + # Python test? + set(python OFF) + if(${INPUTS_FILE} MATCHES ".*\.py$") + set(python ON) + cmake_path(SET INPUTS_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${INPUTS_FILE}") + endif() + + # cannot run Python tests w/o Python support + if(python AND NOT WarpX_PYTHON) + return() + endif() + + # cannot run executable tests w/o WarpX executable application + if(NOT python AND NOT WarpX_APP) + return() + endif() + + # set MPI executable + set(THIS_MPI_TEST_EXE + ${MPIEXEC_EXECUTABLE} + ${MPI_ALLOW_ROOT} + ${MPIEXEC_NUMPROC_FLAG} ${nprocs} + ${MPIEXEC_POSTFLAGS} + ${MPIEXEC_PREFLAGS} + ) + + # set Python executable + if(python) + set(THIS_Python_EXE ${Python_EXECUTABLE}) + else() + set(THIS_Python_EXE "") + endif() + + # test run + if(python) + # for argparse, do not pass command-line arguments as one quoted string + separate_arguments(INPUTS_ARGS UNIX_COMMAND "${INPUTS_ARGS}") + add_test( + NAME ${name}.run + COMMAND + ${THIS_MPI_TEST_EXE} + ${THIS_Python_EXE} + ${INPUTS_FILE} + ${INPUTS_ARGS} + WORKING_DIRECTORY ${THIS_WORKING_DIR} + ) + # FIXME Use helper function to handle Windows exceptions + set_property(TEST ${name}.run APPEND PROPERTY ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_PYTHON_OUTPUT_DIRECTORY}") + else() + # TODO Use these for Python tests too + set(runtime_params + "amrex.abort_on_unused_inputs = 1" + "amrex.throw_exception = 1" + "warpx.always_warn_immediately = 1" + "warpx.do_dynamic_scheduling = 0" + "warpx.serialize_initial_conditions = 1" + # FIXME should go before input file + #"warpx.abort_on_warning_threshold = low" + ) + set(runtime_params_fpetrap "") + if(WarpX_TEST_FPETRAP) + set(runtime_params_fpetrap + "amrex.fpe_trap_invalid = 1" + "amrex.fpe_trap_overflow = 1" + "amrex.fpe_trap_zero = 1" + ) + endif() + if(WarpX_TEST_DEBUGGER) + set(runtime_params_fpetrap "amrex.signal_handling = 0") + endif() + add_test( + NAME ${name}.run + COMMAND + ${THIS_MPI_TEST_EXE} + $ + ${INPUTS_FILE} + ${runtime_params} + ${runtime_params_fpetrap} + ${INPUTS_ARGS} + WORKING_DIRECTORY ${THIS_WORKING_DIR} + ) + endif() + + # AMReX ParmParse prefix: FILE = + set_property(TEST ${name}.run APPEND PROPERTY ENVIRONMENT "AMREX_INPUTS_FILE_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/") + + # run all tests with 1 OpenMP thread by default + set_property(TEST ${name}.run APPEND PROPERTY ENVIRONMENT "OMP_NUM_THREADS=1") + + if(python OR WIN32) + set(THIS_Python_SCRIPT_EXE ${Python_EXECUTABLE}) + else() + set(THIS_Python_SCRIPT_EXE "") + endif() + + # test analysis + if(analysis) + # for argparse, do not pass command-line arguments as one quoted string + separate_arguments(ANALYSIS_ARGS UNIX_COMMAND "${ANALYSIS_ARGS}") + add_test( + NAME ${name}.analysis + COMMAND + ${THIS_Python_SCRIPT_EXE} + ${ANALYSIS_FILE} + ${ANALYSIS_ARGS} + WORKING_DIRECTORY ${THIS_WORKING_DIR} + ) + # test analysis depends on test run + set_property(TEST ${name}.analysis APPEND PROPERTY DEPENDS "${name}.run") + # FIXME Use helper function to handle Windows exceptions + set(PYTHONPATH "$ENV{PYTHONPATH}:${CMAKE_PYTHON_OUTPUT_DIRECTORY}") + # add paths for custom Python modules + set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Regression/PostProcessingUtils") + set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Tools/Parser") + set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Tools/PostProcessing") + set_property(TEST ${name}.analysis APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PYTHONPATH}") + endif() + + # checksum analysis + if(checksum) + # for argparse, do not pass command-line arguments as one quoted string + separate_arguments(CHECKSUM_ARGS UNIX_COMMAND "${CHECKSUM_ARGS}") + add_test( + NAME ${name}.checksum + COMMAND + ${THIS_Python_SCRIPT_EXE} + ${CHECKSUM_FILE} + ${CHECKSUM_ARGS} + WORKING_DIRECTORY ${THIS_WORKING_DIR} + ) + # test analysis depends on test run + set_property(TEST ${name}.checksum APPEND PROPERTY DEPENDS "${name}.run") + if(analysis) + # checksum analysis depends on test analysis + set_property(TEST ${name}.checksum APPEND PROPERTY DEPENDS "${name}.analysis") + endif() + # FIXME Use helper function to handle Windows exceptions + set(PYTHONPATH "$ENV{PYTHONPATH}:${CMAKE_PYTHON_OUTPUT_DIRECTORY}") + # add paths for custom Python modules + set(PYTHONPATH "${PYTHONPATH}:${WarpX_SOURCE_DIR}/Regression/Checksum") + set_property(TEST ${name}.checksum APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PYTHONPATH}") + endif() + + # CI: remove test directory after run + if(WarpX_TEST_CLEANUP) + add_test( + NAME ${name}.cleanup + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/Examples/test_cleanup.cmake ${THIS_WORKING_DIR} + ) + # test cleanup depends on test run + set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.run") + if(analysis) + # test cleanup depends on test analysis + set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.analysis") + endif() + if(checksum) + # test cleanup depends on test analysis + set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.checksum") + endif() + endif() + + # Do we depend on another test? + if(dependency) + # current test depends on dependency test run (and analysis) + set_property(TEST ${name}.run APPEND PROPERTY DEPENDS "${dependency}.run") + if(analysis) + set_property(TEST ${name}.run APPEND PROPERTY DEPENDS "${dependency}.analysis") + endif() + if(checksum) + set_property(TEST ${name}.run APPEND PROPERTY DEPENDS "${dependency}.checksum") + endif() + if(WarpX_TEST_CLEANUP) + # do not clean up dependency test before current test is completed + set_property(TEST ${dependency}.cleanup APPEND PROPERTY DEPENDS "${name}.cleanup") + endif() + endif() +endfunction() + +# Add a CTest label to a WarpX test set. +# +# Labeling it here will add the label to the run test, its analysis and its cleanup. +# +# name: unique name of this test +# label: ctest LABELS property value to be added +# +function(label_warpx_test name label) + set(_test_names "${name}.run;${name}.analysis;${name}.cleanup") + foreach(_test_name IN LISTS _test_names) + if(TEST ${_test_name}) + set_property(TEST ${_test_name} APPEND PROPERTY LABELS "${label}") + endif() + endforeach() +endfunction() + +# Add tests (alphabetical order) ############################################## +# + +add_subdirectory(Tests) +add_subdirectory(Physics_applications) diff --git a/Examples/Physics_applications/CMakeLists.txt b/Examples/Physics_applications/CMakeLists.txt new file mode 100644 index 00000000000..ed06a840501 --- /dev/null +++ b/Examples/Physics_applications/CMakeLists.txt @@ -0,0 +1,13 @@ +# Add tests (alphabetical order) ############################################## +# + +add_subdirectory(beam_beam_collision) +add_subdirectory(capacitive_discharge) +add_subdirectory(free_electron_laser) +add_subdirectory(laser_acceleration) +add_subdirectory(laser_ion) +add_subdirectory(plasma_acceleration) +add_subdirectory(plasma_mirror) +add_subdirectory(spacecraft_charging) +add_subdirectory(uniform_plasma) +add_subdirectory(thomson_parabola_spectrometer) diff --git a/Examples/Physics_applications/beam-beam_collision/README.rst b/Examples/Physics_applications/beam-beam_collision/README.rst deleted file mode 100644 index 559a81277db..00000000000 --- a/Examples/Physics_applications/beam-beam_collision/README.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. _examples-beam-beam_collision: - -Beam-beam collision -==================== - -This example shows how to simulate the collision between two ultra-relativistic particle beams. -This is representative of what happens at the interaction point of a linear collider. -We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. - -We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and -the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). - -To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. -This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. - - -This example is based on the following paper :cite:t:`ex-Yakimenko2019`. - - -Run ---- - -The PICMI input file is not available for this example yet. - -For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. - -.. literalinclude:: inputs - :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs``. - - -Visualize ---------- - -The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). - -We compare different results: -* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; -* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; -* (black) literature results from :cite:t:`ex-Yakimenko2019`. - -The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 128`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: - -.. code-block:: ini - - qed_qs.lookup_table_mode = generate - qed_bw.lookup_table_mode = generate - qed_qs.tab_dndt_chi_min=1e-3 - qed_qs.tab_dndt_chi_max=2e3 - qed_qs.tab_dndt_how_many=512 - qed_qs.tab_em_chi_min=1e-3 - qed_qs.tab_em_chi_max=2e3 - qed_qs.tab_em_chi_how_many=512 - qed_qs.tab_em_frac_how_many=512 - qed_qs.tab_em_frac_min=1e-12 - qed_qs.save_table_in=my_qs_table.txt - qed_bw.tab_dndt_chi_min=1e-2 - qed_bw.tab_dndt_chi_max=2e3 - qed_bw.tab_dndt_how_many=512 - qed_bw.tab_pair_chi_min=1e-2 - qed_bw.tab_pair_chi_max=2e3 - qed_bw.tab_pair_chi_how_many=512 - qed_bw.tab_pair_frac_how_many=512 - qed_bw.save_table_in=my_bw_table.txt - -.. figure:: https://user-images.githubusercontent.com/17280419/291749626-aa61fff2-e6d2-45a3-80ee-84b2851ea0bf.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDMwMzQzNTEsIm5iZiI6MTcwMzAzNDA1MSwicGF0aCI6Ii8xNzI4MDQxOS8yOTE3NDk2MjYtYWE2MWZmZjItZTZkMi00NWEzLTgwZWUtODRiMjg1MWVhMGJmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMjAlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjIwVDAxMDA1MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFiYzY2MGQyYzIyZGIzYzUxOWI3MzNjZTk5ZDM1YzgyNmY4ZDYxOGRlZjAyZTIwNTAyMTc3NTgwN2Q0YjEwNGMmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.I96LQpjqmFXirPDVnBlFQIkCuenR6IuOSY0OIIQvtCo - :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. - :width: 100% - - Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs deleted file mode 100644 index 488e997f895..00000000000 --- a/Examples/Physics_applications/beam-beam_collision/inputs +++ /dev/null @@ -1,231 +0,0 @@ -################################# -########## MY CONSTANTS ######### -################################# -my_constants.mc2 = m_e*clight*clight -my_constants.nano = 1.0e-9 -my_constants.GeV = q_e*1.e9 - -# BEAMS -my_constants.beam_energy = 125.*GeV -my_constants.beam_uz = beam_energy/(mc2) -my_constants.beam_charge = 0.14*nano -my_constants.sigmax = 10*nano -my_constants.sigmay = 10*nano -my_constants.sigmaz = 10*nano -my_constants.beam_uth = 0.1/100.*beam_uz -my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) -my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) -my_constants.mux = 0.0 -my_constants.muy = 0.0 -my_constants.muz = -0.5*Lz+3.2*sigmaz - -# BOX -my_constants.Lx = 100.0*clight/omegab -my_constants.Ly = 100.0*clight/omegab -my_constants.Lz = 180.0*clight/omegab - -# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 -my_constants.nx = 64 -my_constants.ny = 64 -my_constants.nz = 128 - - -# TIME -my_constants.T = 0.7*Lz/clight -my_constants.dt = sigmaz/clight/10. - -# DIAGS -my_constants.every_red = 1. -warpx.used_inputs_file = warpx_used_inputs.txt - -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = T -amr.n_cell = nx ny nz -amr.max_grid_size = 128 -amr.blocking_factor = 2 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz -geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz - -################################# -######## BOUNDARY CONDITION ##### -################################# -boundary.field_lo = PEC PEC PEC -boundary.field_hi = PEC PEC PEC -boundary.particle_lo = Absorbing Absorbing Absorbing -boundary.particle_hi = Absorbing Absorbing Absorbing - -################################# -############ NUMERICS ########### -################################# -warpx.do_electrostatic = relativistic -warpx.const_dt = dt -warpx.grid_type = collocated -algo.particle_shape = 3 -algo.load_balance_intervals=100 -algo.particle_pusher = vay - -################################# -########### PARTICLES ########### -################################# -particles.species_names = beam1 beam2 pho1 pho2 ele1 pos1 ele2 pos2 -particles.photon_species = pho1 pho2 - -beam1.species_type = electron -beam1.injection_style = NUniformPerCell -beam1.num_particles_per_cell_each_dim = 1 1 1 -beam1.profile = parse_density_function -beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" -beam1.density_min = n0 / 1e3 -beam1.momentum_distribution_type = gaussian -beam1.uz_m = beam_uz -beam1.uy_m = 0.0 -beam1.ux_m = 0.0 -beam1.ux_th = beam_uth -beam1.uy_th = beam_uth -beam1.uz_th = beam_uth -beam1.initialize_self_fields = 1 -beam1.self_fields_required_precision = 5e-10 -beam1.self_fields_max_iters = 10000 -beam1.do_qed_quantum_sync = 1 -beam1.qed_quantum_sync_phot_product_species = pho1 -beam1.do_classical_radiation_reaction = 0 - -beam2.species_type = positron -beam2.injection_style = NUniformPerCell -beam2.num_particles_per_cell_each_dim = 1 1 1 -beam2.profile = parse_density_function -beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" -beam2.density_min = n0 / 1e3 -beam2.momentum_distribution_type = gaussian -beam2.uz_m = -beam_uz -beam2.uy_m = 0.0 -beam2.ux_m = 0.0 -beam2.ux_th = beam_uth -beam2.uy_th = beam_uth -beam2.uz_th = beam_uth -beam2.initialize_self_fields = 1 -beam2.self_fields_required_precision = 5e-10 -beam2.self_fields_max_iters = 10000 -beam2.do_qed_quantum_sync = 1 -beam2.qed_quantum_sync_phot_product_species = pho2 -beam2.do_classical_radiation_reaction = 0 - -pho1.species_type = photon -pho1.injection_style = none -pho1.do_qed_breit_wheeler = 1 -pho1.qed_breit_wheeler_ele_product_species = ele1 -pho1.qed_breit_wheeler_pos_product_species = pos1 - -pho2.species_type = photon -pho2.injection_style = none -pho2.do_qed_breit_wheeler = 1 -pho2.qed_breit_wheeler_ele_product_species = ele2 -pho2.qed_breit_wheeler_pos_product_species = pos2 - -ele1.species_type = electron -ele1.injection_style = none -ele1.self_fields_required_precision = 1e-11 -ele1.self_fields_max_iters = 10000 -ele1.do_qed_quantum_sync = 1 -ele1.qed_quantum_sync_phot_product_species = pho1 -ele1.do_classical_radiation_reaction = 0 - -pos1.species_type = positron -pos1.injection_style = none -pos1.self_fields_required_precision = 1e-11 -pos1.self_fields_max_iters = 10000 -pos1.do_qed_quantum_sync = 1 -pos1.qed_quantum_sync_phot_product_species = pho1 -pos1.do_classical_radiation_reaction = 0 - -ele2.species_type = electron -ele2.injection_style = none -ele2.self_fields_required_precision = 1e-11 -ele2.self_fields_max_iters = 10000 -ele2.do_qed_quantum_sync = 1 -ele2.qed_quantum_sync_phot_product_species = pho2 -ele2.do_classical_radiation_reaction = 0 - -pos2.species_type = positron -pos2.injection_style = none -pos2.self_fields_required_precision = 1e-11 -pos2.self_fields_max_iters = 10000 -pos2.do_qed_quantum_sync = 1 -pos2.qed_quantum_sync_phot_product_species = pho2 -pos2.do_classical_radiation_reaction = 0 - -pho1.species_type = photon -pho1.injection_style = none -pho1.do_qed_breit_wheeler = 1 -pho1.qed_breit_wheeler_ele_product_species = ele1 -pho1.qed_breit_wheeler_pos_product_species = pos1 - -pho2.species_type = photon -pho2.injection_style = none -pho2.do_qed_breit_wheeler = 1 -pho2.qed_breit_wheeler_ele_product_species = ele2 -pho2.qed_breit_wheeler_pos_product_species = pos2 - -################################# -############# QED ############### -################################# -qed_qs.photon_creation_energy_threshold = 0. - -qed_qs.lookup_table_mode = builtin -qed_qs.chi_min = 1.e-3 - -qed_bw.lookup_table_mode = builtin -qed_bw.chi_min = 1.e-2 - -# for accurate results use the generated tables with -# the following parameters -# note: must compile with -DWarpX_QED_TABLE_GEN=ON -#qed_qs.lookup_table_mode = generate -#qed_bw.lookup_table_mode = generate -#qed_qs.tab_dndt_chi_min=1e-3 -#qed_qs.tab_dndt_chi_max=2e3 -#qed_qs.tab_dndt_how_many=512 -#qed_qs.tab_em_chi_min=1e-3 -#qed_qs.tab_em_chi_max=2e3 -#qed_qs.tab_em_chi_how_many=512 -#qed_qs.tab_em_frac_how_many=512 -#qed_qs.tab_em_frac_min=1e-12 -#qed_qs.save_table_in=my_qs_table.txt -#qed_bw.tab_dndt_chi_min=1e-2 -#qed_bw.tab_dndt_chi_max=2e3 -#qed_bw.tab_dndt_how_many=512 -#qed_bw.tab_pair_chi_min=1e-2 -#qed_bw.tab_pair_chi_max=2e3 -#qed_bw.tab_pair_chi_how_many=512 -#qed_bw.tab_pair_frac_how_many=512 -#qed_bw.save_table_in=my_bw_table.txt - -warpx.do_qed_schwinger = 0. - -################################# -######### DIAGNOSTICS ########### -################################# -# FULL -diagnostics.diags_names = diag1 - -diag1.intervals = 0 -diag1.diag_type = Full -diag1.write_species = 1 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 -diag1.format = openpmd -diag1.dump_last_timestep = 1 -diag1.species = pho1 pho2 ele1 pos1 ele2 pos2 beam1 beam2 - -# REDUCED -warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 - -ColliderRelevant_beam1_beam2.type = ColliderRelevant -ColliderRelevant_beam1_beam2.intervals = every_red -ColliderRelevant_beam1_beam2.species = beam1 beam2 - -ParticleNumber.type = ParticleNumber -ParticleNumber.intervals = every_red diff --git a/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt b/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt new file mode 100644 index 00000000000..fbdb6dd221f --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt @@ -0,0 +1,13 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_beam_beam_collision # name + 3 # dims + 2 # nprocs + inputs_test_3d_beam_beam_collision # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) +label_warpx_test(test_3d_beam_beam_collision slow) diff --git a/Examples/Physics_applications/beam_beam_collision/README.rst b/Examples/Physics_applications/beam_beam_collision/README.rst new file mode 100644 index 00000000000..d75d43c6d4d --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/README.rst @@ -0,0 +1,145 @@ +.. _examples-beam-beam_collision: + +Beam-beam collision +==================== + +This example shows how to simulate the collision between two ultra-relativistic particle beams. +This is representative of what happens at the interaction point of a linear collider. +We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. + +We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and +the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). + +To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). +This solver accurately reproduces the subtle cancellation that occur for some component of ``E + v x B``, which are crucial in simulations of relativistic particles. + + +This example is based on the following paper :cite:t:`ex-Yakimenko2019`. + + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs_test_3d_beam_beam_collision + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision``. + + +QED tables +---------- + +The quantum synchrotron and nonlinear Breit-Wheeler modules are based on a Monte Carlo algorithm that computes the probabilities of an event from tabulated values. +WarpX comes with `builtin` tables (see the input file above), however these are low resolution tables that may not provide accurate results. +There are two ways to generate your own lookup table: + +* Inside WarpX, at runtime: the tables are generated by WarpX itself at the beginning of the simulation. + This requires to compile WarpX with ``-DWarpX_QED_TABLE_GEN=ON`` and to add the desired tables parameters in WarpX's input file. + `Here `__ are more details. + +* Outside of WarpX, using an external table generator: the tables are pregenerated, before running the actual simulation. + This standalone tool can be compiled at the same time as WarpX using ``-DWarpX_QED_TOOLS=ON``. + The table parameters are then passed to the table generator and do not need to be added to WarpX's input file. + `Here `__ are more details. + +Once the tables have been generated, they can be loaded in the input file using +``qed_qs,bw.lookup_table_mode=load`` and ``qed_qs,bw.load_table_from=/path/to/your/table``. + + +Visualize +--------- + +The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). + +We compare different results for the reduced diagnostics with the literature: +* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; +* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; +* (black) literature results from :cite:t:`ex-Yakimenko2019`. + +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 64`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. +Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. +For the large-scale simulation we have used the following options (added to the input file): + +.. code-block:: ini + + qed_qs.lookup_table_mode = generate + qed_bw.lookup_table_mode = generate + + qed_qs.tab_dndt_chi_min=1e-3 + qed_qs.tab_dndt_chi_max=2e3 + qed_qs.tab_dndt_how_many=512 + qed_qs.tab_em_chi_min=1e-3 + qed_qs.tab_em_chi_max=2e3 + qed_qs.tab_em_chi_how_many=512 + qed_qs.tab_em_frac_how_many=512 + qed_qs.tab_em_frac_min=1e-12 + qed_qs.save_table_in=my_qs_table.txt + + qed_bw.tab_dndt_chi_min=1e-2 + qed_bw.tab_dndt_chi_max=2e3 + qed_bw.tab_dndt_how_many=512 + qed_bw.tab_pair_chi_min=1e-2 + qed_bw.tab_pair_chi_max=2e3 + qed_bw.tab_pair_chi_how_many=512 + qed_bw.tab_pair_frac_how_many=512 + qed_bw.save_table_in=my_bw_table.txt + + +The same table can be also obtained using the table generator with the following lines: + +.. code-block:: ini + + ./qed_table_generator --table QS --mode DP -o my_qs_table.txt \ + --dndt_chi_min 1e-3 --dndt_chi_max 2e3 --dndt_how_many 512 \ + --em_chi_min 1e-3 --em_chi_max 2e3 --em_frac_min 1e-12 --em_chi_how_many 512 --em_frac_how_many 512 + + + ./qed_table_generator --table BW --mode DP -o my_bw_table.txt \ + --dndt_chi_min 1e-2 --dndt_chi_max 2e3 --dndt_how_many 512 --pair_chi_min 1e-2 --pair_chi_max 2e3 --pair_chi_how_many 512 --pair_frac_how_many 512 + + +.. figure:: https://gist.github.com/user-attachments/assets/2dd43782-d039-4faa-9d27-e3cf8fb17352 + :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + :width: 100% + + Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + + +Below are two visualizations scripts that provide examples to graph the field and reduced diagnostics. +They are available in the ``Examples/Physics_applications/beam-beam_collision/`` folder and can be run as simply as ``python3 plot_fields.py`` and ``python3 plot_reduced.py``. + +.. tab-set:: + + .. tab-item:: Field Diagnostics + + This script visualizes the evolution of the fields (:math:`|E|, |B|, \rho`) during the collision between the two ultra-relativistic lepton beams. + The magnitude of E and B and the charge densities of the primary beams and of the secondary pairs are sliced along either one of the two transverse coordinates (:math:`x` and :math:`y`). + + .. literalinclude:: plot_fields.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/plot_fields.py``. + + .. figure:: https://gist.github.com/user-attachments/assets/04c9c0ec-b580-446f-a11a-437c1b244a41 + :alt: Slice across :math:`x` of different fields (:math:`|E|, |B|, \rho`) at timestep 45, in the middle of the collision. + :width: 100% + + Slice across :math:`x` of different fields (:math:`|E|, |B|, \rho`) at timestep 45, in the middle of the collision. + + + .. tab-item:: Reduced Diagnostics + + A similar script to the one below was used to produce the image showing the benchmark against :cite:t:`ex-Yakimenko2019`. + + .. literalinclude:: plot_reduced.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/plot_reduced.py``. + + .. figure:: https://gist.github.com/user-attachments/assets/c280490a-f1f2-4329-ad3c-46817d245dc1 + :alt: Photon and pair production rates in time throughout the collision. + :width: 100% + + Photon and pair production rates in time throughout the collision. diff --git a/Examples/Physics_applications/beam_beam_collision/analysis_default_regression.py b/Examples/Physics_applications/beam_beam_collision/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision new file mode 100644 index 00000000000..1f58f68ba69 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision @@ -0,0 +1,236 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.beam_energy = 125.*GeV +my_constants.beam_uz = beam_energy/(mc2) +my_constants.beam_charge = 0.14*nano +my_constants.sigmax = 10*nano +my_constants.sigmay = 10*nano +my_constants.sigmaz = 10*nano +my_constants.beam_uth = 0.1/100.*beam_uz +my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) +my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = -0.5*Lz+3.2*sigmaz + +# BOX +my_constants.Lx = 100.0*clight/omegab +my_constants.Ly = 100.0*clight/omegab +my_constants.Lz = 180.0*clight/omegab + +# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 64 + +# TIME +my_constants.T = 0.7*Lz/clight +my_constants.dt = sigmaz/clight/10. + +# DIAGS +my_constants.every_red = 1. +warpx.used_inputs_file = warpx_used_inputs.txt + +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 pho1 pho2 ele1 pos1 ele2 pos2 +particles.photon_species = pho1 pho2 + +beam1.species_type = electron +beam1.injection_style = NUniformPerCell +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" +beam1.density_min = n0 / 1e3 +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_uz +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = beam_uth +beam1.uy_th = beam_uth +beam1.uz_th = beam_uth +beam1.initialize_self_fields = 1 +beam1.self_fields_required_precision = 5e-10 +beam1.self_fields_max_iters = 10000 +beam1.do_qed_quantum_sync = 1 +beam1.qed_quantum_sync_phot_product_species = pho1 +beam1.do_classical_radiation_reaction = 0 + +beam2.species_type = positron +beam2.injection_style = NUniformPerCell +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" +beam2.density_min = n0 / 1e3 +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_uz +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = beam_uth +beam2.uy_th = beam_uth +beam2.uz_th = beam_uth +beam2.initialize_self_fields = 1 +beam2.self_fields_required_precision = 5e-10 +beam2.self_fields_max_iters = 10000 +beam2.do_qed_quantum_sync = 1 +beam2.qed_quantum_sync_phot_product_species = pho2 +beam2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +ele1.species_type = electron +ele1.injection_style = none +ele1.self_fields_required_precision = 1e-11 +ele1.self_fields_max_iters = 10000 +ele1.do_qed_quantum_sync = 1 +ele1.qed_quantum_sync_phot_product_species = pho1 +ele1.do_classical_radiation_reaction = 0 + +pos1.species_type = positron +pos1.injection_style = none +pos1.self_fields_required_precision = 1e-11 +pos1.self_fields_max_iters = 10000 +pos1.do_qed_quantum_sync = 1 +pos1.qed_quantum_sync_phot_product_species = pho1 +pos1.do_classical_radiation_reaction = 0 + +ele2.species_type = electron +ele2.injection_style = none +ele2.self_fields_required_precision = 1e-11 +ele2.self_fields_max_iters = 10000 +ele2.do_qed_quantum_sync = 1 +ele2.qed_quantum_sync_phot_product_species = pho2 +ele2.do_classical_radiation_reaction = 0 + +pos2.species_type = positron +pos2.injection_style = none +pos2.self_fields_required_precision = 1e-11 +pos2.self_fields_max_iters = 10000 +pos2.do_qed_quantum_sync = 1 +pos2.qed_quantum_sync_phot_product_species = pho2 +pos2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. + +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 + +qed_bw.lookup_table_mode = builtin +qed_bw.chi_min = 1.e-2 + +# for accurate results use the generated tables with +# the following parameters +# note: must compile with -DWarpX_QED_TABLE_GEN=ON +#qed_qs.lookup_table_mode = generate +#qed_bw.lookup_table_mode = generate +#qed_qs.tab_dndt_chi_min=1e-3 +#qed_qs.tab_dndt_chi_max=2e3 +#qed_qs.tab_dndt_how_many=512 +#qed_qs.tab_em_chi_min=1e-3 +#qed_qs.tab_em_chi_max=2e3 +#qed_qs.tab_em_chi_how_many=512 +#qed_qs.tab_em_frac_how_many=512 +#qed_qs.tab_em_frac_min=1e-12 +#qed_qs.save_table_in=my_qs_table.txt +#qed_bw.tab_dndt_chi_min=1e-2 +#qed_bw.tab_dndt_chi_max=2e3 +#qed_bw.tab_dndt_how_many=512 +#qed_bw.tab_pair_chi_min=1e-2 +#qed_bw.tab_pair_chi_max=2e3 +#qed_bw.tab_pair_chi_how_many=512 +#qed_bw.tab_pair_frac_how_many=512 +#qed_bw.save_table_in=my_bw_table.txt + +# if you wish to use existing tables: +#qed_qs.lookup_table_mode=load +#qed_qs.load_table_from = /path/to/my_qs_table.txt +#qed_bw.lookup_table_mode=load +#qed_bw.load_table_from = /path/to/my_bw_table.txt + +warpx.do_qed_schwinger = 0. + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 15 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 +diag1.format = openpmd +diag1.dump_last_timestep = 1 +diag1.species = pho1 pho2 ele1 pos1 ele2 pos2 beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 + +ColliderRelevant_beam1_beam2.type = ColliderRelevant +ColliderRelevant_beam1_beam2.intervals = every_red +ColliderRelevant_beam1_beam2.species = beam1 beam2 + +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = every_red diff --git a/Examples/Physics_applications/beam_beam_collision/plot_fields.py b/Examples/Physics_applications/beam_beam_collision/plot_fields.py new file mode 100644 index 00000000000..a7ddb2d13e9 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/plot_fields.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import matplotlib.pyplot as plt +import numpy as np +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +from openpmd_viewer import OpenPMDTimeSeries + +plt.rcParams.update({"font.size": 16}) + +series = OpenPMDTimeSeries("./diags/diag1") +steps = series.iterations + + +for slice_axis in ["x", "y"]: # slice the fields along x and y + for n in steps: # loop through the available timesteps + fig, ax = plt.subplots( + ncols=2, nrows=2, figsize=(10, 6), dpi=300, sharex=True, sharey=True + ) + + # get E field + Ex, info = series.get_field( + field="E", coord="x", iteration=n, plot=False, slice_across=slice_axis + ) + Ey, info = series.get_field( + field="E", coord="y", iteration=n, plot=False, slice_across=slice_axis + ) + Ez, info = series.get_field( + field="E", coord="z", iteration=n, plot=False, slice_across=slice_axis + ) + # get B field + Bx, info = series.get_field( + field="B", coord="x", iteration=n, plot=False, slice_across=slice_axis + ) + By, info = series.get_field( + field="B", coord="y", iteration=n, plot=False, slice_across=slice_axis + ) + Bz, info = series.get_field( + field="B", coord="z", iteration=n, plot=False, slice_across=slice_axis + ) + # get charge densities + rho_beam1, info = series.get_field( + field="rho_beam1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_beam2, info = series.get_field( + field="rho_beam2", iteration=n, plot=False, slice_across=slice_axis + ) + rho_ele1, info = series.get_field( + field="rho_ele1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_pos1, info = series.get_field( + field="rho_pos1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_ele2, info = series.get_field( + field="rho_ele2", iteration=n, plot=False, slice_across=slice_axis + ) + rho_pos2, info = series.get_field( + field="rho_pos2", iteration=n, plot=False, slice_across=slice_axis + ) + + xmin = info.z.min() + xmax = info.z.max() + xlabel = "z [m]" + + if slice_axis == "x": + ymin = info.y.min() + ymax = info.y.max() + ylabel = "y [m]" + elif slice_axis == "y": + ymin = info.x.min() + ymax = info.x.max() + ylabel = "x [m]" + + # plot E magnitude + Emag = np.sqrt(Ex**2 + Ey**2 + Ez**2) + im = ax[0, 0].imshow( + np.transpose(Emag), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=0, + vmax=np.max(np.abs(Emag)), + ) + ax[0, 0].set_title("E [V/m]") + divider = make_axes_locatable(ax[0, 0]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot B magnitude + Bmag = np.sqrt(Bx**2 + By**2 + Bz**2) + im = ax[1, 0].imshow( + np.transpose(Bmag), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=0, + vmax=np.max(np.abs(Bmag)), + ) + ax[1, 0].set_title("B [T]") + divider = make_axes_locatable(ax[1, 0]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot beam densities + rho_beams = rho_beam1 + rho_beam2 + im = ax[0, 1].imshow( + np.transpose(rho_beams), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=-np.max(np.abs(rho_beams)), + vmax=np.max(np.abs(rho_beams)), + ) + ax[0, 1].set_title(r"$\rho$ beams [C/m$^3$]") + divider = make_axes_locatable(ax[0, 1]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot secondary densities + rho2 = rho_ele1 + rho_pos1 + rho_ele2 + rho_pos2 + im = ax[1, 1].imshow( + np.transpose(rho2), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=-np.max(np.abs(rho2)), + vmax=np.max(np.abs(rho2)), + ) + ax[1, 1].set_title(r"$\rho$ secondaries [C/m$^3$]") + divider = make_axes_locatable(ax[1, 1]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + for a in ax[-1, :].reshape(-1): + a.set_xlabel(xlabel) + for a in ax[:, 0].reshape(-1): + a.set_ylabel(ylabel) + + fig.suptitle(f"Iteration = {n}, time [s] = {series.current_t}", fontsize=20) + plt.tight_layout() + + image_file_name = "FIELDS_" + slice_axis + f"_{n:03d}.png" + plt.savefig(image_file_name, dpi=100, bbox_inches="tight") + plt.close() diff --git a/Examples/Physics_applications/beam_beam_collision/plot_reduced.py b/Examples/Physics_applications/beam_beam_collision/plot_reduced.py new file mode 100644 index 00000000000..3f59f975519 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/plot_reduced.py @@ -0,0 +1,48 @@ +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from scipy.constants import c, nano, physical_constants + +r_e = physical_constants["classical electron radius"][0] +my_dpi = 300 +sigmaz = 10 * nano + +fig, ax = plt.subplots( + ncols=2, nrows=1, figsize=(2000.0 / my_dpi, 1000.0 / my_dpi), dpi=my_dpi +) + +rdir = "./diags/reducedfiles/" + +df_cr = pd.read_csv(f"{rdir}" + "ColliderRelevant_beam1_beam2.txt", sep=" ", header=0) +df_pn = pd.read_csv(f"{rdir}" + "ParticleNumber.txt", sep=" ", header=0) + + +times = df_cr[[col for col in df_cr.columns if "]time" in col]].to_numpy() +steps = df_cr[[col for col in df_cr.columns if "]step" in col]].to_numpy() + +x = df_cr[[col for col in df_cr.columns if "]dL_dt" in col]].to_numpy() +coll_index = np.argmax(x) +coll_time = times[coll_index] + +# number of photons per beam particle +np1 = df_pn[[col for col in df_pn.columns if "]pho1_weight" in col]].to_numpy() +np2 = df_pn[[col for col in df_pn.columns if "]pho2_weight" in col]].to_numpy() +Ne = df_pn[[col for col in df_pn.columns if "]beam1_weight" in col]].to_numpy()[0] +Np = df_pn[[col for col in df_pn.columns if "]beam2_weight" in col]].to_numpy()[0] + +ax[0].plot((times - coll_time) / (sigmaz / c), (np1 + np2) / (Ne + Np), lw=2) +ax[0].set_title(r"photon number/beam particle") + +# number of NLBW particles per beam particle +e1 = df_pn[[col for col in df_pn.columns if "]ele1_weight" in col]].to_numpy() +e2 = df_pn[[col for col in df_pn.columns if "]ele2_weight" in col]].to_numpy() + +ax[1].plot((times - coll_time) / (sigmaz / c), (e1 + e2) / (Ne + Np), lw=2) +ax[1].set_title(r"NLBW particles/beam particle") + +for a in ax.reshape(-1): + a.set_xlabel(r"time [$\sigma_z/c$]") +image_file_name = "reduced.png" +plt.tight_layout() +plt.savefig(image_file_name, dpi=300, bbox_inches="tight") +plt.close("all") diff --git a/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt b/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt new file mode 100644 index 00000000000..5403e374849 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt @@ -0,0 +1,53 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_background_mcc_picmi # name + 1 # dims + 2 # nprocs + "inputs_base_1d_picmi.py --test --pythonsolver" # inputs + "analysis_1d.py" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_dsmc_picmi # name + 1 # dims + 2 # nprocs + "inputs_base_1d_picmi.py --test --dsmc" # inputs + "analysis_dsmc.py" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_background_mcc # name + 2 # dims + 2 # nprocs + inputs_test_2d_background_mcc # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) + +# FIXME: can we make this single precision for now? +#add_warpx_test( +# test_2d_background_mcc_dp_psp # name +# 2 # dims +# 2 # nprocs +# inputs_test_2d_background_mcc_dp_psp # inputs +# OFF # analysis +# "analysis_default_regression.py --path diags/diag1000050" # checksum +# OFF # dependency +#) + +add_warpx_test( + test_2d_background_mcc_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_background_mcc_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000050 --rtol 5e-3" # checksum + OFF # dependency +) diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py deleted file mode 100644 index 8b58790d2e6..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ /dev/null @@ -1,470 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) -# --- Monte-Carlo Collision script to reproduce the benchmark tests from -# --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 - -import argparse -import sys - -import numpy as np -from scipy.sparse import csc_matrix -from scipy.sparse import linalg as sla - -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi - -constants = picmi.constants - - -class PoissonSolver1D(picmi.ElectrostaticSolver): - """This solver is maintained as an example of the use of Python callbacks. - However, it is not necessarily needed since the 1D code has the direct tridiagonal - solver implemented.""" - - def __init__(self, grid, **kwargs): - """Direct solver for the Poisson equation using superLU. This solver is - useful for 1D cases. - - Arguments: - grid (picmi.Cartesian1DGrid): Instance of the grid on which the - solver will be installed. - """ - # Sanity check that this solver is appropriate to use - if not isinstance(grid, picmi.Cartesian1DGrid): - raise RuntimeError('Direct solver can only be used on a 1D grid.') - - super(PoissonSolver1D, self).__init__( - grid=grid, method=kwargs.pop('method', 'Multigrid'), - required_precision=1, **kwargs - ) - - def solver_initialize_inputs(self): - """Grab geometrical quantities from the grid. The boundary potentials - are also obtained from the grid using 'warpx_potential_zmin' for the - left_voltage and 'warpx_potential_zmax' for the right_voltage. - These can be given as floats or strings that can be parsed by the - WarpX parser. - """ - # grab the boundary potentials from the grid object - self.right_voltage = self.grid.potential_zmax - - # set WarpX boundary potentials to None since we will handle it - # ourselves in this solver - self.grid.potential_xmin = None - self.grid.potential_xmax = None - self.grid.potential_ymin = None - self.grid.potential_ymax = None - self.grid.potential_zmin = None - self.grid.potential_zmax = None - - super(PoissonSolver1D, self).solver_initialize_inputs() - - self.nz = self.grid.number_of_cells[0] - self.dz = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nz - - self.nxguardphi = 1 - self.nzguardphi = 1 - - self.phi = np.zeros(self.nz + 1 + 2*self.nzguardphi) - - self.decompose_matrix() - - callbacks.installpoissonsolver(self._run_solve) - - def decompose_matrix(self): - """Function to build the superLU object used to solve the linear - system.""" - self.nsolve = self.nz + 1 - - # Set up the computation matrix in order to solve A*phi = rho - A = np.zeros((self.nsolve, self.nsolve)) - idx = np.arange(self.nsolve) - A[idx, idx] = -2.0 - A[idx[1:], idx[:-1]] = 1.0 - A[idx[:-1], idx[1:]] = 1.0 - - A[0, 1] = 0.0 - A[-1, -2] = 0.0 - A[0, 0] = 1.0 - A[-1, -1] = 1.0 - - A = csc_matrix(A, dtype=np.float64) - self.lu = sla.splu(A) - - def _run_solve(self): - """Function run on every step to perform the required steps to solve - Poisson's equation.""" - # get rho from WarpX - self.rho_data = fields.RhoFPWrapper(0, False)[...] - # run superLU solver to get phi - self.solve() - # write phi to WarpX - fields.PhiFPWrapper(0, True)[...] = self.phi[:] - - def solve(self): - """The solution step. Includes getting the boundary potentials and - calculating phi from rho.""" - - left_voltage = 0.0 - right_voltage = eval( - self.right_voltage, { - 't': self.sim.extension.warpx.gett_new(0), - 'sin': np.sin, 'pi': np.pi - } - ) - - # Construct b vector - rho = -self.rho_data / constants.ep0 - b = np.zeros(rho.shape[0], dtype=np.float64) - b[:] = rho * self.dz**2 - - b[0] = left_voltage - b[-1] = right_voltage - - phi = self.lu.solve(b) - - self.phi[self.nzguardphi:-self.nzguardphi] = phi - - self.phi[:self.nzguardphi] = left_voltage - self.phi[-self.nzguardphi:] = right_voltage - - -class CapacitiveDischargeExample(object): - '''The following runs a simulation of a parallel plate capacitor seeded - with a plasma in the spacing between the plates. A time varying voltage is - applied across the capacitor. The groups of 4 values below correspond to - the 4 cases simulated by Turner et al. (2013) in their benchmarks of - PIC-MCC codes. - ''' - - gap = 0.067 # m - - freq = 13.56e6 # Hz - voltage = [450.0, 200.0, 150.0, 120.0] # V - - gas_density = [9.64e20, 32.1e20, 96.4e20, 321e20] # m^-3 - gas_temp = 300.0 # K - m_ion = 6.67e-27 # kg - - plasma_density = [2.56e14, 5.12e14, 5.12e14, 3.84e14] # m^-3 - elec_temp = 30000.0 # K - - seed_nppc = 16 * np.array([32, 16, 8, 4]) - - nz = [128, 256, 512, 512] - - dt = 1.0 / (np.array([400, 800, 1600, 3200]) * freq) - - # Total simulation time in seconds - total_time = np.array([1280, 5120, 5120, 15360]) / freq - # Time (in seconds) between diagnostic evaluations - diag_interval = 32 / freq - - def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): - """Get input parameters for the specific case (n) desired.""" - self.n = n - self.test = test - self.pythonsolver = pythonsolver - self.dsmc = dsmc - - # Case specific input parameters - self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" - - self.gas_density = self.gas_density[n] - self.plasma_density = self.plasma_density[n] - self.seed_nppc = self.seed_nppc[n] - - self.nz = self.nz[n] - - self.dt = self.dt[n] - self.max_steps = int(self.total_time[n] / self.dt) - self.diag_steps = int(self.diag_interval / self.dt) - - if self.test: - self.max_steps = 50 - self.diag_steps = 5 - self.mcc_subcycling_steps = 2 - self.rng = np.random.default_rng(23094290) - else: - self.mcc_subcycling_steps = None - self.rng = np.random.default_rng() - - self.ion_density_array = np.zeros(self.nz + 1) - - self.setup_run() - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - self.grid = picmi.Cartesian1DGrid( - number_of_cells=[self.nz], - warpx_max_grid_size=128, - lower_bound=[0], - upper_bound=[self.gap], - lower_boundary_conditions=['dirichlet'], - upper_boundary_conditions=['dirichlet'], - lower_boundary_conditions_particles=['absorbing'], - upper_boundary_conditions_particles=['absorbing'], - warpx_potential_hi_z=self.voltage, - ) - - ####################################################################### - # Field solver # - ####################################################################### - - if self.pythonsolver: - self.solver = PoissonSolver1D(grid=self.grid) - else: - # This will use the tridiagonal solver - self.solver = picmi.ElectrostaticSolver(grid=self.grid) - - ####################################################################### - # Particle types setup # - ####################################################################### - - self.electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=picmi.UniformDistribution( - density=self.plasma_density, - rms_velocity=[np.sqrt(constants.kb * self.elec_temp / constants.m_e)]*3, - ) - ) - self.ions = picmi.Species( - particle_type='He', name='he_ions', - charge='q_e', mass=self.m_ion, - initial_distribution=picmi.UniformDistribution( - density=self.plasma_density, - rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, - ) - ) - if self.dsmc: - self.neutrals = picmi.Species( - particle_type='He', name='neutrals', - charge=0, mass=self.m_ion, - warpx_reflection_model_zlo=1.0, - warpx_reflection_model_zhi=1.0, - warpx_do_resampling=True, - warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc*1.5), - initial_distribution=picmi.UniformDistribution( - density=self.gas_density, - rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, - ) - ) - - ####################################################################### - # Collision initialization # - ####################################################################### - - cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' - electron_colls = picmi.MCCCollisions( - name='coll_elec', - species=self.electrons, - background_density=self.gas_density, - background_temperature=self.gas_temp, - background_mass=self.ions.mass, - ndt=self.mcc_subcycling_steps, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'electron_scattering.dat' - }, - 'excitation1' : { - 'cross_section': cross_sec_direc+'excitation_1.dat', - 'energy' : 19.82 - }, - 'excitation2' : { - 'cross_section': cross_sec_direc+'excitation_2.dat', - 'energy' : 20.61 - }, - 'ionization' : { - 'cross_section' : cross_sec_direc+'ionization.dat', - 'energy' : 24.55, - 'species' : self.ions - }, - } - ) - - ion_scattering_processes={ - 'elastic': {'cross_section': cross_sec_direc+'ion_scattering.dat'}, - 'back': {'cross_section': cross_sec_direc+'ion_back_scatter.dat'}, - # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} - } - if self.dsmc: - ion_colls = picmi.DSMCCollisions( - name='coll_ion', - species=[self.ions, self.neutrals], - ndt=5, scattering_processes=ion_scattering_processes - ) - else: - ion_colls = picmi.MCCCollisions( - name='coll_ion', - species=self.ions, - background_density=self.gas_density, - background_temperature=self.gas_temp, - ndt=self.mcc_subcycling_steps, - scattering_processes=ion_scattering_processes - ) - - ####################################################################### - # Initialize simulation # - ####################################################################### - - self.sim = picmi.Simulation( - solver=self.solver, - time_step_size=self.dt, - max_steps=self.max_steps, - warpx_collisions=[electron_colls, ion_colls], - verbose=self.test - ) - self.solver.sim = self.sim - - self.sim.add_species( - self.electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid - ) - ) - self.sim.add_species( - self.ions, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid - ) - ) - if self.dsmc: - self.sim.add_species( - self.neutrals, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[self.seed_nppc//2], grid=self.grid - ) - ) - self.solver.sim_ext = self.sim.extension - - if self.dsmc: - # Periodically reset neutral density to starting temperature - callbacks.installbeforecollisions(self.rethermalize_neutrals) - - ####################################################################### - # Add diagnostics for the CI test to be happy # - ####################################################################### - - if self.dsmc: - file_prefix = 'Python_dsmc_1d_plt' - else: - if self.pythonsolver: - file_prefix = 'Python_background_mcc_1d_plt' - else: - file_prefix = 'Python_background_mcc_1d_tridiag_plt' - - species = [self.electrons, self.ions] - if self.dsmc: - species.append(self.neutrals) - particle_diag = picmi.ParticleDiagnostic( - species=species, - name='diag1', - period=0, - write_dir='.', - warpx_file_prefix=file_prefix - ) - field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=self.grid, - period=0, - data_list=['rho_electrons', 'rho_he_ions'], - write_dir='.', - warpx_file_prefix=file_prefix - ) - self.sim.add_diagnostic(particle_diag) - self.sim.add_diagnostic(field_diag) - - def rethermalize_neutrals(self): - # When using DSMC the neutral temperature will change due to collisions - # with the ions. This is not captured in the original MCC test. - # Re-thermalize the neutrals every 1000 steps - step = self.sim.extension.warpx.getistep(lev=0) - if step % 1000 != 10: - return - - if not hasattr(self, 'neutral_cont'): - self.neutral_cont = particle_containers.ParticleContainerWrapper( - self.neutrals.name - ) - - ux_arrays = self.neutral_cont.uxp - uy_arrays = self.neutral_cont.uyp - uz_arrays = self.neutral_cont.uzp - - vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) - for ii in range(len(ux_arrays)): - nps = len(ux_arrays[ii]) - ux_arrays[ii][:] = vel_std * self.rng.normal(size=nps) - uy_arrays[ii][:] = vel_std * self.rng.normal(size=nps) - uz_arrays[ii][:] = vel_std * self.rng.normal(size=nps) - - def _get_rho_ions(self): - # deposit the ion density in rho_fp - he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') - he_ions_wrapper.deposit_charge_density(level=0) - - rho_data = self.rho_wrapper[...] - self.ion_density_array += rho_data / constants.q_e / self.diag_steps - - def run_sim(self): - - self.sim.step(self.max_steps - self.diag_steps) - - self.rho_wrapper = fields.RhoFPWrapper(0, False) - callbacks.installafterstep(self._get_rho_ions) - - self.sim.step(self.diag_steps) - - if self.pythonsolver: - # confirm that the external solver was run - assert hasattr(self.solver, 'phi') - - if libwarpx.amr.ParallelDescriptor.MyProc() == 0: - np.save(f'ion_density_case_{self.n+1}.npy', self.ion_density_array) - - # query the particle z-coordinates if this is run during CI testing - # to cover that functionality - if self.test: - he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') - nparts = he_ions_wrapper.get_particle_count(local=True) - z_coords = np.concatenate(he_ions_wrapper.zp) - assert len(z_coords) == nparts - assert np.all(z_coords >= 0.0) and np.all(z_coords <= self.gap) - - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-n', help='Test number to run (1 to 4)', required=False, type=int, - default=1 -) -parser.add_argument( - '--pythonsolver', help='toggle whether to use the Python level solver', - action='store_true' -) -parser.add_argument( - '--dsmc', help='toggle whether to use DSMC for ions in place of MCC', - action='store_true' -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -if args.n < 1 or args.n > 4: - raise AttributeError('Test number must be an integer from 1 to 4.') - -run = CapacitiveDischargeExample( - n=args.n-1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc -) -run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py deleted file mode 100755 index 65baabba605..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file for MCC testing. There is already a test of the MCC -# --- functionality. This tests the PICMI interface for the MCC and -# --- provides an example of how an external Poisson solver can be -# --- used for the field solve step. - -import numpy as np -from scipy.sparse import csc_matrix -from scipy.sparse import linalg as sla - -from pywarpx import callbacks, fields, picmi - -constants = picmi.constants - -########################## -# physics parameters -########################## - -D_CA = 0.067 # m - -N_INERT = 9.64e20 # m^-3 -T_INERT = 300.0 # K - -FREQ = 13.56e6 # Hz - -VOLTAGE = 450.0 - -M_ION = 6.67e-27 # kg - -PLASMA_DENSITY = 2.56e14 # m^-3 -T_ELEC = 30000.0 # K - -DT = 1.0 / (400 * FREQ) - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 50 -diagnostic_intervals = "::50" - -# --- Grid -nx = 128 -ny = 8 - -xmin = 0.0 -ymin = 0.0 -xmax = D_CA -ymax = D_CA / nx * ny - -number_per_cell_each_dim = [32, 16] - -############################# -# specialized Poisson solver -# using superLU decomposition -############################# - -class PoissonSolverPseudo1D(picmi.ElectrostaticSolver): - - def __init__(self, grid, **kwargs): - """Direct solver for the Poisson equation using superLU. This solver is - useful for pseudo 1D cases i.e. diode simulations with small x extent. - - Arguments: - grid (picmi.Cartesian2DGrid): Instance of the grid on which the - solver will be installed. - """ - super(PoissonSolverPseudo1D, self).__init__( - grid=grid, method=kwargs.pop('method', 'Multigrid'), - required_precision=1, **kwargs - ) - self.rho_wrapper = None - self.phi_wrapper = None - self.time_sum = 0.0 - - def solver_initialize_inputs(self): - """Grab geometrical quantities from the grid. - """ - self.right_voltage = self.grid.potential_xmax - - # set WarpX boundary potentials to None since we will handle it - # ourselves in this solver - self.grid.potential_xmin = None - self.grid.potential_xmax = None - self.grid.potential_ymin = None - self.grid.potential_ymax = None - self.grid.potential_zmin = None - self.grid.potential_zmax = None - - super(PoissonSolverPseudo1D, self).solver_initialize_inputs() - - self.nx = self.grid.number_of_cells[0] - self.nz = self.grid.number_of_cells[1] - self.dx = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nx - self.dz = (self.grid.upper_bound[1] - self.grid.lower_bound[1]) / self.nz - - if not np.isclose(self.dx, self.dz): - raise RuntimeError('Direct solver requires dx = dz.') - - self.nxguardrho = 2 - self.nzguardrho = 2 - self.nxguardphi = 1 - self.nzguardphi = 1 - - self.phi = np.zeros( - (self.nx + 1 + 2*self.nxguardphi, - self.nz + 1 + 2*self.nzguardphi) - ) - - self.decompose_matrix() - - callbacks.installpoissonsolver(self._run_solve) - - def decompose_matrix(self): - """Function to build the superLU object used to solve the linear - system.""" - self.nxsolve = self.nx + 1 - self.nzsolve = self.nz + 3 - - # Set up the computation matrix in order to solve A*phi = rho - A = np.zeros( - (self.nzsolve*self.nxsolve, self.nzsolve*self.nxsolve) - ) - kk = 0 - for ii in range(self.nxsolve): - for jj in range(self.nzsolve): - temp = np.zeros((self.nxsolve, self.nzsolve)) - - if ii == 0 or ii == self.nxsolve - 1: - temp[ii, jj] = 1. - elif ii == 1: - temp[ii, jj] = -2.0 - temp[ii-1, jj] = 1.0 - temp[ii+1, jj] = 1.0 - elif ii == self.nxsolve - 2: - temp[ii, jj] = -2.0 - temp[ii+1, jj] = 1.0 - temp[ii-1, jj] = 1.0 - elif jj == 0: - temp[ii, jj] = 1.0 - temp[ii, -3] = -1.0 - elif jj == self.nzsolve - 1: - temp[ii, jj] = 1.0 - temp[ii, 2] = -1.0 - else: - temp[ii, jj] = -4.0 - temp[ii, jj+1] = 1.0 - temp[ii, jj-1] = 1.0 - temp[ii-1, jj] = 1.0 - temp[ii+1, jj] = 1.0 - - A[kk] = temp.flatten() - kk += 1 - - A = csc_matrix(A, dtype=np.float32) - self.lu = sla.splu(A) - - def _run_solve(self): - """Function run on every step to perform the required steps to solve - Poisson's equation.""" - - # get rho from WarpX - if self.rho_wrapper is None: - self.rho_wrapper = fields.RhoFPWrapper(0, True) - self.rho_data = self.rho_wrapper[Ellipsis] - - self.solve() - - if self.phi_wrapper is None: - self.phi_wrapper = fields.PhiFPWrapper(0, True) - self.phi_wrapper[Ellipsis] = self.phi - - def solve(self): - """The solution step. Includes getting the boundary potentials and - calculating phi from rho.""" - right_voltage = eval( - self.right_voltage, - {'t': sim.extension.warpx.gett_new(0), 'sin': np.sin, 'pi': np.pi} - ) - left_voltage = 0.0 - - rho = -self.rho_data[ - self.nxguardrho:-self.nxguardrho, self.nzguardrho:-self.nzguardrho - ] / constants.ep0 - - # Construct b vector - nx, nz = np.shape(rho) - source = np.zeros((nx, nz+2), dtype=np.float32) - source[:,1:-1] = rho * self.dx**2 - - source[0] = left_voltage - source[-1] = right_voltage - - # Construct b vector - b = source.flatten() - - flat_phi = self.lu.solve(b) - self.phi[self.nxguardphi:-self.nxguardphi] = ( - flat_phi.reshape(np.shape(source)) - ) - - self.phi[:self.nxguardphi] = left_voltage - self.phi[-self.nxguardphi:] = right_voltage - - # the electrostatic solver in WarpX keeps the ghost cell values as 0 - self.phi[:,:self.nzguardphi] = 0 - self.phi[:,-self.nzguardphi:] = 0 - -########################## -# physics components -########################## - -v_rms_elec = np.sqrt(constants.kb * T_ELEC / constants.m_e) -v_rms_ion = np.sqrt(constants.kb * T_INERT / M_ION) - -uniform_plasma_elec = picmi.UniformDistribution( - density = PLASMA_DENSITY, - upper_bound = [None] * 3, - rms_velocity = [v_rms_elec] * 3, - directed_velocity = [0.] * 3 -) - -uniform_plasma_ion = picmi.UniformDistribution( - density = PLASMA_DENSITY, - upper_bound = [None] * 3, - rms_velocity = [v_rms_ion] * 3, - directed_velocity = [0.] * 3 -) - -electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=uniform_plasma_elec -) -ions = picmi.Species( - particle_type='He', name='he_ions', - charge='q_e', - initial_distribution=uniform_plasma_ion -) - -# MCC collisions -cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' -mcc_electrons = picmi.MCCCollisions( - name='coll_elec', - species=electrons, - background_density=N_INERT, - background_temperature=T_INERT, - background_mass=ions.mass, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'electron_scattering.dat' - }, - 'excitation1' : { - 'cross_section': cross_sec_direc+'excitation_1.dat', - 'energy' : 19.82 - }, - 'excitation2' : { - 'cross_section': cross_sec_direc+'excitation_2.dat', - 'energy' : 20.61 - }, - 'ionization' : { - 'cross_section' : cross_sec_direc+'ionization.dat', - 'energy' : 24.55, - 'species' : ions - }, - } -) - -mcc_ions = picmi.MCCCollisions( - name='coll_ion', - species=ions, - background_density=N_INERT, - background_temperature=T_INERT, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'ion_scattering.dat' - }, - 'back' : { - 'cross_section' : cross_sec_direc+'ion_back_scatter.dat' - }, - # 'charge_exchange' : { - # 'cross_section' : cross_sec_direc+'charge_exchange.dat' - # } - } -) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, ny], - warpx_max_grid_size=128, - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - bc_xmin = 'dirichlet', - bc_xmax = 'dirichlet', - bc_ymin = 'periodic', - bc_ymax = 'periodic', - warpx_potential_hi_x = "%.1f*sin(2*pi*%.5e*t)" % (VOLTAGE, FREQ), - lower_boundary_conditions_particles=['absorbing', 'periodic'], - upper_boundary_conditions_particles=['absorbing', 'periodic'] -) - -# solver = picmi.ElectrostaticSolver( -# grid=grid, method='Multigrid', required_precision=1e-6 -# ) -solver = PoissonSolverPseudo1D(grid=grid) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = diagnostic_intervals, - write_dir = '.', - warpx_file_prefix = 'Python_background_mcc_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['rho_electrons', 'rho_he_ions'], - write_dir = '.', - warpx_file_prefix = 'Python_background_mcc_plt' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = DT, - max_steps = max_steps, - warpx_collisions=[mcc_electrons, mcc_ions] -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid - ) -) -sim.add_species( - ions, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid - ) -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -########################## -# simulation run -########################## - -sim.step(max_steps) - -# confirm that the external solver was run -assert hasattr(solver, 'phi') diff --git a/Examples/Physics_applications/capacitive_discharge/README.rst b/Examples/Physics_applications/capacitive_discharge/README.rst index 708b4528cd5..13b6b3010b3 100644 --- a/Examples/Physics_applications/capacitive_discharge/README.rst +++ b/Examples/Physics_applications/capacitive_discharge/README.rst @@ -22,17 +22,17 @@ The implementation has been tested against the benchmark results from :cite:t:`e Run --- -The 1D PICMI input file can be used to reproduce the results from Turner et al. for a given case, ``N`` from 1 to 4, by executing ``python3 PICMI_inputs_1d.py -n N``, e.g., +The 1D PICMI input file can be used to reproduce the results from Turner et al. for a given case, ``N`` from 1 to 4, by executing ``python3 inputs_base_1d_picmi.py -n N``, e.g., .. code-block:: bash - python3 PICMI_inputs_1d.py -n 1 + python3 inputs_base_1d_picmi.py -n 1 For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. -.. literalinclude:: PICMI_inputs_1d.py +.. literalinclude:: inputs_base_1d_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py``. + :caption: You can copy this file from ``Examples/Physics_applications/capacitive_discharge/inputs_base_1d_picmi.py``. Analyze diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py index 29b5272d8b1..82d98c38210 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py @@ -4,6 +4,7 @@ import numpy as np +# fmt: off ref_density = np.array([ 1.27989677e+14, 2.23601330e+14, 2.55400265e+14, 2.55664972e+14, 2.55806841e+14, 2.55806052e+14, 2.55815865e+14, 2.55755151e+14, @@ -39,7 +40,8 @@ 2.56041610e+14, 2.56041551e+14, 2.56088641e+14, 2.23853646e+14, 1.27580207e+14 ]) +# fmt: on -density_data = np.load( 'ion_density_case_1.npy' ) +density_data = np.load("ion_density_case_1.npy") print(repr(density_data)) assert np.allclose(density_data, ref_density) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_2d.py b/Examples/Physics_applications/capacitive_discharge/analysis_2d.py deleted file mode 100755 index 472758ec63b..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/analysis_2d.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 Modern Electron - -# This script checks that the PICMI_inputs_2d.py run more-or-less matches the -# results from the non-PICMI run. The PICMI run is using an external Poisson -# solver that directly solves the Poisson equation using matrix inversion -# rather than the iterative approach from the MLMG solver. - -import sys - -sys.path.append('../../../../warpx/Regression/Checksum/') - -import checksumAPI - -my_check = checksumAPI.evaluate_checksum( - 'background_mcc', 'Python_background_mcc_plt000050', - do_particles=True, rtol=5e-3 -) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_default_regression.py b/Examples/Physics_applications/capacitive_discharge/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index 4c46b9e3f10..cdaa6bed58f 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -2,57 +2,46 @@ # 2023 TAE Technologies -import os -import sys - import numpy as np -sys.path.append('../../../../warpx/Regression/Checksum/') - -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] - -my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) - +# fmt: off ref_density = np.array([ - 1.27943881e+14, 2.23583097e+14, 2.55396716e+14, 2.55673406e+14, - 2.55827566e+14, 2.55803446e+14, 2.55798707e+14, 2.55748961e+14, - 2.55906413e+14, 2.56063991e+14, 2.55937018e+14, 2.55841390e+14, - 2.55917724e+14, 2.55988641e+14, 2.56052050e+14, 2.56285151e+14, - 2.56647960e+14, 2.56756264e+14, 2.56430158e+14, 2.56117493e+14, - 2.56065302e+14, 2.56265220e+14, 2.56328575e+14, 2.56031495e+14, - 2.56123757e+14, 2.56431173e+14, 2.56385320e+14, 2.56391170e+14, - 2.56561177e+14, 2.56513926e+14, 2.56332201e+14, 2.56252442e+14, - 2.56238982e+14, 2.56216498e+14, 2.56461281e+14, 2.56863199e+14, - 2.56908100e+14, 2.56926112e+14, 2.57001641e+14, 2.56735963e+14, - 2.56315358e+14, 2.56137028e+14, 2.56101418e+14, 2.56276827e+14, - 2.56425668e+14, 2.56181798e+14, 2.56044925e+14, 2.56330387e+14, - 2.56623150e+14, 2.56445316e+14, 2.56292750e+14, 2.56440918e+14, - 2.56433406e+14, 2.56186982e+14, 2.56236390e+14, 2.56469557e+14, - 2.56349704e+14, 2.56487457e+14, 2.56771823e+14, 2.56614683e+14, - 2.56552210e+14, 2.56850291e+14, 2.56783396e+14, 2.56483187e+14, - 2.56510868e+14, 2.56490408e+14, 2.56656042e+14, 2.56820924e+14, - 2.56640314e+14, 2.56465063e+14, 2.56510264e+14, 2.56917331e+14, - 2.57228490e+14, 2.56960593e+14, 2.56587911e+14, 2.56672682e+14, - 2.56774414e+14, 2.56548335e+14, 2.56225540e+14, 2.56079693e+14, - 2.56062796e+14, 2.56054612e+14, 2.56028683e+14, 2.56068820e+14, - 2.56380975e+14, 2.56654914e+14, 2.56776792e+14, 2.56983661e+14, - 2.56989477e+14, 2.56646250e+14, 2.56589639e+14, 2.56946205e+14, - 2.57091201e+14, 2.56913590e+14, 2.56513535e+14, 2.56122597e+14, - 2.56176340e+14, 2.56808001e+14, 2.57239393e+14, 2.56845066e+14, - 2.56662482e+14, 2.56862583e+14, 2.56518922e+14, 2.56155531e+14, - 2.56362794e+14, 2.57203564e+14, 2.57737938e+14, 2.57252026e+14, - 2.56859277e+14, 2.56658995e+14, 2.56357364e+14, 2.56393454e+14, - 2.56714308e+14, 2.57042200e+14, 2.57551087e+14, 2.57502490e+14, - 2.56641118e+14, 2.56401115e+14, 2.56644629e+14, 2.56673096e+14, - 2.56534659e+14, 2.56357745e+14, 2.56455309e+14, 2.56586850e+14, - 2.56442415e+14, 2.56335971e+14, 2.56411429e+14, 2.24109018e+14, - 1.27678869e+14 + 1.27942709e+14, 2.23579371e+14, 2.55384387e+14, 2.55660663e+14, + 2.55830911e+14, 2.55814337e+14, 2.55798906e+14, 2.55744891e+14, + 2.55915585e+14, 2.56083194e+14, 2.55942354e+14, 2.55833026e+14, + 2.56036175e+14, 2.56234141e+14, 2.56196179e+14, 2.56146141e+14, + 2.56168022e+14, 2.56216909e+14, 2.56119961e+14, 2.56065167e+14, + 2.56194764e+14, 2.56416398e+14, 2.56465239e+14, 2.56234337e+14, + 2.56234503e+14, 2.56316003e+14, 2.56175023e+14, 2.56030269e+14, + 2.56189301e+14, 2.56286379e+14, 2.56130396e+14, 2.56295225e+14, + 2.56474082e+14, 2.56340375e+14, 2.56350864e+14, 2.56462330e+14, + 2.56469391e+14, 2.56412726e+14, 2.56241788e+14, 2.56355650e+14, + 2.56650599e+14, 2.56674748e+14, 2.56642480e+14, 2.56823508e+14, + 2.57025029e+14, 2.57110614e+14, 2.57042364e+14, 2.56950884e+14, + 2.57051822e+14, 2.56952148e+14, 2.56684016e+14, 2.56481130e+14, + 2.56277073e+14, 2.56065774e+14, 2.56190033e+14, 2.56411074e+14, + 2.56202418e+14, 2.56128368e+14, 2.56227002e+14, 2.56083004e+14, + 2.56056768e+14, 2.56343831e+14, 2.56443659e+14, 2.56280541e+14, + 2.56191572e+14, 2.56147304e+14, 2.56342794e+14, 2.56735473e+14, + 2.56994680e+14, 2.56901500e+14, 2.56527131e+14, 2.56490824e+14, + 2.56614730e+14, 2.56382744e+14, 2.56588214e+14, 2.57160270e+14, + 2.57230435e+14, 2.57116530e+14, 2.57065771e+14, 2.57236507e+14, + 2.57112865e+14, 2.56540177e+14, 2.56416828e+14, 2.56648954e+14, + 2.56625594e+14, 2.56411003e+14, 2.56523754e+14, 2.56841108e+14, + 2.56856368e+14, 2.56757912e+14, 2.56895134e+14, 2.57144419e+14, + 2.57001944e+14, 2.56371759e+14, 2.56179404e+14, 2.56541905e+14, + 2.56715727e+14, 2.56851681e+14, 2.57114458e+14, 2.57001739e+14, + 2.56825690e+14, 2.56879682e+14, 2.56699673e+14, 2.56532841e+14, + 2.56479582e+14, 2.56630989e+14, 2.56885996e+14, 2.56694637e+14, + 2.56250819e+14, 2.56045278e+14, 2.56366075e+14, 2.56693733e+14, + 2.56618530e+14, 2.56580918e+14, 2.56812781e+14, 2.56754216e+14, + 2.56444736e+14, 2.56473391e+14, 2.56538398e+14, 2.56626551e+14, + 2.56471950e+14, 2.56274969e+14, 2.56489423e+14, 2.56645266e+14, + 2.56611124e+14, 2.56344324e+14, 2.56244156e+14, 2.24183727e+14, + 1.27909856e+14 ]) +# fmt: on -density_data = np.load( 'ion_density_case_1.npy' ) +density_data = np.load("ion_density_case_1.npy") print(repr(density_data)) assert np.allclose(density_data, ref_density) diff --git a/Examples/Physics_applications/capacitive_discharge/inputs_2d b/Examples/Physics_applications/capacitive_discharge/inputs_2d deleted file mode 100644 index 2b11fd12978..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/inputs_2d +++ /dev/null @@ -1,78 +0,0 @@ -# Input file for MCC testing. This runs an eighth of a voltage input period -# of the first benchmark case of Turner et al. (2013). - -my_constants.Ngas = 9.64e+20 # m^-3 -my_constants.Tgas = 300 # K -my_constants.Nplasma = 2.56e14 # m^-3 -my_constants.freq = 13.56e6 # Hz -my_constants.Mion = 6.67e-27 # kg - -max_step = 50 -warpx.verbose = 0 -warpx.const_dt = 1.0/(400*freq) -warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1e-06 -warpx.use_filter = 0 - -amr.n_cell = 128 8 -amr.max_grid_size = 128 -amr.max_level = 0 - -geometry.dims = 2 -geometry.prob_lo = 0.0 0.0 -geometry.prob_hi = 0.067 0.0041875 - -boundary.field_lo = pec periodic -boundary.field_hi = pec periodic -boundary.potential_hi_x = 450.0*sin(2*pi*freq*t) - -# Order of particle shape factors -algo.particle_shape = 1 - -particles.species_names = electrons he_ions -electrons.species_type = electron -electrons.injection_style = nuniformpercell -electrons.initialize_self_fields = 0 -electrons.num_particles_per_cell_each_dim = 32 16 -electrons.profile = constant -electrons.density = Nplasma -electrons.momentum_distribution_type = maxwell_boltzmann -electrons.theta = (kb*30000/(m_e*clight^2)) - -he_ions.species_type = helium -he_ions.charge = q_e -he_ions.injection_style = nuniformpercell -he_ions.initialize_self_fields = 0 -he_ions.num_particles_per_cell_each_dim = 32 16 -he_ions.profile = constant -he_ions.density = Nplasma -he_ions.momentum_distribution_type = maxwell_boltzmann -he_ions.theta = (kb*Tgas/(Mion*clight^2)) - -collisions.collision_names = coll_elec coll_ion -coll_ion.type = background_mcc -coll_ion.species = he_ions -coll_ion.background_density = Ngas -coll_ion.background_temperature = Tgas -coll_ion.scattering_processes = elastic back -coll_ion.elastic_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ion_scattering.dat -coll_ion.back_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ion_back_scatter.dat - -coll_elec.type = background_mcc -coll_elec.species = electrons -coll_elec.background_density = Ngas -coll_elec.background_temperature = Tgas -coll_elec.scattering_processes = elastic excitation1 excitation2 ionization -coll_elec.elastic_cross_section = ../../../../warpx-data/MCC_cross_sections/He/electron_scattering.dat -coll_elec.excitation1_energy = 19.82 -coll_elec.excitation1_cross_section = ../../../../warpx-data/MCC_cross_sections/He/excitation_1.dat -coll_elec.excitation2_energy = 20.61 -coll_elec.excitation2_cross_section = ../../../../warpx-data/MCC_cross_sections/He/excitation_2.dat -coll_elec.ionization_energy = 24.55 -coll_elec.ionization_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ionization.dat -coll_elec.ionization_species = he_ions - -diagnostics.diags_names = diag1 -diag1.diag_type = Full -diag1.intervals = 50 -diag1.fields_to_plot = rho_electrons rho_he_ions diff --git a/Examples/Physics_applications/capacitive_discharge/inputs_base_1d_picmi.py b/Examples/Physics_applications/capacitive_discharge/inputs_base_1d_picmi.py new file mode 100644 index 00000000000..a03cf1954ad --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/inputs_base_1d_picmi.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# +# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) +# --- Monte-Carlo Collision script to reproduce the benchmark tests from +# --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 + +import argparse +import sys + +import numpy as np +from scipy.sparse import csc_matrix +from scipy.sparse import linalg as sla + +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi +from pywarpx.LoadThirdParty import load_cupy + +constants = picmi.constants + + +class PoissonSolver1D(picmi.ElectrostaticSolver): + """This solver is maintained as an example of the use of Python callbacks. + However, it is not necessarily needed since the 1D code has the direct tridiagonal + solver implemented.""" + + def __init__(self, grid, **kwargs): + """Direct solver for the Poisson equation using superLU. This solver is + useful for 1D cases. + + Arguments: + grid (picmi.Cartesian1DGrid): Instance of the grid on which the + solver will be installed. + """ + # Sanity check that this solver is appropriate to use + if not isinstance(grid, picmi.Cartesian1DGrid): + raise RuntimeError("Direct solver can only be used on a 1D grid.") + + super(PoissonSolver1D, self).__init__( + grid=grid, + method=kwargs.pop("method", "Multigrid"), + required_precision=1, + **kwargs, + ) + + def solver_initialize_inputs(self): + """Grab geometrical quantities from the grid. The boundary potentials + are also obtained from the grid using 'warpx_potential_zmin' for the + left_voltage and 'warpx_potential_zmax' for the right_voltage. + These can be given as floats or strings that can be parsed by the + WarpX parser. + """ + # grab the boundary potentials from the grid object + self.right_voltage = self.grid.potential_zmax + + # set WarpX boundary potentials to None since we will handle it + # ourselves in this solver + self.grid.potential_xmin = None + self.grid.potential_xmax = None + self.grid.potential_ymin = None + self.grid.potential_ymax = None + self.grid.potential_zmin = None + self.grid.potential_zmax = None + + super(PoissonSolver1D, self).solver_initialize_inputs() + + self.nz = self.grid.number_of_cells[0] + self.dz = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nz + + self.nxguardphi = 1 + self.nzguardphi = 1 + + self.phi = np.zeros(self.nz + 1 + 2 * self.nzguardphi) + + self.decompose_matrix() + + callbacks.installpoissonsolver(self._run_solve) + + def decompose_matrix(self): + """Function to build the superLU object used to solve the linear + system.""" + self.nsolve = self.nz + 1 + + # Set up the computation matrix in order to solve A*phi = rho + A = np.zeros((self.nsolve, self.nsolve)) + idx = np.arange(self.nsolve) + A[idx, idx] = -2.0 + A[idx[1:], idx[:-1]] = 1.0 + A[idx[:-1], idx[1:]] = 1.0 + + A[0, 1] = 0.0 + A[-1, -2] = 0.0 + A[0, 0] = 1.0 + A[-1, -1] = 1.0 + + A = csc_matrix(A, dtype=np.float64) + self.lu = sla.splu(A) + + def _run_solve(self): + """Function run on every step to perform the required steps to solve + Poisson's equation.""" + # get rho from WarpX + self.rho_data = fields.RhoFPWrapper(0, False)[...] + # run superLU solver to get phi + self.solve() + # write phi to WarpX + fields.PhiFPWrapper(0, True)[...] = self.phi[:] + + def solve(self): + """The solution step. Includes getting the boundary potentials and + calculating phi from rho.""" + + left_voltage = 0.0 + right_voltage = eval( + self.right_voltage, + {"t": self.sim.extension.warpx.gett_new(0), "sin": np.sin, "pi": np.pi}, + ) + + # Construct b vector + rho = -self.rho_data / constants.ep0 + b = np.zeros(rho.shape[0], dtype=np.float64) + b[:] = rho * self.dz**2 + + b[0] = left_voltage + b[-1] = right_voltage + + phi = self.lu.solve(b) + + self.phi[self.nzguardphi : -self.nzguardphi] = phi + + self.phi[: self.nzguardphi] = left_voltage + self.phi[-self.nzguardphi :] = right_voltage + + +class CapacitiveDischargeExample(object): + """The following runs a simulation of a parallel plate capacitor seeded + with a plasma in the spacing between the plates. A time varying voltage is + applied across the capacitor. The groups of 4 values below correspond to + the 4 cases simulated by Turner et al. (2013) in their benchmarks of + PIC-MCC codes. + """ + + gap = 0.067 # m + + freq = 13.56e6 # Hz + voltage = [450.0, 200.0, 150.0, 120.0] # V + + gas_density = [9.64e20, 32.1e20, 96.4e20, 321e20] # m^-3 + gas_temp = 300.0 # K + m_ion = 6.67e-27 # kg + + plasma_density = [2.56e14, 5.12e14, 5.12e14, 3.84e14] # m^-3 + elec_temp = 30000.0 # K + + seed_nppc = 16 * np.array([32, 16, 8, 4]) + + nz = [128, 256, 512, 512] + + dt = 1.0 / (np.array([400, 800, 1600, 3200]) * freq) + + # Total simulation time in seconds + total_time = np.array([1280, 5120, 5120, 15360]) / freq + # Time (in seconds) between diagnostic evaluations + diag_interval = 32 / freq + + def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): + """Get input parameters for the specific case (n) desired.""" + self.n = n + self.test = test + self.pythonsolver = pythonsolver + self.dsmc = dsmc + + # Case specific input parameters + self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" + + self.gas_density = self.gas_density[n] + self.plasma_density = self.plasma_density[n] + self.seed_nppc = self.seed_nppc[n] + + self.nz = self.nz[n] + + self.dt = self.dt[n] + self.max_steps = int(self.total_time[n] / self.dt) + self.diag_steps = int(self.diag_interval / self.dt) + + if self.test: + self.max_steps = 50 + self.diag_steps = 5 + self.mcc_subcycling_steps = 2 + self.rng = np.random.default_rng(23094290) + else: + self.mcc_subcycling_steps = None + self.rng = np.random.default_rng() + + self.ion_density_array = np.zeros(self.nz + 1) + + self.setup_run() + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + self.grid = picmi.Cartesian1DGrid( + number_of_cells=[self.nz], + warpx_max_grid_size=128, + lower_bound=[0], + upper_bound=[self.gap], + lower_boundary_conditions=["dirichlet"], + upper_boundary_conditions=["dirichlet"], + lower_boundary_conditions_particles=["absorbing"], + upper_boundary_conditions_particles=["absorbing"], + warpx_potential_hi_z=self.voltage, + ) + + ####################################################################### + # Field solver # + ####################################################################### + + if self.pythonsolver: + self.solver = PoissonSolver1D(grid=self.grid) + else: + # This will use the tridiagonal solver + self.solver = picmi.ElectrostaticSolver(grid=self.grid) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=picmi.UniformDistribution( + density=self.plasma_density, + rms_velocity=[np.sqrt(constants.kb * self.elec_temp / constants.m_e)] + * 3, + ), + ) + self.ions = picmi.Species( + particle_type="He", + name="he_ions", + charge="q_e", + mass=self.m_ion, + initial_distribution=picmi.UniformDistribution( + density=self.plasma_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)] * 3, + ), + ) + if self.dsmc: + self.neutrals = picmi.Species( + particle_type="He", + name="neutrals", + charge=0, + mass=self.m_ion, + warpx_reflection_model_zlo=1.0, + warpx_reflection_model_zhi=1.0, + warpx_do_resampling=True, + warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc * 1.5), + initial_distribution=picmi.UniformDistribution( + density=self.gas_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)] + * 3, + ), + ) + + ####################################################################### + # Collision initialization # + ####################################################################### + + cross_sec_direc = "../../../../warpx-data/MCC_cross_sections/He/" + electron_colls = picmi.MCCCollisions( + name="coll_elec", + species=self.electrons, + background_density=self.gas_density, + background_temperature=self.gas_temp, + background_mass=self.ions.mass, + ndt=self.mcc_subcycling_steps, + scattering_processes={ + "elastic": { + "cross_section": cross_sec_direc + "electron_scattering.dat" + }, + "excitation1": { + "cross_section": cross_sec_direc + "excitation_1.dat", + "energy": 19.82, + }, + "excitation2": { + "cross_section": cross_sec_direc + "excitation_2.dat", + "energy": 20.61, + }, + "ionization": { + "cross_section": cross_sec_direc + "ionization.dat", + "energy": 24.55, + "species": self.ions, + }, + }, + ) + + ion_scattering_processes = { + "elastic": {"cross_section": cross_sec_direc + "ion_scattering.dat"}, + "back": {"cross_section": cross_sec_direc + "ion_back_scatter.dat"}, + # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} + } + if self.dsmc: + ion_colls = picmi.DSMCCollisions( + name="coll_ion", + species=[self.ions, self.neutrals], + ndt=5, + scattering_processes=ion_scattering_processes, + ) + else: + ion_colls = picmi.MCCCollisions( + name="coll_ion", + species=self.ions, + background_density=self.gas_density, + background_temperature=self.gas_temp, + ndt=self.mcc_subcycling_steps, + scattering_processes=ion_scattering_processes, + ) + + ####################################################################### + # Initialize simulation # + ####################################################################### + + self.sim = picmi.Simulation( + solver=self.solver, + time_step_size=self.dt, + max_steps=self.max_steps, + warpx_collisions=[electron_colls, ion_colls], + verbose=self.test, + ) + self.solver.sim = self.sim + + self.sim.add_species( + self.electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid + ), + ) + self.sim.add_species( + self.ions, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid + ), + ) + if self.dsmc: + self.sim.add_species( + self.neutrals, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc // 2], grid=self.grid + ), + ) + self.solver.sim_ext = self.sim.extension + + if self.dsmc: + # Periodically reset neutral density to starting temperature + callbacks.installbeforecollisions(self.rethermalize_neutrals) + + ####################################################################### + # Add diagnostics for the CI test to be happy # + ####################################################################### + + species = [self.electrons, self.ions] + if self.dsmc: + species.append(self.neutrals) + particle_diag = picmi.ParticleDiagnostic( + species=species, + name="diag1", + period=0, + ) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=0, + data_list=["rho_electrons", "rho_he_ions"], + ) + self.sim.add_diagnostic(particle_diag) + self.sim.add_diagnostic(field_diag) + + def rethermalize_neutrals(self): + # When using DSMC the neutral temperature will change due to collisions + # with the ions. This is not captured in the original MCC test. + # Re-thermalize the neutrals every 1000 steps + step = self.sim.extension.warpx.getistep(lev=0) + if step % 1000 != 10: + return + + if not hasattr(self, "neutral_cont"): + self.neutral_cont = particle_containers.ParticleContainerWrapper( + self.neutrals.name + ) + + ux_arrays = self.neutral_cont.uxp + uy_arrays = self.neutral_cont.uyp + uz_arrays = self.neutral_cont.uzp + + xp, _ = load_cupy() + + vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) + for ii in range(len(ux_arrays)): + nps = len(ux_arrays[ii]) + ux_arrays[ii][:] = xp.array(vel_std * self.rng.normal(size=nps)) + uy_arrays[ii][:] = xp.array(vel_std * self.rng.normal(size=nps)) + uz_arrays[ii][:] = xp.array(vel_std * self.rng.normal(size=nps)) + + def _get_rho_ions(self): + # deposit the ion density in rho_fp + he_ions_wrapper = particle_containers.ParticleContainerWrapper("he_ions") + he_ions_wrapper.deposit_charge_density(level=0) + + rho_data = self.rho_wrapper[...] + self.ion_density_array += rho_data / constants.q_e / self.diag_steps + + def run_sim(self): + self.sim.step(self.max_steps - self.diag_steps) + + self.rho_wrapper = fields.RhoFPWrapper(0, False) + callbacks.installafterstep(self._get_rho_ions) + + self.sim.step(self.diag_steps) + + if self.pythonsolver: + # confirm that the external solver was run + assert hasattr(self.solver, "phi") + + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: + np.save(f"ion_density_case_{self.n + 1}.npy", self.ion_density_array) + + # query the particle z-coordinates if this is run during CI testing + # to cover that functionality + if self.test: + he_ions_wrapper = particle_containers.ParticleContainerWrapper("he_ions") + nparts = he_ions_wrapper.get_particle_count(local=True) + z_coords = np.concatenate(he_ions_wrapper.zp) + assert len(z_coords) == nparts + assert np.all(z_coords >= 0.0) and np.all(z_coords <= self.gap) + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-n", help="Test number to run (1 to 4)", required=False, type=int, default=1 +) +parser.add_argument( + "--pythonsolver", + help="toggle whether to use the Python level solver", + action="store_true", +) +parser.add_argument( + "--dsmc", + help="toggle whether to use DSMC for ions in place of MCC", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +if args.n < 1 or args.n > 4: + raise AttributeError("Test number must be an integer from 1 to 4.") + +run = CapacitiveDischargeExample( + n=args.n - 1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc +) +run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc b/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc new file mode 100644 index 00000000000..e42e531c9e2 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc @@ -0,0 +1,79 @@ +# Input file for MCC testing. This runs an eighth of a voltage input period +# of the first benchmark case of Turner et al. (2013). + +my_constants.Ngas = 9.64e+20 # m^-3 +my_constants.Tgas = 300 # K +my_constants.Nplasma = 2.56e14 # m^-3 +my_constants.freq = 13.56e6 # Hz +my_constants.Mion = 6.67e-27 # kg + +max_step = 50 +warpx.verbose = 0 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1e-06 +warpx.use_filter = 0 +warpx.abort_on_warning_threshold = high + +amr.n_cell = 128 8 +amr.max_grid_size = 128 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 0.067 0.0041875 + +boundary.field_lo = pec periodic +boundary.field_hi = pec periodic +boundary.potential_hi_x = 450.0*sin(2*pi*freq*t) + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = electrons he_ions +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 32 16 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*30000/(m_e*clight^2)) + +he_ions.species_type = helium +he_ions.charge = q_e +he_ions.injection_style = nuniformpercell +he_ions.initialize_self_fields = 0 +he_ions.num_particles_per_cell_each_dim = 32 16 +he_ions.profile = constant +he_ions.density = Nplasma +he_ions.momentum_distribution_type = maxwell_boltzmann +he_ions.theta = (kb*Tgas/(Mion*clight^2)) + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = he_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back +coll_ion.elastic_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ion_scattering.dat +coll_ion.back_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ion_back_scatter.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 excitation2 ionization +coll_elec.elastic_cross_section = ../../../../warpx-data/MCC_cross_sections/He/electron_scattering.dat +coll_elec.excitation1_energy = 19.82 +coll_elec.excitation1_cross_section = ../../../../warpx-data/MCC_cross_sections/He/excitation_1.dat +coll_elec.excitation2_energy = 20.61 +coll_elec.excitation2_cross_section = ../../../../warpx-data/MCC_cross_sections/He/excitation_2.dat +coll_elec.ionization_energy = 24.55 +coll_elec.ionization_cross_section = ../../../../warpx-data/MCC_cross_sections/He/ionization.dat +coll_elec.ionization_species = he_ions + +diagnostics.diags_names = diag1 +diag1.diag_type = Full +diag1.intervals = 50 +diag1.fields_to_plot = rho_electrons rho_he_ions diff --git a/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc_picmi.py b/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc_picmi.py new file mode 100755 index 00000000000..7879239d5ce --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/inputs_test_2d_background_mcc_picmi.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# +# --- Input file for MCC testing. There is already a test of the MCC +# --- functionality. This tests the PICMI interface for the MCC and +# --- provides an example of how an external Poisson solver can be +# --- used for the field solve step. + +import numpy as np +from scipy.sparse import csc_matrix +from scipy.sparse import linalg as sla + +from pywarpx import callbacks, fields, picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +D_CA = 0.067 # m + +N_INERT = 9.64e20 # m^-3 +T_INERT = 300.0 # K + +FREQ = 13.56e6 # Hz + +VOLTAGE = 450.0 + +M_ION = 6.67e-27 # kg + +PLASMA_DENSITY = 2.56e14 # m^-3 +T_ELEC = 30000.0 # K + +DT = 1.0 / (400 * FREQ) + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 50 +diagnostic_intervals = "::50" + +# --- Grid +nx = 128 +ny = 8 + +xmin = 0.0 +ymin = 0.0 +xmax = D_CA +ymax = D_CA / nx * ny + +number_per_cell_each_dim = [32, 16] + +############################# +# specialized Poisson solver +# using superLU decomposition +############################# + + +class PoissonSolverPseudo1D(picmi.ElectrostaticSolver): + def __init__(self, grid, **kwargs): + """Direct solver for the Poisson equation using superLU. This solver is + useful for pseudo 1D cases i.e. diode simulations with small x extent. + + Arguments: + grid (picmi.Cartesian2DGrid): Instance of the grid on which the + solver will be installed. + """ + super(PoissonSolverPseudo1D, self).__init__( + grid=grid, + method=kwargs.pop("method", "Multigrid"), + required_precision=1, + **kwargs, + ) + self.rho_wrapper = None + self.phi_wrapper = None + self.time_sum = 0.0 + + def solver_initialize_inputs(self): + """Grab geometrical quantities from the grid.""" + self.right_voltage = self.grid.potential_xmax + + # set WarpX boundary potentials to None since we will handle it + # ourselves in this solver + self.grid.potential_xmin = None + self.grid.potential_xmax = None + self.grid.potential_ymin = None + self.grid.potential_ymax = None + self.grid.potential_zmin = None + self.grid.potential_zmax = None + + super(PoissonSolverPseudo1D, self).solver_initialize_inputs() + + self.nx = self.grid.number_of_cells[0] + self.nz = self.grid.number_of_cells[1] + self.dx = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nx + self.dz = (self.grid.upper_bound[1] - self.grid.lower_bound[1]) / self.nz + + if not np.isclose(self.dx, self.dz): + raise RuntimeError("Direct solver requires dx = dz.") + + self.nxguardrho = 2 + self.nzguardrho = 2 + self.nxguardphi = 1 + self.nzguardphi = 1 + + self.phi = np.zeros( + (self.nx + 1 + 2 * self.nxguardphi, self.nz + 1 + 2 * self.nzguardphi) + ) + + self.decompose_matrix() + + callbacks.installpoissonsolver(self._run_solve) + + def decompose_matrix(self): + """Function to build the superLU object used to solve the linear + system.""" + self.nxsolve = self.nx + 1 + self.nzsolve = self.nz + 3 + + # Set up the computation matrix in order to solve A*phi = rho + A = np.zeros((self.nzsolve * self.nxsolve, self.nzsolve * self.nxsolve)) + kk = 0 + for ii in range(self.nxsolve): + for jj in range(self.nzsolve): + temp = np.zeros((self.nxsolve, self.nzsolve)) + + if ii == 0 or ii == self.nxsolve - 1: + temp[ii, jj] = 1.0 + elif ii == 1: + temp[ii, jj] = -2.0 + temp[ii - 1, jj] = 1.0 + temp[ii + 1, jj] = 1.0 + elif ii == self.nxsolve - 2: + temp[ii, jj] = -2.0 + temp[ii + 1, jj] = 1.0 + temp[ii - 1, jj] = 1.0 + elif jj == 0: + temp[ii, jj] = 1.0 + temp[ii, -3] = -1.0 + elif jj == self.nzsolve - 1: + temp[ii, jj] = 1.0 + temp[ii, 2] = -1.0 + else: + temp[ii, jj] = -4.0 + temp[ii, jj + 1] = 1.0 + temp[ii, jj - 1] = 1.0 + temp[ii - 1, jj] = 1.0 + temp[ii + 1, jj] = 1.0 + + A[kk] = temp.flatten() + kk += 1 + + A = csc_matrix(A, dtype=np.float32) + self.lu = sla.splu(A) + + def _run_solve(self): + """Function run on every step to perform the required steps to solve + Poisson's equation.""" + + # get rho from WarpX + if self.rho_wrapper is None: + self.rho_wrapper = fields.RhoFPWrapper(0, True) + self.rho_data = self.rho_wrapper[Ellipsis] + + self.solve() + + if self.phi_wrapper is None: + self.phi_wrapper = fields.PhiFPWrapper(0, True) + self.phi_wrapper[Ellipsis] = self.phi + + def solve(self): + """The solution step. Includes getting the boundary potentials and + calculating phi from rho.""" + right_voltage = eval( + self.right_voltage, + {"t": sim.extension.warpx.gett_new(0), "sin": np.sin, "pi": np.pi}, + ) + left_voltage = 0.0 + + rho = ( + -self.rho_data[ + self.nxguardrho : -self.nxguardrho, self.nzguardrho : -self.nzguardrho + ] + / constants.ep0 + ) + + # Construct b vector + nx, nz = np.shape(rho) + source = np.zeros((nx, nz + 2), dtype=np.float32) + source[:, 1:-1] = rho * self.dx**2 + + source[0] = left_voltage + source[-1] = right_voltage + + # Construct b vector + b = source.flatten() + + flat_phi = self.lu.solve(b) + self.phi[self.nxguardphi : -self.nxguardphi] = flat_phi.reshape( + np.shape(source) + ) + + self.phi[: self.nxguardphi] = left_voltage + self.phi[-self.nxguardphi :] = right_voltage + + # the electrostatic solver in WarpX keeps the ghost cell values as 0 + self.phi[:, : self.nzguardphi] = 0 + self.phi[:, -self.nzguardphi :] = 0 + + +########################## +# physics components +########################## + +v_rms_elec = np.sqrt(constants.kb * T_ELEC / constants.m_e) +v_rms_ion = np.sqrt(constants.kb * T_INERT / M_ION) + +uniform_plasma_elec = picmi.UniformDistribution( + density=PLASMA_DENSITY, + upper_bound=[None] * 3, + rms_velocity=[v_rms_elec] * 3, + directed_velocity=[0.0] * 3, +) + +uniform_plasma_ion = picmi.UniformDistribution( + density=PLASMA_DENSITY, + upper_bound=[None] * 3, + rms_velocity=[v_rms_ion] * 3, + directed_velocity=[0.0] * 3, +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=uniform_plasma_elec +) +ions = picmi.Species( + particle_type="He", + name="he_ions", + charge="q_e", + initial_distribution=uniform_plasma_ion, +) + +# MCC collisions +cross_sec_direc = "../../../../warpx-data/MCC_cross_sections/He/" +mcc_electrons = picmi.MCCCollisions( + name="coll_elec", + species=electrons, + background_density=N_INERT, + background_temperature=T_INERT, + background_mass=ions.mass, + scattering_processes={ + "elastic": {"cross_section": cross_sec_direc + "electron_scattering.dat"}, + "excitation1": { + "cross_section": cross_sec_direc + "excitation_1.dat", + "energy": 19.82, + }, + "excitation2": { + "cross_section": cross_sec_direc + "excitation_2.dat", + "energy": 20.61, + }, + "ionization": { + "cross_section": cross_sec_direc + "ionization.dat", + "energy": 24.55, + "species": ions, + }, + }, +) + +mcc_ions = picmi.MCCCollisions( + name="coll_ion", + species=ions, + background_density=N_INERT, + background_temperature=T_INERT, + scattering_processes={ + "elastic": {"cross_section": cross_sec_direc + "ion_scattering.dat"}, + "back": {"cross_section": cross_sec_direc + "ion_back_scatter.dat"}, + # 'charge_exchange' : { + # 'cross_section' : cross_sec_direc+'charge_exchange.dat' + # } + }, +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + warpx_max_grid_size=128, + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + bc_xmin="dirichlet", + bc_xmax="dirichlet", + bc_ymin="periodic", + bc_ymax="periodic", + warpx_potential_hi_x="%.1f*sin(2*pi*%.5e*t)" % (VOLTAGE, FREQ), + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], +) + +# solver = picmi.ElectrostaticSolver( +# grid=grid, method='Multigrid', required_precision=1e-6 +# ) +solver = PoissonSolverPseudo1D(grid=grid) + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["rho_electrons", "rho_he_ions"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=DT, + max_steps=max_steps, + warpx_collisions=[mcc_electrons, mcc_ions], +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) +sim.add_species( + ions, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +########################## +# simulation run +########################## + +sim.step(max_steps) + +# confirm that the external solver was run +assert hasattr(solver, "phi") diff --git a/Examples/Physics_applications/free_electron_laser/CMakeLists.txt b/Examples/Physics_applications/free_electron_laser/CMakeLists.txt new file mode 100644 index 00000000000..168f06c9859 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_fel # name + 1 # dims + 2 # nprocs + inputs_test_1d_fel # inputs + "analysis_fel.py diags/diag_labframe" # analysis + "analysis_default_regression.py --path diags/diag_labframe" # checksum + OFF # dependency +) diff --git a/Examples/Physics_applications/free_electron_laser/README.rst b/Examples/Physics_applications/free_electron_laser/README.rst new file mode 100644 index 00000000000..00d6ef2758c --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/README.rst @@ -0,0 +1,46 @@ +.. _examples-free-electron-laser: + +Free-electron laser +=================== + +This example shows how to simulate the physics of a free-electron laser (FEL) using WarpX. +In this example, a relativistic electron beam is sent through an undulator (represented by an external, +oscillating magnetic field). The radiation emitted by the beam grows exponentially +as the beam travels through the undulator, due to the Free-Electron-Laser instability. + +The parameters of the simulation are taken from section 5.1 of :cite:t:`ex-Fallahi2020`. + +The simulation is performed in 1D, and uses the boosted-frame technique as described in +:cite:t:`ex-VayFELA2009` and :cite:t:`ex-VayFELB2009` to reduce the computational cost (the Lorentz frame of the simulation is moving at the average speed of the beam in the undulator). +Even though the simulation is run in this boosted frame, the results are reconstructed in the +laboratory frame, using WarpX's ``BackTransformed`` diagnostic. + +The effect of space-charge is intentionally turned off in this example, as it may not be properly modeled in 1D. +This is achieved by initializing two species of opposite charge (electrons and positrons) to +represent the physical electron beam, as discussed in :cite:t:`ex-VayFELB2009`. + +Run +--- + +This example can be run with the WarpX executable using an input file: ``warpx.1d inputs_test_1d_fel``. For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs_test_1d_fel + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel``. + +Visualize +--------- + +The figure below shows the results of the simulation. The left panel shows the exponential growth of the radiation along the undulator (note that the vertical axis is plotted in log scale). The right panel shows a snapshot of the simulation, +1.6 m into the undulator. Microbunching of the beam is visible in the electron density (blue). One can also see the +emitted FEL radiation (red) slipping ahead of the beam. + +.. figure:: https://gist.githubusercontent.com/RemiLehe/871a1e24c69e353c5dbb4625cd636cd1/raw/7f4e3da7e0001cff6c592190fee8622580bbe37a/FEL.png + :alt: Results of the WarpX FEL simulation. + :width: 100% + +This figure was obtained with the script below, which can be run with ``python3 plot_sim.py``. + +.. literalinclude:: plot_sim.py + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/free_electron_laser/plot_sim.py``. diff --git a/Examples/Physics_applications/free_electron_laser/analysis_default_regression.py b/Examples/Physics_applications/free_electron_laser/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/free_electron_laser/analysis_fel.py b/Examples/Physics_applications/free_electron_laser/analysis_fel.py new file mode 100755 index 00000000000..b96ddd47147 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/analysis_fel.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python + +""" +This script tests that the FEL is correctly modelled in the simulation. + +The physical parameters are the same as the ones from section 5.1 +of https://arxiv.org/pdf/2009.13645 + +The simulation uses the boosted-frame technique as described in +https://www.osti.gov/servlets/purl/940581 +In particular, the effect of space-charge is effectively turned off +by initializing an electron and positron beam on top of each other, +each having half the current of the physical beam. + +The script checks that the radiation wavelength and gain length +are the expected ones. The check is performed both in the +lab-frame diagnostics and boosted-frame diagnostics. +""" + +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, e, m_e + +# Physical parameters of the test +gamma_bunch = 100.6 +Bu = 0.5 +lambda_u = 3e-2 +k_u = 2 * np.pi / lambda_u +K = e * Bu / (m_e * c * k_u) # Undulator parameter +gamma_boost = ( + gamma_bunch / (1 + K * K / 2) ** 0.5 +) # Lorentz factor of the ponderomotive frame +beta_boost = (1 - 1.0 / gamma_boost**2) ** 0.5 + + +# Analyze the diagnostics showing quantities in the lab frame +filename = sys.argv[1] +ts_lab = OpenPMDTimeSeries(filename) + + +# Extract the growth of the peak electric field +def extract_peak_E_lab(iteration): + """ + Extract the position of the peak electric field + """ + Ex, info = ts_lab.get_field("E", "x", iteration=iteration) + Ex_max = abs(Ex).max() + z_max = info.z[abs(Ex).argmax()] + return z_max, Ex_max + + +# Loop through all iterations +# Since the radiation power is proportional to the square of the peak electric field, +# the log of the power is equal to the log of the square of the peak electric field, +# up to an additive constant. +z_lab_peak, E_lab_peak = ts_lab.iterate(extract_peak_E_lab) +log_P_peak = np.log(E_lab_peak**2) + +# Pick the iterations between which the growth of the log of the power is linear +# (i.e. the growth of the power is exponential) and fit a line to extract the +# gain length. +i_start = 6 +i_end = 23 +# Perform linear fit +p = np.polyfit(z_lab_peak[i_start:i_end], log_P_peak[i_start:i_end], 1) +# Extract the gain length +Lg = 1 / p[0] +Lg_expected = 0.22 # Expected gain length from https://arxiv.org/pdf/2009.13645 +print(f"Gain length: {Lg}") +assert abs(Lg - Lg_expected) / Lg_expected < 0.15 + +# Check that the radiation wavelength is the expected one +iteration_check = 14 +Ex, info = ts_lab.get_field("E", "x", iteration=iteration_check) +Nz = len(info.z) +fft_E = abs(np.fft.fft(Ex)) +lambd = 1.0 / np.fft.fftfreq(Nz, d=info.dz) +lambda_radiation_lab = lambd[fft_E[: Nz // 2].argmax()] +lambda_expected = lambda_u / (2 * gamma_boost**2) +print(f"lambda_radiation_lab: {lambda_radiation_lab}") +print(f"lambda_expected: {lambda_expected}") +assert abs(lambda_radiation_lab - lambda_expected) / lambda_expected < 0.01 + +# Analyze the diagnostics showing quantities in the boosted frame +ts = OpenPMDTimeSeries("diags/diag_boostedframe") + + +# Extract the growth of the peak electric field +def extract_peak_E_boost(iteration): + """ + Extract the peak electric field in a *boosted-frame* snapshot. + Also return the position of the peak in the lab frame. + """ + Ex, info = ts.get_field("E", "x", iteration=iteration) + By, info = ts.get_field("B", "y", iteration=iteration) + E_lab = gamma_boost * (Ex + c * beta_boost * By) + E_lab_peak = abs(E_lab).max() + z_boost_peak = info.z[abs(E_lab).argmax()] + t_boost_peak = ts.current_t + z_lab_peak = gamma_boost * (z_boost_peak + beta_boost * c * t_boost_peak) + return z_lab_peak, E_lab_peak + + +# Loop through all iterations +z_lab_peak, E_lab_peak = ts.iterate(extract_peak_E_boost) +log_P_peak = np.log(E_lab_peak**2) + +# Pick the iterations between which the growth of the log of the power is linear +# (i.e. the growth of the power is exponential) and fit a line to extract the +# gain length. +i_start = 16 +i_end = 25 +# Perform linear fit +p = np.polyfit(z_lab_peak[i_start:i_end], log_P_peak[i_start:i_end], 1) +# Extract the gain length +Lg = 1 / p[0] +Lg_expected = 0.22 # Expected gain length from https://arxiv.org/pdf/2009.13645 +print(f"Gain length: {Lg}") +assert abs(Lg - Lg_expected) / Lg_expected < 0.15 + +# Check that the radiation wavelength is the expected one +iteration_check = 2000 +Ex, info = ts.get_field("E", "x", iteration=iteration_check) +By, info = ts.get_field("B", "y", iteration=iteration_check) +E_lab = gamma_boost * (Ex + c * beta_boost * By) +Nz = len(info.z) +fft_E = abs(np.fft.fft(E_lab)) +lambd = 1.0 / np.fft.fftfreq(Nz, d=info.dz) +lambda_radiation_boost = lambd[fft_E[: Nz // 2].argmax()] +lambda_radiation_lab = lambda_radiation_boost / (2 * gamma_boost) +lambda_expected = lambda_u / (2 * gamma_boost**2) +assert abs(lambda_radiation_lab - lambda_expected) / lambda_expected < 0.01 diff --git a/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel b/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel new file mode 100644 index 00000000000..79fdadab8ae --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel @@ -0,0 +1,92 @@ +my_constants.gamma_bunch=100.6 +my_constants.Bu = 0.5 +my_constants.lambda_u = 3e-2 +my_constants.k_u= 2*pi/lambda_u +my_constants.K = q_e*Bu/(m_e*clight*k_u) # Undulator parameter + +warpx.gamma_boost = gamma_bunch/sqrt(1+K*K/2) # Lorentz factor of the ponderomotive frame +warpx.boost_direction = z +algo.maxwell_solver = yee +algo.particle_shape = 2 +algo.particle_pusher = vay + +# geometry +geometry.dims = 1 +geometry.prob_hi = 0 +geometry.prob_lo = -192e-6 + +amr.max_grid_size = 1024 +amr.max_level = 0 +amr.n_cell = 1024 + +# boundary +boundary.field_hi = absorbing_silver_mueller +boundary.field_lo = absorbing_silver_mueller +boundary.particle_hi = absorbing +boundary.particle_lo = absorbing + +# diagnostics +diagnostics.diags_names = diag_labframe diag_boostedframe + +# Diagnostic that show quantities in the frame +# of the simulation (boosted-frame) +diag_boostedframe.diag_type = Full +diag_boostedframe.format = openpmd +diag_boostedframe.intervals = 100 + +# Diagnostic that show quantities +# reconstructed in the lab frame +diag_labframe.diag_type = BackTransformed +diag_labframe.num_snapshots_lab = 25 +diag_labframe.dz_snapshots_lab = 0.1 +diag_labframe.format = openpmd +diag_labframe.buffer_size = 64 + +# Run the simulation long enough for +# all backtransformed diagnostic to be complete +warpx.compute_max_step_from_btd = 1 + +particles.species_names = electrons positrons +particles.rigid_injected_species= electrons positrons + +electrons.charge = -q_e +electrons.injection_style = nuniformpercell +electrons.mass = m_e +electrons.momentum_distribution_type = constant +electrons.num_particles_per_cell_each_dim = 8 +electrons.profile = constant +electrons.density = 2.7e19/2 +electrons.ux = 0.0 +electrons.uy = 0.0 +electrons.uz = gamma_bunch +electrons.zmax = -25e-6 +electrons.zmin = -125e-6 +electrons.zinject_plane=0.0 +electrons.rigid_advance=0 + +positrons.charge = q_e +positrons.injection_style = nuniformpercell +positrons.mass = m_e +positrons.momentum_distribution_type = constant +positrons.num_particles_per_cell_each_dim = 8 +positrons.profile = constant +positrons.density = 2.7e19/2 +positrons.ux = 0.0 +positrons.uy = 0.0 +positrons.uz = gamma_bunch +positrons.zmax = -25e-6 +positrons.zmin = -125e-6 +positrons.zinject_plane=0.0 +positrons.rigid_advance=0 + +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = sqrt(1-(1+K*K/2)/(gamma_bunch*gamma_bunch)) + +# Undulator field +particles.B_ext_particle_init_style = parse_B_ext_particle_function +particles.Bx_external_particle_function(x,y,z,t) = 0 +particles.By_external_particle_function(x,y,z,t) = if( z>0, Bu*cos(k_u*z), 0 ) +particles.Bz_external_particle_function(x,y,z,t) =0.0 + +warpx.cfl = 0.99 diff --git a/Examples/Physics_applications/free_electron_laser/plot_sim.py b/Examples/Physics_applications/free_electron_laser/plot_sim.py new file mode 100644 index 00000000000..e7635d65790 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/plot_sim.py @@ -0,0 +1,52 @@ +import matplotlib.pyplot as plt +from openpmd_viewer import OpenPMDTimeSeries + +ts = OpenPMDTimeSeries("./diags/diag_labframe/") + + +def extract_peak_E(iteration): + """ + Extract peak electric field and its position + """ + Ex, info = ts.get_field("E", "x", iteration=iteration) + Ex_max = abs(Ex).max() + z_max = info.z[abs(Ex).argmax()] + return z_max, Ex_max + + +# Loop through the lab-frame snapshots and extract the peak electric field +z_max, Ex_max = ts.iterate(extract_peak_E) + +# Create a figure +plt.figure(figsize=(8, 4)) + +# Plot of the E field growth +plt.subplot(121) # Span all rows in the first column +plt.semilogy(z_max, Ex_max) +plt.ylim(2e7, 2e9) +plt.xlabel("z (m)") +plt.ylabel("Peak $E_x$ (V/m)") +plt.title("Growth of the radiation field\n along the undulator") + +# Plots of snapshot +iteration = 16 +plt.subplot(122) # Upper right panel + + +plt.ylabel("$E_x$ (V/m)") +plt.xlabel("") +ts.get_particle(["z"], iteration=iteration, nbins=300, species="electrons", plot=True) +plt.title("") +plt.ylim(0, 30e12) +plt.ylabel("Electron density (a. u.)", color="b") +plt.twinx() +Ex, info = ts.get_field("E", "x", iteration=iteration, plot=True) +plt.ylabel("$E_x$ (V/m)", color="r") +plt.plot(info.z, Ex, color="r") +plt.ylim(-0.6e9, 0.4e9) +plt.xlabel("z (m)") +plt.title("Snapshot 1.6 m into the undulator") + +plt.tight_layout() + +plt.savefig("FEL.png") diff --git a/Examples/Physics_applications/laser_acceleration/CMakeLists.txt b/Examples/Physics_applications/laser_acceleration/CMakeLists.txt new file mode 100644 index 00000000000..28b0e30c2b4 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/CMakeLists.txt @@ -0,0 +1,134 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_laser_acceleration # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_acceleration # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_laser_acceleration_fluid_boosted # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_acceleration_fluid_boosted # inputs + "analysis_1d_fluid_boosted.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) +label_warpx_test(test_1d_laser_acceleration_fluid_boosted slow) + +add_warpx_test( + test_1d_laser_acceleration_picmi # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_acceleration_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_acceleration_boosted # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_acceleration_boosted # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_acceleration_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_acceleration_mr # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_acceleration_mr_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_acceleration_mr_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_refined_injection # name + 2 # dims + 2 # nprocs + inputs_test_2d_refined_injection # inputs + "analysis_refined_injection.py diags/diag1000200" # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_laser_acceleration # name + 3 # dims + 2 # nprocs + inputs_test_3d_laser_acceleration # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_laser_acceleration_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_laser_acceleration_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_laser_acceleration_single_precision_comms # name + 3 # dims + 2 # nprocs + inputs_test_3d_laser_acceleration_single_precision_comms # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_laser_acceleration # name + RZ # dims + 2 # nprocs + inputs_test_rz_laser_acceleration # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_laser_acceleration_opmd # name + RZ # dims + 2 # nprocs + inputs_test_rz_laser_acceleration_opmd # inputs + "analysis_openpmd_rz.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_laser_acceleration_picmi # name + RZ # dims + 2 # nprocs + inputs_test_rz_laser_acceleration_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) +label_warpx_test(test_rz_laser_acceleration_picmi slow) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py deleted file mode 100755 index d8bdddfaca6..00000000000 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# Number of time steps -max_steps = 100 - -# Number of cells -nz = 256 - -# Physical domain -zmin = -56e-06 -zmax = 12e-06 - -# Domain decomposition -max_grid_size = 64 -blocking_factor = 32 - -# Create grid -grid = picmi.Cartesian1DGrid( - number_of_cells = [nz], - lower_bound = [zmin], - upper_bound = [zmax], - lower_boundary_conditions = ['dirichlet'], - upper_boundary_conditions = ['dirichlet'], - lower_boundary_conditions_particles = ['absorbing'], - upper_boundary_conditions_particles = ['absorbing'], - moving_window_velocity = [c], - warpx_max_grid_size = max_grid_size, - warpx_blocking_factor = blocking_factor) - -# Particles: plasma electrons -plasma_density = 2e23 -plasma_xmin = None -plasma_ymin = None -plasma_zmin = 10e-06 -plasma_xmax = None -plasma_ymax = None -plasma_zmax = None -uniform_distribution = picmi.UniformDistribution( - density = plasma_density, - lower_bound = [plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound = [plasma_xmax, plasma_ymax, plasma_zmax], - fill_in = True) -electrons = picmi.Species( - particle_type = 'electron', - name = 'electrons', - initial_distribution = uniform_distribution) - -# Laser -e_max = 16e12 -position_z = 9e-06 -profile_t_peak = 30.e-15 -profile_focal_distance = 100e-06 -laser = picmi.GaussianLaser( - wavelength = 0.8e-06, - waist = 5e-06, - duration = 15e-15, - focal_position = [0, 0, profile_focal_distance + position_z], - centroid_position = [0, 0, position_z - c*profile_t_peak], - propagation_direction = [0, 0, 1], - polarization_direction = [0, 1, 0], - E0 = e_max, - fill_in = False) -laser_antenna = picmi.LaserAntenna( - position = [0., 0., position_z], - normal_vector = [0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid = grid, - method = 'Yee', - cfl = 0.9, - divE_cleaning = 0) - -# Diagnostics -diag_field_list = ['B', 'E', 'J', 'rho'] -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 100, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAcceleration_1d_plt') -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 100, - data_list = diag_field_list, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAcceleration_1d_plt') - -# Set up simulation -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'cubic', - warpx_use_filter = 1, - warpx_serialize_initial_conditions = 1, - warpx_do_dynamic_scheduling = 0) - -# Add plasma electrons -sim.add_species( - electrons, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [10])) - -# Add laser -sim.add_laser( - laser, - injection_method = laser_antenna) - -# Add diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name = 'inputs_1d_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py deleted file mode 100755 index b50e16bfc0a..00000000000 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# Number of time steps -max_steps = 200 - -# Number of cells -nx = 64 -nz = 512 - -# Physical domain -xmin = -30e-06 -xmax = 30e-06 -zmin = -56e-06 -zmax = 12e-06 -xmin_refined = -5e-06 -xmax_refined = 5e-06 -zmin_refined = -35e-06 -zmax_refined = -25e-06 - -# Domain decomposition -max_grid_size = 64 -blocking_factor = 32 - -# Create grid -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, nz], - lower_bound = [xmin, zmin], - upper_bound = [xmax, zmax], - lower_boundary_conditions = ['open', 'open'], - upper_boundary_conditions = ['open', 'open'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing'], - moving_window_velocity = [0., c], - warpx_max_grid_size = max_grid_size, - warpx_blocking_factor = blocking_factor, - refined_regions = [[1, [xmin_refined, zmin_refined], [xmax_refined, zmax_refined]]]) - -# Particles: plasma electrons -plasma_density = 2e23 -plasma_xmin = -20e-06 -plasma_ymin = None -plasma_zmin = 10e-06 -plasma_xmax = 20e-06 -plasma_ymax = None -plasma_zmax = None -uniform_distribution = picmi.UniformDistribution( - density = plasma_density, - lower_bound = [plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound = [plasma_xmax, plasma_ymax, plasma_zmax], - fill_in = True) -electrons = picmi.Species( - particle_type = 'electron', - name = 'electrons', - initial_distribution = uniform_distribution) - -# Particles: beam electrons -q_tot = 1e-12 -x_m = 0. -y_m = 0. -z_m = -28e-06 -x_rms = 0.5e-06 -y_rms = 0.5e-06 -z_rms = 0.5e-06 -ux_m = 0. -uy_m = 0. -uz_m = 500. -ux_th = 2. -uy_th = 2. -uz_th = 50. -gaussian_bunch_distribution = picmi.GaussianBunchDistribution( - n_physical_particles = q_tot / q_e, - rms_bunch_size = [x_rms, y_rms, z_rms], - rms_velocity = [c*ux_th, c*uy_th, c*uz_th], - centroid_position = [x_m, y_m, z_m], - centroid_velocity = [c*ux_m, c*uy_m, c*uz_m]) -beam = picmi.Species( - particle_type = 'electron', - name = 'beam', - initial_distribution = gaussian_bunch_distribution) - -# Laser -e_max = 16e12 -position_z = 9e-06 -profile_t_peak = 30.e-15 -profile_focal_distance = 100e-06 -laser = picmi.GaussianLaser( - wavelength = 0.8e-06, - waist = 5e-06, - duration = 15e-15, - focal_position = [0, 0, profile_focal_distance + position_z], - centroid_position = [0, 0, position_z - c*profile_t_peak], - propagation_direction = [0, 0, 1], - polarization_direction = [0, 1, 0], - E0 = e_max, - fill_in = False) -laser_antenna = picmi.LaserAntenna( - position = [0., 0., position_z], - normal_vector = [0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid = grid, - method = 'Yee', - cfl = 1., - divE_cleaning = 0) - -# Diagnostics -diag_field_list = ['B', 'E', 'J', 'rho'] -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 200, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAccelerationMR_plt') -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 200, - data_list = diag_field_list, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAccelerationMR_plt') - -# Set up simulation -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'cubic', - warpx_use_filter = 1, - warpx_serialize_initial_conditions = 1) - -# Add plasma electrons -sim.add_species( - electrons, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [1, 1, 1])) - -# Add beam electrons -sim.add_species( - beam, - layout = picmi.PseudoRandomLayout(grid = grid, n_macroparticles = 100)) - -# Add laser -sim.add_laser( - laser, - injection_method = laser_antenna) - -# Add diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name = 'inputs_2d_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py deleted file mode 100755 index 13bf492e203..00000000000 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# Number of time steps -max_steps = 100 - -# Number of cells -nx = 32 -ny = 32 -nz = 256 - -# Physical domain -xmin = -30e-06 -xmax = 30e-06 -ymin = -30e-06 -ymax = 30e-06 -zmin = -56e-06 -zmax = 12e-06 - -# Domain decomposition -max_grid_size = 64 -blocking_factor = 32 - -# Create grid -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['periodic', 'periodic', 'dirichlet'], - upper_boundary_conditions = ['periodic', 'periodic', 'dirichlet'], - lower_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - upper_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - moving_window_velocity = [0., 0., c], - warpx_max_grid_size = max_grid_size, - warpx_blocking_factor = blocking_factor) - -# Particles: plasma electrons -plasma_density = 2e23 -plasma_xmin = -20e-06 -plasma_ymin = -20e-06 -plasma_zmin = 0 -plasma_xmax = 20e-06 -plasma_ymax = 20e-06 -plasma_zmax = None -uniform_distribution = picmi.UniformDistribution( - density = plasma_density, - lower_bound = [plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound = [plasma_xmax, plasma_ymax, plasma_zmax], - fill_in = True) -electrons = picmi.Species( - particle_type = 'electron', - name = 'electrons', - initial_distribution = uniform_distribution, - warpx_add_int_attributes = {'regionofinterest': "(z>12.0e-6) * (z<13.0e-6)"}, - warpx_add_real_attributes = {'initialenergy': "ux*ux + uy*uy + uz*uz"}) - -# Particles: beam electrons -q_tot = 1e-12 -x_m = 0. -y_m = 0. -z_m = -28e-06 -x_rms = 0.5e-06 -y_rms = 0.5e-06 -z_rms = 0.5e-06 -ux_m = 0. -uy_m = 0. -uz_m = 500. -ux_th = 2. -uy_th = 2. -uz_th = 50. -gaussian_bunch_distribution = picmi.GaussianBunchDistribution( - n_physical_particles = q_tot / q_e, - rms_bunch_size = [x_rms, y_rms, z_rms], - rms_velocity = [c*ux_th, c*uy_th, c*uz_th], - centroid_position = [x_m, y_m, z_m], - centroid_velocity = [c*ux_m, c*uy_m, c*uz_m]) -beam = picmi.Species( - particle_type = 'electron', - name = 'beam', - initial_distribution = gaussian_bunch_distribution) - -# Laser -e_max = 16e12 -position_z = 9e-06 -profile_t_peak = 30.e-15 -profile_focal_distance = 100e-06 -laser = picmi.GaussianLaser( - wavelength = 0.8e-06, - waist = 5e-06, - duration = 15e-15, - focal_position = [0, 0, profile_focal_distance + position_z], - centroid_position = [0, 0, position_z - c*profile_t_peak], - propagation_direction = [0, 0, 1], - polarization_direction = [0, 1, 0], - E0 = e_max, - fill_in = False) -laser_antenna = picmi.LaserAntenna( - position = [0., 0., position_z], - normal_vector = [0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid = grid, - method = 'Yee', - cfl = 1., - divE_cleaning = 0) - -# Diagnostics -diag_field_list = ['B', 'E', 'J', 'rho'] -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 100, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAcceleration_plt') -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 100, - data_list = diag_field_list, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAcceleration_plt') - -# Set up simulation -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'cubic', - warpx_use_filter = 1, - warpx_serialize_initial_conditions = 1, - warpx_do_dynamic_scheduling = 0) - -# Add plasma electrons -sim.add_species( - electrons, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [1, 1, 1])) - -# Add beam electrons -sim.add_species( - beam, - layout = picmi.PseudoRandomLayout(grid = grid, n_macroparticles = 100)) - -# Add laser -sim.add_laser( - laser, - injection_method = laser_antenna) - -# Add diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name = 'inputs_3d_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py deleted file mode 100755 index 7f09db8d6b3..00000000000 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# Number of time steps -max_steps = 10 - -# Number of cells -nr = 64 -nz = 512 - -# Physical domain -rmin = 0 -rmax = 30e-06 -zmin = -56e-06 -zmax = 12e-06 - -# Domain decomposition -max_grid_size = 64 -blocking_factor = 32 - -# Create grid -grid = picmi.CylindricalGrid( - number_of_cells = [nr, nz], - n_azimuthal_modes = 2, - lower_bound = [rmin, zmin], - upper_bound = [rmax, zmax], - lower_boundary_conditions = ['none', 'dirichlet'], - upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['none', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing'], - moving_window_velocity = [0., c], - warpx_max_grid_size = max_grid_size, - warpx_blocking_factor = blocking_factor) - -# Particles: plasma electrons -plasma_density = 2e23 -plasma_xmin = -20e-06 -plasma_ymin = None -plasma_zmin = 10e-06 -plasma_xmax = 20e-06 -plasma_ymax = None -plasma_zmax = None -uniform_distribution = picmi.UniformDistribution( - density = plasma_density, - lower_bound = [plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound = [plasma_xmax, plasma_ymax, plasma_zmax], - fill_in = True) -electrons = picmi.Species( - particle_type = 'electron', - name = 'electrons', - initial_distribution = uniform_distribution) - -# Particles: beam electrons -q_tot = 1e-12 -x_m = 0. -y_m = 0. -z_m = -28e-06 -x_rms = 0.5e-06 -y_rms = 0.5e-06 -z_rms = 0.5e-06 -ux_m = 0. -uy_m = 0. -uz_m = 500. -ux_th = 2. -uy_th = 2. -uz_th = 50. -gaussian_bunch_distribution = picmi.GaussianBunchDistribution( - n_physical_particles = q_tot / q_e, - rms_bunch_size = [x_rms, y_rms, z_rms], - rms_velocity = [c*ux_th, c*uy_th, c*uz_th], - centroid_position = [x_m, y_m, z_m], - centroid_velocity = [c*ux_m, c*uy_m, c*uz_m]) -beam = picmi.Species( - particle_type = 'electron', - name = 'beam', - initial_distribution = gaussian_bunch_distribution) - -# Laser -e_max = 16e12 -position_z = 9e-06 -profile_t_peak = 30.e-15 -profile_focal_distance = 100e-06 -laser = picmi.GaussianLaser( - wavelength = 0.8e-06, - waist = 5e-06, - duration = 15e-15, - focal_position = [0, 0, profile_focal_distance + position_z], - centroid_position = [0, 0, position_z - c*profile_t_peak], - propagation_direction = [0, 0, 1], - polarization_direction = [0, 1, 0], - E0 = e_max, - fill_in = False) -laser_antenna = picmi.LaserAntenna( - position = [0., 0., position_z], - normal_vector = [0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid = grid, - method = 'Yee', - cfl = 1., - divE_cleaning = 0) - -# Diagnostics -diag_field_list = ['B', 'E', 'J', 'rho'] -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 10, - data_list = diag_field_list, - warpx_dump_rz_modes = 1, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAccelerationRZ_plt') -diag_particle_list = ['weighting', 'momentum'] -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - species = [electrons, beam], - data_list = diag_particle_list, - write_dir = '.', - warpx_file_prefix = 'Python_LaserAccelerationRZ_plt') - -# Set up simulation -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'cubic', - warpx_use_filter = 0) - -# Add plasma electrons -sim.add_species( - electrons, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [1, 4, 1])) - -# Add beam electrons -sim.add_species( - beam, - layout = picmi.PseudoRandomLayout(grid = grid, n_macroparticles = 100)) - -# Add laser -sim.add_laser( - laser, - injection_method = laser_antenna) - -# Add diagnostics -sim.add_diagnostic(field_diag) -sim.add_diagnostic(particle_diag) - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name = 'inputs_rz_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst index 0ba3b5382f2..66de28de7a7 100644 --- a/Examples/Physics_applications/laser_acceleration/README.rst +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -24,43 +24,43 @@ For `MPI-parallel `__ runs, prefix these lines with ` This example can be run **either** as: - * **Python** script: ``python3 PICMI_inputs_3d.py`` or - * WarpX **executable** using an input file: ``warpx.3d inputs_3d max_step=400`` + * **Python** script: ``python3 inputs_test_3d_laser_acceleration_picmi.py`` or + * WarpX **executable** using an input file: ``warpx.3d inputs_test_3d_laser_acceleration max_step=400`` .. tab-set:: .. tab-item:: Python: Script - .. literalinclude:: PICMI_inputs_3d.py + .. literalinclude:: inputs_test_3d_laser_acceleration_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py``. .. tab-item:: Executable: Input File - .. literalinclude:: inputs_3d + .. literalinclude:: inputs_test_3d_laser_acceleration :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_3d``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration``. .. tab-item:: RZ This example can be run **either** as: - * **Python** script: ``python3 PICMI_inputs_rz.py`` or - * WarpX **executable** using an input file: ``warpx.rz inputs_3d max_step=400`` + * **Python** script: ``python3 inputs_test_rz_laser_acceleration_picmi.py`` or + * WarpX **executable** using an input file: ``warpx.rz inputs_test_rz_laser_acceleration max_step=400`` .. tab-set:: .. tab-item:: Python: Script - .. literalinclude:: PICMI_inputs_rz.py + .. literalinclude:: inputs_test_rz_laser_acceleration_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_picmi.py``. .. tab-item:: Executable: Input File - .. literalinclude:: inputs_rz + .. literalinclude:: inputs_test_rz_laser_acceleration :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_rz``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration``. Analyze ------- diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluid_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluid_boosted.py new file mode 100755 index 00000000000..bd45f30edbb --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluid_boosted.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2023 Grant Johnson, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: +# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 +import sys + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +n0 = 20.0e23 +# Plasma frequency +wp = np.sqrt((n0 * e**2) / (m_e * epsilon_0)) +kp = wp / c +tau = 15.0e-15 +a0 = 2.491668 +e = -e # Electrons +lambda_laser = 0.8e-6 + +zmin = -20e-6 +zmax = 100.0e-6 +Nz = 4864 + +# Compute the theory + +# Call the ode solver +from scipy.integrate import odeint + + +# ODE Function +def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): + phi1, phi2 = phi + a_sq = ( + a0**2 + * np.exp(-2 * (xi - xi_0) ** 2 / (c**2 * tau**2)) + * np.sin(2 * np.pi * (xi - xi_0) / lambda_laser) ** 2 + ) + dphi1_dxi = phi2 + dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1) ** 2) - 0.5) + return [dphi1_dxi, dphi2_dxi] + + +# Call odeint to solve the ODE +xi_span = [-20e-6, 100e-6] +xi_0 = 0e-6 +phi0 = [0.0, 0.0] +dxi = (zmax - zmin) / Nz +xi = zmin + dxi * (0.5 + np.arange(Nz)) +phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) + +# Change array direction to match the simulations +xi = -xi[::-1] +phi = phi[::-1] +xi_0 = -0e-6 +phi2 = phi[:, 0] +Ez = -phi[:, 1] + +# Compute the derived quantities +a_sq = ( + a0**2 + * np.exp(-2 * (xi - xi_0) ** 2 / (c**2 * tau**2)) + * np.sin(2 * np.pi * (xi - xi_0) / lambda_laser) ** 2 +) +gamma_perp_sq = 1 + a_sq +n = n0 * (gamma_perp_sq + (1 + phi2) ** 2) / (2 * (1 + phi2) ** 2) +uz = (gamma_perp_sq - (1 + phi2) ** 2) / (2 * (1 + phi2)) +gamma = (gamma_perp_sq + (1 + phi2) ** 2) / (2 * (1 + phi2)) + +# Theory Components [convert to si] +uz *= c +J_th = np.multiply(np.divide(uz, gamma), n) +J_th *= e +rho_th = e * n +E_th = Ez +E_th *= (m_e * c * c) / e +V_th = np.divide(uz, gamma) +V_th /= c +# Remove the ions +rho_th = rho_th - e * n0 + +# Dictate which region to compare solutions over (cuttoff 0's from BTD extra) +min_i = 200 +max_i = 4864 + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +# Check the validity of the fields +error_rel = 0 +for field in ["Ez"]: + E_sim = data[("mesh", field)].to_ndarray()[:, 0, 0] + # E_th = get_theoretical_field(field, t0) + max_error = ( + abs(E_sim[min_i:max_i] - E_th[min_i:max_i]).max() / abs(E_th[min_i:max_i]).max() + ) + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) + +# Check the validity of the currents +for field in ["Jz"]: + J_sim = data[("mesh", field)].to_ndarray()[:, 0, 0] + # J_th = get_theoretical_J_field(field, t0) + max_error = ( + abs(J_sim[min_i:max_i] - J_th[min_i:max_i]).max() / abs(J_th[min_i:max_i]).max() + ) + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) + +# Check the validity of the charge +for field in ["rho"]: + rho_sim = data[("boxlib", field)].to_ndarray()[:, 0, 0] + # rho_th = get_theoretical_rho_field(field, t0) + max_error = ( + abs(rho_sim[min_i:max_i] - rho_th[min_i:max_i]).max() + / abs(rho_th[min_i:max_i]).max() + ) + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) + +V_sim = np.divide(J_sim, rho_sim) +V_sim /= c + +# Create a figure with 2 rows and 2 columns +fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) + +# Titles and labels +titles = ["Ez", "rho", "Jz", "Vz/c"] +xlabel = r"Xi" +ylabel = ["Ez", "rho", "Jz", "Vz/c"] + +# Plotting loop +for i in range(3): + ax = axes[i // 2, i % 2] # Get the current subplot + + # Plot theoretical data + ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label="Theoretical") + + # Plot simulated data + ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label="Simulated") + + # Set titles and labels + ax.set_title(f"{titles[i]} vs Xi") + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel[i]) + + # Add legend + ax.legend() + +# Adjust subplot layout +plt.tight_layout() + +# Save the figure +plt.savefig("wfa_fluid_nonlinear_1d_analysis.png") + +plt.show() + + +tolerance_rel = 0.30 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert error_rel < tolerance_rel diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py deleted file mode 100755 index a33b82ebc02..00000000000 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2023 Grant Johnson, Remi Lehe -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# This is a script that analyses the simulation results from -# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: -# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 -import os -import sys - -import matplotlib - -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import yt - -yt.funcs.mylog.setLevel(50) - -import numpy as np -from scipy.constants import c, e, epsilon_0, m_e - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# Parameters (these parameters must match the parameters in `inputs.multi.rt`) -n0 = 20.e23 -# Plasma frequency -wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) -kp = wp/c -tau = 15.e-15 -a0 = 2.491668 -e = -e #Electrons -lambda_laser = 0.8e-6 - -zmin = -20e-6; zmax = 100.e-6; Nz = 10240 - -# Compute the theory - -# Call the ode solver -from scipy.integrate import odeint - - -# ODE Function -def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): - phi1, phi2 = phi - a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 - dphi1_dxi = phi2 - dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) - return [dphi1_dxi, dphi2_dxi] - -# Call odeint to solve the ODE -xi_span = [-20e-6, 100e-6] -xi_0 = 0e-6 -phi0 = [0.0, 0.0] -dxi = (zmax-zmin)/Nz -xi = zmin + dxi*( 0.5 + np.arange(Nz) ) -phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) - -# Change array direction to match the simulations -xi = -xi[::-1] -phi = phi[::-1] -xi_0 = -0e-6 -phi2 = phi[:, 0] -Ez = -phi[:, 1] - -# Compute the derived quantities -a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 -gamma_perp_sq = 1 + a_sq -n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) -uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) -gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) - -# Theory Components [convert to si] -uz *= c -J_th = np.multiply( np.divide(uz,gamma), n ) -J_th *= e -rho_th = e*n -E_th = Ez -E_th *= ((m_e*c*c)/e) -V_th = np.divide(uz,gamma) -V_th /= c -# Remove the ions -rho_th = rho_th - e*n0 - -# Dictate which region to compare solutions over -# (Currently this is the full domain) -min_i = 0 -max_i = 10240 - -# Read the file -ds = yt.load(fn) -t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) -# Check the validity of the fields -error_rel = 0 -for field in ['Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,0,0] - #E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -# Check the validity of the currents -for field in ['Jz']: - J_sim = data[('mesh',field)].to_ndarray()[:,0,0] - #J_th = get_theoretical_J_field(field, t0) - max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -# Check the validity of the charge -for field in ['rho']: - rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] - #rho_th = get_theoretical_rho_field(field, t0) - max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -V_sim = np.divide(J_sim,rho_sim) -V_sim /= c - -# Create a figure with 2 rows and 2 columns -fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) - -# Titles and labels -titles = ['Ez', 'rho', 'Jz', 'Vz/c'] -xlabel = r'Xi' -ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] - -# Plotting loop -for i in range(3): - ax = axes[i // 2, i % 2] # Get the current subplot - - # Plot theoretical data - ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') - - # Plot simulated data - ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') - - # Set titles and labels - ax.set_title(f'{titles[i]} vs Xi') - ax.set_xlabel(xlabel) - ax.set_ylabel(ylabel[i]) - - # Add legend - ax.legend() - -# Adjust subplot layout -plt.tight_layout() - -# Save the figure -plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') - -plt.show() - - -tolerance_rel = 0.20 - -print("error_rel : " + str(error_rel)) -print("tolerance_rel: " + str(tolerance_rel)) - -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py deleted file mode 100755 index 30301996921..00000000000 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2023 Grant Johnson, Remi Lehe -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# This is a script that analyses the simulation results from -# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: -# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 -import os -import sys - -import matplotlib - -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import yt - -yt.funcs.mylog.setLevel(50) - -import numpy as np -from scipy.constants import c, e, epsilon_0, m_e - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# Parameters (these parameters must match the parameters in `inputs.multi.rt`) -n0 = 20.e23 -# Plasma frequency -wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) -kp = wp/c -tau = 15.e-15 -a0 = 2.491668 -e = -e #Electrons -lambda_laser = 0.8e-6 - -zmin = -20e-6; zmax = 100.e-6; Nz = 4864 - -# Compute the theory - -# Call the ode solver -from scipy.integrate import odeint - - -# ODE Function -def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): - phi1, phi2 = phi - a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 - dphi1_dxi = phi2 - dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) - return [dphi1_dxi, dphi2_dxi] - -# Call odeint to solve the ODE -xi_span = [-20e-6, 100e-6] -xi_0 = 0e-6 -phi0 = [0.0, 0.0] -dxi = (zmax-zmin)/Nz -xi = zmin + dxi*( 0.5 + np.arange(Nz) ) -phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) - -# Change array direction to match the simulations -xi = -xi[::-1] -phi = phi[::-1] -xi_0 = -0e-6 -phi2 = phi[:, 0] -Ez = -phi[:, 1] - -# Compute the derived quantities -a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 -gamma_perp_sq = 1 + a_sq -n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) -uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) -gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) - -# Theory Components [convert to si] -uz *= c -J_th = np.multiply( np.divide(uz,gamma), n ) -J_th *= e -rho_th = e*n -E_th = Ez -E_th *= ((m_e*c*c)/e) -V_th = np.divide(uz,gamma) -V_th /= c -# Remove the ions -rho_th = rho_th - e*n0 - -# Dictate which region to compare solutions over (cuttoff 0's from BTD extra) -min_i = 200 -max_i = 4864 - -# Read the file -ds = yt.load(fn) -t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) -# Check the validity of the fields -error_rel = 0 -for field in ['Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,0,0] - #E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -# Check the validity of the currents -for field in ['Jz']: - J_sim = data[('mesh',field)].to_ndarray()[:,0,0] - #J_th = get_theoretical_J_field(field, t0) - max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -# Check the validity of the charge -for field in ['rho']: - rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] - #rho_th = get_theoretical_rho_field(field, t0) - max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) - -V_sim = np.divide(J_sim,rho_sim) -V_sim /= c - -# Create a figure with 2 rows and 2 columns -fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) - -# Titles and labels -titles = ['Ez', 'rho', 'Jz', 'Vz/c'] -xlabel = r'Xi' -ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] - -# Plotting loop -for i in range(3): - ax = axes[i // 2, i % 2] # Get the current subplot - - # Plot theoretical data - ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') - - # Plot simulated data - ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') - - # Set titles and labels - ax.set_title(f'{titles[i]} vs Xi') - ax.set_xlabel(xlabel) - ax.set_ylabel(ylabel[i]) - - # Add legend - ax.legend() - -# Adjust subplot layout -plt.tight_layout() - -# Save the figure -plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') - -plt.show() - - -tolerance_rel = 0.30 - -print("error_rel : " + str(error_rel)) -print("tolerance_rel: " + str(tolerance_rel)) - -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_default_openpmd_regression.py b/Examples/Physics_applications/laser_acceleration/analysis_default_openpmd_regression.py new file mode 120000 index 00000000000..73e5ec47001 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_default_openpmd_regression.py @@ -0,0 +1 @@ +../../analysis_default_openpmd_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/laser_acceleration/analysis_default_regression.py b/Examples/Physics_applications/laser_acceleration/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/laser_acceleration/analysis_openpmd_rz.py b/Examples/Physics_applications/laser_acceleration/analysis_openpmd_rz.py new file mode 100755 index 00000000000..1449e54d8ee --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_openpmd_rz.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import sys + +import numpy as np +import openpmd_api as io + +filename = sys.argv[1] +series = io.Series(f"{filename}/openpmd_%T.h5", io.Access.read_only) + +assert len(series.iterations) == 3, "improper number of iterations stored" + +ii = series.iterations[20] + +assert len(ii.meshes) == 8, "improper number of meshes" + +# select j_t +jt = ii.meshes["j"]["t"] + +# this is in C (Python) order; r is the fastest varying index +(Nm, Nz, Nr) = jt.shape + +assert Nm == 3, ( + "Wrong number of angular modes stored or possible incorrect ordering when flushed" +) +assert Nr == 64, ( + "Wrong number of radial points stored or possible incorrect ordering when flushed" +) +assert Nz == 512, ( + "Wrong number of z points stored or possible incorrect ordering when flushed" +) + +assert ii.meshes["part_per_grid"][io.Mesh_Record_Component.SCALAR].shape == [ + 512, + 64, +], "problem with part_per_grid" +assert ii.meshes["rho_electrons"][io.Mesh_Record_Component.SCALAR].shape == [ + 3, + 512, + 64, +], "problem with rho_electrons" + + +### test that openpmd+RZ +### 1. creates rho per species correctly +### 2. orders these appropriately +rhoe_mesh = ii.meshes["rho_electrons"] +rhob_mesh = ii.meshes["rho_beam"] +dz, dr = rhoe_mesh.grid_spacing +zmin, rmin = rhoe_mesh.grid_global_offset + +rhoe = rhoe_mesh[io.Mesh_Record_Component.SCALAR][:] +rhob = rhob_mesh[io.Mesh_Record_Component.SCALAR][:] +series.flush() +nm, nz, nr = rhoe.shape +zlist = zmin + dz * np.arange(nz) +rhoe0 = rhoe[0] # 0 mode +rhob0 = rhob[0] # 0 mode + +electron_meanz = np.sum(np.dot(zlist, rhoe0)) / np.sum(rhoe0) +beam_meanz = np.sum(np.dot(zlist, rhob0)) / np.sum(rhob0) + +assert (electron_meanz > 0) and (beam_meanz < 0), ( + "problem with openPMD+RZ. Maybe openPMD+RZ mixed up the order of rho_ diagnostics?" +) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py index a66c838fe9d..8df5e422ddb 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py @@ -9,16 +9,11 @@ # This script tests the "warpx.refine_plasma=1" option by comparing # the actual number of electrons at step 200 to the expected value -import os import sys import yt yt.funcs.mylog.setLevel(50) -import numpy as np - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # this will be the name of the plot file fn = sys.argv[1] @@ -28,7 +23,7 @@ # count the number of particles ad = ds.all_data() -np = ad['electrons', 'particle_id'].size +ps = ad["electrons", "particle_id"].size # the number of coarse particle streams n_coarse = 10 @@ -46,19 +41,16 @@ # Refined only transversely. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions rr_longitudinal = 1 -np_expected = (n_coarse + n_fine*rr_longitudinal)*(n_0 + n_move) +np_expected = (n_coarse + n_fine * rr_longitudinal) * (n_0 + n_move) -assert( np == np_expected ) +assert ps == np_expected # Test uniformity of rho, by taking a slice of rho that # crosses the edge of the refined injection region # (but is ahead of the mesh refinement patch) ds.force_periodicity() ad = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -rho = ad['rho'].to_ndarray().squeeze() +rho = ad["rho"].to_ndarray().squeeze() rho_slice = rho[13:51, 475] # Test uniformity up to 0.5% relative variation -assert( rho_slice.std() < 0.005*abs(rho_slice.mean()) ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +assert rho_slice.std() < 0.005 * abs(rho_slice.mean()) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids deleted file mode 100644 index 73fa6b7283f..00000000000 --- a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids +++ /dev/null @@ -1,72 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 40000 -amr.n_cell = 10240 -amr.max_grid_size = 512 # maximum size of each AMReX box, used to decompose the domain -amr.blocking_factor = 512 # minimum size of each AMReX box, used to decompose the domain -geometry.dims = 1 -geometry.prob_lo = -120.e-6 # physical domain -geometry.prob_hi = 0.e-6 -amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = pec -boundary.field_hi = pec - -################################# -############ NUMERICS ########### -################################# -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 0 -warpx.cfl = 0.45 #Fluid CFL < 0.5 -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # units of speed of light -warpx.do_dynamic_scheduling = 0 -warpx.serialize_initial_conditions = 1 - -################################# -############ PLASMA ############# -################################# -fluids.species_names = electrons ions - -electrons.species_type = electron -electrons.profile = parse_density_function -electrons.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" -electrons.momentum_distribution_type = "at_rest" - -ions.charge = q_e -ions.mass = m_p -ions.profile = parse_density_function -ions.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" -ions.momentum_distribution_type = "at_rest" - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ LASER ############## -################################# -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. -11.e-6 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 10.e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 5.e-6 # The waist of the laser (in m) -laser1.profile_duration = 15.e-15 # The duration of the laser (in s) -laser1.profile_t_peak = 30.e-15 # Time at which the laser reaches its peak (in s) -laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in m) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) - -# Diagnostics -diagnostics.diags_names = diag1 - -# LAB -diag1.intervals = 20000 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Physics_applications/laser_acceleration/inputs_2d_boost b/Examples/Physics_applications/laser_acceleration/inputs_2d_boost deleted file mode 100644 index c2aa92c3634..00000000000 --- a/Examples/Physics_applications/laser_acceleration/inputs_2d_boost +++ /dev/null @@ -1,131 +0,0 @@ -################################# -######### BOX PARAMETERS ######## -################################# -max_step = 2700 -# stop_time = 1.9e-12 -amr.n_cell = 128 1024 -amr.max_grid_size = 64 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -# physical domain -geometry.prob_lo = -128.e-6 -40.e-6 -geometry.prob_hi = 128.e-6 0.96e-6 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic pec -boundary.field_hi = periodic pec - -################################# -############ NUMERICS ########### -################################# -warpx.verbose = 1 -amrex.v = 1 -algo.current_deposition = esirkepov -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -algo.particle_pusher = vay -algo.maxwell_solver = ckc -warpx.use_filter = 1 -warpx.cfl = 1. -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light -warpx.serialize_initial_conditions = 1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -####### BOOST PARAMETERS ######## -################################# -warpx.gamma_boost = 10. -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons ions beam -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 1 1 -electrons.momentum_distribution_type = "at_rest" -electrons.xmin = -120.e-6 -electrons.xmax = 120.e-6 -electrons.zmin = 0.5e-3 -electrons.zmax = .0035 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 1 1 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -120.e-6 -ions.xmax = 120.e-6 -ions.zmin = 0.5e-3 -ions.zmax = .0035 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 -ions.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 1.e-6 -beam.y_rms = 1.e-6 -beam.z_rms = .2e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -20.e-6 -beam.npart = 1000 -beam.q_tot = -1.e-14 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 200. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 20. -beam.zinject_plane = 0.1e-3 -beam.rigid_advance = true - -################################# -############# LASER ############# -################################# -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. -0.1e-6 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 2.e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 45.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 20.e-15 # The duration of the laser (in seconds) -laser1.profile_t_peak = 40.e-15 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 0.5e-3 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.81e-6 # The wavelength of the laser (in meters) - -# Diagnostics -diagnostics.diags_names = diag1 - -diag1.diag_type = BackTransformed -diag1.do_back_transformed_fields = 1 -diag1.num_snapshots_lab = 3 -diag1.dt_snapshots_lab = 1.6678204759907604e-12 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho -diag1.format = plotfile -diag1.buffer_size = 32 -diag1.write_species = 1 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d b/Examples/Physics_applications/laser_acceleration/inputs_base_1d similarity index 100% rename from Examples/Physics_applications/laser_acceleration/inputs_1d rename to Examples/Physics_applications/laser_acceleration/inputs_base_1d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_2d b/Examples/Physics_applications/laser_acceleration/inputs_base_2d similarity index 100% rename from Examples/Physics_applications/laser_acceleration/inputs_2d rename to Examples/Physics_applications/laser_acceleration/inputs_base_2d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_base_3d similarity index 100% rename from Examples/Physics_applications/laser_acceleration/inputs_3d rename to Examples/Physics_applications/laser_acceleration/inputs_base_3d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_rz b/Examples/Physics_applications/laser_acceleration/inputs_base_rz similarity index 100% rename from Examples/Physics_applications/laser_acceleration/inputs_rz rename to Examples/Physics_applications/laser_acceleration/inputs_base_rz diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration b/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration new file mode 100644 index 00000000000..190b458b397 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_1d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted b/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration_fluid_boosted similarity index 100% rename from Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted rename to Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration_fluid_boosted diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration_picmi.py b/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration_picmi.py new file mode 100755 index 00000000000..b7b86b47821 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration_picmi.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# Number of time steps +max_steps = 100 + +# Number of cells +nz = 256 + +# Physical domain +zmin = -56e-06 +zmax = 12e-06 + +# Domain decomposition +max_grid_size = 64 +blocking_factor = 32 + +# Create grid +grid = picmi.Cartesian1DGrid( + number_of_cells=[nz], + lower_bound=[zmin], + upper_bound=[zmax], + lower_boundary_conditions=["dirichlet"], + upper_boundary_conditions=["dirichlet"], + lower_boundary_conditions_particles=["absorbing"], + upper_boundary_conditions_particles=["absorbing"], + moving_window_velocity=[c], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, +) + +# Particles: plasma electrons +plasma_density = 2e23 +plasma_xmin = None +plasma_ymin = None +plasma_zmin = 10e-06 +plasma_xmax = None +plasma_ymax = None +plasma_zmax = None +uniform_distribution = picmi.UniformDistribution( + density=plasma_density, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + fill_in=True, +) +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_distribution, +) + +# Laser +e_max = 16e12 +position_z = 9e-06 +profile_t_peak = 30.0e-15 +profile_focal_distance = 100e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=5e-06, + duration=15e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[0, 1, 0], + E0=e_max, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=0.9, divE_cleaning=0) + +# Diagnostics +diag_field_list = ["B", "E", "J", "rho"] +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=100, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=100, + data_list=diag_field_list, +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="cubic", + warpx_use_filter=1, + warpx_serialize_initial_conditions=1, + warpx_do_dynamic_scheduling=0, +) + +# Add plasma electrons +sim.add_species( + electrons, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[10]) +) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_1d_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_boosted b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_boosted new file mode 100644 index 00000000000..1997054e885 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_boosted @@ -0,0 +1,131 @@ +################################# +######### BOX PARAMETERS ######## +################################# +max_step = 300 #2700 +# stop_time = 1.9e-12 +amr.n_cell = 64 512 #128 1024 +amr.max_grid_size = 64 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 2 +# physical domain +geometry.prob_lo = -128.e-6 -40.e-6 +geometry.prob_hi = 128.e-6 0.96e-6 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic pec +boundary.field_hi = periodic pec + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +amrex.v = 1 +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.particle_pusher = vay +algo.maxwell_solver = ckc +warpx.use_filter = 1 +warpx.cfl = 1. +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.serialize_initial_conditions = 1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +####### BOOST PARAMETERS ######## +################################# +warpx.gamma_boost = 10. +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons ions beam +particles.use_fdtd_nci_corr = 1 +particles.rigid_injected_species = beam + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = NUniformPerCell +electrons.num_particles_per_cell_each_dim = 1 1 +electrons.momentum_distribution_type = "at_rest" +electrons.xmin = -120.e-6 +electrons.xmax = 120.e-6 +electrons.zmin = 0.5e-3 +electrons.zmax = .0035 +electrons.profile = "predefined" +electrons.predefined_profile_name = "parabolic_channel" +# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 +electrons.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 +electrons.do_continuous_injection = 1 + +ions.charge = q_e +ions.mass = m_p +ions.injection_style = NUniformPerCell +ions.num_particles_per_cell_each_dim = 1 1 +ions.momentum_distribution_type = "at_rest" +ions.xmin = -120.e-6 +ions.xmax = 120.e-6 +ions.zmin = 0.5e-3 +ions.zmax = .0035 +ions.profile = "predefined" +ions.predefined_profile_name = "parabolic_channel" +# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 +ions.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 +ions.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = 1.e-6 +beam.y_rms = 1.e-6 +beam.z_rms = .2e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -20.e-6 +beam.npart = 1000 +beam.q_tot = -1.e-14 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 200. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 20. +beam.zinject_plane = 0.1e-3 +beam.rigid_advance = true + +################################# +############# LASER ############# +################################# +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -0.1e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 2.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 45.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 20.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 40.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 0.5e-3 # Focal distance from the antenna (in meters) +laser1.wavelength = 0.81e-6 # The wavelength of the laser (in meters) + +# Diagnostics +diagnostics.diags_names = diag1 + +diag1.diag_type = BackTransformed +diag1.do_back_transformed_fields = 1 +diag1.num_snapshots_lab = 3 +diag1.dt_snapshots_lab = 1.6678204759907604e-12 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho +diag1.format = plotfile +diag1.buffer_size = 32 +diag1.write_species = 1 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr new file mode 100644 index 00000000000..5a98fa590ee --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_2d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr_picmi.py b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr_picmi.py new file mode 100755 index 00000000000..8d112c0ac09 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_laser_acceleration_mr_picmi.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# Number of time steps +max_steps = 200 + +# Number of cells +nx = 64 +nz = 512 + +# Physical domain +xmin = -30e-06 +xmax = 30e-06 +zmin = -56e-06 +zmax = 12e-06 +xmin_refined = -5e-06 +xmax_refined = 5e-06 +zmin_refined = -35e-06 +zmax_refined = -25e-06 + +# Domain decomposition +max_grid_size = 64 +blocking_factor = 32 + +# Create grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=["open", "open"], + upper_boundary_conditions=["open", "open"], + lower_boundary_conditions_particles=["absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing"], + moving_window_velocity=[0.0, c], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, + refined_regions=[[1, [xmin_refined, zmin_refined], [xmax_refined, zmax_refined]]], +) + +# Particles: plasma electrons +plasma_density = 2e23 +plasma_xmin = -20e-06 +plasma_ymin = None +plasma_zmin = 10e-06 +plasma_xmax = 20e-06 +plasma_ymax = None +plasma_zmax = None +uniform_distribution = picmi.UniformDistribution( + density=plasma_density, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + fill_in=True, +) +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_distribution, +) + +# Particles: beam electrons +q_tot = 1e-12 +x_m = 0.0 +y_m = 0.0 +z_m = -28e-06 +x_rms = 0.5e-06 +y_rms = 0.5e-06 +z_rms = 0.5e-06 +ux_m = 0.0 +uy_m = 0.0 +uz_m = 500.0 +ux_th = 2.0 +uy_th = 2.0 +uz_th = 50.0 +gaussian_bunch_distribution = picmi.GaussianBunchDistribution( + n_physical_particles=q_tot / q_e, + rms_bunch_size=[x_rms, y_rms, z_rms], + rms_velocity=[c * ux_th, c * uy_th, c * uz_th], + centroid_position=[x_m, y_m, z_m], + centroid_velocity=[c * ux_m, c * uy_m, c * uz_m], +) +beam = picmi.Species( + particle_type="electron", + name="beam", + initial_distribution=gaussian_bunch_distribution, +) + +# Laser +e_max = 16e12 +position_z = 9e-06 +profile_t_peak = 30.0e-15 +profile_focal_distance = 100e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=5e-06, + duration=15e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[0, 1, 0], + E0=e_max, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=1.0, divE_cleaning=0) + +# Diagnostics +diag_field_list = ["B", "E", "J", "rho"] +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=200, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=200, + data_list=diag_field_list, +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="cubic", + warpx_use_filter=1, + warpx_serialize_initial_conditions=1, +) + +# Add plasma electrons +sim.add_species( + electrons, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[1, 1, 1]) +) + +# Add beam electrons +sim.add_species(beam, layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=100)) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_2d_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_2d_refined_injection b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_refined_injection new file mode 100644 index 00000000000..ed836e87e6b --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_2d_refined_injection @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +amr.ref_ratio_vect = 2 1 +warpx.refine_plasma = 1 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration new file mode 100644 index 00000000000..7665a846eef --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_3d diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py new file mode 100755 index 00000000000..999c92600e2 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# Number of time steps +max_steps = 100 + +# Number of cells +nx = 32 +ny = 32 +nz = 256 + +# Physical domain +xmin = -30e-06 +xmax = 30e-06 +ymin = -30e-06 +ymax = 30e-06 +zmin = -56e-06 +zmax = 12e-06 + +# Domain decomposition +max_grid_size = 64 +blocking_factor = 32 + +# Create grid +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "dirichlet"], + upper_boundary_conditions=["periodic", "periodic", "dirichlet"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + moving_window_velocity=[0.0, 0.0, c], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, +) + +# Particles: plasma electrons +plasma_density = 2e23 +plasma_xmin = -20e-06 +plasma_ymin = -20e-06 +plasma_zmin = 0 +plasma_xmax = 20e-06 +plasma_ymax = 20e-06 +plasma_zmax = None +uniform_distribution = picmi.UniformDistribution( + density=plasma_density, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + fill_in=True, +) +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_distribution, + warpx_add_int_attributes={"regionofinterest": "(z>12.0e-6) * (z<13.0e-6)"}, + warpx_add_real_attributes={"initialenergy": "ux*ux + uy*uy + uz*uz"}, +) + +# Particles: beam electrons +q_tot = 1e-12 +x_m = 0.0 +y_m = 0.0 +z_m = -28e-06 +x_rms = 0.5e-06 +y_rms = 0.5e-06 +z_rms = 0.5e-06 +ux_m = 0.0 +uy_m = 0.0 +uz_m = 500.0 +ux_th = 2.0 +uy_th = 2.0 +uz_th = 50.0 +gaussian_bunch_distribution = picmi.GaussianBunchDistribution( + n_physical_particles=q_tot / q_e, + rms_bunch_size=[x_rms, y_rms, z_rms], + rms_velocity=[c * ux_th, c * uy_th, c * uz_th], + centroid_position=[x_m, y_m, z_m], + centroid_velocity=[c * ux_m, c * uy_m, c * uz_m], +) +beam = picmi.Species( + particle_type="electron", + name="beam", + initial_distribution=gaussian_bunch_distribution, +) + +# Laser +e_max = 16e12 +position_z = 9e-06 +profile_t_peak = 30.0e-15 +profile_focal_distance = 100e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=5e-06, + duration=15e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[0, 1, 0], + E0=e_max, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=1.0, divE_cleaning=0) + +# Diagnostics +diag_field_list = ["B", "E", "J", "rho"] +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=100, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=100, + data_list=diag_field_list, +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="cubic", + warpx_use_filter=1, + warpx_serialize_initial_conditions=1, + warpx_do_dynamic_scheduling=0, +) + +# Add plasma electrons +sim.add_species( + electrons, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[1, 1, 1]) +) + +# Add beam electrons +sim.add_species(beam, layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=100)) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_3d_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_single_precision_comms b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_single_precision_comms new file mode 100644 index 00000000000..99155ed0ecc --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_single_precision_comms @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +warpx.do_single_precision_comms = 1 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration new file mode 100644 index 00000000000..5879688b00a --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +diag1.dump_rz_modes = 1 +warpx.abort_on_warning_threshold = high diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_opmd b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_opmd new file mode 100644 index 00000000000..16a84950996 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_opmd @@ -0,0 +1,9 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +diag1.fields_to_plot = Er Bt Bz jr jt jz rho part_per_cell part_per_grid rho_beam rho_electrons +diag1.format = openpmd +diag1.openpmd_backend = h5 +max_step = 20 +warpx.abort_on_warning_threshold = high diff --git a/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_picmi.py b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_picmi.py new file mode 100755 index 00000000000..cfbf9879ed4 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_test_rz_laser_acceleration_picmi.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# Number of time steps +max_steps = 10 + +# Number of cells +nr = 64 +nz = 512 + +# Physical domain +rmin = 0 +rmax = 30e-06 +zmin = -56e-06 +zmax = 12e-06 + +# Domain decomposition +max_grid_size = 64 +blocking_factor = 32 + +# Create grid +grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + n_azimuthal_modes=2, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=["none", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["none", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing"], + moving_window_velocity=[0.0, c], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, +) + +# Particles: plasma electrons +plasma_density = 2e23 +plasma_xmin = -20e-06 +plasma_ymin = None +plasma_zmin = 10e-06 +plasma_xmax = 20e-06 +plasma_ymax = None +plasma_zmax = None +uniform_distribution = picmi.UniformDistribution( + density=plasma_density, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + fill_in=True, +) +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_distribution, +) + +# Particles: beam electrons +q_tot = 1e-12 +x_m = 0.0 +y_m = 0.0 +z_m = -28e-06 +x_rms = 0.5e-06 +y_rms = 0.5e-06 +z_rms = 0.5e-06 +ux_m = 0.0 +uy_m = 0.0 +uz_m = 500.0 +ux_th = 2.0 +uy_th = 2.0 +uz_th = 50.0 +gaussian_bunch_distribution = picmi.GaussianBunchDistribution( + n_physical_particles=q_tot / q_e, + rms_bunch_size=[x_rms, y_rms, z_rms], + rms_velocity=[c * ux_th, c * uy_th, c * uz_th], + centroid_position=[x_m, y_m, z_m], + centroid_velocity=[c * ux_m, c * uy_m, c * uz_m], +) +beam = picmi.Species( + particle_type="electron", + name="beam", + initial_distribution=gaussian_bunch_distribution, +) + +# Laser +e_max = 16e12 +position_z = 9e-06 +profile_t_peak = 30.0e-15 +profile_focal_distance = 100e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=5e-06, + duration=15e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[0, 1, 0], + E0=e_max, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=1.0, divE_cleaning=0) + +# Diagnostics +diag_field_list = ["B", "E", "J", "rho"] +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=diag_field_list, + warpx_dump_rz_modes=1, +) +diag_particle_list = ["weighting", "momentum"] +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, + species=[electrons, beam], + data_list=diag_particle_list, +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="cubic", + warpx_use_filter=0, +) + +# Add plasma electrons +sim.add_species( + electrons, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[1, 4, 1]) +) + +# Add beam electrons +sim.add_species(beam, layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=100)) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_rz_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Physics_applications/laser_acceleration/plot_3d.py b/Examples/Physics_applications/laser_acceleration/plot_3d.py index 00222ff43c8..34e3770726b 100755 --- a/Examples/Physics_applications/laser_acceleration/plot_3d.py +++ b/Examples/Physics_applications/laser_acceleration/plot_3d.py @@ -36,5 +36,6 @@ def plot_lwfa(): fig.tight_layout() plt.show() + if __name__ == "__main__": plot_lwfa() diff --git a/Examples/Physics_applications/laser_ion/CMakeLists.txt b/Examples/Physics_applications/laser_ion/CMakeLists.txt new file mode 100644 index 00000000000..cc67bef685c --- /dev/null +++ b/Examples/Physics_applications/laser_ion/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_laser_ion_acc # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_ion_acc # inputs + "analysis_test_laser_ion.py diags/diagInst/" # analysis + "analysis_default_regression.py --path diags/diagInst/" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_ion_acc_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_ion_acc_picmi.py # inputs + "analysis_test_laser_ion.py diags/diagInst/" # analysis + "analysis_default_regression.py --path diags/diagInst/" # checksum + OFF # dependency +) diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py deleted file mode 100755 index 9f7a2aacfca..00000000000 --- a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# We only run 100 steps for tests -# Disable `max_step` below to run until the physical `stop_time`. -max_step = 100 -# time-scale with highly kinetic dynamics -stop_time = 0.2e-12 - -# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length -# (>=6x V100) -# --> choose larger `max_grid_size` and `blocking_factor` for 1 to 8 grids per GPU accordingly -#nx = 7488 -#nz = 14720 - -# Number of cells -nx = 384 -nz = 512 - -# Domain decomposition (deactivate `warpx_numprocs` in `picmi.Simulation` for this to take effect) -max_grid_size = 64 -blocking_factor = 32 - -# Physical domain -xmin = -7.5e-06 -xmax = 7.5e-06 -zmin = -5.0e-06 -zmax = 25.0e-06 - -# Create grid -grid = picmi.Cartesian2DGrid( - number_of_cells=[nx, nz], - lower_bound=[xmin, zmin], - upper_bound=[xmax, zmax], - lower_boundary_conditions=['open', 'open'], - upper_boundary_conditions=['open', 'open'], - lower_boundary_conditions_particles=['absorbing', 'absorbing'], - upper_boundary_conditions_particles=['absorbing', 'absorbing'], - warpx_max_grid_size=max_grid_size, - warpx_blocking_factor=blocking_factor) - -# Particles: plasma parameters -# critical plasma density -nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 -# number density: "fully ionized" electron density as reference -# [material 1] cryogenic H2 -n0 = 30.0 # [n_c] -# [material 2] liquid crystal -# n0 = 192 -# [material 3] PMMA -# n0 = 230 -# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) -# n0 = 1400 -plasma_density = n0 * nc -preplasma_L = 0.05e-6 # [m] scale length (>0) -preplasma_Lcut = 2.0e-6 # [m] hard cutoff from surface -plasma_r0 = 2.5e-6 # [m] radius or half-thickness -plasma_eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) -plasma_creation_limit_z = plasma_r0 + preplasma_Lcut + plasma_eps_z # [m] upper limit in z for particle creation - -plasma_xmin = None -plasma_ymin = None -plasma_zmin = -plasma_creation_limit_z -plasma_xmax = None -plasma_ymax = None -plasma_zmax = plasma_creation_limit_z - -density_expression_str = f'{plasma_density}*((abs(z)<={plasma_r0}) + (abs(z)<{plasma_r0}+{preplasma_Lcut}) * (abs(z)>{plasma_r0}) * exp(-(abs(z)-{plasma_r0})/{preplasma_L}))' - -slab_with_ramp_dist_hydrogen = picmi.AnalyticDistribution( - density_expression=density_expression_str, - lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax] -) - -# thermal velocity spread for electrons in gamma*beta -ux_th = .01 -uz_th = .01 - -slab_with_ramp_dist_electrons = picmi.AnalyticDistribution( - density_expression=density_expression_str, - lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], - upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], - # if `momentum_expressions` and `momentum_spread_expressions` are unset, - # a Gaussian momentum distribution is assumed given that `rms_velocity` has any non-zero elements - rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s -) - -electrons = picmi.Species( - particle_type='electron', - name='electrons', - initial_distribution=slab_with_ramp_dist_electrons, -) - -hydrogen = picmi.Species( - particle_type='proton', - name='hydrogen', - initial_distribution=slab_with_ramp_dist_hydrogen, - warpx_add_real_attributes = {"orig_x": "x", "orig_z": "z"} -) - -# Laser -# e_max = a0 * 3.211e12 / lambda_0[mu] -# a0 = 16, lambda_0 = 0.8mu -> e_max = 64.22 TV/m -e_max = 64.22e12 -position_z = -4.0e-06 -profile_t_peak = 50.e-15 -profile_focal_distance = 4.0e-06 -laser = picmi.GaussianLaser( - wavelength=0.8e-06, - waist=4.e-06, - duration=30.e-15, - focal_position=[0, 0, profile_focal_distance + position_z], - centroid_position=[0, 0, position_z - c * profile_t_peak], - propagation_direction=[0, 0, 1], - polarization_direction=[1, 0, 0], - E0=e_max, - fill_in=False) -laser_antenna = picmi.LaserAntenna( - position=[0., 0., position_z], - normal_vector=[0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid=grid, - method='Yee', - cfl=0.999, - divE_cleaning=0, - #warpx_pml_ncell=10 -) - -# Diagnostics -particle_diag = picmi.ParticleDiagnostic( - name='Python_LaserIonAcc2d_plt', - period=100, - write_dir='./diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5', - # demonstration of a spatial and momentum filter - warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' -) -# reduce resolution of output fields -coarsening_ratio = [4, 4] -ncell_field = [] -for (ncell_comp, cr) in zip([nx,nz], coarsening_ratio): - ncell_field.append(int(ncell_comp/cr)) -field_diag = picmi.FieldDiagnostic( - name='Python_LaserIonAcc2d_plt', - grid=grid, - period=100, - number_of_cells=ncell_field, - data_list=['B', 'E', 'J', 'rho', 'rho_electrons', 'rho_hydrogen'], - write_dir='./diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5' -) - -particle_fw_diag = picmi.ParticleDiagnostic( - name='openPMDfw', - period=100, - write_dir='./diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5', - warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' -) - -particle_bw_diag = picmi.ParticleDiagnostic( - name='openPMDbw', - period=100, - write_dir='./diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5', - warpx_plot_filter_function='(uz<0)' -) - -# histograms with 2.0 degree acceptance angle in fw direction -# 2 deg * pi / 180 : 0.03490658503 rad -# half-angle +/- : 0.017453292515 rad -histuH_rdiag = picmi.ReducedDiagnostic( - diag_type='ParticleHistogram', - name='histuH', - period=100, - species=hydrogen, - bin_number=1000, - bin_min=0.0, - bin_max=0.474, # 100 MeV protons - histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', - filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') - -histue_rdiag = picmi.ReducedDiagnostic( - diag_type='ParticleHistogram', - name='histue', - period=100, - species=electrons, - bin_number=1000, - bin_min=0.0, - bin_max=197.0, # 100 MeV electrons - histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', - filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') - -# just a test entry to make sure that the histogram filter is purely optional: -# this one just records uz of all hydrogen ions, independent of their pointing -histuzAll_rdiag = picmi.ReducedDiagnostic( - diag_type='ParticleHistogram', - name='histuzAll', - period=100, - species=hydrogen, - bin_number=1000, - bin_min=-0.474, - bin_max=0.474, - histogram_function='uz') - -field_probe_z_rdiag = picmi.ReducedDiagnostic( - diag_type='FieldProbe', - name='FieldProbe_Z', - period=100, - integrate=0, - probe_geometry='Line', - x_probe=0.0, - z_probe=-5.0e-6, - x1_probe=0.0, - z1_probe=25.0e-6, - resolution=3712) - -field_probe_scat_point_rdiag = picmi.ReducedDiagnostic( - diag_type='FieldProbe', - name='FieldProbe_ScatPoint', - period=1, - integrate=0, - probe_geometry='Point', - x_probe=0.0, - z_probe=15.0e-6) - -field_probe_scat_line_rdiag = picmi.ReducedDiagnostic( - diag_type='FieldProbe', - name='FieldProbe_ScatLine', - period=100, - integrate=1, - probe_geometry='Line', - x_probe=-2.5e-6, - z_probe=15.0e-6, - x1_probe=2.5e-6, - z1_probe=15e-6, - resolution=201) - -load_balance_costs_rdiag = picmi.ReducedDiagnostic( - diag_type='LoadBalanceCosts', - name='LBC', - period=100) - -# Set up simulation -sim = picmi.Simulation( - solver=solver, - max_time=stop_time, # need to remove `max_step` to run this far - verbose=1, - particle_shape='cubic', - warpx_numprocs=[1, 2], # deactivate `numprocs` for dynamic load balancing - warpx_use_filter=1, - warpx_load_balance_intervals=100, - warpx_load_balance_costs_update='heuristic' -) - -# Add plasma electrons -sim.add_species( - electrons, - layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) - # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c - #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) -) - -# Add hydrogen ions -sim.add_species( - hydrogen, - layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) - # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c - #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) -) - -# Add laser -sim.add_laser( - laser, - injection_method=laser_antenna) - -# Add full diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) -sim.add_diagnostic(particle_fw_diag) -sim.add_diagnostic(particle_bw_diag) -# Add reduced diagnostics -sim.add_diagnostic(histuH_rdiag) -sim.add_diagnostic(histue_rdiag) -sim.add_diagnostic(histuzAll_rdiag) -sim.add_diagnostic(field_probe_z_rdiag) -sim.add_diagnostic(field_probe_scat_point_rdiag) -sim.add_diagnostic(field_probe_scat_line_rdiag) -sim.add_diagnostic(load_balance_costs_rdiag) -# TODO: make ParticleHistogram2D available - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name='inputs_2d_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_step) diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index 29862a30518..c5dc5af3a77 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -21,8 +21,8 @@ Run This example can be run **either** as: -* **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or -* WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` +* **Python** script: ``mpiexec -n 2 python3 inputs_test_2d_laser_ion_acc_picmi.py`` or +* WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_test_2d_laser_ion_acc`` .. tip:: @@ -35,16 +35,16 @@ This example can be run **either** as: .. tab-item:: Python: Script - .. literalinclude:: PICMI_inputs_2d.py + .. literalinclude:: inputs_test_2d_laser_ion_acc_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py``. .. tab-item:: Executable: Input File - .. literalinclude:: inputs_2d + .. literalinclude:: inputs_test_2d_laser_ion_acc :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_2d``. + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc``. Analyze ------- @@ -87,7 +87,7 @@ Visualize :alt: Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). :width: 80% - Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). Particle density output illustrates the evolution of the target in time and space. Logarithmic scales can help to identify where the target becomes transparent for the laser pulse (bottom panel in :numref:`fig-tnsa-densities` ). diff --git a/Examples/Physics_applications/laser_ion/analysis_default_regression.py b/Examples/Physics_applications/laser_ion/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py index a262a2373e5..06d3bb42c8e 100644 --- a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py +++ b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py @@ -9,24 +9,29 @@ import numpy as np from openpmd_viewer import OpenPMDTimeSeries -parser = argparse.ArgumentParser(description='Process a 2D histogram name and an integer.') +parser = argparse.ArgumentParser( + description="Process a 2D histogram name and an integer." +) parser.add_argument("hist2D", help="Folder name of the reduced diagnostic.") -parser.add_argument("iter", help="Iteration number of the simulation that is plotted. Enter a number from the list of iterations or 'All' if you want all plots.") +parser.add_argument( + "iter", + help="Iteration number of the simulation that is plotted. Enter a number from the list of iterations or 'All' if you want all plots.", +) args = parser.parse_args() -path = 'diags/reducedfiles/' + args.hist2D +path = "diags/reducedfiles/" + args.hist2D ts = OpenPMDTimeSeries(path) it = ts.iterations data, info = ts.get_field(field="data", iteration=0, plot=True) -print('The available iterations of the simulation are:', it) -print('The axes of the histogram are (0: ordinate ; 1: abscissa):', info.axes) -print('The data shape is:', data.shape) +print("The available iterations of the simulation are:", it) +print("The axes of the histogram are (0: ordinate ; 1: abscissa):", info.axes) +print("The data shape is:", data.shape) # Add the simulation time to the title once this information # is available in the "info" FieldMetaInformation object. -if args.iter == 'All' : +if args.iter == "All": for it_idx, i in enumerate(it): plt.figure() data, info = ts.get_field(field="data", iteration=i, plot=False) @@ -35,14 +40,20 @@ ordinate_name = info.axes[0] # This might be 'z' or something else ordinate_values = getattr(info, ordinate_name, None) - plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.pcolormesh( + abscissa_values / 1e-6, + ordinate_values, + data, + norm=colors.LogNorm(), + rasterized=True, + ) plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") - plt.xlabel(info.axes[1]+r' ($\mu$m)') - plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') + plt.xlabel(info.axes[1] + r" ($\mu$m)") + plt.ylabel(info.axes[0] + r" ($m_\mathrm{species} c$)") plt.colorbar() plt.tight_layout() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') -else : + plt.savefig("Histogram_2D_" + args.hist2D + "_iteration_" + str(i) + ".png") +else: i = int(args.iter) it_idx = np.where(i == it)[0][0] plt.figure() @@ -52,10 +63,16 @@ ordinate_name = info.axes[0] # This might be 'z' or something else ordinate_values = getattr(info, ordinate_name, None) - plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.pcolormesh( + abscissa_values / 1e-6, + ordinate_values, + data, + norm=colors.LogNorm(), + rasterized=True, + ) plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") - plt.xlabel(info.axes[1]+r' ($\mu$m)') - plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') + plt.xlabel(info.axes[1] + r" ($\mu$m)") + plt.ylabel(info.axes[0] + r" ($m_\mathrm{species} c$)") plt.colorbar() plt.tight_layout() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') + plt.savefig("Histogram_2D_" + args.hist2D + "_iteration_" + str(i) + ".png") diff --git a/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py b/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py new file mode 100755 index 00000000000..360d5d48b5f --- /dev/null +++ b/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +import openpmd_api as io + + +def load_field_from_iteration( + series, iteration: int, field: str, coord: str = None +) -> np.ndarray: + """Load iteration of field data from file.""" + + it = series.iterations[iteration] + field_obj = it.meshes[f"{field}"] + + if field_obj.scalar: + field_data = field_obj[io.Mesh_Record_Component.SCALAR].load_chunk() + elif coord in [item[0] for item in list(field_obj.items())]: + field_data = field_obj[coord].load_chunk() + else: + raise Exception( + f"Specified coordinate: f{coord} is not available for field: f{field}." + ) + series.flush() + + return field_data + + +def compare_time_avg_with_instantaneous_diags(dir_inst: str, dir_avg: str): + """Compare instantaneous data (multiple iterations averaged in post-processing) with in-situ averaged data.""" + + field = "E" + coord = "z" + avg_period_steps = 5 + avg_output_step = 100 + + path_tpl_inst = f"{dir_inst}/openpmd_%T.h5" + path_tpl_avg = f"{dir_avg}/openpmd_%T.h5" + + si = io.Series(path_tpl_inst, io.Access.read_only) + sa = io.Series(path_tpl_avg, io.Access.read_only) + + ii0 = si.iterations[0] + fi0 = ii0.meshes[field][coord] + shape = fi0.shape + + data_inst = np.zeros(shape) + + for i in np.arange(avg_output_step - avg_period_steps + 1, avg_output_step + 1): + data_inst += load_field_from_iteration(si, i, field, coord) + + data_inst = data_inst / avg_period_steps + + data_avg = load_field_from_iteration(sa, avg_output_step, field, coord) + + # Compare the data + if np.allclose(data_inst, data_avg, rtol=1e-12): + print("Test passed: actual data is close to expected data.") + else: + print("Test failed: actual data is not close to expected data.") + sys.exit(1) + + +if __name__ == "__main__": + # TODO: implement intervals parser for PICMI that allows more complex output periods + test_name = os.path.split(os.getcwd())[1] + if "picmi" not in test_name: + # Functionality test for TimeAveragedDiagnostics + compare_time_avg_with_instantaneous_diags( + dir_inst=sys.argv[1], + dir_avg="diags/diagTimeAvg/", + ) diff --git a/Examples/Physics_applications/laser_ion/inputs_2d b/Examples/Physics_applications/laser_ion/inputs_2d deleted file mode 100644 index 5ad8334e9ef..00000000000 --- a/Examples/Physics_applications/laser_ion/inputs_2d +++ /dev/null @@ -1,342 +0,0 @@ -################################# -# Domain, Resolution & Numerics -# - -# We only run 100 steps for tests -# Disable `max_step` below to run until the physical `stop_time`. -max_step = 100 -# time-scale with highly kinetic dynamics -stop_time = 0.2e-12 # [s] -# time-scale for converged ion energy -# notes: - effective acc. time depends on laser pulse -# - ions will start to leave the box -#stop_time = 1.0e-12 # [s] - -# quick tests at ultra-low res. (for CI, and local computer) -amr.n_cell = 384 512 - -# proper resolution for 10 n_c excl. acc. length -# (>=1x V100) -#amr.n_cell = 2688 3712 - -# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length -# (>=6x V100) -#amr.n_cell = 7488 14720 - -# simulation box, no MR -# note: increase z (space & cells) for converging ion energy -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -7.5e-6 -5.e-6 -geometry.prob_hi = 7.5e-6 25.e-6 - -# Boundary condition -boundary.field_lo = pml pml -boundary.field_hi = pml pml - -# Order of particle shape factors -algo.particle_shape = 3 - -# improved plasma stability for 2D with very low initial target temperature -# when using Esirkepov current deposition with energy-conserving field gather -interpolation.galerkin_scheme = 0 - -# numerical tuning -warpx.cfl = 0.999 -warpx.use_filter = 1 # bilinear current/charge filter - - -################################# -# Performance Tuning -# -# simple tuning: -# the numprocs product must be equal to the number of MPI ranks and splits -# the domain on the coarsest level equally into grids; -# slicing in the 2nd dimension is preferred for ideal performance -warpx.numprocs = 1 2 # 2 MPI ranks -#warpx.numprocs = 1 4 # 4 MPI ranks - -# detail tuning instead of warpx.numprocs: -# It is important to have enough cells in a block & grid, otherwise -# performance will suffer. -# Use larger values for GPUs, try to fill a GPU well with memory and place -# few large grids on each device (you can go as low as 1 large grid / device -# if you do not need load balancing). -# Slicing in the 2nd dimension is preferred for ideal performance -#amr.blocking_factor = 64 -#amr.max_grid_size_x = 2688 -#amr.max_grid_size_y = 128 # this is confusingly named and means z in 2D - -# load balancing -# The grid & block parameters above are needed for load balancing: -# an average of ~10 grids per MPI rank (and device) are a good granularity -# to allow efficient load-balancing as the simulation evolves -algo.load_balance_intervals = 100 -algo.load_balance_costs_update = Heuristic - -# particle bin-sorting on GPU (ideal defaults not investigated in 2D) -# Try larger values than the defaults below and report back! :) -#warpx.sort_intervals = 4 # default on CPU: -1 (off); on GPU: 4 -#warpx.sort_bin_size = 1 1 1 - - -################################# -# Target Profile -# - -# definitions for target extent and pre-plasma -my_constants.L = 0.05e-6 # [m] scale length (>0) -my_constants.Lcut = 2.0e-6 # [m] hard cutoff from surface -my_constants.r0 = 2.5e-6 # [m] radius or half-thickness -my_constants.eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) -my_constants.zmax = r0 + Lcut + eps_z # [m] upper limit in z for particle creation - -particles.species_names = electrons hydrogen - -# particle species -hydrogen.species_type = hydrogen -hydrogen.injection_style = NUniformPerCell -hydrogen.num_particles_per_cell_each_dim = 2 2 -# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c -#hydrogen.num_particles_per_cell_each_dim = 4 8 -hydrogen.momentum_distribution_type = at_rest -# minimum and maximum z position between which particles are initialized -# --> should be set for dense targets limit memory consumption during initialization -hydrogen.zmin = -zmax -hydrogen.zmax = zmax -hydrogen.profile = parse_density_function -hydrogen.addRealAttributes = orig_x orig_z -hydrogen.attribute.orig_x(x,y,z,ux,uy,uz,t) = "x" -hydrogen.attribute.orig_z(x,y,z,ux,uy,uz,t) = "z" - -electrons.species_type = electron -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 2 2 -# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c -#electrons.num_particles_per_cell_each_dim = 4 8 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = .01 -electrons.uz_th = .01 -# minimum and maximum z position between which particles are initialized -# --> should be set for dense targets limit memory consumption during initialization -electrons.zmin = -zmax -electrons.zmax = zmax - -# ionization physics (field ionization/ADK) -# [i1] none (fully pre-ionized): -electrons.profile = parse_density_function -# [i2] field ionization (ADK): -#hydrogen.do_field_ionization = 1 -#hydrogen.physical_element = H -#hydrogen.ionization_initial_level = 0 -#hydrogen.ionization_product_species = electrons -#electrons.profile = constant -#electrons.density = 0.0 - -# collisional physics (binary MC model after Nanbu/Perez) -#collisions.collision_names = c_eH c_ee c_HH -#c_eH.species = electrons hydrogen -#c_ee.species = electrons electrons -#c_HH.species = hydrogen hydrogen -#c_eH.CoulombLog = 15.9 -#c_ee.CoulombLog = 15.9 -#c_HH.CoulombLog = 15.9 - -# number density: "fully ionized" electron density as reference -# [material 1] cryogenic H2 -my_constants.nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 -my_constants.n0 = 30.0 # [n_c] -# [material 2] liquid crystal -#my_constants.n0 = 192 -# [material 3] PMMA -#my_constants.n0 = 230 -# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) -#my_constants.n0 = 1400 - -# density profiles (target extent, pre-plasma and cutoffs defined above particle species list) - -# [target 1] flat foil (thickness = 2*r0) -electrons.density_function(x,y,z) = "nc*n0*( - if(abs(z)<=r0, 1.0, if(abs(z) e_max = 64.22 TV/m - - -################################# -# Diagnostics -# -diagnostics.diags_names = diag1 openPMDfw openPMDbw - -diag1.intervals = 100 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen -# reduce resolution of output fields -diag1.coarsening_ratio = 4 4 -# demonstration of a spatial and momentum filter -diag1.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) -diag1.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) -diag1.format = openpmd -diag1.openpmd_backend = h5 - -openPMDfw.intervals = 100 -openPMDfw.diag_type = Full -openPMDfw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen -# reduce resolution of output fields -openPMDfw.coarsening_ratio = 4 4 -openPMDfw.format = openpmd -openPMDfw.openpmd_backend = h5 -# demonstration of a spatial and momentum filter -openPMDfw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) -openPMDfw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) - -openPMDbw.intervals = 100 -openPMDbw.diag_type = Full -openPMDbw.fields_to_plot = rho_hydrogen -# reduce resolution of output fields -openPMDbw.coarsening_ratio = 4 4 -openPMDbw.format = openpmd -openPMDbw.openpmd_backend = h5 -# demonstration of a momentum filter -openPMDbw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) -openPMDbw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) - - -################################# -# Reduced Diagnostics -# - -# histograms with 2.0 degree acceptance angle in fw direction -# 2 deg * pi / 180 : 0.03490658503 rad -# half-angle +/- : 0.017453292515 rad -warpx.reduced_diags_names = histuH histue histuzAll FieldProbe_Z FieldProbe_ScatPoint FieldProbe_ScatLine LBC PhaseSpaceIons PhaseSpaceElectrons - -histuH.type = ParticleHistogram -histuH.intervals = 100 -histuH.species = hydrogen -histuH.bin_number = 1000 -histuH.bin_min = 0.0 -histuH.bin_max = 0.474 # 100 MeV protons -histuH.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" -histuH.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" - -histue.type = ParticleHistogram -histue.intervals = 100 -histue.species = electrons -histue.bin_number = 1000 -histue.bin_min = 0.0 -histue.bin_max = 197 # 100 MeV electrons -histue.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" -histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" - -# just a test entry to make sure that the histogram filter is purely optional: -# this one just records uz of all hydrogen ions, independent of their pointing -histuzAll.type = ParticleHistogram -histuzAll.intervals = 100 -histuzAll.species = hydrogen -histuzAll.bin_number = 1000 -histuzAll.bin_min = -0.474 -histuzAll.bin_max = 0.474 -histuzAll.histogram_function(t,x,y,z,ux,uy,uz) = "uz" - -FieldProbe_Z.type = FieldProbe -FieldProbe_Z.intervals = 100 -FieldProbe_Z.integrate = 0 -FieldProbe_Z.probe_geometry = Line -FieldProbe_Z.x_probe = 0.0 -FieldProbe_Z.z_probe = -5.0e-6 -FieldProbe_Z.x1_probe = 0.0 -FieldProbe_Z.z1_probe = 25.0e-6 -FieldProbe_Z.resolution = 3712 - -FieldProbe_ScatPoint.type = FieldProbe -FieldProbe_ScatPoint.intervals = 1 -FieldProbe_ScatPoint.integrate = 0 -FieldProbe_ScatPoint.probe_geometry = Point -FieldProbe_ScatPoint.x_probe = 0.0 -FieldProbe_ScatPoint.z_probe = 15e-6 - -FieldProbe_ScatLine.type = FieldProbe -FieldProbe_ScatLine.intervals = 100 -FieldProbe_ScatLine.integrate = 1 -FieldProbe_ScatLine.probe_geometry = Line -FieldProbe_ScatLine.x_probe = -2.5e-6 -FieldProbe_ScatLine.z_probe = 15e-6 -FieldProbe_ScatLine.x1_probe = 2.5e-6 -FieldProbe_ScatLine.z1_probe = 15e-6 -FieldProbe_ScatLine.resolution = 201 - -# check computational load per box -LBC.type = LoadBalanceCosts -LBC.intervals = 100 - -PhaseSpaceIons.type = ParticleHistogram2D -PhaseSpaceIons.intervals = 100 -PhaseSpaceIons.species = hydrogen -PhaseSpaceIons.bin_number_abs = 1000 -PhaseSpaceIons.bin_number_ord = 1000 -PhaseSpaceIons.bin_min_abs = -5.e-6 -PhaseSpaceIons.bin_max_abs = 25.e-6 -PhaseSpaceIons.bin_min_ord = -0.474 -PhaseSpaceIons.bin_max_ord = 0.474 -PhaseSpaceIons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" -PhaseSpaceIons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" -PhaseSpaceIons.value_function(t,x,y,z,ux,uy,uz,w) = "w" -# PhaseSpaceIons.filter_function(t,x,y,z,ux,uy,uz,w) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" - -PhaseSpaceElectrons.type = ParticleHistogram2D -PhaseSpaceElectrons.intervals = 100 -PhaseSpaceElectrons.species = electrons -PhaseSpaceElectrons.bin_number_abs = 1000 -PhaseSpaceElectrons.bin_number_ord = 1000 -PhaseSpaceElectrons.bin_min_abs = -5.e-6 -PhaseSpaceElectrons.bin_max_abs = 25.e-6 -PhaseSpaceElectrons.bin_min_ord = -197 -PhaseSpaceElectrons.bin_max_ord = 197 -PhaseSpaceElectrons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" -PhaseSpaceElectrons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" -PhaseSpaceElectrons.value_function(t,x,y,z,ux,uy,uz,w) = "w" -PhaseSpaceElectrons.filter_function(t,x,y,z,ux,uy,uz,w) = "sqrt(x*x+y*y) < 1e-6" - -################################# -# Physical Background -# -# This example is modeled after a target similar to the hydrogen jet here: -# [1] https://doi.org/10.1038/s41598-017-10589-3 -# [2] https://arxiv.org/abs/1903.06428 -# -authors = "Axel Huebl " diff --git a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc new file mode 100644 index 00000000000..d69ed6dc375 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc @@ -0,0 +1,356 @@ +################################# +# Domain, Resolution & Numerics +# + +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 +# time-scale with highly kinetic dynamics +stop_time = 0.2e-12 # [s] +# time-scale for converged ion energy +# notes: - effective acc. time depends on laser pulse +# - ions will start to leave the box +#stop_time = 1.0e-12 # [s] + +# quick tests at ultra-low res. (for CI, and local computer) +amr.n_cell = 384 512 + +# proper resolution for 10 n_c excl. acc. length +# (>=1x V100) +#amr.n_cell = 2688 3712 + +# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length +# (>=6x V100) +#amr.n_cell = 7488 14720 + +# simulation box, no MR +# note: increase z (space & cells) for converging ion energy +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = -7.5e-6 -5.e-6 +geometry.prob_hi = 7.5e-6 25.e-6 + +# Boundary condition +boundary.field_lo = pml pml +boundary.field_hi = pml pml + +# Order of particle shape factors +algo.particle_shape = 3 + +# improved plasma stability for 2D with very low initial target temperature +# when using Esirkepov current deposition with energy-conserving field gather +interpolation.galerkin_scheme = 0 + +# numerical tuning +warpx.cfl = 0.999 +warpx.use_filter = 1 # bilinear current/charge filter + + +################################# +# Performance Tuning +# +# simple tuning: +# the numprocs product must be equal to the number of MPI ranks and splits +# the domain on the coarsest level equally into grids; +# slicing in the 2nd dimension is preferred for ideal performance +warpx.numprocs = 1 2 # 2 MPI ranks +#warpx.numprocs = 1 4 # 4 MPI ranks + +# detail tuning instead of warpx.numprocs: +# It is important to have enough cells in a block & grid, otherwise +# performance will suffer. +# Use larger values for GPUs, try to fill a GPU well with memory and place +# few large grids on each device (you can go as low as 1 large grid / device +# if you do not need load balancing). +# Slicing in the 2nd dimension is preferred for ideal performance +#amr.blocking_factor = 64 +#amr.max_grid_size_x = 2688 +#amr.max_grid_size_y = 128 # this is confusingly named and means z in 2D + +# load balancing +# The grid & block parameters above are needed for load balancing: +# an average of ~10 grids per MPI rank (and device) are a good granularity +# to allow efficient load-balancing as the simulation evolves +algo.load_balance_intervals = 100 +algo.load_balance_costs_update = Heuristic + +# particle bin-sorting on GPU (ideal defaults not investigated in 2D) +# Try larger values than the defaults below and report back! :) +#warpx.sort_intervals = 4 # default on CPU: -1 (off); on GPU: 4 +#warpx.sort_bin_size = 1 1 1 + + +################################# +# Target Profile +# + +# definitions for target extent and pre-plasma +my_constants.L = 0.05e-6 # [m] scale length (>0) +my_constants.Lcut = 2.0e-6 # [m] hard cutoff from surface +my_constants.r0 = 2.5e-6 # [m] radius or half-thickness +my_constants.eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) +my_constants.zmax = r0 + Lcut + eps_z # [m] upper limit in z for particle creation + +particles.species_names = electrons hydrogen + +# particle species +hydrogen.species_type = hydrogen +hydrogen.injection_style = NUniformPerCell +hydrogen.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#hydrogen.num_particles_per_cell_each_dim = 4 8 +hydrogen.momentum_distribution_type = at_rest +# minimum and maximum z position between which particles are initialized +# --> should be set for dense targets limit memory consumption during initialization +hydrogen.zmin = -zmax +hydrogen.zmax = zmax +hydrogen.profile = parse_density_function +hydrogen.addRealAttributes = orig_x orig_z +hydrogen.attribute.orig_x(x,y,z,ux,uy,uz,t) = "x" +hydrogen.attribute.orig_z(x,y,z,ux,uy,uz,t) = "z" + +electrons.species_type = electron +electrons.injection_style = NUniformPerCell +electrons.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#electrons.num_particles_per_cell_each_dim = 4 8 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = .01 +electrons.uz_th = .01 +# minimum and maximum z position between which particles are initialized +# --> should be set for dense targets limit memory consumption during initialization +electrons.zmin = -zmax +electrons.zmax = zmax + +# ionization physics (field ionization/ADK) +# [i1] none (fully pre-ionized): +electrons.profile = parse_density_function +# [i2] field ionization (ADK): +#hydrogen.do_field_ionization = 1 +#hydrogen.physical_element = H +#hydrogen.ionization_initial_level = 0 +#hydrogen.ionization_product_species = electrons +#electrons.profile = constant +#electrons.density = 0.0 + +# collisional physics (binary MC model after Nanbu/Perez) +#collisions.collision_names = c_eH c_ee c_HH +#c_eH.species = electrons hydrogen +#c_ee.species = electrons electrons +#c_HH.species = hydrogen hydrogen +#c_eH.CoulombLog = 15.9 +#c_ee.CoulombLog = 15.9 +#c_HH.CoulombLog = 15.9 + +# number density: "fully ionized" electron density as reference +# [material 1] cryogenic H2 +my_constants.nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 +my_constants.n0 = 30.0 # [n_c] +# [material 2] liquid crystal +#my_constants.n0 = 192 +# [material 3] PMMA +#my_constants.n0 = 230 +# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) +#my_constants.n0 = 1400 + +# density profiles (target extent, pre-plasma and cutoffs defined above particle species list) + +# [target 1] flat foil (thickness = 2*r0) +electrons.density_function(x,y,z) = "nc*n0*( + if(abs(z)<=r0, 1.0, if(abs(z) e_max = 64.22 TV/m + + +################################# +# Diagnostics +# +diagnostics.diags_names = diagInst diagTimeAvg openPMDfw openPMDbw + +# instantaneous field and particle diagnostic +diagInst.intervals = 100,96:100 # second interval only for CI testing the time-averaged diags +diagInst.diag_type = Full +diagInst.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen +# reduce resolution of output fields +diagInst.coarsening_ratio = 4 4 +# demonstration of a spatial and momentum filter +diagInst.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diagInst.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diagInst.format = openpmd +diagInst.openpmd_backend = h5 + +# time-averaged particle and field diagnostic +diagTimeAvg.intervals = 100 +diagTimeAvg.diag_type = TimeAveraged +diagTimeAvg.time_average_mode = dynamic_start +#diagTimeAvg.average_period_time = 2.67e-15 # period of 800 nm light waves +diagTimeAvg.average_period_steps = 5 # use only either `time` or `steps` +diagTimeAvg.write_species = 0 +diagTimeAvg.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen +# reduce resolution of output fields +diagTimeAvg.coarsening_ratio = 4 4 +diagTimeAvg.format = openpmd +diagTimeAvg.openpmd_backend = h5 + +openPMDfw.intervals = 100 +openPMDfw.diag_type = Full +openPMDfw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen +# reduce resolution of output fields +openPMDfw.coarsening_ratio = 4 4 +openPMDfw.format = openpmd +openPMDfw.openpmd_backend = h5 +# demonstration of a spatial and momentum filter +openPMDfw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +openPMDfw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) + +openPMDbw.intervals = 100 +openPMDbw.diag_type = Full +openPMDbw.fields_to_plot = rho_hydrogen +# reduce resolution of output fields +openPMDbw.coarsening_ratio = 4 4 +openPMDbw.format = openpmd +openPMDbw.openpmd_backend = h5 +# demonstration of a momentum filter +openPMDbw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) +openPMDbw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) + + +################################# +# Reduced Diagnostics +# + +# histograms with 2.0 degree acceptance angle in fw direction +# 2 deg * pi / 180 : 0.03490658503 rad +# half-angle +/- : 0.017453292515 rad +warpx.reduced_diags_names = histuH histue histuzAll FieldProbe_Z FieldProbe_ScatPoint FieldProbe_ScatLine LBC PhaseSpaceIons PhaseSpaceElectrons + +histuH.type = ParticleHistogram +histuH.intervals = 100 +histuH.species = hydrogen +histuH.bin_number = 1000 +histuH.bin_min = 0.0 +histuH.bin_max = 0.474 # 100 MeV protons +histuH.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" +histuH.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" + +histue.type = ParticleHistogram +histue.intervals = 100 +histue.species = electrons +histue.bin_number = 1000 +histue.bin_min = 0.0 +histue.bin_max = 197 # 100 MeV electrons +histue.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" +histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" + +# just a test entry to make sure that the histogram filter is purely optional: +# this one just records uz of all hydrogen ions, independent of their pointing +histuzAll.type = ParticleHistogram +histuzAll.intervals = 100 +histuzAll.species = hydrogen +histuzAll.bin_number = 1000 +histuzAll.bin_min = -0.474 +histuzAll.bin_max = 0.474 +histuzAll.histogram_function(t,x,y,z,ux,uy,uz) = "uz" + +FieldProbe_Z.type = FieldProbe +FieldProbe_Z.intervals = 100 +FieldProbe_Z.integrate = 0 +FieldProbe_Z.probe_geometry = Line +FieldProbe_Z.x_probe = 0.0 +FieldProbe_Z.z_probe = -5.0e-6 +FieldProbe_Z.x1_probe = 0.0 +FieldProbe_Z.z1_probe = 25.0e-6 +FieldProbe_Z.resolution = 3712 + +FieldProbe_ScatPoint.type = FieldProbe +FieldProbe_ScatPoint.intervals = 1 +FieldProbe_ScatPoint.integrate = 0 +FieldProbe_ScatPoint.probe_geometry = Point +FieldProbe_ScatPoint.x_probe = 0.0 +FieldProbe_ScatPoint.z_probe = 15e-6 + +FieldProbe_ScatLine.type = FieldProbe +FieldProbe_ScatLine.intervals = 100 +FieldProbe_ScatLine.integrate = 1 +FieldProbe_ScatLine.probe_geometry = Line +FieldProbe_ScatLine.x_probe = -2.5e-6 +FieldProbe_ScatLine.z_probe = 15e-6 +FieldProbe_ScatLine.x1_probe = 2.5e-6 +FieldProbe_ScatLine.z1_probe = 15e-6 +FieldProbe_ScatLine.resolution = 201 + +# check computational load per box +LBC.type = LoadBalanceCosts +LBC.intervals = 100 + +PhaseSpaceIons.type = ParticleHistogram2D +PhaseSpaceIons.intervals = 100 +PhaseSpaceIons.species = hydrogen +PhaseSpaceIons.bin_number_abs = 1000 +PhaseSpaceIons.bin_number_ord = 1000 +PhaseSpaceIons.bin_min_abs = -5.e-6 +PhaseSpaceIons.bin_max_abs = 25.e-6 +PhaseSpaceIons.bin_min_ord = -0.474 +PhaseSpaceIons.bin_max_ord = 0.474 +PhaseSpaceIons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" +PhaseSpaceIons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" +PhaseSpaceIons.value_function(t,x,y,z,ux,uy,uz,w) = "w" +# PhaseSpaceIons.filter_function(t,x,y,z,ux,uy,uz,w) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" + +PhaseSpaceElectrons.type = ParticleHistogram2D +PhaseSpaceElectrons.intervals = 100 +PhaseSpaceElectrons.species = electrons +PhaseSpaceElectrons.bin_number_abs = 1000 +PhaseSpaceElectrons.bin_number_ord = 1000 +PhaseSpaceElectrons.bin_min_abs = -5.e-6 +PhaseSpaceElectrons.bin_max_abs = 25.e-6 +PhaseSpaceElectrons.bin_min_ord = -197 +PhaseSpaceElectrons.bin_max_ord = 197 +PhaseSpaceElectrons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" +PhaseSpaceElectrons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" +PhaseSpaceElectrons.value_function(t,x,y,z,ux,uy,uz,w) = "w" +PhaseSpaceElectrons.filter_function(t,x,y,z,ux,uy,uz,w) = "sqrt(x*x+y*y) < 1e-6" + +################################# +# Physical Background +# +# This example is modeled after a target similar to the hydrogen jet here: +# [1] https://doi.org/10.1038/s41598-017-10589-3 +# [2] https://arxiv.org/abs/1903.06428 +# +authors = "Axel Huebl " diff --git a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py new file mode 100755 index 00000000000..c869c770b99 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 +# time-scale with highly kinetic dynamics +stop_time = 0.2e-12 + +# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length +# (>=6x V100) +# --> choose larger `max_grid_size` and `blocking_factor` for 1 to 8 grids per GPU accordingly +# nx = 7488 +# nz = 14720 + +# Number of cells +nx = 384 +nz = 512 + +# Domain decomposition (deactivate `warpx_numprocs` in `picmi.Simulation` for this to take effect) +max_grid_size = 64 +blocking_factor = 32 + +# Physical domain +xmin = -7.5e-06 +xmax = 7.5e-06 +zmin = -5.0e-06 +zmax = 25.0e-06 + +# Create grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=["open", "open"], + upper_boundary_conditions=["open", "open"], + lower_boundary_conditions_particles=["absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing"], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, +) + +# Particles: plasma parameters +# critical plasma density +nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 +# number density: "fully ionized" electron density as reference +# [material 1] cryogenic H2 +n0 = 30.0 # [n_c] +# [material 2] liquid crystal +# n0 = 192 +# [material 3] PMMA +# n0 = 230 +# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) +# n0 = 1400 +plasma_density = n0 * nc +preplasma_L = 0.05e-6 # [m] scale length (>0) +preplasma_Lcut = 2.0e-6 # [m] hard cutoff from surface +plasma_r0 = 2.5e-6 # [m] radius or half-thickness +plasma_eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) +plasma_creation_limit_z = ( + plasma_r0 + preplasma_Lcut + plasma_eps_z +) # [m] upper limit in z for particle creation + +plasma_xmin = None +plasma_ymin = None +plasma_zmin = -plasma_creation_limit_z +plasma_xmax = None +plasma_ymax = None +plasma_zmax = plasma_creation_limit_z + +density_expression_str = f"{plasma_density}*((abs(z)<={plasma_r0}) + (abs(z)<{plasma_r0}+{preplasma_Lcut}) * (abs(z)>{plasma_r0}) * exp(-(abs(z)-{plasma_r0})/{preplasma_L}))" + +slab_with_ramp_dist_hydrogen = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], +) + +# thermal velocity spread for electrons in gamma*beta +ux_th = 0.01 +uz_th = 0.01 + +slab_with_ramp_dist_electrons = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + # if `momentum_expressions` and `momentum_spread_expressions` are unset, + # a Gaussian momentum distribution is assumed given that `rms_velocity` has any non-zero elements + rms_velocity=[c * ux_th, 0.0, c * uz_th], # thermal velocity spread in m/s +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=slab_with_ramp_dist_electrons, +) + +hydrogen = picmi.Species( + particle_type="proton", + name="hydrogen", + initial_distribution=slab_with_ramp_dist_hydrogen, + warpx_add_real_attributes={"orig_x": "x", "orig_z": "z"}, +) + +# Laser +# e_max = a0 * 3.211e12 / lambda_0[mu] +# a0 = 16, lambda_0 = 0.8mu -> e_max = 64.22 TV/m +e_max = 64.22e12 +position_z = -4.0e-06 +profile_t_peak = 50.0e-15 +profile_focal_distance = 4.0e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=4.0e-06, + duration=30.0e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[1, 0, 0], + E0=e_max, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver( + grid=grid, + method="Yee", + cfl=0.999, + divE_cleaning=0, + # warpx_pml_ncell=10 +) + +# Diagnostics +particle_diag = picmi.ParticleDiagnostic( + name="diagInst", + period=100, + warpx_format="openpmd", + warpx_openpmd_backend="h5", + # demonstration of a spatial and momentum filter + warpx_plot_filter_function="(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)", +) +# reduce resolution of output fields +coarsening_ratio = [4, 4] +ncell_field = [] +for ncell_comp, cr in zip([nx, nz], coarsening_ratio): + ncell_field.append(int(ncell_comp / cr)) +field_diag = picmi.FieldDiagnostic( + name="diagInst", + grid=grid, + period=100, + number_of_cells=ncell_field, + data_list=["B", "E", "J", "rho", "rho_electrons", "rho_hydrogen"], + warpx_format="openpmd", + warpx_openpmd_backend="h5", +) + +field_time_avg_diag = picmi.TimeAveragedFieldDiagnostic( + name="diagTimeAvg", + grid=grid, + period=100, + number_of_cells=ncell_field, + data_list=["B", "E", "J", "rho", "rho_electrons", "rho_hydrogen"], + warpx_format="openpmd", + warpx_openpmd_backend="h5", + warpx_time_average_mode="dynamic_start", + warpx_average_period_time=2.67e-15, +) + +particle_fw_diag = picmi.ParticleDiagnostic( + name="openPMDfw", + period=100, + warpx_format="openpmd", + warpx_openpmd_backend="h5", + warpx_plot_filter_function="(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)", +) + +particle_bw_diag = picmi.ParticleDiagnostic( + name="openPMDbw", + period=100, + warpx_format="openpmd", + warpx_openpmd_backend="h5", + warpx_plot_filter_function="(uz<0)", +) + +# histograms with 2.0 degree acceptance angle in fw direction +# 2 deg * pi / 180 : 0.03490658503 rad +# half-angle +/- : 0.017453292515 rad +histuH_rdiag = picmi.ReducedDiagnostic( + diag_type="ParticleHistogram", + name="histuH", + species=hydrogen, + bin_number=1000, + bin_min=0.0, + bin_max=0.474, # 100 MeV protons + histogram_function="u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)", + filter_function="u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)", +) + +histue_rdiag = picmi.ReducedDiagnostic( + diag_type="ParticleHistogram", + name="histue", + species=electrons, + bin_number=1000, + bin_min=0.0, + bin_max=197.0, # 100 MeV electrons + histogram_function="u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)", + filter_function="u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)", +) + +# just a test entry to make sure that the histogram filter is purely optional: +# this one just records uz of all hydrogen ions, independent of their pointing +histuzAll_rdiag = picmi.ReducedDiagnostic( + diag_type="ParticleHistogram", + name="histuzAll", + species=hydrogen, + bin_number=1000, + bin_min=-0.474, + bin_max=0.474, + histogram_function="uz", +) + +field_probe_z_rdiag = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + name="FieldProbe_Z", + integrate=0, + probe_geometry="Line", + x_probe=0.0, + z_probe=-5.0e-6, + x1_probe=0.0, + z1_probe=25.0e-6, + resolution=3712, +) + +field_probe_scat_point_rdiag = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + name="FieldProbe_ScatPoint", + integrate=0, + probe_geometry="Point", + x_probe=0.0, + z_probe=15.0e-6, +) + +field_probe_scat_line_rdiag = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + name="FieldProbe_ScatLine", + integrate=1, + probe_geometry="Line", + x_probe=-2.5e-6, + z_probe=15.0e-6, + x1_probe=2.5e-6, + z1_probe=15e-6, + resolution=201, +) + +load_balance_costs_rdiag = picmi.ReducedDiagnostic( + diag_type="LoadBalanceCosts", + name="LBC", +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_time=stop_time, # need to remove `max_step` to run this far + verbose=1, + particle_shape="cubic", + warpx_numprocs=[1, 2], # deactivate `numprocs` for dynamic load balancing + warpx_use_filter=1, + warpx_reduced_diags_intervals=100, + warpx_load_balance_intervals=100, + warpx_load_balance_costs_update="heuristic", +) + +# Add plasma electrons +sim.add_species( + electrons, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2, 2]), + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + # layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add hydrogen ions +sim.add_species( + hydrogen, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2, 2]), + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + # layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add full diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(field_time_avg_diag) +sim.add_diagnostic(particle_fw_diag) +sim.add_diagnostic(particle_bw_diag) +# Add reduced diagnostics +sim.add_diagnostic(histuH_rdiag) +sim.add_diagnostic(histue_rdiag) +sim.add_diagnostic(histuzAll_rdiag) +sim.add_diagnostic(field_probe_z_rdiag) +sim.add_diagnostic(field_probe_scat_point_rdiag) +sim.add_diagnostic(field_probe_scat_line_rdiag) +sim.add_diagnostic(load_balance_costs_rdiag) +# TODO: make ParticleHistogram2D available + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_2d_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_step) diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py index 736203e85ea..87b2d76c8f7 100644 --- a/Examples/Physics_applications/laser_ion/plot_2d.py +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -21,7 +21,8 @@ from matplotlib.colors import TwoSlopeNorm from openpmd_viewer import OpenPMDTimeSeries -plt.rcParams.update({'font.size':16}) +plt.rcParams.update({"font.size": 16}) + def create_analysis_dir(directory): if not os.path.exists(directory): @@ -40,7 +41,9 @@ def visualize_density_iteration(ts, iteration, out_dir): # Physics parameters lambda_L = 800e-9 # Laser wavelength in meters omega_L = 2 * np.pi * sc.c / lambda_L # Laser frequency in seconds - n_c = sc.m_e * sc.epsilon_0 * omega_L**2 / sc.elementary_charge**2 # Critical plasma density in meters^(-3) + n_c = ( + sc.m_e * sc.epsilon_0 * omega_L**2 / sc.elementary_charge**2 + ) # Critical plasma density in meters^(-3) micron = 1e-6 # Simulation parameters @@ -66,24 +69,49 @@ def visualize_density_iteration(ts, iteration, out_dir): # Plotting # Electron density - im0 = axs[0].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, -rho_e.T[::nr, ::nr], - vmin=0, vmax=n_max, cmap="Reds", rasterized=True) + im0 = axs[0].pcolormesh( + zax[::nr] / micron, + xax[::nr] / micron, + -rho_e.T[::nr, ::nr], + vmin=0, + vmax=n_max, + cmap="Reds", + rasterized=True, + ) plt.colorbar(im0, ax=axs[0], label=r"$n_\mathrm{\,e}\ (n_\mathrm{c})$") # Hydrogen density - im1 = axs[1].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, rho_d.T[::nr, ::nr], - vmin=0, vmax=n_max, cmap="Blues", rasterized=True) + im1 = axs[1].pcolormesh( + zax[::nr] / micron, + xax[::nr] / micron, + rho_d.T[::nr, ::nr], + vmin=0, + vmax=n_max, + cmap="Blues", + rasterized=True, + ) plt.colorbar(im1, ax=axs[1], label=r"$n_\mathrm{\,H}\ (n_\mathrm{c})$") # Masked electron density - divnorm = TwoSlopeNorm(vmin=-7., vcenter=0., vmax=2) + divnorm = TwoSlopeNorm(vmin=-7.0, vcenter=0.0, vmax=2) masked_data = np.ma.masked_where(rho_e.T == 0, rho_e.T) my_cmap = plt.cm.PiYG_r.copy() - my_cmap.set_bad(color='black') - im2 = axs[2].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, np.log(-masked_data[::nr, ::nr]), - norm=divnorm, cmap=my_cmap, rasterized=True) - plt.colorbar(im2, ax=axs[2], ticks=[-6, -3, 0, 1, 2], extend='both', - label=r"$\log n_\mathrm{\,e}\ (n_\mathrm{c})$") + my_cmap.set_bad(color="black") + im2 = axs[2].pcolormesh( + zax[::nr] / micron, + xax[::nr] / micron, + np.log(-masked_data[::nr, ::nr]), + norm=divnorm, + cmap=my_cmap, + rasterized=True, + ) + plt.colorbar( + im2, + ax=axs[2], + ticks=[-6, -3, 0, 1, 2], + extend="both", + label=r"$\log n_\mathrm{\,e}\ (n_\mathrm{c})$", + ) # Axis labels and title for ax in axs: @@ -92,14 +120,14 @@ def visualize_density_iteration(ts, iteration, out_dir): for ax in axs[:-1]: ax.set_xticklabels([]) axs[2].set_xlabel(r"$z$ ($\mu$m)") - fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + fig.suptitle(f"Iteration: {it}, Time: {time / 1e-15:.1f} fs") plt.tight_layout() plt.savefig(f"{out_dir}/densities_{it:06d}.png") -def visualize_field_iteration(ts, iteration, out_dir): +def visualize_field_iteration(ts, iteration, out_dir): # Additional parameters nr = 1 # Number to decrease resolution micron = 1e-6 @@ -110,35 +138,50 @@ def visualize_field_iteration(ts, iteration, out_dir): time = ts.t[ii] Ex, Ex_info = ts.get_field(field="E", coord="x", iteration=it) - Exmax = np.max(np.abs([np.min(Ex),np.max(Ex)])) + Exmax = np.max(np.abs([np.min(Ex), np.max(Ex)])) By, By_info = ts.get_field(field="B", coord="y", iteration=it) - Bymax = np.max(np.abs([np.min(By),np.max(By)])) + Bymax = np.max(np.abs([np.min(By), np.max(By)])) Ez, Ez_info = ts.get_field(field="E", coord="z", iteration=it) - Ezmax = np.max(np.abs([np.min(Ez),np.max(Ez)])) + Ezmax = np.max(np.abs([np.min(Ez), np.max(Ez)])) # Axes setup - fig,axs = plt.subplots(3, 1, figsize=(5, 8)) + fig, axs = plt.subplots(3, 1, figsize=(5, 8)) xax, zax = Ex_info.x, Ex_info.z # Plotting im0 = axs[0].pcolormesh( - zax[::nr]/micron,xax[::nr]/micron,Ex.T[::nr,::nr], - vmin=-Exmax, vmax=Exmax, - cmap="RdBu", rasterized=True) + zax[::nr] / micron, + xax[::nr] / micron, + Ex.T[::nr, ::nr], + vmin=-Exmax, + vmax=Exmax, + cmap="RdBu", + rasterized=True, + ) - plt.colorbar(im0,ax=axs[00], label=r"$E_x$ (V/m)") + plt.colorbar(im0, ax=axs[00], label=r"$E_x$ (V/m)") im1 = axs[1].pcolormesh( - zax[::nr]/micron,xax[::nr]/micron,By.T[::nr,::nr], - vmin=-Bymax, vmax=Bymax, - cmap="RdBu", rasterized=True) - plt.colorbar(im1,ax=axs[1], label=r"$B_y$ (T)") + zax[::nr] / micron, + xax[::nr] / micron, + By.T[::nr, ::nr], + vmin=-Bymax, + vmax=Bymax, + cmap="RdBu", + rasterized=True, + ) + plt.colorbar(im1, ax=axs[1], label=r"$B_y$ (T)") im2 = axs[2].pcolormesh( - zax[::nr]/micron,xax[::nr]/micron,Ez.T[::nr,::nr], - vmin=-Ezmax, vmax=Ezmax, - cmap="RdBu", rasterized=True) - plt.colorbar(im2,ax=axs[2],label=r"$E_z$ (V/m)") + zax[::nr] / micron, + xax[::nr] / micron, + Ez.T[::nr, ::nr], + vmin=-Ezmax, + vmax=Ezmax, + cmap="RdBu", + rasterized=True, + ) + plt.colorbar(im2, ax=axs[2], label=r"$E_z$ (V/m)") # Axis labels and title for ax in axs: @@ -147,48 +190,54 @@ def visualize_field_iteration(ts, iteration, out_dir): for ax in axs[:-1]: ax.set_xticklabels([]) axs[2].set_xlabel(r"$z$ ($\mu$m)") - fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + fig.suptitle(f"Iteration: {it}, Time: {time / 1e-15:.1f} fs") plt.tight_layout() plt.savefig(f"{out_dir}/fields_{it:06d}.png") -def visualize_particle_histogram_iteration(diag_name="histuH", species="hydrogen", iteration=1000, out_dir="./analysis"): +def visualize_particle_histogram_iteration( + diag_name="histuH", species="hydrogen", iteration=1000, out_dir="./analysis" +): it = iteration if species == "hydrogen": # proton rest energy in eV - mc2 = sc.m_p/sc.electron_volt * sc.c**2 + mc2 = sc.m_p / sc.electron_volt * sc.c**2 elif species == "electron": - mc2 = sc.m_e/sc.electron_volt * sc.c**2 + mc2 = sc.m_e / sc.electron_volt * sc.c**2 else: - raise NotImplementedError("The only implemented presets for this analysis script are `electron` or `hydrogen`.") + raise NotImplementedError( + "The only implemented presets for this analysis script are `electron` or `hydrogen`." + ) - fs = 1.e-15 - MeV = 1.e6 + fs = 1.0e-15 + MeV = 1.0e6 - df = pd.read_csv(f"./diags/reducedfiles/{diag_name}.txt",delimiter=r'\s+') + df = pd.read_csv(f"./diags/reducedfiles/{diag_name}.txt", delimiter=r"\s+") # the columns look like this: # #[0]step() [1]time(s) [2]bin1=0.000220() [3]bin2=0.000660() [4]bin3=0.001100() # matches words, strings surrounded by " ' ", dots, minus signs and e for scientific notation in numbers - nested_list = [re.findall(r"[\w'\.]+",col) for col in df.columns] + nested_list = [re.findall(r"[\w'\.]+", col) for col in df.columns] - index = pd.MultiIndex.from_tuples(nested_list, names=('column#', 'name', 'bin value')) + index = pd.MultiIndex.from_tuples( + nested_list, names=("column#", "name", "bin value") + ) - df.columns = (index) + df.columns = index steps = df.values[:, 0].astype(int) ii = np.where(steps == it)[0][0] time = df.values[:, 1] data = df.values[:, 2:] edge_vals = np.array([float(row[2]) for row in df.columns[2:]]) - edges_MeV = (np.sqrt(edge_vals**2 + 1)-1) * mc2 / MeV + edges_MeV = (np.sqrt(edge_vals**2 + 1) - 1) * mc2 / MeV time_fs = time / fs - fig,ax = plt.subplots(1,1) + fig, ax = plt.subplots(1, 1) ax.plot(edges_MeV, data[ii, :]) ax.set_yscale("log") @@ -202,17 +251,42 @@ def visualize_particle_histogram_iteration(diag_name="histuH", species="hydrogen if __name__ == "__main__": - # Argument parsing - parser = argparse.ArgumentParser(description='Visualize Laser-Ion Accelerator Densities and Fields') - parser.add_argument('-d', '--diag_dir', type=str, default='./diags/diag1', help='Directory containing density and field diagnostics') - parser.add_argument('-i', '--iteration', type=int, default=None, help='Specific iteration to visualize') - parser.add_argument('-hn', '--histogram_name', type=str, default='histuH', help='Name of histogram diagnostic to visualize') - parser.add_argument('-hs', '--histogram_species', type=str, default='hydrogen', help='Particle species in the visualized histogram diagnostic') + parser = argparse.ArgumentParser( + description="Visualize Laser-Ion Accelerator Densities and Fields" + ) + parser.add_argument( + "-d", + "--diag_dir", + type=str, + default="./diags/diagInst", + help="Directory containing density and field diagnostics", + ) + parser.add_argument( + "-i", + "--iteration", + type=int, + default=None, + help="Specific iteration to visualize", + ) + parser.add_argument( + "-hn", + "--histogram_name", + type=str, + default="histuH", + help="Name of histogram diagnostic to visualize", + ) + parser.add_argument( + "-hs", + "--histogram_species", + type=str, + default="hydrogen", + help="Particle species in the visualized histogram diagnostic", + ) args = parser.parse_args() # Create analysis directory - analysis_dir = 'analysis' + analysis_dir = "analysis" create_analysis_dir(analysis_dir) # Loading the time series @@ -221,9 +295,13 @@ def visualize_particle_histogram_iteration(diag_name="histuH", species="hydrogen if args.iteration is not None: visualize_density_iteration(ts, args.iteration, analysis_dir) visualize_field_iteration(ts, args.iteration, analysis_dir) - visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, args.iteration, analysis_dir) + visualize_particle_histogram_iteration( + args.histogram_name, args.histogram_species, args.iteration, analysis_dir + ) else: for it in ts.iterations: visualize_density_iteration(ts, it, analysis_dir) visualize_field_iteration(ts, it, analysis_dir) - visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, it, analysis_dir) + visualize_particle_histogram_iteration( + args.histogram_name, args.histogram_species, it, analysis_dir + ) diff --git a/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt b/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt new file mode 100644 index 00000000000..68e81e4b9e4 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt @@ -0,0 +1,82 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_plasma_acceleration_picmi # name + 1 # dims + 2 # nprocs + inputs_test_1d_plasma_acceleration_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1001000" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_plasma_acceleration_boosted # name + 2 # dims + 2 # nprocs + inputs_test_2d_plasma_acceleration_boosted # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_plasma_acceleration_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_plasma_acceleration_mr # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000400" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_plasma_acceleration_mr_momentum_conserving # name + 2 # dims + 2 # nprocs + inputs_test_2d_plasma_acceleration_mr_momentum_conserving # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000400" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_acceleration_boosted # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_acceleration_boosted # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000005" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_acceleration_boosted_hybrid # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_acceleration_boosted_hybrid # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000025" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_acceleration_mr_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_acceleration_mr_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_acceleration_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_acceleration_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) diff --git a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py deleted file mode 100755 index 296aea48b35..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -#from warp import picmi - -constants = picmi.constants - -nx = 64 -ny = 64 -nz = 64 - -xmin = -200.e-6 -xmax = +200.e-6 -ymin = -200.e-6 -ymax = +200.e-6 -zmin = -200.e-6 -zmax = +200.e-6 - -moving_window_velocity = [0., 0., constants.c] - -number_per_cell_each_dim = [2, 2, 1] - -max_steps = 10 - -grid = picmi.Cartesian3DGrid(number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['periodic', 'periodic', 'open'], - upper_boundary_conditions = ['periodic', 'periodic', 'open'], - lower_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - upper_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - moving_window_velocity = moving_window_velocity, - warpx_max_grid_size=32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1) - -beam_distribution = picmi.UniformDistribution(density = 1.e23, - lower_bound = [-20.e-6, -20.e-6, -150.e-6], - upper_bound = [+20.e-6, +20.e-6, -100.e-6], - directed_velocity = [0., 0., 1.e9]) - -plasma_distribution = picmi.UniformDistribution(density = 1.e22, - lower_bound = [-200.e-6, -200.e-6, 0.], - upper_bound = [+200.e-6, +200.e-6, None], - fill_in = True) - -beam = picmi.Species(particle_type='electron', name='beam', initial_distribution=beam_distribution) -plasma = picmi.Species(particle_type='electron', name='plasma', initial_distribution=plasma_distribution) - -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - warpx_current_deposition_algo = 'esirkepov', - warpx_use_filter = 0) - -sim.add_species(beam, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) -sim.add_species(plasma, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) - -field_diag = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = max_steps, - data_list = ['Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz', 'part_per_cell'], - write_dir = '.', - warpx_file_prefix = 'Python_PlasmaAcceleration_plt') - -part_diag = picmi.ParticleDiagnostic(name = 'diag1', - period = max_steps, - species = [beam, plasma], - data_list = ['ux', 'uy', 'uz', 'weighting']) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(part_diag) - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_1d.py b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_1d.py deleted file mode 100755 index 27f7236204e..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_1d.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -#from warp import picmi - -constants = picmi.constants - -nz = 64 - -zmin = -200.e-6 -zmax = +200.e-6 - -moving_window_velocity = [constants.c] - -number_per_cell_each_dim = [10] - -max_steps = 1000 - -grid = picmi.Cartesian1DGrid(number_of_cells = [nz], - lower_bound = [zmin], - upper_bound = [zmax], - lower_boundary_conditions = ['dirichlet'], - upper_boundary_conditions = ['dirichlet'], - lower_boundary_conditions_particles = ['absorbing'], - upper_boundary_conditions_particles = ['absorbing'], - moving_window_velocity = moving_window_velocity, - warpx_max_grid_size=32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=0.999) - -beam_distribution = picmi.UniformDistribution(density = 1.e23, - lower_bound = [None, None, -150.e-6], - upper_bound = [None, None, -100.e-6], - directed_velocity = [0., 0., 1.e9]) - -plasma_distribution = picmi.UniformDistribution(density = 1.e22, - lower_bound = [None, None, 0.], - upper_bound = [None, None, None], - fill_in = True) - -beam = picmi.Species(particle_type='electron', name='beam', initial_distribution=beam_distribution) -plasma = picmi.Species(particle_type='electron', name='plasma', initial_distribution=plasma_distribution) - -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - warpx_current_deposition_algo = 'esirkepov', - warpx_use_filter = 0) - -sim.add_species(beam, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) -sim.add_species(plasma, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) - -field_diag = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = max_steps, - data_list = ['Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz', 'part_per_cell'], - write_dir = '.', - warpx_file_prefix = 'Python_PlasmaAcceleration1d_plt') - -part_diag = picmi.ParticleDiagnostic(name = 'diag1', - period = max_steps, - species = [beam, plasma], - data_list = ['ux', 'uy', 'uz', 'weighting']) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(part_diag) - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py deleted file mode 100755 index 52a9729a1fb..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -#from warp import picmi - -constants = picmi.constants - -nx = 64 -ny = 64 -nz = 64 - -xmin = -200.e-6 -xmax = +200.e-6 -ymin = -200.e-6 -ymax = +200.e-6 -zmin = -200.e-6 -zmax = +200.e-6 - -moving_window_velocity = [0., 0., constants.c] - -number_per_cell_each_dim = [4, 4, 4] - -grid = picmi.Cartesian3DGrid(number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['periodic', 'periodic', 'open'], - upper_boundary_conditions = ['periodic', 'periodic', 'open'], - lower_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - upper_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - moving_window_velocity = moving_window_velocity, - #refined_regions = [[1, [-25e-6, -25e-6, -200.e-6], [25e-6, 25e-6, 200.e-6]]], # as argument - warpx_max_grid_size=128, warpx_blocking_factor=16) - -# --- As a separate function call (instead of refined_regions argument) -grid.add_refined_region(level = 1, - lo = [-25e-6, -25e-6, -200.e-6], - hi = [25e-6, 25e-6, 200.e-6]) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1, - warpx_pml_ncell = 10) - -beam_distribution = picmi.UniformDistribution(density = 1.e23, - lower_bound = [-20.e-6, -20.e-6, -150.e-6], - upper_bound = [+20.e-6, +20.e-6, -100.e-6], - directed_velocity = [0., 0., 1.e9]) - -plasma_distribution = picmi.UniformDistribution(density = 1.e22, - lower_bound = [-200.e-6, -200.e-6, 0.], - upper_bound = [+200.e-6, +200.e-6, None], - fill_in = True) - -beam = picmi.Species(particle_type='electron', name='beam', initial_distribution=beam_distribution) -plasma = picmi.Species(particle_type='electron', name='plasma', initial_distribution=plasma_distribution) - -sim = picmi.Simulation(solver = solver, - max_steps = 2, - verbose = 1, - warpx_current_deposition_algo = 'esirkepov', - warpx_use_filter = 0) - -sim.add_species(beam, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) -sim.add_species(plasma, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim)) - -field_diag = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = 2, - data_list = ['Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz', 'part_per_cell'], - write_dir = '.', - warpx_file_prefix = 'Python_PlasmaAccelerationMR_plt') - -part_diag = picmi.ParticleDiagnostic(name = 'diag1', - period = 2, - species = [beam, plasma], - data_list = ['ux', 'uy', 'uz', 'weighting']) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(part_diag) - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI.mr') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst index d5775e93aa8..0c1b1819cab 100644 --- a/Examples/Physics_applications/plasma_acceleration/README.rst +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -16,7 +16,7 @@ In the Beam, Plasma & Accelerator Simulation Toolkit (BLAST), `HiPACE++ `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. @@ -35,17 +35,17 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. note:: - TODO: This input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + TODO: This input file should use the boosted frame method, like the ``inputs_test_3d_plasma_acceleration_boosted`` file. - .. literalinclude:: PICMI_inputs_plasma_acceleration.py + .. literalinclude:: inputs_test_3d_plasma_acceleration_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py``. + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_picmi.py``. .. tab-item:: Executable: Input File - .. literalinclude:: inputs_3d_boost + .. literalinclude:: inputs_test_3d_plasma_acceleration_boosted :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_3d_boost``. + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted``. Analyze ------- diff --git a/Examples/Physics_applications/plasma_acceleration/analysis_default_regression.py b/Examples/Physics_applications/plasma_acceleration/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_2d b/Examples/Physics_applications/plasma_acceleration/inputs_2d deleted file mode 100644 index 7e11ae7b3de..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/inputs_2d +++ /dev/null @@ -1,95 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 3.7e-12 -amr.n_cell = 64 128 -amr.max_grid_size = 128 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -125.e-6 -149.e-6 -geometry.prob_hi = 125.e-6 1.e-6 -warpx.fine_tag_lo = -12.e-6 -110.e-6 -warpx.fine_tag_hi = 12.e-6 -100.e-6 - -################################# -######## Boundary condition ##### -################################# -boundary.field_lo = pml pml -boundary.field_hi = pml pml -# PML -warpx.pml_ncell = 10 - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = ckc -warpx.use_filter = 1 -warpx.cfl = .99 -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1. # in units of the speed of light - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ PLASMA ############# -################################# -particles.species_names = plasma_e beam driver - -driver.charge = -q_e -driver.mass = m_e -driver.injection_style = "gaussian_beam" -driver.x_rms = 2.e-6 -driver.y_rms = 2.e-6 -driver.z_rms = 4.e-6 -driver.x_m = 0. -driver.y_m = 0. -driver.z_m = -20.e-6 -driver.npart = 1000 -driver.q_tot = -3.e-11 -driver.momentum_distribution_type = "gaussian" -driver.ux_m = 0.0 -driver.uy_m = 0.0 -driver.uz_m = 2000. -driver.ux_th = 2. -driver.uy_th = 2. -driver.uz_th = 200. - -plasma_e.charge = -q_e -plasma_e.mass = m_e -plasma_e.injection_style = "NUniformPerCell" -plasma_e.zmin = 0.e-6 -plasma_e.zmax = 1.e-3 -plasma_e.xmin = -70.e-6 -plasma_e.xmax = 70.e-6 -plasma_e.profile = constant -plasma_e.density = 1.e23 -plasma_e.num_particles_per_cell_each_dim = 1 1 -plasma_e.momentum_distribution_type = "at_rest" -plasma_e.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = .5e-6 -beam.y_rms = .5e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -105.e-6 -beam.npart = 1000 -beam.q_tot = -1.e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 2000. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 200. - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 100 -diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_2d_boost b/Examples/Physics_applications/plasma_acceleration/inputs_2d_boost deleted file mode 100644 index 76dcd3ee286..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/inputs_2d_boost +++ /dev/null @@ -1,116 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 2500 -amr.n_cell = 64 640 -amr.max_grid_size = 128 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -125.e-6 -149.e-6 -geometry.prob_hi = 125.e-6 1.e-6 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = pml pml -boundary.field_hi = pml pml - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = ckc -warpx.verbose = 1 -warpx.use_filter = 1 -warpx.pml_ncell = 10 -warpx.cfl = .99 -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1. # in units of the speed of light -my_constants.lramp = 8.e-3 -my_constants.dens = 1e+23 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -######### BOOSTED FRAME ######### -################################# -warpx.gamma_boost = 10.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = driver plasma_e plasma_p beam -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = driver beam - -driver.species_type = electron -driver.injection_style = "gaussian_beam" -driver.x_rms = 2.e-6 -driver.y_rms = 2.e-6 -driver.z_rms = 4.e-6 -driver.x_m = 0. -driver.y_m = 0. -driver.z_m = -20.e-6 -driver.npart = 1000 -driver.q_tot = -3.e-11 -driver.momentum_distribution_type = "gaussian" -driver.ux_m = 0.0 -driver.uy_m = 0.0 -driver.uz_m = 200000. -driver.ux_th = 2. -driver.uy_th = 2. -driver.uz_th = 20000. -driver.zinject_plane = 0. -driver.rigid_advance = true - -plasma_e.species_type = electron -plasma_e.injection_style = "NUniformPerCell" -plasma_e.zmin = 0.e-6 -plasma_e.zmax = 0.2 -plasma_e.xmin = -70.e-6 -plasma_e.xmax = 70.e-6 -plasma_e.profile = parse_density_function -plasma_e.density_function(x,y,z) = "(zlramp)*dens" -plasma_e.num_particles_per_cell_each_dim = 1 1 -plasma_e.momentum_distribution_type = "at_rest" -plasma_e.do_continuous_injection = 1 - -plasma_p.species_type = hydrogen -plasma_p.injection_style = "NUniformPerCell" -plasma_p.zmin = 0.e-6 -plasma_p.zmax = 0.2 -plasma_p.profile = parse_density_function -plasma_p.density_function(x,y,z) = "(zlramp)*dens" -plasma_p.xmin = -70.e-6 -plasma_p.xmax = 70.e-6 -plasma_p.num_particles_per_cell_each_dim = 1 1 -plasma_p.momentum_distribution_type = "at_rest" -plasma_p.do_continuous_injection = 1 - -beam.species_type = electron -beam.injection_style = "gaussian_beam" -beam.x_rms = .5e-6 -beam.y_rms = .5e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -105.e-6 -beam.npart = 1000 -beam.q_tot = -3.e-13 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 2000. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 200. -beam.zinject_plane = 0. -beam.rigid_advance = true - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 500 -diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_3d_boost b/Examples/Physics_applications/plasma_acceleration/inputs_3d_boost deleted file mode 100644 index 2264872ec43..00000000000 --- a/Examples/Physics_applications/plasma_acceleration/inputs_3d_boost +++ /dev/null @@ -1,148 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 3.93151387287e-11 -amr.n_cell = 32 32 256 -amr.max_grid_size = 64 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -0.00015 -0.00015 -0.00012 -geometry.prob_hi = 0.00015 0.00015 1.e-06 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic pml -boundary.field_hi = periodic periodic pml - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = ckc -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 1 -warpx.cfl = .99 -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1. # in units of the speed of light -my_constants.lramp = 8.e-3 -my_constants.dens = 1e+23 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -######### BOOSTED FRAME ######### -################################# -warpx.gamma_boost = 10.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = driver plasma_e plasma_p beam driverback -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = driver beam - -driver.charge = -q_e -driver.mass = 1.e10 -driver.injection_style = "gaussian_beam" -driver.x_rms = 2.e-6 -driver.y_rms = 2.e-6 -driver.z_rms = 4.e-6 -driver.x_m = 0. -driver.y_m = 0. -driver.z_m = -20.e-6 -driver.npart = 1000 -driver.q_tot = -1.e-9 -driver.momentum_distribution_type = "gaussian" -driver.ux_m = 0.0 -driver.uy_m = 0.0 -driver.uz_m = 200000. -driver.ux_th = 2. -driver.uy_th = 2. -driver.uz_th = 20000. -driver.zinject_plane = 0. -driver.rigid_advance = true - -driverback.charge = q_e -driverback.mass = 1.e10 -driverback.injection_style = "gaussian_beam" -driverback.x_rms = 2.e-6 -driverback.y_rms = 2.e-6 -driverback.z_rms = 4.e-6 -driverback.x_m = 0. -driverback.y_m = 0. -driverback.z_m = -20.e-6 -driverback.npart = 1000 -driverback.q_tot = 1.e-9 -driverback.momentum_distribution_type = "gaussian" -driverback.ux_m = 0.0 -driverback.uy_m = 0.0 -driverback.uz_m = 200000. -driverback.ux_th = 2. -driverback.uy_th = 2. -driverback.uz_th = 20000. -driverback.do_backward_propagation = true - -plasma_e.charge = -q_e -plasma_e.mass = m_e -plasma_e.injection_style = "NUniformPerCell" -plasma_e.zmin = -100.e-6 # 0.e-6 -plasma_e.zmax = 0.2 -plasma_e.xmin = -70.e-6 -plasma_e.xmax = 70.e-6 -plasma_e.ymin = -70.e-6 -plasma_e.ymax = 70.e-6 -# plasma_e.profile = constant -# plasma_e.density = 1.e23 -plasma_e.profile = parse_density_function -plasma_e.density_function(x,y,z) = "(zlramp)*dens" -plasma_e.num_particles_per_cell_each_dim = 1 1 1 -plasma_e.momentum_distribution_type = "at_rest" -plasma_e.do_continuous_injection = 1 - -plasma_p.charge = q_e -plasma_p.mass = m_p -plasma_p.injection_style = "NUniformPerCell" -plasma_p.zmin = -100.e-6 # 0.e-6 -plasma_p.zmax = 0.2 -# plasma_p.profile = "constant" -# plasma_p.density = 1.e23 -plasma_p.profile = parse_density_function -plasma_p.density_function(x,y,z) = "(zlramp)*dens" -plasma_p.xmin = -70.e-6 -plasma_p.xmax = 70.e-6 -plasma_p.ymin = -70.e-6 -plasma_p.ymax = 70.e-6 -plasma_p.num_particles_per_cell_each_dim = 1 1 1 -plasma_p.momentum_distribution_type = "at_rest" -plasma_p.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = .5e-6 -beam.y_rms = .5e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -100.e-6 -beam.npart = 1000 -beam.q_tot = -5.e-10 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 2000. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 200. -beam.zinject_plane = .8e-3 -beam.rigid_advance = true - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 10000 -diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_base_2d b/Examples/Physics_applications/plasma_acceleration/inputs_base_2d new file mode 100644 index 00000000000..769e1ebce37 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_base_2d @@ -0,0 +1,96 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +#stop_time = 3.7e-12 +max_step = 400 +amr.n_cell = 32 512 #64 128 +amr.max_grid_size = 128 +amr.blocking_factor = 32 +amr.max_level = 1 +geometry.dims = 2 +geometry.prob_lo = -125.e-6 -149.e-6 +geometry.prob_hi = 125.e-6 1.e-6 +warpx.fine_tag_lo = -12.e-6 -110.e-6 +warpx.fine_tag_hi = 12.e-6 -100.e-6 + +################################# +######## Boundary condition ##### +################################# +boundary.field_lo = pml pml +boundary.field_hi = pml pml +# PML +warpx.pml_ncell = 10 + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.use_filter = 1 +warpx.cfl = .99 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1. # in units of the speed of light + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ PLASMA ############# +################################# +particles.species_names = plasma_e beam driver + +driver.charge = -q_e +driver.mass = m_e +driver.injection_style = "gaussian_beam" +driver.x_rms = 2.e-6 +driver.y_rms = 2.e-6 +driver.z_rms = 4.e-6 +driver.x_m = 0. +driver.y_m = 0. +driver.z_m = -20.e-6 +driver.npart = 1000 +driver.q_tot = -3.e-11 +driver.momentum_distribution_type = "gaussian" +driver.ux_m = 0.0 +driver.uy_m = 0.0 +driver.uz_m = 2000. +driver.ux_th = 2. +driver.uy_th = 2. +driver.uz_th = 200. + +plasma_e.charge = -q_e +plasma_e.mass = m_e +plasma_e.injection_style = "NUniformPerCell" +plasma_e.zmin = 0.e-6 +plasma_e.zmax = 1.e-3 +plasma_e.xmin = -70.e-6 +plasma_e.xmax = 70.e-6 +plasma_e.profile = constant +plasma_e.density = 1.e23 +plasma_e.num_particles_per_cell_each_dim = 1 1 +plasma_e.momentum_distribution_type = "at_rest" +plasma_e.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = .5e-6 +beam.y_rms = .5e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -105.e-6 +beam.npart = 1000 +beam.q_tot = -1.e-12 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 2000. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 200. + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_base_3d b/Examples/Physics_applications/plasma_acceleration/inputs_base_3d new file mode 100644 index 00000000000..66debc4f99f --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_base_3d @@ -0,0 +1,148 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 3.93151387287e-11 +amr.n_cell = 64 64 128 #32 32 256 +amr.max_grid_size = 64 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.00015 -0.00015 -0.00012 +geometry.prob_hi = 0.00015 0.00015 1.e-06 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic pml +boundary.field_hi = periodic periodic pml + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 1 +warpx.cfl = .99 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1. # in units of the speed of light +my_constants.lramp = 8.e-3 +my_constants.dens = 1e+23 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +######### BOOSTED FRAME ######### +################################# +warpx.gamma_boost = 10.0 +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +particles.species_names = driver plasma_e plasma_p beam driverback +particles.use_fdtd_nci_corr = 1 +particles.rigid_injected_species = driver beam + +driver.charge = -q_e +driver.mass = 1.e10 +driver.injection_style = "gaussian_beam" +driver.x_rms = 2.e-6 +driver.y_rms = 2.e-6 +driver.z_rms = 4.e-6 +driver.x_m = 0. +driver.y_m = 0. +driver.z_m = -20.e-6 +driver.npart = 1000 +driver.q_tot = -1.e-9 +driver.momentum_distribution_type = "gaussian" +driver.ux_m = 0.0 +driver.uy_m = 0.0 +driver.uz_m = 200000. +driver.ux_th = 2. +driver.uy_th = 2. +driver.uz_th = 20000. +driver.zinject_plane = 0. +driver.rigid_advance = true + +driverback.charge = q_e +driverback.mass = 1.e10 +driverback.injection_style = "gaussian_beam" +driverback.x_rms = 2.e-6 +driverback.y_rms = 2.e-6 +driverback.z_rms = 4.e-6 +driverback.x_m = 0. +driverback.y_m = 0. +driverback.z_m = -20.e-6 +driverback.npart = 1000 +driverback.q_tot = 1.e-9 +driverback.momentum_distribution_type = "gaussian" +driverback.ux_m = 0.0 +driverback.uy_m = 0.0 +driverback.uz_m = 200000. +driverback.ux_th = 2. +driverback.uy_th = 2. +driverback.uz_th = 20000. +driverback.do_backward_propagation = true + +plasma_e.charge = -q_e +plasma_e.mass = m_e +plasma_e.injection_style = "NUniformPerCell" +plasma_e.zmin = -100.e-6 # 0.e-6 +plasma_e.zmax = 0.2 +plasma_e.xmin = -70.e-6 +plasma_e.xmax = 70.e-6 +plasma_e.ymin = -70.e-6 +plasma_e.ymax = 70.e-6 +# plasma_e.profile = constant +# plasma_e.density = 1.e23 +plasma_e.profile = parse_density_function +plasma_e.density_function(x,y,z) = "(zlramp)*dens" +plasma_e.num_particles_per_cell_each_dim = 1 1 1 +plasma_e.momentum_distribution_type = "at_rest" +plasma_e.do_continuous_injection = 1 + +plasma_p.charge = q_e +plasma_p.mass = m_p +plasma_p.injection_style = "NUniformPerCell" +plasma_p.zmin = -100.e-6 # 0.e-6 +plasma_p.zmax = 0.2 +# plasma_p.profile = "constant" +# plasma_p.density = 1.e23 +plasma_p.profile = parse_density_function +plasma_p.density_function(x,y,z) = "(zlramp)*dens" +plasma_p.xmin = -70.e-6 +plasma_p.xmax = 70.e-6 +plasma_p.ymin = -70.e-6 +plasma_p.ymax = 70.e-6 +plasma_p.num_particles_per_cell_each_dim = 1 1 1 +plasma_p.momentum_distribution_type = "at_rest" +plasma_p.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = .5e-6 +beam.y_rms = .5e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -100.e-6 +beam.npart = 1000 +beam.q_tot = -5.e-10 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 2000. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 200. +beam.zinject_plane = .8e-3 +beam.rigid_advance = true + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 10000 +diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_1d_plasma_acceleration_picmi.py b/Examples/Physics_applications/plasma_acceleration/inputs_test_1d_plasma_acceleration_picmi.py new file mode 100755 index 00000000000..4bde8cb1343 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_1d_plasma_acceleration_picmi.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# from warp import picmi + +constants = picmi.constants + +nz = 64 + +zmin = -200.0e-6 +zmax = +200.0e-6 + +moving_window_velocity = [constants.c] + +number_per_cell_each_dim = [10] + +max_steps = 1000 + +grid = picmi.Cartesian1DGrid( + number_of_cells=[nz], + lower_bound=[zmin], + upper_bound=[zmax], + lower_boundary_conditions=["dirichlet"], + upper_boundary_conditions=["dirichlet"], + lower_boundary_conditions_particles=["absorbing"], + upper_boundary_conditions_particles=["absorbing"], + moving_window_velocity=moving_window_velocity, + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=0.999) + +beam_distribution = picmi.UniformDistribution( + density=1.0e23, + lower_bound=[None, None, -150.0e-6], + upper_bound=[None, None, -100.0e-6], + directed_velocity=[0.0, 0.0, 1.0e9], +) + +plasma_distribution = picmi.UniformDistribution( + density=1.0e22, + lower_bound=[None, None, 0.0], + upper_bound=[None, None, None], + fill_in=True, +) + +beam = picmi.Species( + particle_type="electron", name="beam", initial_distribution=beam_distribution +) +plasma = picmi.Species( + particle_type="electron", name="plasma", initial_distribution=plasma_distribution +) + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + warpx_current_deposition_algo="esirkepov", + warpx_use_filter=0, +) + +sim.add_species( + beam, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) +sim.add_species( + plasma, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=max_steps, + data_list=["Ex", "Ey", "Ez", "Jx", "Jy", "Jz", "part_per_cell"], +) + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=max_steps, + species=[beam, plasma], + data_list=["ux", "uy", "uz", "weighting"], +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_boosted b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_boosted new file mode 100644 index 00000000000..5d65649353c --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_boosted @@ -0,0 +1,116 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 #2500 +amr.n_cell = 64 256 #64 640 +amr.max_grid_size = 128 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = -125.e-6 -149.e-6 +geometry.prob_hi = 125.e-6 1.e-6 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pml pml +boundary.field_hi = pml pml + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.verbose = 1 +warpx.use_filter = 1 +warpx.pml_ncell = 10 +warpx.cfl = .99 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1. # in units of the speed of light +my_constants.lramp = 8.e-3 +my_constants.dens = 1e+23 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +######### BOOSTED FRAME ######### +################################# +warpx.gamma_boost = 10.0 +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +particles.species_names = driver plasma_e plasma_p beam +particles.use_fdtd_nci_corr = 1 +particles.rigid_injected_species = driver beam + +driver.species_type = electron +driver.injection_style = "gaussian_beam" +driver.x_rms = 2.e-6 +driver.y_rms = 2.e-6 +driver.z_rms = 4.e-6 +driver.x_m = 0. +driver.y_m = 0. +driver.z_m = -20.e-6 +driver.npart = 1000 +driver.q_tot = -3.e-11 +driver.momentum_distribution_type = "gaussian" +driver.ux_m = 0.0 +driver.uy_m = 0.0 +driver.uz_m = 200000. +driver.ux_th = 2. +driver.uy_th = 2. +driver.uz_th = 20000. +driver.zinject_plane = 0. +driver.rigid_advance = true + +plasma_e.species_type = electron +plasma_e.injection_style = "NUniformPerCell" +plasma_e.zmin = 0.e-6 +plasma_e.zmax = 0.2 +plasma_e.xmin = -70.e-6 +plasma_e.xmax = 70.e-6 +plasma_e.profile = parse_density_function +plasma_e.density_function(x,y,z) = "(zlramp)*dens" +plasma_e.num_particles_per_cell_each_dim = 1 1 +plasma_e.momentum_distribution_type = "at_rest" +plasma_e.do_continuous_injection = 1 + +plasma_p.species_type = hydrogen +plasma_p.injection_style = "NUniformPerCell" +plasma_p.zmin = 0.e-6 +plasma_p.zmax = 0.2 +plasma_p.profile = parse_density_function +plasma_p.density_function(x,y,z) = "(zlramp)*dens" +plasma_p.xmin = -70.e-6 +plasma_p.xmax = 70.e-6 +plasma_p.num_particles_per_cell_each_dim = 1 1 +plasma_p.momentum_distribution_type = "at_rest" +plasma_p.do_continuous_injection = 1 + +beam.species_type = electron +beam.injection_style = "gaussian_beam" +beam.x_rms = .5e-6 +beam.y_rms = .5e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -105.e-6 +beam.npart = 1000 +beam.q_tot = -3.e-13 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 2000. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 200. +beam.zinject_plane = 0. +beam.rigid_advance = true + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 500 +diag1.diag_type = Full diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr new file mode 100644 index 00000000000..5a98fa590ee --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_2d diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr_momentum_conserving b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr_momentum_conserving new file mode 100644 index 00000000000..c21068325a0 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_2d_plasma_acceleration_mr_momentum_conserving @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.field_gathering = momentum-conserving diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted new file mode 100644 index 00000000000..62abe8e9df8 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +max_step = 5 diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted_hybrid b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted_hybrid new file mode 100644 index 00000000000..3c085b64b1b --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_boosted_hybrid @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +max_step = 25 +warpx.do_current_centering = 0 +warpx.grid_type = hybrid diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_mr_picmi.py b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_mr_picmi.py new file mode 100755 index 00000000000..9eb640ade95 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_mr_picmi.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# from warp import picmi + +constants = picmi.constants + +nx = 64 +ny = 64 +nz = 64 + +xmin = -200.0e-6 +xmax = +200.0e-6 +ymin = -200.0e-6 +ymax = +200.0e-6 +zmin = -200.0e-6 +zmax = +200.0e-6 + +moving_window_velocity = [0.0, 0.0, constants.c] + +number_per_cell_each_dim = [4, 4, 4] + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "open"], + upper_boundary_conditions=["periodic", "periodic", "open"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + moving_window_velocity=moving_window_velocity, + # refined_regions = [[1, [-25e-6, -25e-6, -200.e-6], [25e-6, 25e-6, 200.e-6]]], # as argument + warpx_max_grid_size=128, + warpx_blocking_factor=16, +) + +# --- As a separate function call (instead of refined_regions argument) +grid.add_refined_region( + level=1, lo=[-25e-6, -25e-6, -200.0e-6], hi=[25e-6, 25e-6, 200.0e-6] +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1, warpx_pml_ncell=10) + +beam_distribution = picmi.UniformDistribution( + density=1.0e23, + lower_bound=[-20.0e-6, -20.0e-6, -150.0e-6], + upper_bound=[+20.0e-6, +20.0e-6, -100.0e-6], + directed_velocity=[0.0, 0.0, 1.0e9], +) + +plasma_distribution = picmi.UniformDistribution( + density=1.0e22, + lower_bound=[-200.0e-6, -200.0e-6, 0.0], + upper_bound=[+200.0e-6, +200.0e-6, None], + fill_in=True, +) + +beam = picmi.Species( + particle_type="electron", name="beam", initial_distribution=beam_distribution +) +plasma = picmi.Species( + particle_type="electron", name="plasma", initial_distribution=plasma_distribution +) + +sim = picmi.Simulation( + solver=solver, + max_steps=2, + verbose=1, + warpx_current_deposition_algo="esirkepov", + warpx_use_filter=0, +) + +sim.add_species( + beam, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) +sim.add_species( + plasma, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=2, + data_list=["Ex", "Ey", "Ez", "Jx", "Jy", "Jz", "part_per_cell"], +) + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=2, + species=[beam, plasma], + data_list=["ux", "uy", "uz", "weighting"], +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI.mr') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_picmi.py b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_picmi.py new file mode 100755 index 00000000000..d5b99dbed97 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/inputs_test_3d_plasma_acceleration_picmi.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# from warp import picmi + +constants = picmi.constants + +nx = 64 +ny = 64 +nz = 64 + +xmin = -200.0e-6 +xmax = +200.0e-6 +ymin = -200.0e-6 +ymax = +200.0e-6 +zmin = -200.0e-6 +zmax = +200.0e-6 + +moving_window_velocity = [0.0, 0.0, constants.c] + +number_per_cell_each_dim = [2, 2, 1] + +max_steps = 10 + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "open"], + upper_boundary_conditions=["periodic", "periodic", "open"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + moving_window_velocity=moving_window_velocity, + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1) + +beam_distribution = picmi.UniformDistribution( + density=1.0e23, + lower_bound=[-20.0e-6, -20.0e-6, -150.0e-6], + upper_bound=[+20.0e-6, +20.0e-6, -100.0e-6], + directed_velocity=[0.0, 0.0, 1.0e9], +) + +plasma_distribution = picmi.UniformDistribution( + density=1.0e22, + lower_bound=[-200.0e-6, -200.0e-6, 0.0], + upper_bound=[+200.0e-6, +200.0e-6, None], + fill_in=True, +) + +beam = picmi.Species( + particle_type="electron", name="beam", initial_distribution=beam_distribution +) +plasma = picmi.Species( + particle_type="electron", name="plasma", initial_distribution=plasma_distribution +) + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + warpx_current_deposition_algo="esirkepov", + warpx_use_filter=0, +) + +sim.add_species( + beam, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) +sim.add_species( + plasma, + layout=picmi.GriddedLayout( + grid=grid, n_macroparticle_per_cell=number_per_cell_each_dim + ), +) + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=max_steps, + data_list=["Ex", "Ey", "Ez", "Jx", "Jy", "Jz", "part_per_cell"], +) + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=max_steps, + species=[beam, plasma], + data_list=["ux", "uy", "uz", "weighting"], +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Physics_applications/plasma_mirror/CMakeLists.txt b/Examples/Physics_applications/plasma_mirror/CMakeLists.txt new file mode 100644 index 00000000000..0d183ebbf4c --- /dev/null +++ b/Examples/Physics_applications/plasma_mirror/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_plasma_mirror # name + 2 # dims + 2 # nprocs + inputs_test_2d_plasma_mirror # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst index 8741db09699..56f1da7db46 100644 --- a/Examples/Physics_applications/plasma_mirror/README.rst +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -33,9 +33,9 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. tab-item:: Executable: Input File - .. literalinclude:: inputs_2d + .. literalinclude:: inputs_test_2d_plasma_mirror :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/inputs_2d``. + :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/inputs_test_2d_plasma_mirror``. Analyze diff --git a/Examples/Physics_applications/plasma_mirror/analysis_default_regression.py b/Examples/Physics_applications/plasma_mirror/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/plasma_mirror/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/plasma_mirror/inputs_2d b/Examples/Physics_applications/plasma_mirror/inputs_2d deleted file mode 100644 index 714af80affe..00000000000 --- a/Examples/Physics_applications/plasma_mirror/inputs_2d +++ /dev/null @@ -1,82 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 1000 -amr.n_cell = 1024 512 -amr.max_grid_size = 128 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -100.e-6 0. # physical domain -geometry.prob_hi = 100.e-6 100.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = pml pml -boundary.field_hi = pml pml - -################################# -############ NUMERICS ########### -################################# -my_constants.zc = 20.e-6 -my_constants.zp = 20.05545177444479562e-6 -my_constants.lgrad = .08e-6 -my_constants.nc = 1.74e27 -my_constants.zp2 = 24.e-6 -my_constants.zc2 = 24.05545177444479562e-6 -warpx.cfl = 1.0 -warpx.use_filter = 1 -algo.load_balance_intervals = 66 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 2 2 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = .01 -electrons.uz_th = .01 -electrons.zmin = "zc-lgrad*log(400)" -electrons.zmax = 25.47931e-6 -electrons.profile = parse_density_function -electrons.density_function(x,y,z) = "if(z`__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs_test_3d_thomson_parabola_spectrometer + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer``. + +Visualize +--------- + +This figure below shows the ion trajectories starting from the pinhole (black star), entering the E and B field regions (purple box), up to the detector (gray plane). +The colors represent the different species: protons in blue, C :sup:`+4` in red, and C :sup:`+6` in green. +The particles are accelerated and deflected through the TPS. + +.. figure:: https://gist.github.com/assets/17280419/3e45e5aa-d1fc-46e3-aa24-d9e0d6a74d1a + :alt: Ion trajectories through a synthetic TPS. + :width: 100% + +In our simulation, the virtual detector stores all the particle data once entering it (i.e. exiting the simulation box). +The figure below shows the ions colored according to their species (same as above) and shaded according to their initial energy. +The :math:`x` coordinate represents the electric deflection, while :math:`y` the magnetic deflection. + +.. figure:: https://gist.github.com/assets/17280419/4dd1adb7-b4ab-481d-bc24-8a7ca51471d9 + :alt: Synthetic TPS screen. + :width: 100% + +.. literalinclude:: analysis.py + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py``. diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py new file mode 100755 index 00000000000..6f61ed92c72 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, eV + +mpl.use("Agg") +mpl.rcParams.update({"font.size": 18}) + +MeV = 1e6 * eV + +# open the BoundaryScrapingDiagnostic that represents the detector +series = OpenPMDTimeSeries("./diags/screen/particles_at_zhi/") +# open the Full diagnostic at time zero +series0 = OpenPMDTimeSeries("./diags/diag0/") +# we use the data at time 0 to retrieve the initial energy +# of all the particles the boundary + +# timesteps and real times +it = series.iterations +time = series.t # s +N_iterations = len(it) + +# list of species names +species = series.avail_species +N_species = len(species) + +fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 8), dpi=300) + +# some stuff for plotting +vmin = 0 +vmax = 50 +cmap = ["Reds", "Greens", "Blues"] + +# loop through the species +for s in range(N_species): + print(species[s]) + + # arrays of positions and energies + X, Y, E = [], [], [] + for i in range(N_iterations): + # get particles at detector location + x, y, z, ids = series.get_particle( + ["x", "y", "z", "id"], iteration=it[i], species=species[s], plot=False + ) + # get particles at initialization + uz0, ids0, m = series0.get_particle( + ["uz", "id", "mass"], + iteration=series0.iterations[0], + species=species[s], + plot=False, + ) + + indeces = np.where(np.in1d(ids0, ids))[0] + + E = np.append(E, 0.5 * m[indeces] * (uz0[indeces] * c) ** 2 / MeV) + X = np.append(X, x) + Y = np.append(Y, y) + print(np.min(E), np.max(E)) + + # sort particles according to energy for nicer plot + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + +# dummy plot just to have a neutral colorbar +im = ax.scatter(np.nan, np.nan, c=np.nan, cmap="Greys_r", vmin=vmin, vmax=vmax) +plt.colorbar(im, label="E [MeV]") +ax.set_xlabel("x [m]") +ax.set_ylabel("y [m]") + +plt.tight_layout() +fig.savefig("detect.png", dpi=300) +plt.close() diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_regression.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer new file mode 100644 index 00000000000..04e238da86d --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer @@ -0,0 +1,192 @@ +############## +#### CONSTANTS +############## +my_constants.MeV = 1e6*q_e + +# distance between pinhole and electric field +my_constants.d1 = 0.1 # m +# length of the electric field region +my_constants.d2 = 0.19 # m +# length of the magnetic field region +my_constants.d3 = 0.12 # m +# distance between the magnetic field and the screen +my_constants.d4 = 0.2 # m + +# constant fields in the TPS +my_constants.E0 = 1e5 # V/m +my_constants.B0 = 0.872 # T + +# transverse domain +my_constants.xmin = -0.4 # m +my_constants.xmax = 0.4 # m +my_constants.ymin = -0.4 # m +my_constants.ymax = 0.4 # m + +# longitudinal domain +my_constants.zmin= -1e-3 # m +my_constants.zmax = d1+d2+d3+d4 + +# each macroparticle corresponds to 1 real particle +my_constants.N_real_particles = 1e3 +my_constants.N_macro_particles = 1e3 + +# maximum energy of the different species +# we assume that all the species have a +# uniform energy distribution in [0.5*Emax,Emax] +my_constants.Emax_hydrogen1_1 = 40*MeV +my_constants.Emax_carbon12_6 = 20*MeV +my_constants.Emax_carbon12_4 = 20*MeV + +# velocity of a very slow particle +# used to estimate the simulation time +my_constants.vz = sqrt(2*1*MeV/(12*m_p)) +my_constants.max_steps = 400 +my_constants.max_time = (-zmin+d1+d2+d3+d4) / vz +my_constants.dt = max_time / max_steps + +############# +#### NUMERICS +############# +algo.particle_shape = 1 +algo.maxwell_solver = none +algo.particle_pusher = boris +amr.max_level = 0 +warpx.verbose = 1 + +######## +#### BOX +######## +amr.n_cell = 8 8 8 +geometry.dims = 3 +geometry.prob_hi = xmax ymax zmax +geometry.prob_lo = xmin ymin zmin + +######### +#### TIME +######### +stop_time = max_time +warpx.const_dt = dt + +############# +#### BOUNDARY +############# +boundary.particle_hi = absorbing absorbing absorbing +boundary.particle_lo = absorbing absorbing absorbing + +############## +#### PARTICLES +############## +particles.species_names = hydrogen1_1 carbon12_6 carbon12_4 + +hydrogen1_1.charge = q_e +hydrogen1_1.initialize_self_fields = 0 +hydrogen1_1.injection_style = gaussian_beam +hydrogen1_1.mass = m_p +hydrogen1_1.momentum_distribution_type = uniform +hydrogen1_1.npart = N_macro_particles +hydrogen1_1.q_tot = N_real_particles*q_e +hydrogen1_1.ux_min = 0 +hydrogen1_1.uy_min = 0 +hydrogen1_1.uz_min = sqrt(Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.ux_max = 0 +hydrogen1_1.uy_max = 0 +hydrogen1_1.uz_max = sqrt(2*Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.x_m = 0 +hydrogen1_1.x_rms = 0 +hydrogen1_1.y_m = 0 +hydrogen1_1.y_rms = 0 +hydrogen1_1.z_m = 0 +hydrogen1_1.z_rms = 0 +hydrogen1_1.do_not_gather = 1 +hydrogen1_1.do_not_deposit = 1 + +# carbon12_6 means carbon ions with 12 nucleons, of which 6 protons +carbon12_6.charge = 6*q_e +carbon12_6.initialize_self_fields = 0 +carbon12_6.injection_style = gaussian_beam +carbon12_6.mass = 12*m_p +carbon12_6.momentum_distribution_type = uniform +carbon12_6.npart = N_macro_particles +carbon12_6.q_tot = N_real_particles*6*q_e +carbon12_6.ux_min = 0 +carbon12_6.uy_min = 0 +carbon12_6.uz_min = sqrt(Emax_carbon12_6/(12*m_p))/clight +carbon12_6.ux_max = 0 +carbon12_6.uy_max = 0 +carbon12_6.uz_max = sqrt(2*Emax_carbon12_6/(12*m_p))/clight +carbon12_6.x_m = 0 +carbon12_6.x_rms = 0 +carbon12_6.y_m = 0 +carbon12_6.y_rms = 0 +carbon12_6.z_m = 0 +carbon12_6.z_rms = 0 +carbon12_6.do_not_gather = 1 +carbon12_6.do_not_deposit = 1 + +carbon12_4.charge = 4*q_e +carbon12_4.initialize_self_fields = 0 +carbon12_4.injection_style = gaussian_beam +carbon12_4.mass = 12*m_p +carbon12_4.momentum_distribution_type = uniform +carbon12_4.npart = N_macro_particles +carbon12_4.q_tot = N_real_particles*4*q_e +carbon12_4.ux_min = 0 +carbon12_4.uy_min = 0 +carbon12_4.uz_min = sqrt(Emax_carbon12_4/(12*m_p))/clight +carbon12_4.ux_max = 0 +carbon12_4.uy_max = 0 +carbon12_4.uz_max = sqrt(2*Emax_carbon12_4/(12*m_p))/clight +carbon12_4.x_m = 0 +carbon12_4.x_rms = 0 +carbon12_4.y_m = 0 +carbon12_4.y_rms = 0 +carbon12_4.z_m = 0 +carbon12_4.z_rms = 0 +carbon12_4.do_not_gather = 1 +carbon12_4.do_not_deposit = 1 + +########### +#### FIELDS +########### +particles.E_ext_particle_init_style = parse_E_ext_particle_function +particles.Ex_external_particle_function(x,y,z,t) = "E0*(z>d1)*(z<(d1+d2))" +particles.Ey_external_particle_function(x,y,z,t) = 0 +particles.Ez_external_particle_function(x,y,z,t) = 0 + +particles.B_ext_particle_init_style = parse_B_ext_particle_function +particles.Bx_external_particle_function(x,y,z,t) = "B0*(z>d1+d2)*(z<(d1+d2+d3))" +particles.By_external_particle_function(x,y,z,t) = 0 +particles.Bz_external_particle_function(x,y,z,t) = 0 + +################ +#### DIAGNOSTICS +################ +diagnostics.diags_names = diag0 screen diag1 + +diag0.diag_type = Full +diag0.fields_to_plot = none +diag0.format = openpmd +diag0.intervals = 0:0 +diag0.write_species = 1 +diag0.species = hydrogen1_1 carbon12_6 carbon12_4 +diag0.dump_last_timestep = 0 + +# diagnostic that collects the particles at the detector's position, +# i.e. when a particle exits the domain from z_max = zhi +# we store it in the screen diagnostic +# we are assuming that most particles will exit the domain at z_max +# which requires a large enough transverse box +screen.diag_type = BoundaryScraping +screen.format = openpmd +screen.intervals = 1 +hydrogen1_1.save_particles_at_zhi = 1 +carbon12_6.save_particles_at_zhi = 1 +carbon12_4.save_particles_at_zhi = 1 + +diag1.diag_type = Full +diag1.fields_to_plot = rho_hydrogen1_1 rho_carbon12_6 rho_carbon12_4 +diag1.format = openpmd +diag1.intervals = 50:50 +diag1.write_species = 1 +diag1.species = hydrogen1_1 carbon12_6 carbon12_4 +diag1.dump_last_timestep = 0 diff --git a/Examples/Physics_applications/uniform_plasma/CMakeLists.txt b/Examples/Physics_applications/uniform_plasma/CMakeLists.txt new file mode 100644 index 00000000000..6d0f37ab726 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_uniform_plasma # name + 2 # dims + 2 # nprocs + inputs_test_2d_uniform_plasma # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_uniform_plasma # name + 3 # dims + 2 # nprocs + inputs_test_3d_uniform_plasma # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_uniform_plasma_restart # name + 3 # dims + 2 # nprocs + inputs_test_3d_uniform_plasma_restart # inputs + "analysis_default_restart.py diags/diag1000010" # analysis + "analysis_default_regression.py --path diags/diag1000010 --rtol 1e-12" # checksum + test_3d_uniform_plasma # dependency +) diff --git a/Examples/Physics_applications/uniform_plasma/README.rst b/Examples/Physics_applications/uniform_plasma/README.rst index 50d132712c6..04a0fb4555c 100644 --- a/Examples/Physics_applications/uniform_plasma/README.rst +++ b/Examples/Physics_applications/uniform_plasma/README.rst @@ -21,15 +21,15 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. note:: - TODO: This input file should be created following the ``inputs_3d`` file. + TODO: This input file should be created following the ``inputs_test_3d_uniform_plasma`` file. .. tab-item:: Executable: Input File - This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_test_3d_uniform_plasma`` - .. literalinclude:: inputs_3d + .. literalinclude:: inputs_test_3d_uniform_plasma :language: ini - :caption: You can copy this file from ``usage/examples/lwfa/inputs_3d``. + :caption: You can copy this file from ``usage/examples/lwfa/inputs_test_3d_uniform_plasma``. .. tab-item:: 2D @@ -39,15 +39,15 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. note:: - TODO: This input file should be created following the ``inputs_2d`` file. + TODO: This input file should be created following the ``inputs_test_2d_uniform_plasma`` file. .. tab-item:: Executable: Input File - This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_test_2d_uniform_plasma`` - .. literalinclude:: inputs_2d + .. literalinclude:: inputs_test_2d_uniform_plasma :language: ini - :caption: You can copy this file from ``usage/examples/lwfa/inputs_2d``. + :caption: You can copy this file from ``usage/examples/lwfa/inputs_test_2d_uniform_plasma``. Analyze ------- diff --git a/Examples/Physics_applications/uniform_plasma/analysis_default_regression.py b/Examples/Physics_applications/uniform_plasma/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/uniform_plasma/analysis_default_restart.py b/Examples/Physics_applications/uniform_plasma/analysis_default_restart.py new file mode 120000 index 00000000000..0459986eebc --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/analysis_default_restart.py @@ -0,0 +1 @@ +../../analysis_default_restart.py \ No newline at end of file diff --git a/Examples/Physics_applications/uniform_plasma/inputs_3d b/Examples/Physics_applications/uniform_plasma/inputs_base_3d similarity index 100% rename from Examples/Physics_applications/uniform_plasma/inputs_3d rename to Examples/Physics_applications/uniform_plasma/inputs_base_3d diff --git a/Examples/Physics_applications/uniform_plasma/inputs_2d b/Examples/Physics_applications/uniform_plasma/inputs_test_2d_uniform_plasma similarity index 100% rename from Examples/Physics_applications/uniform_plasma/inputs_2d rename to Examples/Physics_applications/uniform_plasma/inputs_test_2d_uniform_plasma diff --git a/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma b/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma new file mode 100644 index 00000000000..7665a846eef --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_3d diff --git a/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma_restart b/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma_restart new file mode 100644 index 00000000000..4711ece3843 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/inputs_test_3d_uniform_plasma_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_3d_uniform_plasma + +# test input parameters +amr.restart = "../test_3d_uniform_plasma/diags/chk000006" diff --git a/Examples/Tests/AcceleratorLattice/analysis.py b/Examples/Tests/AcceleratorLattice/analysis.py deleted file mode 100755 index 5f3de4543d6..00000000000 --- a/Examples/Tests/AcceleratorLattice/analysis.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 David Grote -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -""" -This script tests the quad lattice -The input file sets up a series of quadrupoles and propagates two particles through them. -One particle is in the X plane, the other the Y plane. -The final positions are compared to the analytic solutions. -The motion is slow enough that relativistic effects are ignored. -""" - -import os -import sys - -import numpy as np -import yt -from scipy.constants import c, e, m_e - -yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] -ds = yt.load( filename ) -ad = ds.all_data() - -gamma_boost = float(ds.parameters.get('warpx.gamma_boost', 1.)) -uz_boost = np.sqrt(gamma_boost*gamma_boost - 1.)*c - -# Fetch the final particle position -xx_sim = ad['electron', 'particle_position_x'].v[0] -zz_sim = ad['electron', 'particle_position_z'].v[0] -ux_sim = ad['electron', 'particle_momentum_x'].v[0]/m_e - -if gamma_boost > 1.: - # The simulation data is in the boosted frame. - # Transform the z position to the lab frame. - time = ds.current_time.value - zz_sim = gamma_boost*zz_sim + uz_boost*time; - -# Fetch the quadrupole lattice data -quad_starts = [] -quad_lengths = [] -quad_strengths_E = [] -z_location = 0. -def read_lattice(rootname, z_location): - lattice_elements = ds.parameters.get(f'{rootname}.elements').split() - for element in lattice_elements: - element_type = ds.parameters.get(f'{element}.type') - if element_type == 'drift': - length = float(ds.parameters.get(f'{element}.ds')) - z_location += length - elif element_type == 'quad': - length = float(ds.parameters.get(f'{element}.ds')) - quad_starts.append(z_location) - quad_lengths.append(length) - quad_strengths_E.append(float(ds.parameters.get(f'{element}.dEdx'))) - z_location += length - elif element_type == 'line': - z_location = read_lattice(element, z_location) - return z_location - -read_lattice('lattice', z_location) - -# Fetch the initial position of the particle -x0 = [float(x) for x in ds.parameters.get('electron.single_particle_pos').split()] -ux0 = [float(x)*c for x in ds.parameters.get('electron.single_particle_u').split()] - -xx = x0[0] -zz = x0[2] -ux = ux0[0] -uz = ux0[2] - -gamma = np.sqrt(uz**2/c**2 + 1.) -vz = uz/gamma - -def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): - """Use analytic solution of a particle with a transverse dependent field""" - kb0 = np.sqrt(e/(m_e*gamma*vz0**2)*abs(lens_strength)) - if lens_strength >= 0.: - x1 = x0*np.cos(kb0*lens_length) + (vx0/vz0)/kb0*np.sin(kb0*lens_length) - vx1 = vz0*(-kb0*x0*np.sin(kb0*lens_length) + (vx0/vz0)*np.cos(kb0*lens_length)) - else: - x1 = x0*np.cosh(kb0*lens_length) + (vx0/vz0)/kb0*np.sinh(kb0*lens_length) - vx1 = vz0*(+kb0*x0*np.sinh(kb0*lens_length) + (vx0/vz0)*np.cosh(kb0*lens_length)) - return x1, vx1 - -# Integrate the particle using the analytic solution -for i in range(len(quad_starts)): - z_lens = quad_starts[i] - vx = ux/gamma - dt = (z_lens - zz)/vz - xx = xx + dt*vx - xx, vx = applylens(xx, vx, vz, gamma, quad_lengths[i], quad_strengths_E[i]) - ux = gamma*vx - zz = z_lens + quad_lengths[i] - -dt = (zz_sim - zz)/vz -vx = ux/gamma -xx = xx + dt*vx - -# Compare the analytic to the simulated final values -print(f'Error in x position is {abs(np.abs((xx - xx_sim)/xx))}, which should be < 0.01') -print(f'Error in x velocity is {abs(np.abs((ux - ux_sim)/ux))}, which should be < 0.002') - -assert abs(np.abs((xx - xx_sim)/xx)) < 0.01, Exception('error in x particle position') -assert abs(np.abs((ux - ux_sim)/ux)) < 0.002, Exception('error in x particle velocity') - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_boosted_3d b/Examples/Tests/AcceleratorLattice/inputs_quad_boosted_3d deleted file mode 100644 index 668ec73d2dd..00000000000 --- a/Examples/Tests/AcceleratorLattice/inputs_quad_boosted_3d +++ /dev/null @@ -1,45 +0,0 @@ -max_step = 50 -amr.n_cell = 16 16 8 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -0.2 -0.2 -0.1 -geometry.prob_hi = +0.2 +0.2 +0.1 - -# Boundary condition -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec - -warpx.gamma_boost = 2. -warpx.boost_direction = z - -# Order of particle shape factors -algo.particle_shape = 1 - -particles.species_names = electron -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.05 0.0 0.0 -electron.single_particle_u = 0.0 0.0 2.0 # gamma*beta -electron.single_particle_weight = 1.0 - -lattice.elements = drift1 quad1 drift2 quad2 - -drift1.type = drift -drift1.ds = 0.2 - -quad1.type = quad -quad1.ds = 0.2 -quad1.dEdx = 1.e4 - -drift2.type = drift -drift2.ds = 0.6 - -quad2.type = quad -quad2.ds = 0.4 -quad2.dEdx = -1.e4 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 50 -diag1.diag_type = Full diff --git a/Examples/Tests/CMakeLists.txt b/Examples/Tests/CMakeLists.txt new file mode 100644 index 00000000000..b80e6158f49 --- /dev/null +++ b/Examples/Tests/CMakeLists.txt @@ -0,0 +1,82 @@ +# Add tests (alphabetical order) ############################################## +# + +add_subdirectory(accelerator_lattice) +add_subdirectory(boosted_diags) +add_subdirectory(boundaries) +add_subdirectory(btd_rz) +add_subdirectory(collider_relevant_diags) +add_subdirectory(collision) +add_subdirectory(diff_lumi_diag) +add_subdirectory(divb_cleaning) +add_subdirectory(dive_cleaning) +add_subdirectory(electrostatic_dirichlet_bc) +add_subdirectory(electrostatic_sphere) +add_subdirectory(electrostatic_sphere_eb) +add_subdirectory(embedded_boundary_cube) +add_subdirectory(embedded_boundary_diffraction) +add_subdirectory(embedded_boundary_em_particle_absorption) +add_subdirectory(embedded_boundary_python_api) +add_subdirectory(embedded_boundary_rotated_cube) +add_subdirectory(embedded_circle) +add_subdirectory(energy_conserving_thermal_plasma) +add_subdirectory(field_probe) +add_subdirectory(flux_injection) +add_subdirectory(gaussian_beam) +add_subdirectory(implicit) +add_subdirectory(initial_distribution) +add_subdirectory(initial_plasma_profile) +add_subdirectory(field_ionization) +add_subdirectory(ion_stopping) +add_subdirectory(langmuir) +add_subdirectory(langmuir_fluids) +add_subdirectory(larmor) +add_subdirectory(laser_injection) +add_subdirectory(laser_injection_from_file) +add_subdirectory(laser_on_fine) +add_subdirectory(load_external_field) +add_subdirectory(magnetostatic_eb) +add_subdirectory(maxwell_hybrid_qed) +add_subdirectory(nci_fdtd_stability) +add_subdirectory(nci_psatd_stability) +add_subdirectory(nodal_electrostatic) +add_subdirectory(nuclear_fusion) +add_subdirectory(ohm_solver_cylinder_compression) +add_subdirectory(ohm_solver_em_modes) +add_subdirectory(ohm_solver_ion_beam_instability) +add_subdirectory(ohm_solver_ion_Landau_damping) +add_subdirectory(ohm_solver_magnetic_reconnection) +add_subdirectory(open_bc_poisson_solver) +add_subdirectory(particle_boundary_interaction) +add_subdirectory(particle_boundary_process) +add_subdirectory(particle_boundary_scrape) +add_subdirectory(particle_data_python) +add_subdirectory(particle_fields_diags) +add_subdirectory(particle_pusher) +add_subdirectory(particle_thermal_boundary) +add_subdirectory(particles_in_pml) +add_subdirectory(pass_mpi_communicator) +add_subdirectory(pec) +add_subdirectory(photon_pusher) +add_subdirectory(plasma_lens) +add_subdirectory(pml) +add_subdirectory(point_of_contact_eb) +add_subdirectory(projection_divb_cleaner) +add_subdirectory(python_wrappers) +add_subdirectory(qed) +add_subdirectory(radiation_reaction) +add_subdirectory(reduced_diags) +add_subdirectory(relativistic_space_charge_initialization) +add_subdirectory(repelling_particles) +add_subdirectory(resampling) +add_subdirectory(restart) +add_subdirectory(restart_eb) +add_subdirectory(rigid_injection) +add_subdirectory(secondary_ion_emission) +add_subdirectory(scraping) +add_subdirectory(effective_potential_electrostatic) +add_subdirectory(silver_mueller) +add_subdirectory(single_particle) +add_subdirectory(space_charge_initialization) +add_subdirectory(subcycling) +add_subdirectory(vay_deposition) diff --git a/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py deleted file mode 100755 index 2f919124e13..00000000000 --- a/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Tests the python interface to the Implicit solver - -import numpy as np - -from pywarpx import picmi - -constants = picmi.constants - -########################## -# physics parameters -########################## - -n0 = 1.e30 # m^-3 -Ti = 100. # eV -Te = 100. # eV -wpe = constants.q_e*np.sqrt(n0/(constants.m_e*constants.ep0)) -de0 = constants.c/wpe -nppcz = 10 # number of particles/cell in z -dt = 0.1/wpe # s - -vthe = np.sqrt(Te*constants.q_e/constants.m_e) -vthi = np.sqrt(Ti*constants.q_e/constants.m_p) - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 20 -diagnostic_intervals = "::20" - -# --- Grid -nx = 40 -ny = 40 - -xmin = 0. -ymin = 0. -xmax = 10.0*de0 -ymax = 10.0*de0 - -number_per_cell_each_dim = [nppcz, nppcz] - -########################## -# physics components -########################## - -electrons_uniform_plasma = picmi.UniformDistribution(density = n0, - rms_velocity = [vthe, vthe, vthe]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=electrons_uniform_plasma) - -protons_uniform_plasma = picmi.UniformDistribution(density = n0, - rms_velocity = [vthi, vthi, vthi]) - -protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=protons_uniform_plasma) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['periodic', 'periodic'], - upper_boundary_conditions = ['periodic', 'periodic'], - warpx_max_grid_size = 8, - warpx_blocking_factor = 8) - -solver = picmi.ElectromagneticSolver(grid = grid, - method = 'Yee') - -GMRES_solver = picmi.GMRESLinearSolver(verbose_int = 2, - max_iterations = 1000, - relative_tolerance = 1.0e-8, - absolute_tolerance = 0.0) - -newton_solver = picmi.NewtonNonlinearSolver(verbose = True, - max_iterations = 20, - relative_tolerance = 1.0e-12, - absolute_tolerance = 0.0, - require_convergence = False, - linear_solver = GMRES_solver, - max_particle_iterations = 21, - particle_tolerance = 1.0e-12) - -evolve_scheme = picmi.ThetaImplicitEMEvolveScheme(theta = 0.5, - nonlinear_solver = newton_solver) - -########################## -# diagnostics -########################## - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Jx', 'Jy', 'Jz', 'rho', 'divE'], - write_dir = '.', - warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') - -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = diagnostic_intervals, - species = [electrons, protons], - data_list = ['weighting', 'position', 'momentum'], - write_dir = '.', - warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') - -particle_energy_diag = picmi.ReducedDiagnostic(diag_type = 'ParticleEnergy', - name = 'particle_energy', - period = 1) - -field_energy_diag = picmi.ReducedDiagnostic(diag_type = 'FieldEnergy', - name = 'field_energy', - period = 1) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation(solver = solver, - particle_shape = 2, - time_step_size = dt, - max_steps = max_steps, - verbose = 1, - warpx_evolve_scheme = evolve_scheme, - warpx_current_deposition_algo = 'villasenor', - warpx_particle_pusher_algo = 'boris', - warpx_serialize_initial_conditions = 1, - warpx_use_filter = 0) - -sim.add_species(electrons, - layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) -sim.add_species(protons, - layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) - -sim.add_diagnostic(field_diag1) -sim.add_diagnostic(part_diag1) -sim.add_diagnostic(particle_energy_diag) -sim.add_diagnostic(field_energy_diag) - -########################## -# simulation run -########################## - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -sim.write_input_file(file_name = 'inputs2d_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py deleted file mode 100755 index 0e20b925df5..00000000000 --- a/Examples/Tests/Implicit/analysis_1d.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2023 David Grote -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# This is a script that analyses the simulation results from -# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. -import os -import re -import sys - -import numpy as np - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) -particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) - -total_energy = field_energy[:,2] + particle_energy[:,2] - -delta_E = (total_energy - total_energy[0])/total_energy[0] -max_delta_E = np.abs(delta_E).max() - -if re.match('SemiImplicitPicard_1d', fn): - tolerance_rel = 2.5e-5 -elif re.match('ThetaImplicitPicard_1d', fn): - # This case should have near machine precision conservation of energy - tolerance_rel = 1.e-14 - -print(f"max change in energy: {max_delta_E}") -print(f"tolerance: {tolerance_rel}") - -assert( max_delta_E < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py b/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py deleted file mode 100755 index 85faab61fcc..00000000000 --- a/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2024 Justin Angus -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -# -# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. -# This simulates a 2D periodic plasma using the implicit solver -# with the Villasenor deposition using shape factor 2. -import os -import sys - -import numpy as np -import yt -from scipy.constants import e, epsilon_0 - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) -particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) - -total_energy = field_energy[:,2] + particle_energy[:,2] - -delta_E = (total_energy - total_energy[0])/total_energy[0] -max_delta_E = np.abs(delta_E).max() - -# This case should have near machine precision conservation of energy -tolerance_rel_energy = 2.e-14 -tolerance_rel_charge = 2.e-15 - -print(f"max change in energy: {max_delta_E}") -print(f"tolerance: {tolerance_rel_energy}") - -assert( max_delta_E < tolerance_rel_energy ) - -# check for machine precision conservation of charge density -n0 = 1.e30 - -pltdir = sys.argv[1] -ds = yt.load(pltdir) -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) - -divE = data['boxlib', 'divE'].value -rho = data['boxlib', 'rho'].value - -# compute local error in Gauss's law -drho = (rho - epsilon_0*divE)/e/n0 - -# compute RMS on in error on the grid -nX = drho.shape[0] -nZ = drho.shape[1] -drho2_avg = (drho**2).sum()/(nX*nZ) -drho_rms = np.sqrt(drho2_avg) - -print(f"rms error in charge conservation: {drho_rms}") -print(f"tolerance: {tolerance_rel_charge}") - -assert( drho_rms < tolerance_rel_charge ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d deleted file mode 100644 index 3e57689b723..00000000000 --- a/Examples/Tests/Implicit/inputs_1d +++ /dev/null @@ -1,89 +0,0 @@ -################################# -############ CONSTANTS ############# -################################# - -my_constants.n0 = 1.e30 # plasma densirty, m^-3 -my_constants.nz = 40 # number of grid cells -my_constants.Ti = 100. # ion temperature, eV -my_constants.Te = 100. # electron temperature, eV -my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s -my_constants.de0 = clight/wpe # skin depth, m -my_constants.nppcz = 100 # number of particles/cell in z -my_constants.dt = 0.1/wpe # time step size, s - -################################# -####### GENERAL PARAMETERS ###### -################################# - -max_step = 100 -amr.n_cell = nz -amr.max_level = 0 - -geometry.dims = 1 -geometry.prob_lo = 0.0 -geometry.prob_hi = 10.*de0 -boundary.field_lo = periodic -boundary.field_hi = periodic -boundary.particle_lo = periodic -boundary.particle_hi = periodic - -################################# -############ NUMERICS ########### -################################# - -warpx.verbose = 1 -warpx.const_dt = dt -algo.evolve_scheme = theta_implicit_em - -implicit_evolve.nonlinear_solver = "picard" - -picard.verbose = true -picard.max_iterations = 31 -picard.relative_tolerance = 0.0 -picard.absolute_tolerance = 0.0 -picard.require_convergence = false - -algo.current_deposition = esirkepov -algo.field_gathering = energy-conserving -algo.particle_shape = 2 -warpx.use_filter = 0 - -################################# -############ PLASMA ############# -################################# - -particles.species_names = electrons protons - -electrons.species_type = electron -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = nppcz -electrons.profile = constant -electrons.density = n0 -electrons.momentum_distribution_type = gaussian -electrons.ux_th = sqrt(Te*q_e/m_e)/clight -electrons.uy_th = sqrt(Te*q_e/m_e)/clight -electrons.uz_th = sqrt(Te*q_e/m_e)/clight - -protons.species_type = proton -protons.injection_style = "NUniformPerCell" -protons.num_particles_per_cell_each_dim = nppcz -protons.profile = constant -protons.density = n0 -protons.momentum_distribution_type = gaussian -protons.ux_th = sqrt(Ti*q_e/m_p)/clight -protons.uy_th = sqrt(Ti*q_e/m_p)/clight -protons.uz_th = sqrt(Ti*q_e/m_p)/clight - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 100 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = z w ux uy uz -diag1.protons.variables = z w ux uy uz - -warpx.reduced_diags_names = particle_energy field_energy -particle_energy.type = ParticleEnergy -particle_energy.intervals = 1 -field_energy.type = FieldEnergy -field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit deleted file mode 100644 index 07460e08be8..00000000000 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ /dev/null @@ -1,89 +0,0 @@ -################################# -############ CONSTANTS ############# -################################# - -my_constants.n0 = 1.e30 # plasma densirty, m^-3 -my_constants.nz = 40 # number of grid cells -my_constants.Ti = 100. # ion temperature, eV -my_constants.Te = 100. # electron temperature, eV -my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s -my_constants.de0 = clight/wpe # skin depth, m -my_constants.nppcz = 100 # number of particles/cell in z -my_constants.dt = 0.1/wpe # time step size, s - -################################# -####### GENERAL PARAMETERS ###### -################################# - -max_step = 100 -amr.n_cell = nz -amr.max_level = 0 - -geometry.dims = 1 -geometry.prob_lo = 0.0 -geometry.prob_hi = 10.*de0 -boundary.field_lo = periodic -boundary.field_hi = periodic -boundary.particle_lo = periodic -boundary.particle_hi = periodic - -################################# -############ NUMERICS ########### -################################# - -warpx.verbose = 1 -warpx.const_dt = dt -algo.evolve_scheme = semi_implicit_em - -implicit_evolve.nonlinear_solver = "picard" - -picard.verbose = true -picard.max_iterations = 5 -picard.relative_tolerance = 0.0 -picard.absolute_tolerance = 0.0 -picard.require_convergence = false - -algo.current_deposition = esirkepov -algo.field_gathering = energy-conserving -algo.particle_shape = 2 -warpx.use_filter = 0 - -################################# -############ PLASMA ############# -################################# - -particles.species_names = electrons protons - -electrons.species_type = electron -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = nppcz -electrons.profile = constant -electrons.density = n0 -electrons.momentum_distribution_type = gaussian -electrons.ux_th = sqrt(Te*q_e/m_e)/clight -electrons.uy_th = sqrt(Te*q_e/m_e)/clight -electrons.uz_th = sqrt(Te*q_e/m_e)/clight - -protons.species_type = proton -protons.injection_style = "NUniformPerCell" -protons.num_particles_per_cell_each_dim = nppcz -protons.profile = constant -protons.density = n0 -protons.momentum_distribution_type = gaussian -protons.ux_th = sqrt(Ti*q_e/m_p)/clight -protons.uy_th = sqrt(Ti*q_e/m_p)/clight -protons.uz_th = sqrt(Ti*q_e/m_p)/clight - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 100 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = z w ux uy uz -diag1.protons.variables = z w ux uy uz - -warpx.reduced_diags_names = particle_energy field_energy -particle_energy.type = ParticleEnergy -particle_energy.intervals = 1 -field_energy.type = FieldEnergy -field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_vandb_jfnk_2d b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d deleted file mode 100644 index 393a9d90330..00000000000 --- a/Examples/Tests/Implicit/inputs_vandb_jfnk_2d +++ /dev/null @@ -1,114 +0,0 @@ -################################# -########## CONSTANTS ############ -################################# - -my_constants.n0 = 1.e30 # m^-3 -my_constants.Ti = 100. # eV -my_constants.Te = 100. # eV -my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) -my_constants.de0 = clight/wpe -my_constants.nppcz = 10 # number of particles/cell in z -my_constants.dt = 0.1/wpe # s - -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 20 -amr.n_cell = 40 40 -amr.max_grid_size = 8 -amr.blocking_factor = 8 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = 0.0 0.0 # physical domain -geometry.prob_hi = 10.0*de0 10.0*de0 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -warpx.serialize_initial_conditions = 1 -warpx.verbose = 1 -warpx.const_dt = dt -#warpx.cfl = 0.5656 -warpx.use_filter = 0 - -algo.maxwell_solver = Yee -algo.evolve_scheme = "theta_implicit_em" -#algo.evolve_scheme = "semi_implicit_em" - -implicit_evolve.theta = 0.5 -implicit_evolve.max_particle_iterations = 21 -implicit_evolve.particle_tolerance = 1.0e-12 - -#implicit_evolve.nonlinear_solver = "picard" -#picard.verbose = true -#picard.max_iterations = 25 -#picard.relative_tolerance = 0.0 #1.0e-12 -#picard.absolute_tolerance = 0.0 #1.0e-24 -#picard.require_convergence = false - -implicit_evolve.nonlinear_solver = "newton" -newton.verbose = true -newton.max_iterations = 20 -newton.relative_tolerance = 1.0e-12 -newton.absolute_tolerance = 0.0 -newton.require_convergence = false - -gmres.verbose_int = 2 -gmres.max_iterations = 1000 -gmres.relative_tolerance = 1.0e-8 -gmres.absolute_tolerance = 0.0 - -algo.particle_pusher = "boris" -#algo.particle_pusher = "higuera" - -algo.particle_shape = 2 -#algo.current_deposition = "direct" -#algo.current_deposition = "esirkepov" -algo.current_deposition = "villasenor" - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons protons - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = nppcz nppcz -electrons.profile = constant -electrons.density = 1.e30 # number per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = sqrt(Te*q_e/m_e)/clight -electrons.uy_th = sqrt(Te*q_e/m_e)/clight -electrons.uz_th = sqrt(Te*q_e/m_e)/clight - -protons.charge = q_e -protons.mass = m_p -protons.injection_style = "NUniformPerCell" -protons.num_particles_per_cell_each_dim = nppcz nppcz -protons.profile = constant -protons.density = 1.e30 # number per m^3 -protons.momentum_distribution_type = "gaussian" -protons.ux_th = sqrt(Ti*q_e/m_p)/clight -protons.uy_th = sqrt(Ti*q_e/m_p)/clight -protons.uz_th = sqrt(Ti*q_e/m_p)/clight - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 20 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = x z w ux uy uz -diag1.protons.variables = x z w ux uy uz - -warpx.reduced_diags_names = particle_energy field_energy -particle_energy.type = ParticleEnergy -particle_energy.intervals = 1 -field_energy.type = FieldEnergy -field_energy.intervals = 1 diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py deleted file mode 100644 index d128d9c10e0..00000000000 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file for loading initial field from openPMD file. - -from pywarpx import picmi - -constants = picmi.constants - -################################# -####### GENERAL PARAMETERS ###### -################################# - -max_steps = 300 - -max_grid_size = 40 -nx = max_grid_size -ny = max_grid_size -nz = max_grid_size - -xmin = -1 -xmax = 1 -ymin = xmin -ymax = xmax -zmin = 0 -zmax = 5 - -number_per_cell = 200 - -################################# -############ NUMERICS ########### -################################# - -verbose = 1 -dt = 4.4e-7 -use_filter = 0 - -# Order of particle shape factors -particle_shape = 1 - -################################# -############ PLASMA ############# -################################# - -ion_dist = picmi.ParticleListDistribution( - x=0.0, - y=0.2, - z=2.5, - ux=9.5e-05*constants.c, - uy=0.0*constants.c, - uz=1.34e-4*constants.c, - weight=1.0 - ) - -ions = picmi.Species( - particle_type='H', - name='proton', charge='q_e',mass="m_p", - warpx_do_not_deposit=1, - initial_distribution=ion_dist - ) - -################################# -######## INITIAL FIELD ########## -################################# - -initial_field = picmi.LoadInitialField( - read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", - load_E=False - ) - -################################# -###### GRID AND SOLVER ########## -################################# - -grid = picmi.Cartesian3DGrid( - number_of_cells=[nx, ny, nz], - warpx_max_grid_size=max_grid_size, - lower_bound=[xmin, ymin, zmin], - upper_bound=[xmax, ymax, zmax], - lower_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], - upper_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], - lower_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'], - upper_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'] - ) -solver = picmi.ElectrostaticSolver(grid=grid) - -################################# -######### DIAGNOSTICS ########### -################################# - -particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=300, - species=[ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], - write_dir='.', - warpx_file_prefix='Python_LoadExternalGridField3D_plt' - ) -field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=grid, - period=300, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir='.', - warpx_file_prefix='Python_LoadExternalGridField3D_plt' - ) - -################################# -####### SIMULATION SETUP ######## -################################# - -sim = picmi.Simulation( - solver=solver, - max_steps=max_steps, - verbose=verbose, - warpx_serialize_initial_conditions=False, - warpx_grid_type='collocated', - warpx_do_dynamic_scheduling=False, - warpx_use_filter=use_filter, - time_step_size=dt, - particle_shape=particle_shape - ) - -sim.add_applied_field(initial_field) - -sim.add_species( - ions, - layout=picmi.PseudoRandomLayout( - n_macroparticles_per_cell=number_per_cell, grid=grid - ) - ) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(particle_diag) - -################################# -##### SIMULATION EXECUTION ###### -################################# - -#sim.write_input_file('PICMI_inputs_3d') -sim.step(max_steps) diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py deleted file mode 100644 index 7bf7c5a084c..00000000000 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file for loading initial field from openPMD file. - -from pywarpx import picmi - -constants = picmi.constants - -################################# -####### GENERAL PARAMETERS ###### -################################# - -max_steps = 300 - -max_grid_size = 40 -nx = max_grid_size -ny = max_grid_size -nz = max_grid_size - -xmin = -1 -xmax = 1 -ymin = xmin -ymax = xmax -zmin = 0 -zmax = 5 - -number_per_cell = 200 - -################################# -############ NUMERICS ########### -################################# - -verbose = 1 -dt = 4.4e-7 -use_filter = 0 - -# Order of particle shape factors -particle_shape = 1 - -################################# -############ PLASMA ############# -################################# - -ion_dist = picmi.ParticleListDistribution( - x=0.0, - y=0.2, - z=2.5, - ux=9.5e-05*constants.c, - uy=0.0*constants.c, - uz=1.34e-4*constants.c, - weight=1.0 - ) - -ions = picmi.Species( - particle_type='H', - name='proton', charge='q_e',mass="m_p", - warpx_do_not_deposit=1, - initial_distribution=ion_dist - ) - -################################# -######## INITIAL FIELD ########## -################################# - -applied_field = picmi.LoadAppliedField( - read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", - load_E=False - ) - -################################# -###### GRID AND SOLVER ########## -################################# - -grid = picmi.Cartesian3DGrid( - number_of_cells=[nx, ny, nz], - warpx_max_grid_size=max_grid_size, - lower_bound=[xmin, ymin, zmin], - upper_bound=[xmax, ymax, zmax], - lower_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], - upper_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], - lower_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'], - upper_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'] - ) -solver = picmi.ElectrostaticSolver(grid=grid) - -################################# -######### DIAGNOSTICS ########### -################################# - -particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=300, - species=[ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], - write_dir='.', - warpx_file_prefix='Python_LoadExternalParticleField3D_plt' - ) -field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=grid, - period=300, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir='.', - warpx_file_prefix='Python_LoadExternalParticleField3D_plt' - ) - -################################# -####### SIMULATION SETUP ######## -################################# - -sim = picmi.Simulation( - solver=solver, - max_steps=max_steps, - verbose=verbose, - warpx_serialize_initial_conditions=False, - warpx_grid_type='collocated', - warpx_do_dynamic_scheduling=False, - warpx_use_filter=use_filter, - time_step_size=dt, - particle_shape=particle_shape - ) - -sim.add_applied_field(applied_field) - -sim.add_species( - ions, - layout=picmi.PseudoRandomLayout( - n_macroparticles_per_cell=number_per_cell, grid=grid - ) - ) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(particle_diag) - -################################# -##### SIMULATION EXECUTION ###### -################################# - -#sim.write_input_file('PICMI_inputs_3d') -sim.step(max_steps) diff --git a/Examples/Tests/LoadExternalField/analysis_3d.py b/Examples/Tests/LoadExternalField/analysis_3d.py deleted file mode 100755 index 0539448f873..00000000000 --- a/Examples/Tests/LoadExternalField/analysis_3d.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 Yinjian Zhao -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This test tests the external field loading feature. -# A magnetic mirror field is loaded, and a single particle -# in the mirror will be reflected by the magnetic mirror effect. -# At the end of the simulation, the position of the particle -# is compared with known correct results. - -# Possible errors: 3.915833656984999e-9 -# tolerance: 1.0e-8 -# Possible running time: 2.756646401 s - -import os -import sys - -import numpy as np -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -tolerance = 1.0e-8 -x0 = 0.12238072 -y0 = 0.00965394 -z0 = 4.31754477 - -filename = sys.argv[1] - -ds = yt.load( filename ) -ad = ds.all_data() -x = ad['proton','particle_position_x'].to_ndarray() -y = ad['proton','particle_position_y'].to_ndarray() -z = ad['proton','particle_position_z'].to_ndarray() - -error = np.min(np.sqrt((x-x0)**2+(y-y0)**2+(z-z0)**2)) - -print('error = ', error) -print('tolerance = ', tolerance) -assert(error < tolerance) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/LoadExternalField/analysis_rz.py b/Examples/Tests/LoadExternalField/analysis_rz.py deleted file mode 100755 index fd82fbbdac6..00000000000 --- a/Examples/Tests/LoadExternalField/analysis_rz.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 Yinjian Zhao -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This test tests the external field loading feature. -# A magnetic mirror field is loaded, and a single particle -# in the mirror will be reflected by the magnetic mirror effect. -# At the end of the simulation, the position of the particle -# is compared with known correct results. - -# Possible errors: 6.235230443866285e-9 -# tolerance: 1.0e-8 -# Possible running time: 0.327827743 s - -import os -import sys - -import numpy as np -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -tolerance = 1.0e-8 -r0 = 0.12402005 -z0 = 4.3632492 - -filename = sys.argv[1] -ds = yt.load( filename ) -ad = ds.all_data() -r = ad['proton','particle_position_x'].to_ndarray() -z = ad['proton','particle_position_y'].to_ndarray() - -error = np.min(np.sqrt((r-r0)**2+(z-z0)**2)) - -print('error = ', error) -print('tolerance = ', tolerance) -assert(error < tolerance) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/LoadExternalField/inputs_rz_grid_fields b/Examples/Tests/LoadExternalField/inputs_rz_grid_fields deleted file mode 100644 index 2e22ca299ea..00000000000 --- a/Examples/Tests/LoadExternalField/inputs_rz_grid_fields +++ /dev/null @@ -1,65 +0,0 @@ -warpx.serialize_initial_conditions = 0 -warpx.do_dynamic_scheduling = 0 -particles.do_tiling = 0 - -warpx.B_ext_grid_init_style = "read_from_file" -warpx.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" - -warpx.grid_type = collocated -warpx.do_electrostatic = labframe - -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 300 -amr.n_cell = 40 40 -warpx.numprocs = 1 1 -amr.max_level = 0 -geometry.dims = RZ - -geometry.prob_lo = 0.0 0.0 -geometry.prob_hi = 1.0 5.0 - -################################# -###### Boundary Condition ####### -################################# -boundary.field_lo = none pec -boundary.field_hi = pec pec -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 - -################################# -############ NUMERICS ########### -################################# -warpx.serialize_initial_conditions = 1 -warpx.verbose = 1 -warpx.const_dt = 4.40917904849092e-7 -warpx.use_filter = 0 - -# Order of particle shape factors -algo.particle_shape = 1 - -################################# -############ PLASMA ############# -################################# -particles.species_names = proton -proton.injection_style = "SingleParticle" -proton.single_particle_pos = 0.0 0.2 2.5 -proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 -proton.single_particle_weight = 1.0 -proton.do_not_deposit = 1 -proton.mass = m_p -proton.charge = q_e - -# Diagnostics -diagnostics.diags_names = diag1 chk -diag1.intervals = 300 -diag1.diag_type = Full - -chk.intervals = 150 -chk.diag_type = Full -chk.format = checkpoint diff --git a/Examples/Tests/LoadExternalField/inputs_rz_particle_fields b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields deleted file mode 100644 index b76d4cb7efc..00000000000 --- a/Examples/Tests/LoadExternalField/inputs_rz_particle_fields +++ /dev/null @@ -1,65 +0,0 @@ -warpx.serialize_initial_conditions = 0 -warpx.do_dynamic_scheduling = 0 -particles.do_tiling = 0 - -particles.B_ext_particle_init_style = "read_from_file" -particles.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" - -warpx.grid_type = collocated -warpx.do_electrostatic = labframe - -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 300 -amr.n_cell = 40 40 -warpx.numprocs = 1 1 -amr.max_level = 0 -geometry.dims = RZ - -geometry.prob_lo = 0.0 0.0 -geometry.prob_hi = 1.0 5.0 - -################################# -###### Boundary Condition ####### -################################# -boundary.field_lo = none pec -boundary.field_hi = pec pec -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 - -################################# -############ NUMERICS ########### -################################# -warpx.serialize_initial_conditions = 1 -warpx.verbose = 1 -warpx.const_dt = 4.40917904849092e-7 -warpx.use_filter = 0 - -# Order of particle shape factors -algo.particle_shape = 1 - -################################# -############ PLASMA ############# -################################# -particles.species_names = proton -proton.injection_style = "SingleParticle" -proton.single_particle_pos = 0.0 0.2 2.5 -proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 -proton.single_particle_weight = 1.0 -proton.do_not_deposit = 1 -proton.mass = m_p -proton.charge = q_e - -# Diagnostics -diagnostics.diags_names = diag1 chk -diag1.intervals = 300 -diag1.diag_type = Full - -chk.intervals = 150 -chk.diag_type = Full -chk.format = checkpoint diff --git a/Examples/Tests/accelerator_lattice/CMakeLists.txt b/Examples/Tests/accelerator_lattice/CMakeLists.txt new file mode 100644 index 00000000000..accccde34d0 --- /dev/null +++ b/Examples/Tests/accelerator_lattice/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_hard_edged_quadrupoles # name + 3 # dims + 2 # nprocs + inputs_test_3d_hard_edged_quadrupoles # inputs + "analysis.py diags/diag1000050" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_hard_edged_quadrupoles_boosted # name + 3 # dims + 2 # nprocs + inputs_test_3d_hard_edged_quadrupoles_boosted # inputs + "analysis.py diags/diag1000050" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_hard_edged_quadrupoles_moving # name + 3 # dims + 2 # nprocs + inputs_test_3d_hard_edged_quadrupoles_moving # inputs + "analysis.py diags/diag1000050" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) diff --git a/Examples/Tests/accelerator_lattice/analysis.py b/Examples/Tests/accelerator_lattice/analysis.py new file mode 100755 index 00000000000..331c5322e03 --- /dev/null +++ b/Examples/Tests/accelerator_lattice/analysis.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +# Copyright 2022 David Grote +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +""" +This script tests the quad lattice +The input file sets up a series of quadrupoles and propagates two particles through them. +One particle is in the X plane, the other the Y plane. +The final positions are compared to the analytic solutions. +The motion is slow enough that relativistic effects are ignored. +""" + +import sys + +import numpy as np +import yt +from scipy.constants import c, e, m_e + +yt.funcs.mylog.setLevel(0) + +filename = sys.argv[1] +ds = yt.load(filename) +ad = ds.all_data() + +gamma_boost = float(ds.parameters.get("warpx.gamma_boost", 1.0)) +uz_boost = np.sqrt(gamma_boost * gamma_boost - 1.0) * c + +# Fetch the final particle position +xx_sim = ad["electron", "particle_position_x"].v[0] +zz_sim = ad["electron", "particle_position_z"].v[0] +ux_sim = ad["electron", "particle_momentum_x"].v[0] / m_e + +if gamma_boost > 1.0: + # The simulation data is in the boosted frame. + # Transform the z position to the lab frame. + time = ds.current_time.value + zz_sim = gamma_boost * zz_sim + uz_boost * time + +# Fetch the quadrupole lattice data +quad_starts = [] +quad_lengths = [] +quad_strengths_E = [] +z_location = 0.0 + + +def read_lattice(rootname, z_location): + lattice_elements = ds.parameters.get(f"{rootname}.elements").split() + for element in lattice_elements: + element_type = ds.parameters.get(f"{element}.type") + if element_type == "drift": + length = float(ds.parameters.get(f"{element}.ds")) + z_location += length + elif element_type == "quad": + length = float(ds.parameters.get(f"{element}.ds")) + quad_starts.append(z_location) + quad_lengths.append(length) + quad_strengths_E.append(float(ds.parameters.get(f"{element}.dEdx"))) + z_location += length + elif element_type == "line": + z_location = read_lattice(element, z_location) + return z_location + + +read_lattice("lattice", z_location) + +# Fetch the initial position of the particle +x0 = [float(x) for x in ds.parameters.get("electron.single_particle_pos").split()] +ux0 = [float(x) * c for x in ds.parameters.get("electron.single_particle_u").split()] + +xx = x0[0] +zz = x0[2] +ux = ux0[0] +uz = ux0[2] + +gamma = np.sqrt(uz**2 / c**2 + 1.0) +vz = uz / gamma + + +def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): + """Use analytic solution of a particle with a transverse dependent field""" + kb0 = np.sqrt(e / (m_e * gamma * vz0**2) * abs(lens_strength)) + if lens_strength >= 0.0: + x1 = x0 * np.cos(kb0 * lens_length) + (vx0 / vz0) / kb0 * np.sin( + kb0 * lens_length + ) + vx1 = vz0 * ( + -kb0 * x0 * np.sin(kb0 * lens_length) + + (vx0 / vz0) * np.cos(kb0 * lens_length) + ) + else: + x1 = x0 * np.cosh(kb0 * lens_length) + (vx0 / vz0) / kb0 * np.sinh( + kb0 * lens_length + ) + vx1 = vz0 * ( + +kb0 * x0 * np.sinh(kb0 * lens_length) + + (vx0 / vz0) * np.cosh(kb0 * lens_length) + ) + return x1, vx1 + + +# Integrate the particle using the analytic solution +for i in range(len(quad_starts)): + z_lens = quad_starts[i] + vx = ux / gamma + dt = (z_lens - zz) / vz + xx = xx + dt * vx + xx, vx = applylens(xx, vx, vz, gamma, quad_lengths[i], quad_strengths_E[i]) + ux = gamma * vx + zz = z_lens + quad_lengths[i] + +dt = (zz_sim - zz) / vz +vx = ux / gamma +xx = xx + dt * vx + +# Compare the analytic to the simulated final values +print( + f"Error in x position is {abs(np.abs((xx - xx_sim) / xx))}, which should be < 0.01" +) +print( + f"Error in x velocity is {abs(np.abs((ux - ux_sim) / ux))}, which should be < 0.002" +) + +assert abs(np.abs((xx - xx_sim) / xx)) < 0.01, Exception("error in x particle position") +assert abs(np.abs((ux - ux_sim) / ux)) < 0.002, Exception( + "error in x particle velocity" +) diff --git a/Examples/Tests/accelerator_lattice/analysis_default_regression.py b/Examples/Tests/accelerator_lattice/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/accelerator_lattice/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_3d b/Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles similarity index 100% rename from Examples/Tests/AcceleratorLattice/inputs_quad_3d rename to Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles diff --git a/Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles_boosted b/Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles_boosted new file mode 100644 index 00000000000..c056ff1fc66 --- /dev/null +++ b/Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles_boosted @@ -0,0 +1,45 @@ +max_step = 50 +amr.n_cell = 16 16 8 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.2 -0.2 -0.1866 +geometry.prob_hi = +0.2 +0.2 +0.1866 + +# Boundary condition +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec + +warpx.gamma_boost = 2. +warpx.boost_direction = z + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.05 0.0 0.0 +electron.single_particle_u = 0.0 0.0 2.0 # gamma*beta +electron.single_particle_weight = 1.0 + +lattice.elements = drift1 quad1 drift2 quad2 + +drift1.type = drift +drift1.ds = 0.2 + +quad1.type = quad +quad1.ds = 0.2 +quad1.dEdx = 1.e4 + +drift2.type = drift +drift2.ds = 0.6 + +quad2.type = quad +quad2.ds = 0.4 +quad2.dEdx = -1.e4 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 50 +diag1.diag_type = Full diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_moving_3d b/Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles_moving similarity index 100% rename from Examples/Tests/AcceleratorLattice/inputs_quad_moving_3d rename to Examples/Tests/accelerator_lattice/inputs_test_3d_hard_edged_quadrupoles_moving diff --git a/Examples/Tests/boosted_diags/CMakeLists.txt b/Examples/Tests/boosted_diags/CMakeLists.txt new file mode 100644 index 00000000000..b749d7153ea --- /dev/null +++ b/Examples/Tests/boosted_diags/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_laser_acceleration_btd # name + 3 # dims + 2 # nprocs + inputs_test_3d_laser_acceleration_btd # inputs + "analysis.py diags/diag1000003" # analysis + "analysis_default_regression.py --path diags/diag1000003" # checksum + OFF # dependency +) diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index 2b21184dc3d..3c26b343d78 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -7,16 +7,15 @@ # License: BSD-3-Clause-LBNL -''' +""" Analysis script of a WarpX simulation in a boosted frame. The simulation runs in a boosted frame, and the analysis is done in the lab frame, i.e., on the back-transformed diagnostics for the full 3D simulation and an x-z slice at y=y_center. The field-data, Ez, along z, at (x_center,y_center,:) is compared between the full back-transformed diagnostic and the reduced diagnostic (i.e., x-z slice) . -''' +""" -import os import sys import numpy as np @@ -26,9 +25,6 @@ yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - filename = sys.argv[1] # Tolerances to check consistency between legacy BTD and new BTD @@ -38,24 +34,20 @@ # Read data from new back-transformed diagnostics (plotfile) ds_plotfile = yt.load(filename) data = ds_plotfile.covering_grid( - level=0, - left_edge=ds_plotfile.domain_left_edge, - dims=ds_plotfile.domain_dimensions) -Ez_plotfile = data[('mesh', 'Ez')].to_ndarray() + level=0, left_edge=ds_plotfile.domain_left_edge, dims=ds_plotfile.domain_dimensions +) +Ez_plotfile = data[("mesh", "Ez")].to_ndarray() # Read data from new back-transformed diagnostics (openPMD) series = io.Series("./diags/diag2/openpmd_%T.h5", io.Access.read_only) ds_openpmd = series.iterations[3] -Ez_openpmd = ds_openpmd.meshes['E']['z'].load_chunk() +Ez_openpmd = ds_openpmd.meshes["E"]["z"].load_chunk() Ez_openpmd = Ez_openpmd.transpose() series.flush() # Compare arrays to check consistency between new BTD formats (plotfile and openPMD) -assert(np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol)) +assert np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol) # Check that particle random sub-selection has been applied -ts = OpenPMDTimeSeries('./diags/diag2/') -w, = ts.get_particle(['w'], species='beam', iteration=3) +ts = OpenPMDTimeSeries("./diags/diag2/") +(w,) = ts.get_particle(["w"], species="beam", iteration=3) assert (400 < len(w)) & (len(w) < 600) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/boosted_diags/analysis_default_regression.py b/Examples/Tests/boosted_diags/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/boosted_diags/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/boosted_diags/inputs_3d b/Examples/Tests/boosted_diags/inputs_test_3d_laser_acceleration_btd similarity index 100% rename from Examples/Tests/boosted_diags/inputs_3d rename to Examples/Tests/boosted_diags/inputs_test_3d_laser_acceleration_btd diff --git a/Examples/Tests/boundaries/CMakeLists.txt b/Examples/Tests/boundaries/CMakeLists.txt new file mode 100644 index 00000000000..00a53742cb9 --- /dev/null +++ b/Examples/Tests/boundaries/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_particle_boundaries # name + 3 # dims + 2 # nprocs + inputs_test_3d_particle_boundaries # inputs + "analysis.py diags/diag1000008" # analysis + "analysis_default_regression.py --path diags/diag1000008" # checksum + OFF # dependency +) diff --git a/Examples/Tests/boundaries/analysis.py b/Examples/Tests/boundaries/analysis.py index 9c108b16196..29de2bb37cb 100755 --- a/Examples/Tests/boundaries/analysis.py +++ b/Examples/Tests/boundaries/analysis.py @@ -14,7 +14,6 @@ and checks that they end up in the correct place (or are deleted). """ -import os import sys import numpy as np @@ -22,21 +21,19 @@ from scipy.constants import c, m_e yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # The min and max size of the box along the three axis. -dmin = -1. -dmax = +1. +dmin = -1.0 +dmax = +1.0 # Open plotfile specified in command line filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) ad = ds.all_data() time = ds.current_time.to_value() -filename0 = filename[:-5] + '00000' -ds0 = yt.load( filename0 ) +filename0 = filename[:-5] + "00000" +ds0 = yt.load(filename0) ad0 = ds0.all_data() # Read in the particle initial values and the current values. @@ -44,42 +41,44 @@ # differently in the diagnostic files. # For the absorbing particles, an extra particle was added that won't be absorbed # so that there will be something to read in here. -r_id0 = ad0['reflecting_particles', 'particle_id'].v -a_id0 = ad0['absorbing_particles', 'particle_id'].v -p_id0 = ad0['periodic_particles', 'particle_id'].v - -xx0 = ad0['reflecting_particles', 'particle_position_x'].v[np.argsort(r_id0)] -zz0 = ad0['periodic_particles', 'particle_position_z'].v[np.argsort(p_id0)] - -ux0 = ad0['reflecting_particles', 'particle_momentum_x'].v[np.argsort(r_id0)]/m_e/c -uz0 = ad0['periodic_particles', 'particle_momentum_z'].v[np.argsort(p_id0)]/m_e/c -gx0 = np.sqrt(1. + ux0**2) -gz0 = np.sqrt(1. + uz0**2) -vx0 = ux0/gx0*c -vz0 = uz0/gz0*c - -r_id = ad['reflecting_particles', 'particle_id'].v -a_id = ad['absorbing_particles', 'particle_id'].v -p_id = ad['periodic_particles', 'particle_id'].v - -xx = ad['reflecting_particles', 'particle_position_x'].v[np.argsort(r_id)] -zz = ad['periodic_particles', 'particle_position_z'].v[np.argsort(p_id)] - -ux = ad['reflecting_particles', 'particle_momentum_x'].v[np.argsort(r_id)]/m_e/c -uz = ad['periodic_particles', 'particle_momentum_z'].v[np.argsort(p_id)]/m_e/c -gx = np.sqrt(1. + ux**2) -gz = np.sqrt(1. + uz**2) -vx = ux/gx*c -vz = uz/gz*c +r_id0 = ad0["reflecting_particles", "particle_id"].v +a_id0 = ad0["absorbing_particles", "particle_id"].v +p_id0 = ad0["periodic_particles", "particle_id"].v + +xx0 = ad0["reflecting_particles", "particle_position_x"].v[np.argsort(r_id0)] +zz0 = ad0["periodic_particles", "particle_position_z"].v[np.argsort(p_id0)] + +ux0 = ad0["reflecting_particles", "particle_momentum_x"].v[np.argsort(r_id0)] / m_e / c +uz0 = ad0["periodic_particles", "particle_momentum_z"].v[np.argsort(p_id0)] / m_e / c +gx0 = np.sqrt(1.0 + ux0**2) +gz0 = np.sqrt(1.0 + uz0**2) +vx0 = ux0 / gx0 * c +vz0 = uz0 / gz0 * c + +r_id = ad["reflecting_particles", "particle_id"].v +a_id = ad["absorbing_particles", "particle_id"].v +p_id = ad["periodic_particles", "particle_id"].v + +xx = ad["reflecting_particles", "particle_position_x"].v[np.argsort(r_id)] +zz = ad["periodic_particles", "particle_position_z"].v[np.argsort(p_id)] + +ux = ad["reflecting_particles", "particle_momentum_x"].v[np.argsort(r_id)] / m_e / c +uz = ad["periodic_particles", "particle_momentum_z"].v[np.argsort(p_id)] / m_e / c +gx = np.sqrt(1.0 + ux**2) +gz = np.sqrt(1.0 + uz**2) +vx = ux / gx * c +vz = uz / gz * c + def do_reflect(x): if x < dmin: - return 2.*dmin - x + return 2.0 * dmin - x elif x > dmax: - return 2.*dmax - x + return 2.0 * dmax - x else: return x + def do_periodic(x): if x < dmin: return x + (dmax - dmin) @@ -88,21 +87,23 @@ def do_periodic(x): else: return x + # Calculate the analytic value of the current particle locations and # apply the appropriate boundary conditions. -xxa = xx0 + vx0*time +xxa = xx0 + vx0 * time xxa[0] = do_reflect(xxa[0]) xxa[1] = do_reflect(xxa[1]) -zza = zz0 + vz0*time +zza = zz0 + vz0 * time zza[0] = do_periodic(zza[0]) zza[1] = do_periodic(zza[1]) -assert (len(a_id) == 1), 'Absorbing particles not absorbed' -assert (np.all(vx == -vx0)), 'Reflecting particle velocity not correct' -assert (np.all(vz == +vz0)), 'Periodic particle velocity not correct' -assert (np.all(np.abs((xx - xxa)/xx) < 1.e-15)), 'Reflecting particle position not correct' -assert (np.all(np.abs((zz - zza)/zz) < 1.e-15)), 'Periodic particle position not correct' - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +assert len(a_id) == 1, "Absorbing particles not absorbed" +assert np.all(vx == -vx0), "Reflecting particle velocity not correct" +assert np.all(vz == +vz0), "Periodic particle velocity not correct" +assert np.all(np.abs((xx - xxa) / xx) < 1.0e-15), ( + "Reflecting particle position not correct" +) +assert np.all(np.abs((zz - zza) / zz) < 1.0e-15), ( + "Periodic particle position not correct" +) diff --git a/Examples/Tests/boundaries/analysis_default_regression.py b/Examples/Tests/boundaries/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/boundaries/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/boundaries/inputs_3d b/Examples/Tests/boundaries/inputs_test_3d_particle_boundaries similarity index 100% rename from Examples/Tests/boundaries/inputs_3d rename to Examples/Tests/boundaries/inputs_test_3d_particle_boundaries diff --git a/Examples/Tests/btd_rz/CMakeLists.txt b/Examples/Tests/btd_rz/CMakeLists.txt new file mode 100644 index 00000000000..3c4bfffb609 --- /dev/null +++ b/Examples/Tests/btd_rz/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_rz_btd # name + RZ # dims + 2 # nprocs + inputs_test_rz_btd # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000289" # checksum + OFF # dependency +) diff --git a/Examples/Tests/btd_rz/analysis.py b/Examples/Tests/btd_rz/analysis.py new file mode 100755 index 00000000000..c3f4f0243fa --- /dev/null +++ b/Examples/Tests/btd_rz/analysis.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright 2022 +# Authors: Revathi Jambunathan, Remi Lehe +# +# This tests checks the backtransformed diagnostics by emitting a laser +# (with the antenna) in the boosted-frame and then checking that the +# fields recorded by the backtransformed diagnostics have the right amplitude, +# wavelength, and envelope (i.e. gaussian envelope with the right duration. + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, e, m_e +from scipy.optimize import curve_fit + + +def gaussian_laser(z, a0, z0_phase, z0_prop, ctau, lambda0): + """ + Returns a Gaussian laser profile + """ + k0 = 2 * np.pi / lambda0 + E0 = a0 * m_e * c**2 * k0 / e + return E0 * np.exp(-((z - z0_prop) ** 2) / ctau**2) * np.cos(k0 * (z - z0_phase)) + + +# Fit the on-axis profile to extract the phase (a.k.a. CEP) +def fit_function(z, z0_phase): + return gaussian_laser(z, a0, z0_phase, z0_b + Lprop_b, ctau0, lambda0) + + +# The values must be consistent with the values provided in the simulation input +t_current = 80e-15 # Time of the snapshot1 +z0_antenna = -1.0e-6 # position of laser +lambda0 = 0.8e-6 # wavelength of the signal +tau0 = 10e-15 # duration of the signal +ctau0 = tau0 * c +a0 = 15 # amplitude +t_peak = 20e-15 # Time at which laser reaches its peak +Lprop_b = c * t_current +z0_b = z0_antenna - c * t_peak + +ts = OpenPMDTimeSeries("./diags/back_rz") +Ex, info = ts.get_field("E", "x", iteration=1, slice_across="r") + +fit_result = curve_fit(fit_function, info.z, Ex, p0=np.array([z0_b + Lprop_b])) +z0_fit = fit_result[0] + +Ex_fit = gaussian_laser(info.z, a0, z0_fit, z0_b + Lprop_b, ctau0, lambda0) + +## Check that the a0 agrees within 5% of the predicted value +assert np.allclose(Ex, Ex_fit, atol=0.18 * Ex.max()) diff --git a/Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py deleted file mode 100755 index e2f174d6e29..00000000000 --- a/Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 -# Authors: Revathi Jambunathan, Remi Lehe -# -# This tests checks the backtransformed diagnostics by emitting a laser -# (with the antenna) in the boosted-frame and then checking that the -# fields recorded by the backtransformed diagnostics have the right amplitude, -# wavelength, and envelope (i.e. gaussian envelope with the right duration. - -import os -import sys - -import numpy as np -from openpmd_viewer import OpenPMDTimeSeries -from scipy.constants import c, e, m_e -from scipy.optimize import curve_fit - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - - -def gaussian_laser( z, a0, z0_phase, z0_prop, ctau, lambda0 ): - """ - Returns a Gaussian laser profile - """ - k0 = 2*np.pi/lambda0 - E0 = a0*m_e*c**2*k0/e - return( E0*np.exp( - (z-z0_prop)**2/ctau**2 ) \ - *np.cos( k0*(z-z0_phase) ) ) - -# Fit the on-axis profile to extract the phase (a.k.a. CEP) -def fit_function(z, z0_phase): - return( gaussian_laser( z, a0, z0_phase, - z0_b+Lprop_b, ctau0, lambda0 ) ) - -plotfile = sys.argv[1] - -# The values must be consistent with the values provided in the simulation input -t_current = 80e-15 # Time of the snapshot1 -c = 299792458; -z0_antenna = -1.e-6 # position of laser -lambda0 = 0.8e-6 # wavelength of the signal -tau0 = 10e-15 # duration of the signal -ctau0 = tau0 * c -a0 = 15 # amplitude -t_peak = 20e-15 # Time at which laser reaches its peak -Lprop_b = c*t_current -z0_b = z0_antenna - c * t_peak - -ts = OpenPMDTimeSeries('./diags/back_rz') -Ex, info = ts.get_field('E', 'x', iteration=1, slice_across='r') - -fit_result = curve_fit( fit_function, info.z, Ex, - p0=np.array([z0_b+Lprop_b]) ) -z0_fit = fit_result[0] - -Ex_fit = gaussian_laser( info.z, a0, z0_fit, z0_b+Lprop_b, ctau0, lambda0) - -## Check that the a0 agrees within 5% of the predicted value -assert np.allclose( Ex, Ex_fit, atol=0.18*Ex.max() ) - -# Checksum regression analysis -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, plotfile) diff --git a/Examples/Tests/btd_rz/analysis_default_regression.py b/Examples/Tests/btd_rz/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/btd_rz/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/btd_rz/inputs_rz_z_boosted_BTD b/Examples/Tests/btd_rz/inputs_test_rz_btd similarity index 100% rename from Examples/Tests/btd_rz/inputs_rz_z_boosted_BTD rename to Examples/Tests/btd_rz/inputs_test_rz_btd diff --git a/Examples/Tests/collider_relevant_diags/CMakeLists.txt b/Examples/Tests/collider_relevant_diags/CMakeLists.txt new file mode 100644 index 00000000000..d7bd38a9475 --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_collider_diagnostics # name + 3 # dims + 2 # nprocs + inputs_test_3d_collider_diagnostics # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/collider_relevant_diags/analysis.py b/Examples/Tests/collider_relevant_diags/analysis.py new file mode 100755 index 00000000000..17e63e69076 --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/analysis.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +import sys + +import numpy as np +import openpmd_api as io +import pandas as pd +from scipy.constants import c, e, hbar, m_e + +sys.path.append("../../../../warpx/Tools/Parser/") +from input_file_parser import parse_input_file + +E_crit = m_e**2 * c**3 / (e * hbar) +B_crit = m_e**2 * c**2 / (e * hbar) + + +def chi(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz): + gamma = np.sqrt(1.0 + ux**2 + uy**2 + uz**2) + vx = ux / gamma * c + vy = uy / gamma * c + vz = uz / gamma * c + tmp1x = Ex + vy * Bz - vz * By + tmp1y = Ey - vx * Bz + vz * Bx + tmp1z = Ez + vx * By - vy * Bx + tmp2 = (Ex * vx + Ey * vy + Ez * vz) / c + chi = gamma / E_crit * np.sqrt(tmp1x**2 + tmp1y**2 + tmp1z**2 - tmp2**2) + return chi + + +def dL_dt(): + series = io.Series("diags/diag2/openpmd_%T.h5", io.Access.read_only) + iterations = np.asarray(series.iterations) + lumi = [] + for n, ts in enumerate(iterations): + it = series.iterations[ts] + rho1 = it.meshes["rho_beam_e"] + dV = np.prod(rho1.grid_spacing) + rho1 = it.meshes["rho_beam_e"][io.Mesh_Record_Component.SCALAR].load_chunk() + rho2 = it.meshes["rho_beam_p"][io.Mesh_Record_Component.SCALAR].load_chunk() + beam_e_charge = it.particles["beam_e"]["charge"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + beam_p_charge = it.particles["beam_p"]["charge"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + q1 = beam_e_charge[0] + if not np.all(beam_e_charge == q1): + sys.exit("beam_e particles do not have the same charge") + q2 = beam_p_charge[0] + if not np.all(beam_p_charge == q2): + sys.exit("beam_p particles do not have the same charge") + series.flush() + n1 = rho1 / q1 + n2 = rho2 / q2 + ln = 2 * np.sum(n1 * n2) * dV * c + lumi.append(ln) + return lumi + + +input_dict = parse_input_file("warpx_used_inputs") +Ex, Ey, Ez = [float(w) for w in input_dict["particles.E_external_particle"]] +Bx, By, Bz = [float(w) for w in input_dict["particles.B_external_particle"]] + +CollDiagFname = "diags/reducedfiles/ColliderRelevant_beam_e_beam_p.txt" +df = pd.read_csv(CollDiagFname, sep=" ", header=0) + +for species in ["beam_p", "beam_e"]: + ux1, ux2, ux3 = [float(w) for w in input_dict[f"{species}.multiple_particles_ux"]] + uy1, uy2, uy3 = [float(w) for w in input_dict[f"{species}.multiple_particles_uy"]] + uz1, uz2, uz3 = [float(w) for w in input_dict[f"{species}.multiple_particles_uz"]] + + x = np.array([float(w) for w in input_dict[f"{species}.multiple_particles_pos_x"]]) + y = np.array([float(w) for w in input_dict[f"{species}.multiple_particles_pos_y"]]) + + w = np.array([float(w) for w in input_dict[f"{species}.multiple_particles_weight"]]) + + CHI_ANALYTICAL = np.array( + [ + chi(ux1, uy1, uz1, Ex, Ey, Ez, Bx, By, Bz), + chi(ux2, uy2, uz2, Ex, Ey, Ez, Bx, By, Bz), + chi(ux3, uy3, uz3, Ex, Ey, Ez, Bx, By, Bz), + ] + ) + THETAX = np.array( + [np.arctan2(ux1, uz1), np.arctan2(ux2, uz2), np.arctan2(ux3, uz3)] + ) + THETAY = np.array( + [np.arctan2(uy1, uz1), np.arctan2(uy2, uz2), np.arctan2(uy3, uz3)] + ) + + # CHI MAX + fname = f"diags/reducedfiles/ParticleExtrema_{species}.txt" + chimax_pe = np.loadtxt(fname)[:, 19] + chimax_cr = df[ + [col for col in df.columns if f"chi_max_{species}" in col] + ].to_numpy() + assert np.allclose(np.max(CHI_ANALYTICAL), chimax_cr, rtol=1e-8) + assert np.allclose(chimax_pe, chimax_cr, rtol=1e-8) + + # CHI MIN + fname = f"diags/reducedfiles/ParticleExtrema_{species}.txt" + chimin_pe = np.loadtxt(fname)[:, 18] + chimin_cr = df[ + [col for col in df.columns if f"chi_min_{species}" in col] + ].to_numpy() + assert np.allclose(np.min(CHI_ANALYTICAL), chimin_cr, rtol=1e-8) + assert np.allclose(chimin_pe, chimin_cr, rtol=1e-8) + + # CHI AVERAGE + chiave_cr = df[ + [col for col in df.columns if f"chi_ave_{species}" in col] + ].to_numpy() + assert np.allclose(np.average(CHI_ANALYTICAL, weights=w), chiave_cr, rtol=1e-8) + + # X AVE STD + x_ave_cr = df[[col for col in df.columns if f"]x_ave_{species}" in col]].to_numpy() + x_std_cr = df[[col for col in df.columns if f"]x_std_{species}" in col]].to_numpy() + x_ave = np.average(x, weights=w) + x_std = np.sqrt(np.average((x - x_ave) ** 2, weights=w)) + assert np.allclose(x_ave, x_ave_cr, rtol=1e-8) + assert np.allclose(x_std, x_std_cr, rtol=1e-8) + + # Y AVE STD + y_ave_cr = df[[col for col in df.columns if f"]y_ave_{species}" in col]].to_numpy() + y_std_cr = df[[col for col in df.columns if f"]y_std_{species}" in col]].to_numpy() + y_ave = np.average(y, weights=w) + y_std = np.sqrt(np.average((y - y_ave) ** 2, weights=w)) + assert np.allclose(y_ave, y_ave_cr, rtol=1e-8) + assert np.allclose(y_std, y_std_cr, rtol=1e-8) + + # THETA X MIN AVE MAX STD + thetax_min_cr = df[ + [col for col in df.columns if f"theta_x_min_{species}" in col] + ].to_numpy() + thetax_ave_cr = df[ + [col for col in df.columns if f"theta_x_ave_{species}" in col] + ].to_numpy() + thetax_max_cr = df[ + [col for col in df.columns if f"theta_x_max_{species}" in col] + ].to_numpy() + thetax_std_cr = df[ + [col for col in df.columns if f"theta_x_std_{species}" in col] + ].to_numpy() + thetax_min = np.min(THETAX) + thetax_ave = np.average(THETAX, weights=w) + thetax_max = np.max(THETAX) + thetax_std = np.sqrt(np.average((THETAX - thetax_ave) ** 2, weights=w)) + assert np.allclose(thetax_min, thetax_min_cr, rtol=1e-8) + assert np.allclose(thetax_ave, thetax_ave_cr, rtol=1e-8) + assert np.allclose(thetax_max, thetax_max_cr, rtol=1e-8) + assert np.allclose(thetax_std, thetax_std_cr, rtol=1e-8) + + # THETA Y MIN AVE MAX STD + thetay_min_cr = df[ + [col for col in df.columns if f"theta_y_min_{species}" in col] + ].to_numpy() + thetay_ave_cr = df[ + [col for col in df.columns if f"theta_y_ave_{species}" in col] + ].to_numpy() + thetay_max_cr = df[ + [col for col in df.columns if f"theta_y_max_{species}" in col] + ].to_numpy() + thetay_std_cr = df[ + [col for col in df.columns if f"theta_y_std_{species}" in col] + ].to_numpy() + thetay_min = np.min(THETAY) + thetay_ave = np.average(THETAY, weights=w) + thetay_max = np.max(THETAY) + thetay_std = np.sqrt(np.average((THETAY - thetay_ave) ** 2, weights=w)) + assert np.allclose(thetay_min, thetay_min_cr, rtol=1e-8) + assert np.allclose(thetay_ave, thetay_ave_cr, rtol=1e-8) + assert np.allclose(thetay_max, thetay_max_cr, rtol=1e-8) + assert np.allclose(thetay_std, thetay_std_cr, rtol=1e-8) + + # dL/dt + dL_dt_cr = df[[col for col in df.columns if "dL_dt" in col]].to_numpy() + assert np.allclose(dL_dt_cr, dL_dt(), rtol=1e-8) diff --git a/Examples/Tests/collider_relevant_diags/analysis_default_regression.py b/Examples/Tests/collider_relevant_diags/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py b/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py deleted file mode 100755 index b23bb69d52c..00000000000 --- a/Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -import numpy as np -import openpmd_api as io -import pandas as pd -from scipy.constants import c, e, hbar, m_e - -sys.path.append('../../../../warpx/Regression/Checksum/') -import checksumAPI - -sys.path.append('../../../../warpx/Tools/Parser/') -from input_file_parser import parse_input_file - -E_crit = m_e**2*c**3/(e*hbar) -B_crit = m_e**2*c**2/(e*hbar) - -def chi(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz): - gamma = np.sqrt(1.+ux**2+uy**2+uz**2) - vx = ux / gamma * c - vy = uy / gamma * c - vz = uz / gamma * c - tmp1x = Ex + vy*Bz - vz*By - tmp1y = Ey - vx*Bz + vz*Bx - tmp1z = Ez + vx*By - vy*Bx - tmp2 = (Ex*vx + Ey*vy + Ez*vz)/c - chi = gamma/E_crit*np.sqrt(tmp1x**2+tmp1y**2+tmp1z**2 - tmp2**2) - return chi - -def dL_dt(): - series = io.Series("diags/diag2/openpmd_%T.h5",io.Access.read_only) - iterations = np.asarray(series.iterations) - lumi = [] - for n,ts in enumerate(iterations): - it = series.iterations[ts] - rho1 = it.meshes["rho_beam_e"] - dV = np.prod(rho1.grid_spacing) - rho1 = it.meshes["rho_beam_e"][io.Mesh_Record_Component.SCALAR].load_chunk() - rho2 = it.meshes["rho_beam_p"][io.Mesh_Record_Component.SCALAR].load_chunk() - beam_e_charge = it.particles["beam_e"]["charge"][io.Mesh_Record_Component.SCALAR].load_chunk() - beam_p_charge = it.particles["beam_p"]["charge"][io.Mesh_Record_Component.SCALAR].load_chunk() - q1 = beam_e_charge[0] - if not np.all(beam_e_charge == q1): - sys.exit('beam_e particles do not have the same charge') - q2 = beam_p_charge[0] - if not np.all(beam_p_charge == q2): - sys.exit('beam_p particles do not have the same charge') - series.flush() - n1 = rho1/q1 - n2 = rho2/q2 - l = 2*np.sum(n1*n2)*dV*c - lumi.append(l) - return lumi - -input_dict = parse_input_file('inputs_3d_multiple_particles') -Ex, Ey, Ez = [float(w) for w in input_dict['particles.E_external_particle']] -Bx, By, Bz = [float(w) for w in input_dict['particles.B_external_particle']] - -CollDiagFname='diags/reducedfiles/ColliderRelevant_beam_e_beam_p.txt' -df = pd.read_csv(CollDiagFname, sep=" ", header=0) - -for species in ['beam_p', 'beam_e']: - - ux1, ux2, ux3 = [float(w) for w in input_dict[f'{species}.multiple_particles_ux']] - uy1, uy2, uy3 = [float(w) for w in input_dict[f'{species}.multiple_particles_uy']] - uz1, uz2, uz3 = [float(w) for w in input_dict[f'{species}.multiple_particles_uz']] - - x = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_pos_x']]) - y = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_pos_y']]) - - w = np.array([float(w) for w in input_dict[f'{species}.multiple_particles_weight']]) - - CHI_ANALYTICAL = np.array([chi(ux1, uy1, uz1, Ex, Ey, Ez, Bx, By, Bz), - chi(ux2, uy2, uz2, Ex, Ey, Ez, Bx, By, Bz), - chi(ux3, uy3, uz3, Ex, Ey, Ez, Bx, By, Bz)]) - THETAX = np.array([np.arctan2(ux1, uz1), np.arctan2(ux2, uz2), np.arctan2(ux3, uz3)]) - THETAY = np.array([np.arctan2(uy1, uz1), np.arctan2(uy2, uz2), np.arctan2(uy3, uz3)]) - - # CHI MAX - fname=f'diags/reducedfiles/ParticleExtrema_{species}.txt' - chimax_pe = np.loadtxt(fname)[:,19] - chimax_cr = df[[col for col in df.columns if f'chi_max_{species}' in col]].to_numpy() - assert np.allclose(np.max(CHI_ANALYTICAL), chimax_cr, rtol=1e-8) - assert np.allclose(chimax_pe, chimax_cr, rtol=1e-8) - - # CHI MIN - fname=f'diags/reducedfiles/ParticleExtrema_{species}.txt' - chimin_pe = np.loadtxt(fname)[:,18] - chimin_cr = df[[col for col in df.columns if f'chi_min_{species}' in col]].to_numpy() - assert np.allclose(np.min(CHI_ANALYTICAL), chimin_cr, rtol=1e-8) - assert np.allclose(chimin_pe, chimin_cr, rtol=1e-8) - - # CHI AVERAGE - chiave_cr = df[[col for col in df.columns if f'chi_ave_{species}' in col]].to_numpy() - assert np.allclose(np.average(CHI_ANALYTICAL, weights=w), chiave_cr, rtol=1e-8) - - # X AVE STD - x_ave_cr = df[[col for col in df.columns if f']x_ave_{species}' in col]].to_numpy() - x_std_cr = df[[col for col in df.columns if f']x_std_{species}' in col]].to_numpy() - x_ave = np.average(x, weights=w) - x_std = np.sqrt(np.average((x-x_ave)**2, weights=w)) - assert np.allclose(x_ave, x_ave_cr, rtol=1e-8) - assert np.allclose(x_std, x_std_cr, rtol=1e-8) - - # Y AVE STD - y_ave_cr = df[[col for col in df.columns if f']y_ave_{species}' in col]].to_numpy() - y_std_cr = df[[col for col in df.columns if f']y_std_{species}' in col]].to_numpy() - y_ave = np.average(y, weights=w) - y_std = np.sqrt(np.average((y-y_ave)**2, weights=w)) - assert np.allclose(y_ave, y_ave_cr, rtol=1e-8) - assert np.allclose(y_std, y_std_cr, rtol=1e-8) - - # THETA X MIN AVE MAX STD - thetax_min_cr = df[[col for col in df.columns if f'theta_x_min_{species}' in col]].to_numpy() - thetax_ave_cr = df[[col for col in df.columns if f'theta_x_ave_{species}' in col]].to_numpy() - thetax_max_cr = df[[col for col in df.columns if f'theta_x_max_{species}' in col]].to_numpy() - thetax_std_cr = df[[col for col in df.columns if f'theta_x_std_{species}' in col]].to_numpy() - thetax_min = np.min(THETAX) - thetax_ave = np.average(THETAX, weights=w) - thetax_max = np.max(THETAX) - thetax_std = np.sqrt(np.average((THETAX-thetax_ave)**2, weights=w)) - assert np.allclose(thetax_min, thetax_min_cr, rtol=1e-8) - assert np.allclose(thetax_ave, thetax_ave_cr, rtol=1e-8) - assert np.allclose(thetax_max, thetax_max_cr, rtol=1e-8) - assert np.allclose(thetax_std, thetax_std_cr, rtol=1e-8) - - # THETA Y MIN AVE MAX STD - thetay_min_cr = df[[col for col in df.columns if f'theta_y_min_{species}' in col]].to_numpy() - thetay_ave_cr = df[[col for col in df.columns if f'theta_y_ave_{species}' in col]].to_numpy() - thetay_max_cr = df[[col for col in df.columns if f'theta_y_max_{species}' in col]].to_numpy() - thetay_std_cr = df[[col for col in df.columns if f'theta_y_std_{species}' in col]].to_numpy() - thetay_min = np.min(THETAY) - thetay_ave = np.average(THETAY, weights=w) - thetay_max = np.max(THETAY) - thetay_std = np.sqrt(np.average((THETAY-thetay_ave)**2, weights=w)) - assert np.allclose(thetay_min, thetay_min_cr, rtol=1e-8) - assert np.allclose(thetay_ave, thetay_ave_cr, rtol=1e-8) - assert np.allclose(thetay_max, thetay_max_cr, rtol=1e-8) - assert np.allclose(thetay_std, thetay_std_cr, rtol=1e-8) - - # dL/dt - dL_dt_cr = df[[col for col in df.columns if 'dL_dt' in col]].to_numpy() - assert np.allclose(dL_dt_cr, dL_dt(), rtol=1e-8) - -# Checksum analysis -plotfile = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, plotfile) diff --git a/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles b/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles deleted file mode 100644 index 1efc68c33b0..00000000000 --- a/Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles +++ /dev/null @@ -1,130 +0,0 @@ -################################# -########## MY CONSTANTS ######### -################################# -my_constants.nx = 8 -my_constants.ny = 8 -my_constants.nz = 8 - -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 1 -amr.n_cell = nx ny nz -amr.max_grid_size = 4 -amr.blocking_factor = 4 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = 0 0 0 -geometry.prob_hi = 8 8 8 -particles.do_tiling = 0 -warpx.use_filter = 0 - -################################# -######## BOUNDARY CONDITION ##### -################################# -boundary.field_lo = periodic periodic periodic -boundary.field_hi = periodic periodic periodic -boundary.particle_lo = periodic periodic periodic -boundary.particle_hi = periodic periodic periodic - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = ckc -warpx.cfl = 0.99 -algo.particle_shape = 1 - -################################# -############ FIELDS ############# -################################# -particles.E_ext_particle_init_style = constant -particles.B_ext_particle_init_style = constant -particles.E_external_particle = 10000. 0. 0. -particles.B_external_particle = 0. 5000. 0. - -################################# -########### PARTICLES ########### -################################# -particles.species_names = pho beam_p beam_e -particles.photon_species = pho - -beam_e.species_type = electron -beam_e.injection_style = MultipleParticles -beam_e.multiple_particles_pos_x = 4.5 3.5 0.5 -beam_e.multiple_particles_pos_y = 4.5 2.5 1.5 -beam_e.multiple_particles_pos_z = 4.5 1.5 1.5 -beam_e.multiple_particles_ux = 0.3 0.2 0.1 -beam_e.multiple_particles_uy = 0.4 -0.3 -0.1 -beam_e.multiple_particles_uz = 0.3 0.1 -10. -beam_e.multiple_particles_weight = 1. 2 3 -beam_e.initialize_self_fields = 0 -beam_e.self_fields_required_precision = 5e-10 -beam_e.do_qed_quantum_sync = 1 -beam_e.qed_quantum_sync_phot_product_species = pho -beam_e.do_not_push = 1 -beam_e.do_not_deposit = 1 - -beam_p.species_type = positron -beam_p.injection_style = MultipleParticles -beam_p.multiple_particles_pos_x = 4.5 3.5 0.5 -beam_p.multiple_particles_pos_y = 4.5 2.5 1.5 -beam_p.multiple_particles_pos_z = 4.5 1.5 1.5 -beam_p.multiple_particles_ux = 0.3 0.2 0.1 -beam_p.multiple_particles_uy = 0.4 -0.3 -0.1 -beam_p.multiple_particles_uz = 0.3 0.1 -10. -beam_p.multiple_particles_weight = 1. 2 3 -beam_p.initialize_self_fields = 0 -beam_p.self_fields_required_precision = 5e-10 -beam_p.do_qed_quantum_sync = 1 -beam_p.qed_quantum_sync_phot_product_species = pho -beam_p.do_not_push = 1 -beam_p.do_not_deposit = 1 - -pho.species_type = photon -pho.injection_style = none - -################################# -############# QED ############### -################################# -qed_qs.photon_creation_energy_threshold = 0. -qed_qs.lookup_table_mode = builtin -qed_qs.chi_min = 1.e-3 -warpx.do_qed_schwinger = 0 - -################################# -######### DIAGNOSTICS ########### -################################# -# FULL -diagnostics.diags_names = diag1 diag2 - -diag1.intervals = 1 -diag1.diag_type = Full -diag1.write_species = 1 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho -diag1.species = pho beam_e beam_p -diag1.format = plotfile -#diag1.dump_last_timestep = 1 - -diag2.intervals = 1 -diag2.diag_type = Full -diag2.write_species = 1 -diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho -diag2.species = pho beam_e beam_p -diag2.format = openpmd -diag2.openpmd_backend = h5 -#diag2.dump_last_timestep = 1 - -# REDUCED -warpx.reduced_diags_names = ParticleExtrema_beam_e ParticleExtrema_beam_p ColliderRelevant_beam_e_beam_p - -ColliderRelevant_beam_e_beam_p.type = ColliderRelevant -ColliderRelevant_beam_e_beam_p.intervals = 1 -ColliderRelevant_beam_e_beam_p.species =beam_e beam_p - -ParticleExtrema_beam_e.type = ParticleExtrema -ParticleExtrema_beam_e.intervals = 1 -ParticleExtrema_beam_e.species = beam_e - -ParticleExtrema_beam_p.type = ParticleExtrema -ParticleExtrema_beam_p.intervals = 1 -ParticleExtrema_beam_p.species = beam_p diff --git a/Examples/Tests/collider_relevant_diags/inputs_test_3d_collider_diagnostics b/Examples/Tests/collider_relevant_diags/inputs_test_3d_collider_diagnostics new file mode 100644 index 00000000000..d88e0b767d6 --- /dev/null +++ b/Examples/Tests/collider_relevant_diags/inputs_test_3d_collider_diagnostics @@ -0,0 +1,131 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.nx = 8 +my_constants.ny = 8 +my_constants.nz = 8 + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 1 +amr.n_cell = nx ny nz +amr.max_grid_size = 4 +amr.blocking_factor = 4 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 8 8 8 +particles.do_tiling = 0 +warpx.use_filter = 0 +warpx.abort_on_warning_threshold = high + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic +boundary.particle_lo = periodic periodic periodic +boundary.particle_hi = periodic periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.cfl = 0.99 +algo.particle_shape = 1 + +################################# +############ FIELDS ############# +################################# +particles.E_ext_particle_init_style = constant +particles.B_ext_particle_init_style = constant +particles.E_external_particle = 10000. 0. 0. +particles.B_external_particle = 0. 5000. 0. + +################################# +########### PARTICLES ########### +################################# +particles.species_names = pho beam_p beam_e +particles.photon_species = pho + +beam_e.species_type = electron +beam_e.injection_style = MultipleParticles +beam_e.multiple_particles_pos_x = 4.5 3.5 0.5 +beam_e.multiple_particles_pos_y = 4.5 2.5 1.5 +beam_e.multiple_particles_pos_z = 4.5 1.5 1.5 +beam_e.multiple_particles_ux = 0.3 0.2 0.1 +beam_e.multiple_particles_uy = 0.4 -0.3 -0.1 +beam_e.multiple_particles_uz = 0.3 0.1 -10. +beam_e.multiple_particles_weight = 1. 2 3 +beam_e.initialize_self_fields = 0 +beam_e.self_fields_required_precision = 5e-10 +beam_e.do_qed_quantum_sync = 1 +beam_e.qed_quantum_sync_phot_product_species = pho +beam_e.do_not_push = 1 +beam_e.do_not_deposit = 1 + +beam_p.species_type = positron +beam_p.injection_style = MultipleParticles +beam_p.multiple_particles_pos_x = 4.5 3.5 0.5 +beam_p.multiple_particles_pos_y = 4.5 2.5 1.5 +beam_p.multiple_particles_pos_z = 4.5 1.5 1.5 +beam_p.multiple_particles_ux = 0.3 0.2 0.1 +beam_p.multiple_particles_uy = 0.4 -0.3 -0.1 +beam_p.multiple_particles_uz = 0.3 0.1 -10. +beam_p.multiple_particles_weight = 1. 2 3 +beam_p.initialize_self_fields = 0 +beam_p.self_fields_required_precision = 5e-10 +beam_p.do_qed_quantum_sync = 1 +beam_p.qed_quantum_sync_phot_product_species = pho +beam_p.do_not_push = 1 +beam_p.do_not_deposit = 1 + +pho.species_type = photon +pho.injection_style = none + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 +warpx.do_qed_schwinger = 0 + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho +diag1.species = pho beam_e beam_p +diag1.format = plotfile +#diag1.dump_last_timestep = 1 + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.write_species = 1 +diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho_beam_e rho_beam_p rho +diag2.species = pho beam_e beam_p +diag2.format = openpmd +diag2.openpmd_backend = h5 +#diag2.dump_last_timestep = 1 + +# REDUCED +warpx.reduced_diags_names = ParticleExtrema_beam_e ParticleExtrema_beam_p ColliderRelevant_beam_e_beam_p + +ColliderRelevant_beam_e_beam_p.type = ColliderRelevant +ColliderRelevant_beam_e_beam_p.intervals = 1 +ColliderRelevant_beam_e_beam_p.species =beam_e beam_p + +ParticleExtrema_beam_e.type = ParticleExtrema +ParticleExtrema_beam_e.intervals = 1 +ParticleExtrema_beam_e.species = beam_e + +ParticleExtrema_beam_p.type = ParticleExtrema +ParticleExtrema_beam_p.intervals = 1 +ParticleExtrema_beam_p.species = beam_p diff --git a/Examples/Tests/collision/CMakeLists.txt b/Examples/Tests/collision/CMakeLists.txt new file mode 100644 index 00000000000..522dafbfbfb --- /dev/null +++ b/Examples/Tests/collision/CMakeLists.txt @@ -0,0 +1,62 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_collision_z # name + 1 # dims + 2 # nprocs + inputs_test_1d_collision_z # inputs + "analysis_collision_1d.py diags/diag1000600" # analysis + "analysis_default_regression.py --path diags/diag1000600" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_collision_xz # name + 2 # dims + 1 # nprocs + inputs_test_2d_collision_xz # inputs + "analysis_collision_2d.py diags/diag1000150" # analysis + "analysis_default_regression.py --path diags/diag1000150" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_collision_xz_picmi # name + 2 # dims + 1 # nprocs + inputs_test_2d_collision_xz_picmi.py # inputs + "analysis_collision_2d.py diags/diag1000150" # analysis + "analysis_default_regression.py --path diags/diag1000150" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_collision_iso # name + 3 # dims + 1 # nprocs + inputs_test_3d_collision_iso # inputs + "analysis_collision_3d_isotropization.py diags/diag1000100" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_collision_xyz # name + 3 # dims + 1 # nprocs + inputs_test_3d_collision_xyz # inputs + "analysis_collision_3d.py diags/diag1000150" # analysis + "analysis_default_regression.py --path diags/diag1000150" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_collision # name + RZ # dims + 1 # nprocs + inputs_test_rz_collision # inputs + "analysis_collision_rz.py diags/diag1000150" # analysis + "analysis_default_regression.py --path diags/diag1000150 --skip-particles" # checksum + OFF # dependency +) diff --git a/Examples/Tests/collision/PICMI_inputs_2d.py b/Examples/Tests/collision/PICMI_inputs_2d.py deleted file mode 100755 index 99e217b0afc..00000000000 --- a/Examples/Tests/collision/PICMI_inputs_2d.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file for binary Coulomb collision testing. This input script -# --- runs the same test as inputs_2d but via Python, therefore the input -# --- values where directly copied from inputs_2d. - -from pywarpx import picmi - -constants = picmi.constants - -################################# -####### GENERAL PARAMETERS ###### -################################# - -max_steps = 150 - -max_grid_size = 8 -max_level = 0 -nx = max_grid_size -ny = max_grid_size - -xmin = 0 -xmax = 4.154046151855669e2 -ymin = xmin -ymax = xmax - -plasma_density = 1e21 -elec_rms_velocity = 0.044237441120300*constants.c -ion_rms_velocity = 0.006256118919701*constants.c -number_per_cell = 200 - -################################# -############ NUMERICS ########### -################################# -serialize_initial_conditions = 1 -verbose = 1 -cfl = 1.0 - -# Order of particle shape factors -particle_shape = 1 - -################################# -############ PLASMA ############# -################################# - -elec_dist = picmi.UniformDistribution( - density=plasma_density, - rms_velocity=[elec_rms_velocity]*3, - directed_velocity=[elec_rms_velocity, 0., 0.] -) - -ion_dist = picmi.UniformDistribution( - density=plasma_density, - rms_velocity=[ion_rms_velocity]*3, -) - -electrons = picmi.Species( - particle_type='electron', name='electron', - warpx_do_not_deposit=1, - initial_distribution=elec_dist, -) -ions = picmi.Species( - particle_type='H', - name='ion', charge='q_e', - mass="5*m_e", - warpx_do_not_deposit=1, - initial_distribution=ion_dist -) - -################################# -########## COLLISIONS ########### -################################# - -collision1 = picmi.CoulombCollisions( - name='collisions1', - species=[electrons, ions], - CoulombLog=15.9 -) -collision2 = picmi.CoulombCollisions( - name='collisions2', - species=[electrons, electrons], - CoulombLog=15.9 -) -collision3 = picmi.CoulombCollisions( - name='collisions3', - species=[ions, ions], - CoulombLog=15.9 -) - -################################# -###### GRID AND SOLVER ########## -################################# - -grid = picmi.Cartesian2DGrid( - number_of_cells=[nx, ny], - warpx_max_grid_size=max_grid_size, - warpx_blocking_factor=max_grid_size, - lower_bound=[xmin, ymin], - upper_bound=[xmax, ymax], - lower_boundary_conditions=['periodic', 'periodic'], - upper_boundary_conditions=['periodic', 'periodic'], -) -solver = picmi.ElectromagneticSolver(grid=grid, cfl=cfl) - -################################# -######### DIAGNOSTICS ########### -################################# - -particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=10, - write_dir='.', - warpx_file_prefix='Python_collisionXZ_plt' -) -field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=grid, - period=10, - data_list=[], - write_dir='.', - warpx_file_prefix='Python_collisionXZ_plt' -) - -################################# -####### SIMULATION SETUP ######## -################################# - -sim = picmi.Simulation( - solver=solver, - max_steps=max_steps, - verbose=verbose, - warpx_serialize_initial_conditions=serialize_initial_conditions, - warpx_collisions=[collision1, collision2, collision3] -) - -sim.add_species( - electrons, - layout=picmi.PseudoRandomLayout( - n_macroparticles_per_cell=number_per_cell, grid=grid - ) -) -sim.add_species( - ions, - layout=picmi.PseudoRandomLayout( - n_macroparticles_per_cell=number_per_cell, grid=grid - ) -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -################################# -##### SIMULATION EXECUTION ###### -################################# - -#sim.write_input_file('PICMI_inputs_2d') -sim.step(max_steps) diff --git a/Examples/Tests/collision/analysis_collision_1d.py b/Examples/Tests/collision/analysis_collision_1d.py index 7775a476dae..d5cf8b1cebd 100755 --- a/Examples/Tests/collision/analysis_collision_1d.py +++ b/Examples/Tests/collision/analysis_collision_1d.py @@ -15,45 +15,43 @@ # Both populations belong to the same carbon12 ion species. # See test T1b from JCP 413 (2020) by D. Higginson, et al. # -import os import sys import numpy as np import yt from scipy.constants import e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file -fn = sys.argv[1] -ds = yt.load(fn) -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) +last_fn = sys.argv[1] +ds = yt.load(last_fn) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # carbon 12 ion (mass = 12*amu - 6*me) mass = 1.992100316897910e-26 # Separate macroparticles from group A (low weight) and group B (high weight) # by sorting based on weight -sorted_indices = data['ions','particle_weight'].argsort() -sorted_wp = data['ions', 'particle_weight'][sorted_indices].value -sorted_px = data['ions', 'particle_momentum_x'][sorted_indices].value -sorted_py = data['ions', 'particle_momentum_y'][sorted_indices].value -sorted_pz = data['ions', 'particle_momentum_z'][sorted_indices].value +sorted_indices = data["ions", "particle_weight"].argsort() +sorted_wp = data["ions", "particle_weight"][sorted_indices].value +sorted_px = data["ions", "particle_momentum_x"][sorted_indices].value +sorted_py = data["ions", "particle_momentum_y"][sorted_indices].value +sorted_pz = data["ions", "particle_momentum_z"][sorted_indices].value # Find the index 'Npmin' that separates macroparticles from group A and group B Np = len(sorted_wp) -wpmin = sorted_wp.min(); -wpmax = sorted_wp.max(); +wpmin = sorted_wp.min() +wpmax = sorted_wp.max() for i in range(len(sorted_wp)): if sorted_wp[i] > wpmin: Npmin = i break NpA = Npmin -wpA = wpmin; +wpA = wpmin NpB = Np - Npmin -wpB = wpmax; +wpB = wpmax NpAs = 0 NpAe = Npmin NpBs = Npmin @@ -61,66 +59,63 @@ ############# -sorted_px_sum = np.abs(sorted_px).sum(); -sorted_py_sum = np.abs(sorted_py).sum(); -sorted_pz_sum = np.abs(sorted_pz).sum(); -sorted_wp_sum = np.abs(sorted_wp).sum(); +sorted_px_sum = np.abs(sorted_px).sum() +sorted_py_sum = np.abs(sorted_py).sum() +sorted_pz_sum = np.abs(sorted_pz).sum() +sorted_wp_sum = np.abs(sorted_wp).sum() # compute mean velocities -wAtot = wpA*NpA -wBtot = wpB*NpB - -uBx = uBy = uBz = 0. -for i in range(NpBs,NpBe): - uBx += wpB*sorted_px[i] - uBy += wpB*sorted_py[i] - uBz += wpB*sorted_pz[i] -uBx /= (mass*wBtot) # [m/s] -uBy /= (mass*wBtot) # [m/s] -uBz /= (mass*wBtot) # [m/s] - -uAx = uAy = uAz = 0. -for i in range(NpAs,NpAe): - uAx += wpA*sorted_px[i] - uAy += wpA*sorted_py[i] - uAz += wpA*sorted_pz[i] -uAx /= (mass*wAtot) # [m/s] -uAy /= (mass*wAtot) # [m/s] -uAz /= (mass*wAtot) # [m/s] +wAtot = wpA * NpA +wBtot = wpB * NpB + +uBx = uBy = uBz = 0.0 +for i in range(NpBs, NpBe): + uBx += wpB * sorted_px[i] + uBy += wpB * sorted_py[i] + uBz += wpB * sorted_pz[i] +uBx /= mass * wBtot # [m/s] +uBy /= mass * wBtot # [m/s] +uBz /= mass * wBtot # [m/s] + +uAx = uAy = uAz = 0.0 +for i in range(NpAs, NpAe): + uAx += wpA * sorted_px[i] + uAy += wpA * sorted_py[i] + uAz += wpA * sorted_pz[i] +uAx /= mass * wAtot # [m/s] +uAy /= mass * wAtot # [m/s] +uAz /= mass * wAtot # [m/s] # compute temperatures -TBx = TBy = TBz = 0. -for i in range(NpBs,NpBe): - TBx += wpB*(sorted_px[i]/mass - uBx)**2 - TBy += wpB*(sorted_py[i]/mass - uBy)**2 - TBz += wpB*(sorted_pz[i]/mass - uBz)**2 -TBx *= mass/(e*wBtot) -TBy *= mass/(e*wBtot) -TBz *= mass/(e*wBtot) - -TAx = TAy = TAz = 0. -for i in range(NpAs,NpAe): - TAx += wpA*(sorted_px[i]/mass - uAx)**2 - TAy += wpA*(sorted_py[i]/mass - uAy)**2 - TAz += wpA*(sorted_pz[i]/mass - uAz)**2 -TAx *= mass/(e*wAtot) -TAy *= mass/(e*wAtot) -TAz *= mass/(e*wAtot) +TBx = TBy = TBz = 0.0 +for i in range(NpBs, NpBe): + TBx += wpB * (sorted_px[i] / mass - uBx) ** 2 + TBy += wpB * (sorted_py[i] / mass - uBy) ** 2 + TBz += wpB * (sorted_pz[i] / mass - uBz) ** 2 +TBx *= mass / (e * wBtot) +TBy *= mass / (e * wBtot) +TBz *= mass / (e * wBtot) + +TAx = TAy = TAz = 0.0 +for i in range(NpAs, NpAe): + TAx += wpA * (sorted_px[i] / mass - uAx) ** 2 + TAy += wpA * (sorted_py[i] / mass - uAy) ** 2 + TAz += wpA * (sorted_pz[i] / mass - uAz) ** 2 +TAx *= mass / (e * wAtot) +TAy *= mass / (e * wAtot) +TAz *= mass / (e * wAtot) TApar = TAz -TAperp = (TAx + TAy)/2.0 -TA = (TAx + TAy + TAz)/3.0 +TAperp = (TAx + TAy) / 2.0 +TA = (TAx + TAy + TAz) / 3.0 TBpar = TBz -TBperp = (TBx + TBy)/2.0 -TB = (TBx + TBy + TBz)/3.0 +TBperp = (TBx + TBy) / 2.0 +TB = (TBx + TBy + TBz) / 3.0 -TApar_30ps_soln = 6.15e3 # TA parallel solution at t = 30 ps -error = np.abs(TApar-TApar_30ps_soln)/TApar_30ps_soln +TApar_30ps_soln = 6.15e3 # TA parallel solution at t = 30 ps +error = np.abs(TApar - TApar_30ps_soln) / TApar_30ps_soln tolerance = 0.02 -print('TApar at 30ps error = ', error); -print('tolerance = ', tolerance); +print("TApar at 30ps error = ", error) +print("tolerance = ", tolerance) assert error < tolerance - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/analysis_collision_2d.py b/Examples/Tests/collision/analysis_collision_2d.py index 29482909b2e..7e2746be752 100755 --- a/Examples/Tests/collision/analysis_collision_2d.py +++ b/Examples/Tests/collision/analysis_collision_2d.py @@ -26,15 +26,13 @@ import glob import math import os -import re import sys import numpy import post_processing_utils import yt -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +test_name = os.path.split(os.getcwd())[1] tolerance = 0.001 @@ -43,7 +41,7 @@ ni = ng * 200 np = ne + ni -c = 299792458.0 +c = 299792458.0 me = 9.10938356e-31 mi = me * 5.0 @@ -51,48 +49,52 @@ ## fit. # exponential fit coefficients -a = 0.04330638981264072 +a = 0.04330638981264072 b = -0.11588277796546632 last_fn = sys.argv[1] -# Remove trailing '/' from file name, if necessary -last_fn.rstrip('/') -# Find last iteration in file name, such as 'test_name_plt000001' (last_it = '000001') -last_it = re.search('\d+', last_fn).group() -# Find output prefix in file name, such as 'test_name_plt000001' (prefix = 'test_name_plt') -prefix = last_fn[:-len(last_it)] +if last_fn[-1] == "/": + last_fn = last_fn[:-1] +last_it = last_fn[-6:] # i.e., 000150 +prefix = last_fn[:-6] # i.e., diags/diag1 + # Collect all output files in fn_list (names match pattern prefix + arbitrary number) -fn_list = glob.glob(prefix + '*[0-9]') +fn_list = glob.glob(prefix + "*[0-9]") + +print(last_fn) +print(last_it) +print(prefix) +print(fn_list) error = 0.0 nt = 0 for fn in fn_list: # load file - ds = yt.load( fn ) - ad = ds.all_data() - px = ad[('all', 'particle_momentum_x')].to_ndarray() + ds = yt.load(fn) + ad = ds.all_data() + px = ad[("all", "particle_momentum_x")].to_ndarray() # get time index j j = int(fn[-5:]) # compute error - vxe = numpy.mean(px[ 0:ne])/me/c - vxi = numpy.mean(px[ne:np])/mi/c + vxe = numpy.mean(px[0:ne]) / me / c + vxi = numpy.mean(px[ne:np]) / mi / c vxd = vxe - vxi - fit = a*math.exp(b*j) - error = error + abs(fit-vxd) + fit = a * math.exp(b * j) + error = error + abs(fit - vxd) nt = nt + 1 error = error / nt -print('error = ', error) -print('tolerance = ', tolerance) -assert(error < tolerance) +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance # The second part of the analysis is not done for the Python test # since the particle filter function is not accessible from PICMI yet -if "Python" in last_fn: +if "picmi" in test_name: exit() -## In the second past of the test, we verify that the diagnostic particle filter function works as +## In the second part of the test, we verify that the diagnostic particle filter function works as ## expected. For this, we only use the last simulation timestep. dim = "2d" @@ -100,18 +102,18 @@ parser_filter_fn = "diags/diag_parser_filter" + last_it parser_filter_expression = "(x>200) * (z<200) * (px-3*pz>0)" -post_processing_utils.check_particle_filter(last_fn, parser_filter_fn, parser_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + last_fn, parser_filter_fn, parser_filter_expression, dim, species_name +) uniform_filter_fn = "diags/diag_uniform_filter" + last_it uniform_filter_expression = "ids%6 == 0" -post_processing_utils.check_particle_filter(last_fn, uniform_filter_fn, uniform_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + last_fn, uniform_filter_fn, uniform_filter_expression, dim, species_name +) random_filter_fn = "diags/diag_random_filter" + last_it random_fraction = 0.77 -post_processing_utils.check_random_filter(last_fn, random_filter_fn, random_fraction, - dim, species_name) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +post_processing_utils.check_random_filter( + last_fn, random_filter_fn, random_fraction, dim, species_name +) diff --git a/Examples/Tests/collision/analysis_collision_3d.py b/Examples/Tests/collision/analysis_collision_3d.py index 86a434caab2..c160d020cdc 100755 --- a/Examples/Tests/collision/analysis_collision_3d.py +++ b/Examples/Tests/collision/analysis_collision_3d.py @@ -25,17 +25,12 @@ import glob import math -import os -import re import sys import numpy import post_processing_utils import yt -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - tolerance = 0.001 ng = 512 @@ -43,7 +38,7 @@ ni = ng * 200 np = ne + ni -c = 299792458.0 +c = 299792458.0 me = 9.10938356e-31 mi = me * 5.0 @@ -51,45 +46,44 @@ ## fit. # exponential fit coefficients -a = 0.041817463099883 +a = 0.041817463099883 b = -0.083851393560288 last_fn = sys.argv[1] -# Remove trailing '/' from file name, if necessary -last_fn.rstrip('/') -# Find last iteration in file name, such as 'test_name_plt000001' (last_it = '000001') -last_it = re.search('\d+', last_fn).group() -# Find output prefix in file name, such as 'test_name_plt000001' (prefix = 'test_name_plt') -prefix = last_fn[:-len(last_it)] +if last_fn[-1] == "/": + last_fn = last_fn[:-1] +last_it = last_fn[-6:] # i.e., 000150 +prefix = last_fn[:-6] # i.e., diags/diag1 + # Collect all output files in fn_list (names match pattern prefix + arbitrary number) -fn_list = glob.glob(prefix + '*[0-9]') +fn_list = glob.glob(prefix + "*[0-9]") error = 0.0 nt = 0 for fn in fn_list: # load file - ds = yt.load( fn ) - ad = ds.all_data() - pxe = ad['electron', 'particle_momentum_x'].to_ndarray() - pxi = ad['ion', 'particle_momentum_x'].to_ndarray() + ds = yt.load(fn) + ad = ds.all_data() + pxe = ad["electron", "particle_momentum_x"].to_ndarray() + pxi = ad["ion", "particle_momentum_x"].to_ndarray() # get time index j j = int(fn[-5:]) # compute error - vxe = numpy.mean(pxe)/me/c - vxi = numpy.mean(pxi)/mi/c + vxe = numpy.mean(pxe) / me / c + vxi = numpy.mean(pxi) / mi / c vxd = vxe - vxi - fit = a*math.exp(b*j) - error = error + abs(fit-vxd) + fit = a * math.exp(b * j) + error = error + abs(fit - vxd) nt = nt + 1 error = error / nt -print('error = ', error) -print('tolerance = ', tolerance) -assert(error < tolerance) +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance -## In the second past of the test, we verify that the diagnostic particle filter function works as +## In the second part of the test, we verify that the diagnostic particle filter function works as ## expected. For this, we only use the last simulation timestep. dim = "3d" @@ -97,18 +91,18 @@ parser_filter_fn = "diags/diag_parser_filter" + last_it parser_filter_expression = "(px*py*pz < 0) * (np.sqrt(x**2+y**2+z**2)<100)" -post_processing_utils.check_particle_filter(last_fn, parser_filter_fn, parser_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + last_fn, parser_filter_fn, parser_filter_expression, dim, species_name +) uniform_filter_fn = "diags/diag_uniform_filter" + last_it uniform_filter_expression = "ids%11 == 0" -post_processing_utils.check_particle_filter(last_fn, uniform_filter_fn, uniform_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + last_fn, uniform_filter_fn, uniform_filter_expression, dim, species_name +) random_filter_fn = "diags/diag_random_filter" + last_it random_fraction = 0.88 -post_processing_utils.check_random_filter(last_fn, random_filter_fn, random_fraction, - dim, species_name) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +post_processing_utils.check_random_filter( + last_fn, random_filter_fn, random_fraction, dim, species_name +) diff --git a/Examples/Tests/collision/analysis_collision_3d_isotropization.py b/Examples/Tests/collision/analysis_collision_3d_isotropization.py index ba029760e8b..2656c5bac4d 100755 --- a/Examples/Tests/collision/analysis_collision_3d_isotropization.py +++ b/Examples/Tests/collision/analysis_collision_3d_isotropization.py @@ -11,16 +11,12 @@ # https://smileipic.github.io/tutorials/advanced_collisions.html # https://smileipic.github.io/Smilei/Understand/collisions.html#test-cases-for-collisions -import os import sys import numpy as np import scipy.constants as sc import yt -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - e = sc.e pi = sc.pi ep0 = sc.epsilon_0 @@ -29,34 +25,37 @@ dt = 1.4e-17 ne = 1.116e28 log = 2.0 -T_par = 5.62*e -T_per = 5.1*e - -A = 1.0 - T_per/T_par -mu = (e**4*ne*log/(8.0*pi**1.5*ep0**2*m**0.5*T_par**1.5) - *A**(-2)*(-3.0+(3.0-A)*np.arctanh(A**0.5)/A**0.5)) +T_par = 5.62 * e +T_per = 5.1 * e + +A = 1.0 - T_per / T_par +mu = ( + e**4 + * ne + * log + / (8.0 * pi**1.5 * ep0**2 * m**0.5 * T_par**1.5) + * A ** (-2) + * (-3.0 + (3.0 - A) * np.arctanh(A**0.5) / A**0.5) +) fn = sys.argv[1] ds = yt.load(fn) ad = ds.all_data() -vx = ad['electron', 'particle_momentum_x'].to_ndarray()/m -vy = ad['electron', 'particle_momentum_y'].to_ndarray()/m -Tx = np.mean(vx**2)*m/e -Ty = np.mean(vy**2)*m/e +vx = ad["electron", "particle_momentum_x"].to_ndarray() / m +vy = ad["electron", "particle_momentum_y"].to_ndarray() / m +Tx = np.mean(vx**2) * m / e +Ty = np.mean(vy**2) * m / e nt = 100 Tx0 = T_par Ty0 = T_per -for _ in range(nt-1): - Tx0 = Tx0 + dt*mu*(Ty0-Tx0)*2.0 - Ty0 = Ty0 + dt*mu*(Tx0-Ty0) +for _ in range(nt - 1): + Tx0 = Tx0 + dt * mu * (Ty0 - Tx0) * 2.0 + Ty0 = Ty0 + dt * mu * (Tx0 - Ty0) tolerance = 0.05 -error = np.maximum(abs(Tx-Tx0/e)/Tx, abs(Ty-Ty0/e)/Ty) - -print(f'error = {error}') -print(f'tolerance = {tolerance}') -assert(error < tolerance) +error = np.maximum(abs(Tx - Tx0 / e) / Tx, abs(Ty - Ty0 / e) / Ty) -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +print(f"error = {error}") +print(f"tolerance = {tolerance}") +assert error < tolerance diff --git a/Examples/Tests/collision/analysis_collision_rz.py b/Examples/Tests/collision/analysis_collision_rz.py index b206b2eba7b..b37887943f8 100755 --- a/Examples/Tests/collision/analysis_collision_rz.py +++ b/Examples/Tests/collision/analysis_collision_rz.py @@ -16,43 +16,37 @@ # tolerance: 1.0e-30 # Possible running time: ~ 1.0 s -import os import sys from glob import glob import numpy as np import yt -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - tolerance = 1.0e-15 last_fn = sys.argv[1] -if (last_fn[-1] == "/"): last_fn = last_fn[:-1] +if last_fn[-1] == "/": + last_fn = last_fn[:-1] fn_list = glob(last_fn[:-5] + "?????") for fn in fn_list: # get time index j j = int(fn[-5:]) - if j==0: + if j == 0: # load file - ds = yt.load( fn ) + ds = yt.load(fn) ad = ds.all_data() - px1 = ad['particle_momentum_x'].to_ndarray() - py1 = ad['particle_momentum_y'].to_ndarray() - if j==150: + px1 = ad["particle_momentum_x"].to_ndarray() + py1 = ad["particle_momentum_y"].to_ndarray() + if j == 150: # load file - ds = yt.load( fn ) + ds = yt.load(fn) ad = ds.all_data() - px2 = ad['particle_momentum_x'].to_ndarray() - py2 = ad['particle_momentum_y'].to_ndarray() - -error = np.max( abs(px1-px2)+abs(py1-py2) ) + px2 = ad["particle_momentum_x"].to_ndarray() + py2 = ad["particle_momentum_y"].to_ndarray() -print('error = ', error) -print('tolerance = ', tolerance) -assert(error < tolerance) +error = np.max(abs(px1 - px2) + abs(py1 - py2)) -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/collision/analysis_default_regression.py b/Examples/Tests/collision/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/collision/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/collision/inputs_1d b/Examples/Tests/collision/inputs_test_1d_collision_z similarity index 100% rename from Examples/Tests/collision/inputs_1d rename to Examples/Tests/collision/inputs_test_1d_collision_z diff --git a/Examples/Tests/collision/inputs_2d b/Examples/Tests/collision/inputs_test_2d_collision_xz similarity index 100% rename from Examples/Tests/collision/inputs_2d rename to Examples/Tests/collision/inputs_test_2d_collision_xz diff --git a/Examples/Tests/collision/inputs_test_2d_collision_xz_picmi.py b/Examples/Tests/collision/inputs_test_2d_collision_xz_picmi.py new file mode 100755 index 00000000000..7bc83a1e801 --- /dev/null +++ b/Examples/Tests/collision/inputs_test_2d_collision_xz_picmi.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# +# --- Input file for binary Coulomb collision testing. This input script +# --- runs the same test as inputs_test_2d_collision_xz but via Python, therefore the input +# --- values where directly copied from inputs_2d. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 150 + +max_grid_size = 8 +max_level = 0 +nx = max_grid_size +ny = max_grid_size + +xmin = 0 +xmax = 4.154046151855669e2 +ymin = xmin +ymax = xmax + +plasma_density = 1e21 +elec_rms_velocity = 0.044237441120300 * constants.c +ion_rms_velocity = 0.006256118919701 * constants.c +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# +serialize_initial_conditions = 1 +verbose = 1 +cfl = 1.0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +elec_dist = picmi.UniformDistribution( + density=plasma_density, + rms_velocity=[elec_rms_velocity] * 3, + directed_velocity=[elec_rms_velocity, 0.0, 0.0], +) + +ion_dist = picmi.UniformDistribution( + density=plasma_density, + rms_velocity=[ion_rms_velocity] * 3, +) + +electrons = picmi.Species( + particle_type="electron", + name="electron", + warpx_do_not_deposit=1, + initial_distribution=elec_dist, +) +ions = picmi.Species( + particle_type="H", + name="ion", + charge="q_e", + mass="5*m_e", + warpx_do_not_deposit=1, + initial_distribution=ion_dist, +) + +################################# +########## COLLISIONS ########### +################################# + +collision1 = picmi.CoulombCollisions( + name="collisions1", species=[electrons, ions], CoulombLog=15.9 +) +collision2 = picmi.CoulombCollisions( + name="collisions2", species=[electrons, electrons], CoulombLog=15.9 +) +collision3 = picmi.CoulombCollisions( + name="collisions3", species=[ions, ions], CoulombLog=15.9 +) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=max_grid_size, + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic"], +) +solver = picmi.ElectromagneticSolver(grid=grid, cfl=cfl) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic(name="diag1", period=10) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=[], +) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=serialize_initial_conditions, + warpx_collisions=[collision1, collision2, collision3], +) + +sim.add_species( + electrons, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ), +) +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ), +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +# sim.write_input_file('inputs_test_2d_collision_xz') +sim.step(max_steps) diff --git a/Examples/Tests/collision/inputs_3d_isotropization b/Examples/Tests/collision/inputs_test_3d_collision_iso similarity index 100% rename from Examples/Tests/collision/inputs_3d_isotropization rename to Examples/Tests/collision/inputs_test_3d_collision_iso diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_test_3d_collision_xyz similarity index 100% rename from Examples/Tests/collision/inputs_3d rename to Examples/Tests/collision/inputs_test_3d_collision_xyz diff --git a/Examples/Tests/collision/inputs_rz b/Examples/Tests/collision/inputs_test_rz_collision similarity index 100% rename from Examples/Tests/collision/inputs_rz rename to Examples/Tests/collision/inputs_test_rz_collision diff --git a/Examples/Tests/comoving/inputs_2d_hybrid b/Examples/Tests/comoving/inputs_2d_hybrid deleted file mode 100644 index 393e18d2077..00000000000 --- a/Examples/Tests/comoving/inputs_2d_hybrid +++ /dev/null @@ -1,115 +0,0 @@ -max_step = 400 - -amr.max_level = 0 -amr.n_cell = 48 704 -warpx.numprocs = 1 2 - -geometry.dims = 2 -geometry.prob_lo = -70.e-6 -50.e-6 -geometry.prob_hi = 70.e-6 0.e-6 - -################################# -###### Boundary Condition ####### -################################# -boundary.field_lo = periodic damped -boundary.field_hi = periodic damped - -algo.maxwell_solver = psatd -algo.current_deposition = direct -algo.charge_deposition = standard -algo.particle_pusher = vay - -# Order of particle shape factors -algo.particle_shape = 3 - -psatd.use_default_v_comoving = 1 - -warpx.cfl = 1. - -warpx.grid_type = hybrid -warpx.do_current_centering = 0 - -warpx.gamma_boost = 13. -warpx.boost_direction = z - -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 - -warpx.use_filter = 1 - -warpx.serialize_initial_conditions = 1 -warpx.verbose = 1 - -particles.species_names = electrons ions beam -particles.use_fdtd_nci_corr = 0 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 2 2 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.0001 -electrons.uy_th = 0.0001 -electrons.uz_th = 0.0001 -electrons.xmin = -45.e-6 -electrons.xmax = 45.e-6 -electrons.zmin = 0. -electrons.zmax = 15000.e-6 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -electrons.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 2 2 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -45.e-6 -ions.xmax = 45.e-6 -ions.zmin = 0. -ions.zmax = 15000.e-6 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -ions.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 -ions.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 2.e-6 -beam.y_rms = 2.e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -42.e-6 -beam.npart = 1000 -beam.q_tot = -1.e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 200. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 0. -beam.zinject_plane = 0.5e-3 -beam.rigid_advance = true - -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. -0.1e-6 -laser1.direction = 0. 0. 1. -laser1.polarization = 0. 1. 0. -laser1.e_max = 6.e12 -laser1.profile_waist = 30.e-6 -laser1.profile_duration = 26.e-15 -laser1.profile_t_peak = 66.e-15 -laser1.profile_focal_distance = 0.5e-3 -laser1.wavelength = 0.81e-6 - -diagnostics.diags_names = diag1 -diag1.intervals = 400 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Tests/diff_lumi_diag/CMakeLists.txt b/Examples/Tests/diff_lumi_diag/CMakeLists.txt new file mode 100644 index 00000000000..9a4e58d0e62 --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/CMakeLists.txt @@ -0,0 +1,25 @@ +# Add tests (alphabetical order) ############################################## +# +if(WarpX_FFT) +add_warpx_test( + test_3d_diff_lumi_diag_leptons # name + 3 # dims + 2 # nprocs + inputs_test_3d_diff_lumi_diag_leptons # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000080 --rtol 1e-2" # checksum + OFF # dependency +) +endif() + +if(WarpX_FFT) +add_warpx_test( + test_3d_diff_lumi_diag_photons # name + 3 # dims + 2 # nprocs + inputs_test_3d_diff_lumi_diag_photons # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000080 --rtol 1e-2" # checksum + OFF # dependency +) +endif() diff --git a/Examples/Tests/diff_lumi_diag/analysis.py b/Examples/Tests/diff_lumi_diag/analysis.py new file mode 100755 index 00000000000..f8ed5f79779 --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/analysis.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# This test checks the differential luminosity of the beam-beam interaction +# in the case of two Gaussian beams crossing rigidly at the interaction point. +# The beams have a Gaussian distribution both in energy and in transverse positions. +# In that case, the differential luminosity can be calculated analytically. + +import os +import re + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries + +# Extract the 1D differential luminosity from the file +filename = "./diags/reducedfiles/DifferentialLuminosity_beam1_beam2.txt" +with open(filename) as f: + # First line: header, contains the energies + line = f.readline() + E_bin = np.array(list(map(float, re.findall("=(.*?)\(", line)))) +data = np.loadtxt(filename) +dE_bin = E_bin[1] - E_bin[0] +dL_dE_sim = data[-1, 2:] # Differential luminosity at the end of the simulation + +# Beam parameters +N = 1.2e10 +E_beam = 125e9 # in eV +sigma_x = 500e-9 +sigma_y = 10e-9 + +# Compute the analytical differential luminosity for 2 Gaussian beams +sigma_E1 = 0.02 * E_beam +sigma_E2 = 0.03 * E_beam +sigma_E = np.sqrt( + sigma_E1**2 + sigma_E2**2 +) # Expected width of the differential luminosity +dL_dE_th = ( + N**2 + / (2 * (2 * np.pi) ** 1.5 * sigma_x * sigma_y * sigma_E) + * np.exp(-((E_bin - 2 * E_beam) ** 2) / (2 * sigma_E**2)) +) + +# Extract the 2D differential luminosity from the file +series = OpenPMDTimeSeries("./diags/reducedfiles/DifferentialLuminosity2d_beam1_beam2/") +d2L_dE1_dE2_sim, info = series.get_field("d2L_dE1_dE2", iteration=80) + +# Compute the analytical 2D differential luminosity for 2 Gaussian beams +assert info.axes[0] == "E2" +assert info.axes[1] == "E1" +E2, E1 = np.meshgrid(info.E2, info.E1, indexing="ij") +d2L_dE1_dE2_th = ( + N**2 + / (2 * (2 * np.pi) ** 2 * sigma_x * sigma_y * sigma_E1 * sigma_E2) + * np.exp( + -((E1 - E_beam) ** 2) / (2 * sigma_E1**2) + - (E2 - E_beam) ** 2 / (2 * sigma_E2**2) + ) +) + +# Extract test name from path +test_name = os.path.split(os.getcwd())[1] +print("test_name", test_name) + +# Pick tolerance +if "leptons" in test_name: + tol1 = 0.02 + tol2 = 0.04 +elif "photons" in test_name: + # In the photons case, the particles are + # initialized from a density distribution ; + # tolerance is larger due to lower particle statistics + tol1 = 0.021 + tol2 = 0.06 + +# Check that the 1D diagnostic and analytical result match +error1 = abs(dL_dE_sim - dL_dE_th).max() / abs(dL_dE_th).max() +print("Relative error: ", error1) +print("Tolerance: ", tol1) + +# Check that the 2D and 1D diagnostics match +error2 = abs(d2L_dE1_dE2_sim - d2L_dE1_dE2_th).max() / abs(d2L_dE1_dE2_th).max() +print("Relative error: ", error2) +print("Tolerance: ", tol2) + +assert error1 < tol1 +assert error2 < tol2 diff --git a/Examples/Tests/diff_lumi_diag/analysis_default_regression.py b/Examples/Tests/diff_lumi_diag/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/diff_lumi_diag/inputs_base_3d b/Examples/Tests/diff_lumi_diag/inputs_base_3d new file mode 100644 index 00000000000..0c65850e82b --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/inputs_base_3d @@ -0,0 +1,114 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2_eV = m_e*clight*clight/q_e + +# BEAMS +my_constants.beam_energy_eV = 125.e9 +my_constants.beam_gamma = beam_energy_eV/(mc2_eV) +my_constants.beam_N = 1.2e10 +my_constants.sigmax = 500e-9 +my_constants.sigmay = 10e-9 +my_constants.sigmaz = 300e-3 +my_constants.muz = 4*sigmaz + +# BOX +my_constants.Lx = 8*sigmax +my_constants.Ly = 8*sigmay +my_constants.Lz = 16*sigmaz + +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 128 + +# TIME +my_constants.T = 0.5*Lz/clight +my_constants.dt = sigmaz/clight/10. + +################################# +####### GENERAL PARAMETERS ###### +################################# + +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = open open open +boundary.field_hi = open open open +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay +warpx.poisson_solver = fft + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 + +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_gamma +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = 0 +beam1.uy_th = 0 +beam1.uz_th = 0.02*beam_gamma +beam1.do_not_deposit = 1 + +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_gamma +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = 0 +beam2.uy_th = 0 +beam2.uz_th = 0.03*beam_gamma +beam2.do_not_deposit = 1 + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 80 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = rho_beam1 rho_beam2 +diag1.dump_last_timestep = 1 +diag1.species = beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = DifferentialLuminosity_beam1_beam2 DifferentialLuminosity2d_beam1_beam2 + +DifferentialLuminosity_beam1_beam2.type = DifferentialLuminosity +DifferentialLuminosity_beam1_beam2.intervals = 80 +DifferentialLuminosity_beam1_beam2.species = beam1 beam2 +DifferentialLuminosity_beam1_beam2.bin_number = 128 +DifferentialLuminosity_beam1_beam2.bin_max = 2.1*beam_energy_eV +DifferentialLuminosity_beam1_beam2.bin_min = 0 + +DifferentialLuminosity2d_beam1_beam2.type = DifferentialLuminosity2D +DifferentialLuminosity2d_beam1_beam2.intervals = 80 +DifferentialLuminosity2d_beam1_beam2.species = beam1 beam2 +DifferentialLuminosity2d_beam1_beam2.bin_number_1 = 128 +DifferentialLuminosity2d_beam1_beam2.bin_max_1 = 1.45*beam_energy_eV +DifferentialLuminosity2d_beam1_beam2.bin_min_1 = 0 +DifferentialLuminosity2d_beam1_beam2.bin_number_2 = 128 +DifferentialLuminosity2d_beam1_beam2.bin_max_2 = 1.45*beam_energy_eV +DifferentialLuminosity2d_beam1_beam2.bin_min_2 = 0 diff --git a/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_leptons b/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_leptons new file mode 100644 index 00000000000..1cded30d3af --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_leptons @@ -0,0 +1,31 @@ +# base input parameters +FILE = inputs_base_3d + +# Test with electrons/positrons: use gaussian beam distribution +# by providing the total charge (q_tot) + +my_constants.nmacropart = 2e5 + +beam1.species_type = electron +beam1.injection_style = gaussian_beam +beam1.x_rms = sigmax +beam1.y_rms = sigmay +beam1.z_rms = sigmaz +beam1.x_m = 0 +beam1.y_m = 0 +beam1.z_m = -muz +beam1.npart = nmacropart +beam1.q_tot = -beam_N*q_e +beam1.z_cut = 4 + +beam2.species_type = positron +beam2.injection_style = gaussian_beam +beam2.x_rms = sigmax +beam2.y_rms = sigmay +beam2.z_rms = sigmaz +beam2.x_m = 0 +beam2.y_m = 0 +beam2.z_m = muz +beam2.npart = nmacropart +beam2.q_tot = beam_N*q_e +beam2.z_cut = 4 diff --git a/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_photons b/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_photons new file mode 100644 index 00000000000..90d66938a4a --- /dev/null +++ b/Examples/Tests/diff_lumi_diag/inputs_test_3d_diff_lumi_diag_photons @@ -0,0 +1,30 @@ +# base input parameters +FILE = inputs_base_3d + +# Test with electrons/positrons: use parse_density_function + +particles.photon_species = beam1 beam2 + +beam1.species_type = photon +beam1.injection_style = "NUniformPerCell" +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "beam_N/(sqrt(2*pi)*2*pi*sigmax*sigmay*sigmaz)*exp(-x*x/(2*sigmax*sigmax)-y*y/(2*sigmay*sigmay)-(z+muz)*(z+muz)/(2*sigmaz*sigmaz))" +beam1.xmin = -4*sigmax +beam1.xmax = 4*sigmax +beam1.ymin = -4*sigmay +beam1.ymax = 4*sigmay +beam1.zmin =-muz-4*sigmaz +beam1.zmax =-muz+4*sigmaz + +beam2.species_type = photon +beam2.injection_style = "NUniformPerCell" +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.xmin = -4*sigmax +beam2.xmax = 4*sigmax +beam2.ymin = -4*sigmay +beam2.ymax = 4*sigmay +beam2.zmin = muz-4*sigmaz +beam2.zmax = muz+4*sigmaz +beam2.density_function(x,y,z) = "beam_N/(sqrt(2*pi)*2*pi*sigmax*sigmay*sigmaz)*exp(-x*x/(2*sigmax*sigmax)-y*y/(2*sigmay*sigmay)-(z-muz)*(z-muz)/(2*sigmaz*sigmaz))" diff --git a/Examples/Tests/divb_cleaning/CMakeLists.txt b/Examples/Tests/divb_cleaning/CMakeLists.txt new file mode 100644 index 00000000000..d851a7ca322 --- /dev/null +++ b/Examples/Tests/divb_cleaning/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_divb_cleaning # name + 3 # dims + 2 # nprocs + inputs_test_3d_divb_cleaning # inputs + "analysis.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400" # checksum + OFF # dependency +) diff --git a/Examples/Tests/divb_cleaning/analysis.py b/Examples/Tests/divb_cleaning/analysis.py index a3523218ca6..6fcd8f6f755 100755 --- a/Examples/Tests/divb_cleaning/analysis.py +++ b/Examples/Tests/divb_cleaning/analysis.py @@ -8,33 +8,34 @@ import sys -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import os - import numpy as np import yt yt.funcs.mylog.setLevel(50) -import re -import checksumAPI from scipy.constants import c # Name of the last plotfile fn = sys.argv[1] # Load yt data -ds_old = yt.load('divb_cleaning_3d_plt000398') -ds_mid = yt.load('divb_cleaning_3d_plt000399') -ds_new = yt.load(fn) # this is the last plotfile - -ad_old = ds_old.covering_grid(level = 0, left_edge = ds_old.domain_left_edge, dims = ds_old.domain_dimensions) -ad_mid = ds_mid.covering_grid(level = 0, left_edge = ds_mid.domain_left_edge, dims = ds_mid.domain_dimensions) -ad_new = ds_new.covering_grid(level = 0, left_edge = ds_new.domain_left_edge, dims = ds_new.domain_dimensions) - -G_old = ad_old['boxlib', 'G'].v.squeeze() -G_new = ad_new['boxlib', 'G'].v.squeeze() -divB = ad_mid['boxlib', 'divB'].v.squeeze() +ds_old = yt.load("diags/diag1000398") +ds_mid = yt.load("diags/diag1000399") +ds_new = yt.load(fn) # this is the last plotfile + +ad_old = ds_old.covering_grid( + level=0, left_edge=ds_old.domain_left_edge, dims=ds_old.domain_dimensions +) +ad_mid = ds_mid.covering_grid( + level=0, left_edge=ds_mid.domain_left_edge, dims=ds_mid.domain_dimensions +) +ad_new = ds_new.covering_grid( + level=0, left_edge=ds_new.domain_left_edge, dims=ds_new.domain_dimensions +) + +G_old = ad_old["boxlib", "G"].v.squeeze() +G_new = ad_new["boxlib", "G"].v.squeeze() +divB = ad_mid["boxlib", "divB"].v.squeeze() # Check max norm of error on c2 * div(B) = dG/dt # (the time interval between old and new is 2*dt) @@ -45,11 +46,4 @@ rel_error = np.amax(abs(x - y)) / np.amax(abs(y)) tolerance = 1e-1 -assert(rel_error < tolerance) - -test_name = os.path.split(os.getcwd())[1] - -if re.search('single_precision', fn): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +assert rel_error < tolerance diff --git a/Examples/Tests/divb_cleaning/analysis_default_regression.py b/Examples/Tests/divb_cleaning/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/divb_cleaning/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/divb_cleaning/inputs_3d b/Examples/Tests/divb_cleaning/inputs_test_3d_divb_cleaning similarity index 100% rename from Examples/Tests/divb_cleaning/inputs_3d rename to Examples/Tests/divb_cleaning/inputs_test_3d_divb_cleaning diff --git a/Examples/Tests/dive_cleaning/CMakeLists.txt b/Examples/Tests/dive_cleaning/CMakeLists.txt new file mode 100644 index 00000000000..c5fe87baad0 --- /dev/null +++ b/Examples/Tests/dive_cleaning/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_dive_cleaning # name + 2 # dims + 2 # nprocs + inputs_test_2d_dive_cleaning # inputs + "analysis.py diags/diag1000128" # analysis + "analysis_default_regression.py --path diags/diag1000128" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_dive_cleaning # name + 3 # dims + 2 # nprocs + inputs_test_3d_dive_cleaning # inputs + "analysis.py diags/diag1000128" # analysis + "analysis_default_regression.py --path diags/diag1000128" # checksum + OFF # dependency +) diff --git a/Examples/Tests/dive_cleaning/analysis.py b/Examples/Tests/dive_cleaning/analysis.py index a9b52455baa..9d92767fa05 100755 --- a/Examples/Tests/dive_cleaning/analysis.py +++ b/Examples/Tests/dive_cleaning/analysis.py @@ -14,11 +14,12 @@ This script verifies that the field at the end of the simulation corresponds to the theoretical field of a Gaussian beam. """ + import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc @@ -28,88 +29,101 @@ yt.funcs.mylog.setLevel(0) # Parameters from the Simulation -Qtot = -1.e-20 -r0 = 2.e-6 +Qtot = -1.0e-20 +r0 = 2.0e-6 # Open data file filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. -if 'force_periodicity' in dir(ds): ds.force_periodicity() +if "force_periodicity" in dir(ds): + ds.force_periodicity() # Extract data -ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) Ex_array = ad0[("mesh", "Ex")].to_ndarray().squeeze() if ds.dimensionality == 2: # Rename the z dimension as y, so as to make this script work for 2d and 3d Ey_array = ad0[("mesh", "Ez")].to_ndarray().squeeze() - E_array = ( Ex_array**2 + Ey_array**2 )**.5 + E_array = (Ex_array**2 + Ey_array**2) ** 0.5 relative_tolerance = 0.1 elif ds.dimensionality == 3: Ey_array = ad0[("mesh", "Ey")].to_ndarray() Ez_array = ad0[("mesh", "Ez")].to_ndarray() - E_array = ( Ex_array**2 + Ey_array**2 + Ez_array**2 )**.5 + E_array = (Ex_array**2 + Ey_array**2 + Ez_array**2) ** 0.5 relative_tolerance = 0.165 # Extract grid coordinates -Nx, Ny, Nz = ds.domain_dimensions +Nx, Ny, Nz = ds.domain_dimensions xmin, ymin, zmin = ds.domain_left_edge.v Lx, Ly, Lz = ds.domain_width.v -x = xmin + Lx/Nx*(0.5+np.arange(Nx)) -y = ymin + Ly/Ny*(0.5+np.arange(Ny)) -z = zmin + Lz/Nz*(0.5+np.arange(Nz)) +x = xmin + Lx / Nx * (0.5 + np.arange(Nx)) +y = ymin + Ly / Ny * (0.5 + np.arange(Ny)) +z = zmin + Lz / Nz * (0.5 + np.arange(Nz)) # Compute theoretical field if ds.dimensionality == 2: - x_2d, y_2d = np.meshgrid(x, y, indexing='ij') + x_2d, y_2d = np.meshgrid(x, y, indexing="ij") r2 = x_2d**2 + y_2d**2 - factor = (Qtot/r0)/(2*np.pi*scc.epsilon_0*r2) * (1-np.exp(-r2/(2*r0**2))) + factor = ( + (Qtot / r0) / (2 * np.pi * scc.epsilon_0 * r2) * (1 - np.exp(-r2 / (2 * r0**2))) + ) Ex_th = x_2d * factor Ey_th = y_2d * factor - E_th = ( Ex_th**2 + Ey_th**2 )**.5 + E_th = (Ex_th**2 + Ey_th**2) ** 0.5 elif ds.dimensionality == 3: - x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') + x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing="ij") r2 = x_2d**2 + y_2d**2 + z_2d**2 - factor = Qtot/(4*np.pi*scc.epsilon_0*r2**1.5) * gammainc(3./2, r2/(2.*r0**2)) - Ex_th = factor*x_2d - Ey_th = factor*y_2d - Ez_th = factor*z_2d - E_th = ( Ex_th**2 + Ey_th**2 + Ez_th**2 )**.5 + factor = ( + Qtot + / (4 * np.pi * scc.epsilon_0 * r2**1.5) + * gammainc(3.0 / 2, r2 / (2.0 * r0**2)) + ) + Ex_th = factor * x_2d + Ey_th = factor * y_2d + Ez_th = factor * z_2d + E_th = (Ex_th**2 + Ey_th**2 + Ez_th**2) ** 0.5 + # Plot theory and data def make_2d(arr): if arr.ndim == 3: - return arr[:,:,Nz//2] + return arr[:, :, Nz // 2] else: return arr -plt.figure(figsize=(10,10)) + + +plt.figure(figsize=(10, 10)) plt.subplot(221) -plt.title('E: Theory') +plt.title("E: Theory") plt.imshow(make_2d(E_th)) plt.colorbar() plt.subplot(222) -plt.title('E: Simulation') +plt.title("E: Simulation") plt.imshow(make_2d(E_array)) plt.colorbar() plt.subplot(223) -plt.title('E: Diff') -plt.imshow(make_2d(E_th-E_array)) +plt.title("E: Diff") +plt.imshow(make_2d(E_th - E_array)) plt.colorbar() plt.subplot(224) -plt.title('E: Relative diff') -plt.imshow(make_2d((E_th-E_array)/E_th)) +plt.title("E: Relative diff") +plt.imshow(make_2d((E_th - E_array) / E_th)) plt.colorbar() -plt.savefig('Comparison.png') +plt.savefig("Comparison.png") + # Automatically check the results def check(E, E_th, label): - print( 'Relative error in %s: %.3f'%( - label, abs(E-E_th).max()/E_th.max())) - assert np.allclose( E, E_th, atol=relative_tolerance*E_th.max() ) + print("Relative error in %s: %.3f" % (label, abs(E - E_th).max() / E_th.max())) + assert np.allclose(E, E_th, atol=relative_tolerance * E_th.max()) + -check( Ex_array, Ex_th, 'Ex' ) -check( Ey_array, Ey_th, 'Ey' ) +check(Ex_array, Ex_th, "Ex") +check(Ey_array, Ey_th, "Ey") if ds.dimensionality == 3: - check( Ez_array, Ez_th, 'Ez' ) + check(Ez_array, Ez_th, "Ez") diff --git a/Examples/Tests/dive_cleaning/analysis_default_regression.py b/Examples/Tests/dive_cleaning/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/dive_cleaning/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/dive_cleaning/inputs_3d b/Examples/Tests/dive_cleaning/inputs_3d deleted file mode 100644 index c3f83ddbdd9..00000000000 --- a/Examples/Tests/dive_cleaning/inputs_3d +++ /dev/null @@ -1,35 +0,0 @@ -max_step = 128 -amr.n_cell = 64 64 64 -amr.max_grid_size = 32 -amr.max_level = 0 - -geometry.dims = 3 -geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 -geometry.prob_hi = 50.e-6 50.e-6 50.e-6 - -boundary.field_lo = pml pml pml -boundary.field_hi = pml pml pml - -warpx.do_dive_cleaning = 1 -warpx.use_filter = 0 - -# Order of particle shape factors -algo.particle_shape = 1 - -particles.species_names = beam -beam.charge = -q_e -beam.mass = 1.e30 -beam.injection_style = "gaussian_beam" -beam.x_rms = 2.e-6 -beam.y_rms = 2.e-6 -beam.z_rms = 2.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = 0.e-6 -beam.npart = 20000 -beam.q_tot = -1.e-20 -beam.momentum_distribution_type = "at_rest" - -diagnostics.diags_names = diag1 -diag1.intervals = 8 -diag1.diag_type = Full diff --git a/Examples/Tests/dive_cleaning/inputs_test_2d_dive_cleaning b/Examples/Tests/dive_cleaning/inputs_test_2d_dive_cleaning new file mode 100644 index 00000000000..48cabee4495 --- /dev/null +++ b/Examples/Tests/dive_cleaning/inputs_test_2d_dive_cleaning @@ -0,0 +1,35 @@ +max_step = 128 +amr.n_cell = 64 64 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.prob_lo = -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 +geometry.dims = 2 + +boundary.field_lo = pml pml +boundary.field_hi = pml pml + +warpx.do_dive_cleaning = 1 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = beam +beam.charge = -q_e +beam.mass = 1.e30 +beam.injection_style = "gaussian_beam" +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "at_rest" + +diagnostics.diags_names = diag1 +diag1.intervals = 8 +diag1.diag_type = Full diff --git a/Examples/Tests/dive_cleaning/inputs_test_3d_dive_cleaning b/Examples/Tests/dive_cleaning/inputs_test_3d_dive_cleaning new file mode 100644 index 00000000000..3f22a2206cf --- /dev/null +++ b/Examples/Tests/dive_cleaning/inputs_test_3d_dive_cleaning @@ -0,0 +1,35 @@ +max_step = 128 +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 50.e-6 +geometry.dims = 3 + +boundary.field_lo = pml pml pml +boundary.field_hi = pml pml pml + +warpx.do_dive_cleaning = 1 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = beam +beam.charge = -q_e +beam.mass = 1.e30 +beam.injection_style = "gaussian_beam" +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "at_rest" + +diagnostics.diags_names = diag1 +diag1.intervals = 8 +diag1.diag_type = Full diff --git a/Examples/Tests/effective_potential_electrostatic/CMakeLists.txt b/Examples/Tests/effective_potential_electrostatic/CMakeLists.txt new file mode 100644 index 00000000000..528ee6d1e08 --- /dev/null +++ b/Examples/Tests/effective_potential_electrostatic/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_effective_potential_electrostatic_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_effective_potential_electrostatic_picmi.py # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/field_diag/" # checksum + OFF # dependency +) diff --git a/Examples/Tests/effective_potential_electrostatic/analysis.py b/Examples/Tests/effective_potential_electrostatic/analysis.py new file mode 100755 index 00000000000..20998e0a066 --- /dev/null +++ b/Examples/Tests/effective_potential_electrostatic/analysis.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# +# --- Analysis script for the effective potential Poisson solver test. This test +# --- is based on the adiabatic plasma expansion benchmark from Connor et al. (2021) +# --- doi.org/10.1109/TPS.2021.3072353. +# --- The electron density distribution (as a function of radius) is compared +# --- with the analytically calculated density based on the input parameters +# --- of the test simulation at each output timestep. + +import dill +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.interpolate import RegularGridInterpolator + +from pywarpx import picmi + +constants = picmi.constants + +# load simulation parameters +with open("sim_parameters.dpkl", "rb") as f: + sim = dill.load(f) + +# characteristic expansion time +tau = sim["sigma_0"] * np.sqrt(sim["M"] / (constants.kb * (sim["T_e"] + sim["T_i"]))) + + +def get_analytic_density(r, t): + expansion_factor = 1.0 + t**2 / tau**2 + T = sim["T_e"] / expansion_factor + sigma = sim["sigma_0"] * np.sqrt(expansion_factor) + return ( + sim["n_plasma"] * (T / sim["T_e"]) ** 1.5 * np.exp(-(r**2) / (2.0 * sigma**2)) + ) + + +def get_radial_function(field, info): + """Helper function to transform Cartesian data to polar data and average + over theta and phi.""" + r_grid = np.linspace(0, np.max(info.z) - 1e-3, 30) + theta_grid = np.linspace(0, 2 * np.pi, 40, endpoint=False) + phi_grid = np.linspace(0, np.pi, 20) + + r, t, p = np.meshgrid(r_grid, theta_grid, phi_grid, indexing="ij") + x_sp = r * np.sin(p) * np.cos(t) + y_sp = r * np.sin(p) * np.sin(t) + z_sp = r * np.cos(p) + + interp = RegularGridInterpolator((info.x, info.y, info.z), field) + field_sp = interp((x_sp, y_sp, z_sp)) + return r_grid, np.mean(field_sp, axis=(1, 2)) + + +diag_dir = "diags/field_diag" +ts = OpenPMDTimeSeries(diag_dir, check_all_files=True) + +rms_errors = np.zeros(len(ts.iterations)) + +for ii, it in enumerate(ts.iterations): + rho_e, info = ts.get_field(field="rho_electrons", iteration=it) + r_grid, n_e = get_radial_function(-rho_e / constants.q_e, info) + + n_e_analytic = get_analytic_density(r_grid, ts.t[ii]) + rms_errors[ii] = ( + np.sqrt(np.mean(np.sum((n_e - n_e_analytic) ** 2))) / n_e_analytic[0] + ) + + plt.plot(r_grid, n_e_analytic, "k--", alpha=0.6) + plt.plot(r_grid, n_e, label=f"t = {ts.t[ii] * 1e6:.2f} $\mu$s") + +print("RMS error (%) in density: ", rms_errors) +assert np.all(rms_errors < 0.05) + +plt.ylabel("$n_e$ (m$^{-3}$)") +plt.xlabel("r (m)") +plt.grid() +plt.legend() +plt.show() diff --git a/Examples/Tests/effective_potential_electrostatic/analysis_default_regression.py b/Examples/Tests/effective_potential_electrostatic/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/effective_potential_electrostatic/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/effective_potential_electrostatic/inputs_test_3d_effective_potential_electrostatic_picmi.py b/Examples/Tests/effective_potential_electrostatic/inputs_test_3d_effective_potential_electrostatic_picmi.py new file mode 100644 index 00000000000..018739aa682 --- /dev/null +++ b/Examples/Tests/effective_potential_electrostatic/inputs_test_3d_effective_potential_electrostatic_picmi.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# --- Test script for the effective potential Poisson solver. This test is based +# --- on the adiabatic plasma expansion benchmark from Connor et al. (2021) +# --- doi.org/10.1109/TPS.2021.3072353. +# --- In the benchmark an expanding plasma ball with Gaussian density distribution +# --- in the radial direction is simulated and the time evolution of the +# --- density of the electron species is compared to an approximate analytic solution. +# --- The example is modified slightly in the following ways: +# --- 1) The original example used an electromagnetic solver with absorbing +# --- boundaries while the present case encloses the plasma in a conducting +# --- sphere. +# --- 2) The domain and plasma parameters for this case has been modified to +# --- set the cell-size and time step such that the explicit electrostatic +# --- solver is unstable. + +import dill +import numpy as np +from mpi4py import MPI as mpi +from scipy.special import erf + +from pywarpx import picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=0) + +m_ion = 25 # Ion mass (electron masses) + +# Plasma parameters +n_plasma = 5e12 # Plasma density (m^-3) +sigma_0 = 22 # Initial Gaussian distribution standard deviation (Debye lengths) +T_e = 100.0 # Electron temperature (K) +T_i = 10.0 # Ion temperature (K) + +# Spatial domain +R = 86 # Radius of metallic sphere (Debye lengths) +NZ = 72 # Number of cells in each direction + +# Temporal domain (if not run as a CI test) +LT = 0.6e-6 # Simulation temporal length (s) + +# Numerical parameters +NPARTS = 500000 # Seed number of particles +DT = 0.8 # Time step (electron streaming) + +# Solver parameter +C_EP = 1.0 # Effective potential factor + +####################################################################### +# Calculate various plasma parameters based on the simulation input # +####################################################################### + +# Ion mass (kg) +M = m_ion * constants.m_e + +# Electron plasma frequency (Hz) +f_pe = np.sqrt(constants.q_e**2 * n_plasma / (constants.m_e * constants.ep0)) / ( + 2.0 * np.pi +) + +# Debye length (m) +lambda_e = np.sqrt(constants.ep0 * constants.kb * T_e / (n_plasma * constants.q_e**2)) + +# Thermal velocities (m/s) from v_th = np.sqrt(kT / m) +v_ti = np.sqrt(T_i * constants.kb / M) +v_te = np.sqrt(T_e * constants.kb / constants.m_e) + +R *= lambda_e +sigma_0 *= lambda_e + +dz = 2.0 * R / (NZ - 4) +Lz = dz * NZ +dt = DT * dz / v_te + +total_steps = int(LT / dt) +diag_steps = total_steps // 3 +total_steps = diag_steps * 3 + +# dump attributes needed for analysis to a dill pickle file +if comm.rank == 0: + parameter_dict = { + "sigma_0": sigma_0, + "M": M, + "T_i": T_i, + "T_e": T_e, + "n_plasma": n_plasma, + } + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(parameter_dict, f) + +# print out plasma parameters +if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT_e = {T_e:.1f} K\n" + f"\tT_i = {T_i:.1f} K\n" + f"\tn = {n_plasma:.1e} m^-3\n" + ) + print( + f"Plasma parameters:\n" + f"\tlambda_e = {lambda_e:.1e} m\n" + f"\tt_pe = {1.0 / f_pe:.1e} s\n" + f"\tv_ti = {v_ti:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz/lambda_e = {dz / lambda_e:.2f}\n" + f"\tdt*w_pe = {dt * f_pe * 2.0 * np.pi:.2f}\n" + f"\tdiag steps = {diag_steps:d}\n" + f"\ttotal steps = {total_steps:d}\n" + ) + + +####################################################################### +# Set geometry and boundary conditions # +####################################################################### + +grid = picmi.Cartesian3DGrid( + number_of_cells=[NZ] * 3, + lower_bound=[-Lz / 2.0] * 3, + upper_bound=[Lz / 2.0] * 3, + lower_boundary_conditions=["neumann"] * 3, + upper_boundary_conditions=["neumann"] * 3, + lower_boundary_conditions_particles=["absorbing"] * 3, + upper_boundary_conditions_particles=["absorbing"] * 3, + warpx_max_grid_size=NZ // 2, +) +simulation.time_step_size = dt +simulation.max_steps = total_steps +simulation.current_deposition_algo = "direct" +simulation.particle_shape = 1 +simulation.verbose = 1 + +####################################################################### +# Insert spherical boundary as EB # +####################################################################### + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function=f"(x**2+y**2+z**2-{R**2})", + potential=0.0, +) +simulation.embedded_boundary = embedded_boundary + +####################################################################### +# Field solver # +####################################################################### + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + warpx_effective_potential=True, + warpx_effective_potential_factor=C_EP, + warpx_self_fields_verbosity=0, +) +simulation.solver = solver + +####################################################################### +# Particle types setup # +####################################################################### + +total_parts = ( + n_plasma + * sigma_0**2 + * ( + (2.0 * np.pi) ** 1.5 * sigma_0 * erf(R / (np.sqrt(2) * sigma_0)) + + 4.0 * np.pi * R * np.exp(-(R**2) / (2.0 * sigma_0**2)) + ) +) + +electrons = picmi.Species( + name="electrons", + particle_type="electron", + initial_distribution=picmi.GaussianBunchDistribution( + n_physical_particles=total_parts, + rms_bunch_size=[sigma_0] * 3, + rms_velocity=[v_te] * 3, + ), +) +simulation.add_species( + electrons, + layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=NPARTS), +) + +ions = picmi.Species( + name="ions", + charge="q_e", + mass=M, + initial_distribution=picmi.GaussianBunchDistribution( + n_physical_particles=total_parts, + rms_bunch_size=[sigma_0] * 3, + rms_velocity=[v_ti] * 3, + ), +) +simulation.add_species( + ions, + layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=NPARTS), +) + +####################################################################### +# Add diagnostics # +####################################################################### + +field_diag = picmi.FieldDiagnostic( + name="field_diag", + grid=grid, + period=diag_steps, + data_list=[ + "E", + "J", + "T_electrons", + "T_ions", + "phi", + "rho_electrons", + "rho_ions", + ], + write_dir="diags", + warpx_format="openpmd", + warpx_openpmd_backend="h5", +) +simulation.add_diagnostic(field_diag) + +####################################################################### +# Initialize simulation # +####################################################################### + +simulation.initialize_inputs() +simulation.initialize_warpx() + +####################################################################### +# Execute simulation # +####################################################################### + +simulation.step() diff --git a/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt b/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt new file mode 100644 index 00000000000..039181096a8 --- /dev/null +++ b/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_dirichlet_bc # name + 2 # dims + 2 # nprocs + inputs_test_2d_dirichlet_bc # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_dirichlet_bc_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_dirichlet_bc_picmi.py # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) diff --git a/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py b/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py deleted file mode 100755 index e4dd530c3bc..00000000000 --- a/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -########################## -# physics parameters -########################## - -V_xmin = "150.0*sin(2*pi*6.78e6*t)" -V_xmax = "450.0*sin(2*pi*13.56e6*t)" - - -########################## -# numerics parameters -########################## - -dt = 7.5e-10 - -# --- Nb time steps - -max_steps = 100 - -# --- grid - -nx = 64 -ny = 8 - -xmin = 0 -xmax = 0.032 -ymin = 0 -ymax = 0.004 - - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - warpx_potential_lo_x = V_xmin, - warpx_potential_hi_x = V_xmax, - moving_window_velocity = None, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6 -) - - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 4, - write_dir = '.', - warpx_file_prefix = 'Python_dirichletbc_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 4, - data_list = ['phi'], - write_dir = '.', - warpx_file_prefix = 'Python_dirichletbc_plt' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - particle_shape = None, - verbose = 0 -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -########################## -# simulation run -########################## - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step(max_steps) diff --git a/Examples/Tests/electrostatic_dirichlet_bc/analysis.py b/Examples/Tests/electrostatic_dirichlet_bc/analysis.py index eae2f17243c..82fe061c3a8 100755 --- a/Examples/Tests/electrostatic_dirichlet_bc/analysis.py +++ b/Examples/Tests/electrostatic_dirichlet_bc/analysis.py @@ -18,9 +18,7 @@ import numpy as np import yt -files = sorted(glob.glob('dirichletbc_plt*'))[1:] -if len(files) == 0: - files = sorted(glob.glob('Python_dirichletbc_plt*'))[1:] +files = sorted(glob.glob("diags/diag1*"))[1:] assert len(files) > 0 times = np.ones(len(files)) @@ -28,15 +26,13 @@ potentials_hi = np.zeros(len(files)) for ii, file in enumerate(files): - ds = yt.load( file ) - times[ii] = ( - ds.current_time.item() - ) + ds = yt.load(file) + times[ii] = ds.current_time.item() data = ds.covering_grid( level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions ) - potentials_lo[ii] = np.mean(data['phi'].to_ndarray()[0]) - potentials_hi[ii] = np.mean(data['phi'].to_ndarray()[-1]) + potentials_lo[ii] = np.mean(data["phi"].to_ndarray()[0]) + potentials_hi[ii] = np.mean(data["phi"].to_ndarray()[-1]) expected_potentials_lo = 150.0 * np.sin(2.0 * np.pi * 6.78e6 * times) expected_potentials_hi = 450.0 * np.sin(2.0 * np.pi * 13.56e6 * times) diff --git a/Examples/Tests/electrostatic_dirichlet_bc/analysis_default_regression.py b/Examples/Tests/electrostatic_dirichlet_bc/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/electrostatic_dirichlet_bc/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/electrostatic_dirichlet_bc/inputs_2d b/Examples/Tests/electrostatic_dirichlet_bc/inputs_2d deleted file mode 100644 index d501dac7d0c..00000000000 --- a/Examples/Tests/electrostatic_dirichlet_bc/inputs_2d +++ /dev/null @@ -1,22 +0,0 @@ -max_step = 100 -warpx.verbose = 0 -warpx.const_dt = 7.5e-10 -warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1e-06 -warpx.use_filter = 0 -amr.n_cell = 64 8 -amr.max_grid_size = 32 -amr.max_level = 0 - -geometry.dims = 2 -geometry.prob_lo = 0.0 0.0 -geometry.prob_hi = 0.032 0.004 -boundary.field_lo = pec periodic -boundary.field_hi = pec periodic -boundary.potential_lo_x = 150.0*sin(2*pi*6.78e+06*t) -boundary.potential_hi_x = 450.0*sin(2*pi*13.56e+06*t) - -diagnostics.diags_names = diag1 -diag1.diag_type = Full -diag1.intervals = ::4 -diag1.fields_to_plot = phi diff --git a/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc b/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc new file mode 100644 index 00000000000..46b00819926 --- /dev/null +++ b/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc @@ -0,0 +1,23 @@ +max_step = 100 +warpx.verbose = 0 +warpx.abort_on_warning_threshold = medium +warpx.const_dt = 7.5e-10 +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1e-06 +warpx.use_filter = 0 +amr.n_cell = 64 8 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 0.032 0.004 +boundary.field_lo = pec periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 150.0*sin(2*pi*6.78e+06*t) +boundary.potential_hi_x = 450.0*sin(2*pi*13.56e+06*t) + +diagnostics.diags_names = diag1 +diag1.diag_type = Full +diag1.intervals = ::4 +diag1.fields_to_plot = phi diff --git a/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc_picmi.py b/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc_picmi.py new file mode 100755 index 00000000000..0c30e84d781 --- /dev/null +++ b/Examples/Tests/electrostatic_dirichlet_bc/inputs_test_2d_dirichlet_bc_picmi.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +########################## +# physics parameters +########################## + +V_xmin = "150.0*sin(2*pi*6.78e6*t)" +V_xmax = "450.0*sin(2*pi*13.56e6*t)" + + +########################## +# numerics parameters +########################## + +dt = 7.5e-10 + +# --- Nb time steps + +max_steps = 100 + +# --- grid + +nx = 64 +ny = 8 + +xmin = 0 +xmax = 0.032 +ymin = 0 +ymax = 0.004 + + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + warpx_potential_lo_x=V_xmin, + warpx_potential_hi_x=V_xmax, + moving_window_velocity=None, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, method="Multigrid", required_precision=1e-6 +) + + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic(name="diag1", period=4) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=4, + data_list=["phi"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=dt, + max_steps=max_steps, + particle_shape=None, + verbose=0, +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step(max_steps) diff --git a/Examples/Tests/electrostatic_sphere/CMakeLists.txt b/Examples/Tests/electrostatic_sphere/CMakeLists.txt new file mode 100644 index 00000000000..fc69ac8ba6e --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/CMakeLists.txt @@ -0,0 +1,62 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_electrostatic_sphere # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere # inputs + "analysis_electrostatic_sphere.py diags/diag1000030" # analysis + "analysis_default_regression.py --path diags/diag1000030" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_electrostatic_sphere_lab_frame # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_lab_frame # inputs + "analysis_electrostatic_sphere.py diags/diag1000030" # analysis + "analysis_default_regression.py --path diags/diag1000030" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_electrostatic_sphere_lab_frame_mr_emass_10 # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_lab_frame_mr_emass_10 # inputs + "analysis_electrostatic_sphere.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_electrostatic_sphere_rel_nodal # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_rel_nodal # inputs + "analysis_electrostatic_sphere.py diags/diag1000030" # analysis + "analysis_default_regression.py --path diags/diag1000030" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_electrostatic_sphere_adaptive # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_adaptive # inputs + "analysis_electrostatic_sphere.py diags/diag1000054" # analysis + "analysis_default_regression.py --path diags/diag1000054" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_electrostatic_sphere # name + RZ # dims + 2 # nprocs + inputs_test_rz_electrostatic_sphere # inputs + "analysis_electrostatic_sphere.py diags/diag1000030" # analysis + "analysis_default_regression.py --path diags/diag1000030" # checksum + OFF # dependency +) diff --git a/Examples/Tests/electrostatic_sphere/analysis_default_regression.py b/Examples/Tests/electrostatic_sphere/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index e63a3f02ba8..2176dcbd7c4 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -17,6 +17,7 @@ known analytic solution. While the radius r(t) is not analytically known, its inverse t(r) can be solved for exactly. """ + import os import re import sys @@ -27,53 +28,52 @@ from scipy.constants import c from scipy.optimize import fsolve -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - yt.funcs.mylog.setLevel(0) # Open plotfile specified in command line +test_name = os.path.split(os.getcwd())[1] filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) t_max = ds.current_time.item() # time of simulation # Parse test name and check if particle_shape = 4 is used -emass_10 = True if re.search('emass_10', filename) else False +emass_10 = True if re.search("emass_10", test_name) else False if emass_10: l2_tolerance = 0.096 m_e = 10 else: l2_tolerance = 0.05 - m_e = 9.10938356e-31 #Electron mass in kg + m_e = 9.10938356e-31 # Electron mass in kg ndims = np.count_nonzero(ds.domain_dimensions > 1) if ndims == 2: - xmin, zmin = [float(x) for x in ds.parameters.get('geometry.prob_lo').split()] - xmax, zmax = [float(x) for x in ds.parameters.get('geometry.prob_hi').split()] - nx, nz = [int(n) for n in ds.parameters['amr.n_cell'].split()] + xmin, zmin = [float(x) for x in ds.parameters.get("geometry.prob_lo").split()] + xmax, zmax = [float(x) for x in ds.parameters.get("geometry.prob_hi").split()] + nx, nz = [int(n) for n in ds.parameters["amr.n_cell"].split()] ymin, ymax = xmin, xmax ny = nx else: - xmin, ymin, zmin = [float(x) for x in ds.parameters.get('geometry.prob_lo').split()] - xmax, ymax, zmax = [float(x) for x in ds.parameters.get('geometry.prob_hi').split()] - nx, ny, nz = [int(n) for n in ds.parameters['amr.n_cell'].split()] + xmin, ymin, zmin = [float(x) for x in ds.parameters.get("geometry.prob_lo").split()] + xmax, ymax, zmax = [float(x) for x in ds.parameters.get("geometry.prob_hi").split()] + nx, ny, nz = [int(n) for n in ds.parameters["amr.n_cell"].split()] -dx = (xmax - xmin)/nx -dy = (ymax - ymin)/ny -dz = (zmax - zmin)/nz +dx = (xmax - xmin) / nx +dy = (ymax - ymin) / ny +dz = (zmax - zmin) / nz # Grid location of the axis -ix0 = round((0. - xmin)/dx) -iy0 = round((0. - ymin)/dy) -iz0 = round((0. - zmin)/dz) +ix0 = round((0.0 - xmin) / dx) +iy0 = round((0.0 - ymin) / dy) +iz0 = round((0.0 - zmin) / dz) # Constants -eps_0 = 8.8541878128e-12 #Vacuum Permittivity in C/(V*m) -q_e = -1.60217662e-19 #Electron charge in C -pi = np.pi #Circular constant of the universe -r_0 = 0.1 #Initial radius of sphere -q_tot = -1e-15 #Total charge of sphere in C +eps_0 = 8.8541878128e-12 # Vacuum Permittivity in C/(V*m) +q_e = -1.60217662e-19 # Electron charge in C +pi = np.pi # Circular constant of the universe +r_0 = 0.1 # Initial radius of sphere +q_tot = -1e-15 # Total charge of sphere in C + # Define functions for exact forms of v(r), t(r), Er(r) with r as the radius of # the sphere. The sphere starts with initial radius r_0 and this radius expands @@ -84,44 +84,61 @@ # r(0) = r_0, r'(0) = 0, and a = q_e*q_tot/(4*pi*eps_0*m_e) # # The E was calculated at the end of the last time step -v_exact = lambda r: np.sqrt((q_e*q_tot)/(2*pi*m_e*eps_0)*(1/r_0-1/r)) -t_exact = lambda r: np.sqrt(r_0**3*2*pi*m_e*eps_0/(q_e*q_tot)) \ - * (np.sqrt(r/r_0-1)*np.sqrt(r/r_0) \ - + np.log(np.sqrt(r/r_0-1)+np.sqrt(r/r_0))) -func = lambda rho: t_exact(rho) - t_max #Objective function to find r(t_max) -r_end = fsolve(func,r_0)[0] #Numerically solve for r(t_max) -E_exact = lambda r: np.sign(r)*(q_tot/(4*pi*eps_0*r**2)*(abs(r)>=r_end) \ - + q_tot*abs(r)/(4*pi*eps_0*r_end**3)*(abs(r)= r_end) + + q_tot * abs(r) / (4 * pi * eps_0 * r_end**3) * (abs(r) < r_end) + ) + # Load data pertaining to fields -data = ds.covering_grid(level=0, - left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Extract the E field along the axes # if ndims == 2: -if ds.parameters['geometry.dims'] == 'RZ': - Ex = data[('boxlib','Er')].to_ndarray() - Ex_axis = Ex[:,iz0,0] +if ds.parameters["geometry.dims"] == "RZ": + Ex = data[("boxlib", "Er")].to_ndarray() + Ex_axis = Ex[:, iz0, 0] Ey_axis = Ex_axis - Ez = data[('boxlib','Ez')].to_ndarray() - Ez_axis = Ez[ix0,:,0] + Ez = data[("boxlib", "Ez")].to_ndarray() + Ez_axis = Ez[ix0, :, 0] else: - Ex = data[('mesh','Ex')].to_ndarray() - Ex_axis = Ex[:,iy0,iz0] - Ey = data[('mesh','Ey')].to_ndarray() - Ey_axis = Ey[ix0,:,iz0] - Ez = data[('mesh','Ez')].to_ndarray() - Ez_axis = Ez[ix0,iy0,:] + Ex = data[("mesh", "Ex")].to_ndarray() + Ex_axis = Ex[:, iy0, iz0] + Ey = data[("mesh", "Ey")].to_ndarray() + Ey_axis = Ey[ix0, :, iz0] + Ez = data[("mesh", "Ez")].to_ndarray() + Ez_axis = Ez[ix0, iy0, :] + def calculate_error(E_axis, xmin, dx, nx): # Compute cell centers for grid - x_cell_centers = np.linspace(xmin+dx/2.,xmax-dx/2.,nx) + x_cell_centers = np.linspace(xmin + dx / 2.0, xmax - dx / 2.0, nx) # Extract subgrid away from boundary (exact solution assumes infinite/open # domain but WarpX solution assumes perfect conducting walls) - ix1 = round((xmin/2. - xmin)/dx) - ix2 = round((xmax/2. - xmin)/dx) + ix1 = round((xmin / 2.0 - xmin) / dx) + ix2 = round((xmax / 2.0 - xmin) / dx) x_sub_grid = x_cell_centers[ix1:ix2] # Exact solution of field along Cartesian axes @@ -131,37 +148,44 @@ def calculate_error(E_axis, xmin, dx, nx): E_grid = E_axis[ix1:ix2] # Define approximate L2 norm error between exact and numerical solutions - L2_error = (np.sqrt(sum((E_exact_grid - E_grid)**2)) - / np.sqrt(sum((E_exact_grid)**2))) + L2_error = np.sqrt(sum((E_exact_grid - E_grid) ** 2)) / np.sqrt( + sum((E_exact_grid) ** 2) + ) return L2_error + L2_error_x = calculate_error(Ex_axis, xmin, dx, nx) L2_error_y = calculate_error(Ey_axis, ymin, dy, ny) L2_error_z = calculate_error(Ez_axis, zmin, dz, nz) -print("L2 error along x-axis = %s" %L2_error_x) -print("L2 error along y-axis = %s" %L2_error_y) -print("L2 error along z-axis = %s" %L2_error_z) +print("L2 error along x-axis = %s" % L2_error_x) +print("L2 error along y-axis = %s" % L2_error_y) +print("L2 error along z-axis = %s" % L2_error_z) assert L2_error_x < l2_tolerance assert L2_error_y < l2_tolerance assert L2_error_z < l2_tolerance + # Check conservation of energy def return_energies(iteration): - ux, uy, uz, phi, m, q, w = ts.get_particle(['ux', 'uy', 'uz', 'phi', 'mass', 'charge', 'w'], iteration=iteration) - E_kinetic = (w*m*c**2 * (np.sqrt(1 + ux**2 + uy**2 + uz**2) - 1)).sum() - E_potential = 0.5*(w*q*phi).sum() # potential energy of particles in their own space-charge field: includes factor 1/2 + ux, uy, uz, phi, m, q, w = ts.get_particle( + ["ux", "uy", "uz", "phi", "mass", "charge", "w"], iteration=iteration + ) + E_kinetic = (w * m * c**2 * (np.sqrt(1 + ux**2 + uy**2 + uz**2) - 1)).sum() + E_potential = ( + 0.5 * (w * q * phi).sum() + ) # potential energy of particles in their own space-charge field: includes factor 1/2 return E_kinetic, E_potential -ts = OpenPMDTimeSeries('./diags/diag2') -if 'phi' in ts.avail_record_components['electron']: + + +ts = OpenPMDTimeSeries("./diags/diag2") +if "phi" in ts.avail_record_components["electron"]: # phi is only available when this script is run with the labframe poisson solver - print('Checking conservation of energy') + print("Checking conservation of energy") Ek_i, Ep_i = return_energies(0) Ek_f, Ep_f = return_energies(30) - assert Ep_f < 0.7*Ep_i # Check that potential energy changes significantly - assert abs( (Ek_i + Ep_i) - (Ek_f + Ep_f) ) < 0.003 * (Ek_i + Ep_i) # Check conservation of energy - -# Checksum regression analysis -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) + assert Ep_f < 0.7 * Ep_i # Check that potential energy changes significantly + assert abs((Ek_i + Ep_i) - (Ek_f + Ep_f)) < 0.003 * ( + Ek_i + Ep_i + ) # Check conservation of energy diff --git a/Examples/Tests/electrostatic_sphere/catalyst_pipeline.py b/Examples/Tests/electrostatic_sphere/catalyst_pipeline.py new file mode 100644 index 00000000000..f0c55fa4e11 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/catalyst_pipeline.py @@ -0,0 +1,241 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.11.1-1332-ga0a402a54e +# and validated with 5.12 +import paraview + +paraview.compatibility.major = 5 +paraview.compatibility.minor = 12 + +#### import the simple module from the paraview +from paraview.simple import * # noqa: F403 + +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# ---------------------------------------------------------------- +# setup views used in the visualization +# ---------------------------------------------------------------- + +# Create a new 'Render View' +renderView1 = CreateView("RenderView") +renderView1.ViewSize = [889, 820] +renderView1.AxesGrid = "Grid Axes 3D Actor" +renderView1.OrientationAxesVisibility = 0 +renderView1.StereoType = "Crystal Eyes" +renderView1.CameraPosition = [ + 0.537460346333711, + -0.7885714264869157, + 0.47540803206975096, +] +renderView1.CameraFocalPoint = [ + -0.002360633658114411, + -0.0033755520058871905, + -0.0028772574955099155, +] +renderView1.CameraViewUp = [ + -0.010196376455648859, + 0.5150674559093057, + 0.8570889975786005, +] +renderView1.CameraFocalDisk = 1.0 +renderView1.CameraParallelScale = 0.8660254037844386 +renderView1.LegendGrid = "Legend Grid Actor" +renderView1.UseColorPaletteForBackground = 0 +renderView1.Background = [0.08627450980392157, 0.06274509803921569, 0.11372549019607843] + +SetActiveView(None) + +# ---------------------------------------------------------------- +# setup view layouts +# ---------------------------------------------------------------- + +# create new layout object 'Layout #1' +layout1 = CreateLayout(name="Layout #1") +layout1.AssignView(0, renderView1) +layout1.SetSize(889, 820) + +# ---------------------------------------------------------------- +# restore active view +# ---------------------------------------------------------------- +SetActiveView(renderView1) + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create a new 'XML Partitioned Dataset Collection Reader' +mesh = PVTrivialProducer(registrationName="mesh") + +# create a new 'Calculator' +calculator1 = Calculator(registrationName="Calculator1", Input=mesh) +calculator1.AttributeType = "Cell Data" +calculator1.ResultArrayName = "E" +calculator1.Function = "sqrt(Ex^2 + Ey^2 + Ez^2)" + +# create a new 'Threshold' +threshold1 = Threshold(registrationName="Threshold1", Input=calculator1) +threshold1.Scalars = ["CELLS", "E"] +threshold1.LowerThreshold = 0.00014553574512481541 +threshold1.UpperThreshold = 0.00033841915322276836 + +# create a new 'Clip' +clip1 = Clip(registrationName="Clip1", Input=threshold1) +clip1.ClipType = "Plane" +clip1.HyperTreeGridClipper = "Plane" +clip1.Scalars = ["CELLS", "E"] +clip1.Value = 0.00034051798664982454 + +# ---------------------------------------------------------------- +# setup the visualization in view 'renderView1' +# ---------------------------------------------------------------- + +# show data from clip1 +clip1Display = Show(clip1, renderView1, "UnstructuredGridRepresentation") + +# get 2D transfer function for 'E' +eTF2D = GetTransferFunction2D("E") +eTF2D.ScalarRangeInitialized = 1 +eTF2D.Range = [2.7209191271959258e-08, 0.00033841915322276836, 0.0, 1.0] + +# get color transfer function/color map for 'E' +eLUT = GetColorTransferFunction("E") +eLUT.TransferFunction2D = eTF2D +eLUT.RGBPoints = [ + 0.00014565122778316538, + 0.23137254902, + 0.298039215686, + 0.752941176471, + 0.0002596781039235302, + 0.865, + 0.865, + 0.865, + 0.00037370498006389504, + 0.705882352941, + 0.0156862745098, + 0.149019607843, +] +eLUT.ScalarRangeInitialized = 1.0 + +# get opacity transfer function/opacity map for 'E' +ePWF = GetOpacityTransferFunction("E") +ePWF.Points = [ + 0.00014565122778316538, + 0.0, + 0.5, + 0.0, + 0.00037370498006389504, + 1.0, + 0.5, + 0.0, +] +ePWF.ScalarRangeInitialized = 1 + +# trace defaults for the display properties. +clip1Display.Representation = "Surface" +clip1Display.ColorArrayName = ["CELLS", "E"] +clip1Display.LookupTable = eLUT +clip1Display.SelectTCoordArray = "None" +clip1Display.SelectNormalArray = "None" +clip1Display.SelectTangentArray = "None" +clip1Display.OSPRayScaleFunction = "Piecewise Function" +clip1Display.Assembly = "Hierarchy" +clip1Display.SelectOrientationVectors = "None" +clip1Display.ScaleFactor = 0.078125 +clip1Display.SelectScaleArray = "E" +clip1Display.GlyphType = "Arrow" +clip1Display.GlyphTableIndexArray = "E" +clip1Display.GaussianRadius = 0.00390625 +clip1Display.SetScaleArray = [None, ""] +clip1Display.ScaleTransferFunction = "Piecewise Function" +clip1Display.OpacityArray = [None, ""] +clip1Display.OpacityTransferFunction = "Piecewise Function" +clip1Display.DataAxesGrid = "Grid Axes Representation" +clip1Display.PolarAxes = "Polar Axes Representation" +clip1Display.ScalarOpacityFunction = ePWF +clip1Display.ScalarOpacityUnitDistance = 0.03870829665815266 +clip1Display.OpacityArrayName = ["CELLS", "E"] +clip1Display.SelectInputVectors = [None, ""] +clip1Display.WriteLog = "" + +# setup the color legend parameters for each legend in this view + +# get color legend/bar for eLUT in view renderView1 +eLUTColorBar = GetScalarBar(eLUT, renderView1) +eLUTColorBar.WindowLocation = "Any Location" +eLUTColorBar.Position = [0.7840269966254219, 0.3353658536585365] +eLUTColorBar.Title = "E" +eLUTColorBar.ComponentTitle = "" +eLUTColorBar.ScalarBarLength = 0.3300000000000001 + +# set color bar visibility +eLUTColorBar.Visibility = 1 + +# show color legend +clip1Display.SetScalarBarVisibility(renderView1, True) + +# ---------------------------------------------------------------- +# setup color maps and opacity maps used in the visualization +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# ---------------------------------------------------------------- +# setup animation scene, tracks and keyframes +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get time animation track +timeAnimationCue1 = GetTimeTrack() + +# initialize the animation scene + +# get the time-keeper +timeKeeper1 = GetTimeKeeper() + +# initialize the timekeeper + +# initialize the animation track + +# get animation scene +animationScene1 = GetAnimationScene() + +# initialize the animation scene +animationScene1.ViewModules = renderView1 +animationScene1.Cues = timeAnimationCue1 +animationScene1.AnimationTime = 3.000000000000001e-05 +animationScene1.EndTime = 3.000000000000001e-05 +animationScene1.PlayMode = "Snap To TimeSteps" + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +pNG1 = CreateExtractor("PNG", renderView1, registrationName="PNG1") +# trace defaults for the extractor. +pNG1.Trigger = "Time Step" + +# init the 'PNG' selected for 'Writer' +pNG1.Writer.FileName = "RenderView1_{timestep:06d}{camera}.png" +pNG1.Writer.ImageResolution = [889, 820] +pNG1.Writer.Format = "PNG" + +# ---------------------------------------------------------------- +# restore active source +# ---------------------------------------------------------------- +SetActiveSource(threshold1) + +# ------------------------------------------------------------------------------ +# Catalyst options +# ------------------------------------------------------------------------------ +from paraview import catalyst + +options = catalyst.Options() +options.GlobalTrigger = "Time Step" +options.CatalystLiveTrigger = "Time Step" + +if __name__ == "__main__": + from paraview.simple import SaveExtractsUsingCatalystOptions + + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options) diff --git a/Examples/Tests/electrostatic_sphere/inputs_3d b/Examples/Tests/electrostatic_sphere/inputs_base_3d similarity index 100% rename from Examples/Tests/electrostatic_sphere/inputs_3d rename to Examples/Tests/electrostatic_sphere/inputs_base_3d diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz deleted file mode 100644 index 2b6151e6d8c..00000000000 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ /dev/null @@ -1,43 +0,0 @@ -max_step = 30 -amr.n_cell = 32 64 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 128 -geometry.dims = RZ -geometry.prob_lo = 0. -0.5 -geometry.prob_hi = 0.5 0.5 -boundary.field_lo = none pec -boundary.field_hi = pec pec -warpx.const_dt = 1e-6 -warpx.do_electrostatic = labframe -warpx.use_filter = 0 - -particles.species_names = electron - -algo.field_gathering = momentum-conserving - -# Order of particle shape factors -algo.particle_shape = 1 - -my_constants.n0 = 1.49e6 -my_constants.R0 = 0.1 - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "NUniformPerCell" -electron.num_particles_per_cell_each_dim = 2 2 2 -electron.profile = parse_density_function -electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" -electron.momentum_distribution_type = at_rest - -diagnostics.diags_names = diag1 diag2 - -diag1.intervals = 30 -diag1.diag_type = Full -diag1.fields_to_plot = Er Et Ez rho - -diag2.intervals = 30 -diag2.diag_type = Full -diag2.fields_to_plot = none -diag2.electron.variables = x y z ux uy uz w phi -diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere new file mode 100644 index 00000000000..d89395e9d74 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive new file mode 100644 index 00000000000..f64f6de08ee --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive @@ -0,0 +1,47 @@ +stop_time = 60e-6 +warpx.cfl = 0.2 +warpx.dt_update_interval = 10 +warpx.max_dt = 1.5e-6 +amr.n_cell = 64 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +geometry.dims = 3 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +warpx.do_electrostatic = relativistic + +particles.species_names = electron + +algo.field_gathering = momentum-conserving + +# Order of particle shape factors +algo.particle_shape = 1 + +my_constants.n0 = 1.49e6 +my_constants.R0 = 0.1 + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" +electron.momentum_distribution_type = at_rest + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 30 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.format = openpmd + +warpx.reduced_diags_names = timestep +timestep.intervals = 1 +timestep.type = Timestep diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame new file mode 100644 index 00000000000..da97ae8afe7 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +diag2.electron.variables = x y z ux uy uz w phi +warpx.do_electrostatic = labframe diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame_mr_emass_10 b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame_mr_emass_10 new file mode 100644 index 00000000000..481cc65f030 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_lab_frame_mr_emass_10 @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +amr.max_level = 1 +amr.ref_ratio_vect = 2 2 2 +diag2.electron.variables = x y z ux uy uz w +electron.mass = 10 +max_step = 2 +warpx.abort_on_warning_threshold = medium +warpx.do_electrostatic = labframe +warpx.fine_tag_hi = 0.5 0.5 0.5 +warpx.fine_tag_lo = -0.5 -0.5 -0.5 diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_rel_nodal b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_rel_nodal new file mode 100644 index 00000000000..96bff8aa9c7 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_rel_nodal @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +warpx.abort_on_warning_threshold = medium +warpx.grid_type = collocated diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_rz_electrostatic_sphere b/Examples/Tests/electrostatic_sphere/inputs_test_rz_electrostatic_sphere new file mode 100644 index 00000000000..a1c71c58fc3 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_rz_electrostatic_sphere @@ -0,0 +1,44 @@ +max_step = 30 +amr.n_cell = 32 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +geometry.dims = RZ +geometry.prob_lo = 0. -0.5 +geometry.prob_hi = 0.5 0.5 +boundary.field_lo = none pec +boundary.field_hi = pec pec +warpx.const_dt = 1e-6 +warpx.do_electrostatic = labframe +warpx.use_filter = 0 +warpx.abort_on_warning_threshold = medium + +particles.species_names = electron + +algo.field_gathering = momentum-conserving + +# Order of particle shape factors +algo.particle_shape = 1 + +my_constants.n0 = 1.49e6 +my_constants.R0 = 0.1 + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" +electron.momentum_distribution_type = at_rest + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 30 +diag1.diag_type = Full +diag1.fields_to_plot = Er Et Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.electron.variables = x y z ux uy uz w phi +diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt b/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt new file mode 100644 index 00000000000..0511212c4d5 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt @@ -0,0 +1,62 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_electrostatic_sphere_eb # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_eb # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_electrostatic_sphere_eb_mixed_bc # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_eb_mixed_bc # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_electrostatic_sphere_eb_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_eb_picmi.py # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_electrostatic_sphere_eb # name + RZ # dims + 2 # nprocs + inputs_test_rz_electrostatic_sphere_eb # inputs + "analysis_rz.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_electrostatic_sphere_eb_mr # name + RZ # dims + 2 # nprocs + inputs_test_rz_electrostatic_sphere_eb_mr # inputs + "analysis_rz_mr.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py b/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py deleted file mode 100755 index 55fbc87bd9e..00000000000 --- a/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -########################## -# physics parameters -########################## - -V_domain_boundary = 0.0 -V_embedded_boundary = 1.0 - - -########################## -# numerics parameters -########################## - -dt = 1e-6 - -# --- Nb time steps - -max_steps = 2 - -# --- grid - -nx = 64 -ny = 64 -nz = 64 - -xmin = -0.5 -xmax = 0.5 -ymin = -0.5 -ymax = 0.5 -zmin = -0.5 -zmax = 0.5 - - -########################## -# numerics components -########################## - -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['dirichlet', 'dirichlet', 'dirichlet'], - upper_boundary_conditions = ['dirichlet', 'dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing'], - warpx_potential_lo_x = V_domain_boundary, - warpx_potential_hi_x = V_domain_boundary, - warpx_potential_lo_y = V_domain_boundary, - warpx_potential_hi_y = V_domain_boundary, - warpx_potential_lo_z = V_domain_boundary, - warpx_potential_hi_z = V_domain_boundary, - warpx_blocking_factor=8, - warpx_max_grid_size = 128 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-7 -) - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="-(x**2+y**2+z**2-radius**2)", - potential=V_embedded_boundary, - radius = 0.1 -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 1, - write_dir = '.', - warpx_file_prefix = 'Python_ElectrostaticSphereEB_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 1, - data_list = ['Ex', 'Ey', 'Ez', 'phi', 'rho'], - write_dir = '.', - warpx_file_prefix = 'Python_ElectrostaticSphereEB_plt' -) - -reduced_diag = picmi.ReducedDiagnostic( - diag_type = 'ChargeOnEB', - name = 'eb_charge', - period = 1) - -reduced_diag_one_eighth = picmi.ReducedDiagnostic( - diag_type = 'ChargeOnEB', - name = 'eb_charge_one_eighth', - weighting_function = '(x>0)*(y>0)*(z>0)', - period = 1) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - warpx_embedded_boundary=embedded_boundary, - warpx_field_gathering_algo='momentum-conserving' -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) -sim.add_diagnostic(reduced_diag) -sim.add_diagnostic(reduced_diag_one_eighth) - -########################## -# simulation run -########################## - -sim.step(1) - -sim.extension.warpx.set_potential_on_eb("2.") - -sim.step(1) diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis.py b/Examples/Tests/electrostatic_sphere_eb/analysis.py index bf2725616b5..114db2871ee 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis.py @@ -4,31 +4,21 @@ # using the same reference file as for the non-PICMI test since the two # tests are otherwise the same. -import os -import sys - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # Check reduced diagnostics for charge on EB import numpy as np from scipy.constants import epsilon_0 # Theoretical charge on the embedded boundary, for sphere at potential phi_0 -phi_0 = 1. # V -R = 0.1 # m -q_th = 4*np.pi*epsilon_0*phi_0*R -print('Theoretical charge: ', q_th) - -data = np.loadtxt('diags/reducedfiles/eb_charge.txt') -q_sim = data[1,2] -print('Simulation charge: ', q_sim) -assert abs((q_sim-q_th)/q_th) < 0.06 - -data_eighth = np.loadtxt('diags/reducedfiles/eb_charge_one_eighth.txt') -q_sim_eighth = data_eighth[1,2] -assert abs((q_sim_eighth-q_th/8)/(q_th/8)) < 0.06 - -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +phi_0 = 1.0 # V +R = 0.1 # m +q_th = 4 * np.pi * epsilon_0 * phi_0 * R +print("Theoretical charge: ", q_th) + +data = np.loadtxt("diags/reducedfiles/eb_charge.txt") +q_sim = data[1, 2] +print("Simulation charge: ", q_sim) +assert abs((q_sim - q_th) / q_th) < 0.06 + +data_eighth = np.loadtxt("diags/reducedfiles/eb_charge_one_eighth.txt") +q_sim_eighth = data_eighth[1, 2] +assert abs((q_sim_eighth - q_th / 8) / (q_th / 8)) < 0.06 diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_default_regression.py b/Examples/Tests/electrostatic_sphere_eb/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py index 18aba4322f0..be9033e2b14 100755 --- a/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz.py @@ -16,54 +16,49 @@ # tolerance: 0.004 # Possible running time: < 1 s -import os import sys import numpy as np import yt from unyt import m -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - tolerance = 0.0041 fn = sys.argv[1] -ds = yt.load( fn ) +ds = yt.load(fn) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -phi = all_data_level_0['boxlib', 'phi'].v.squeeze() -Er = all_data_level_0['boxlib', 'Er'].v.squeeze() +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +phi = all_data_level_0["boxlib", "phi"].v.squeeze() +Er = all_data_level_0["boxlib", "Er"].v.squeeze() -Dx = ds.domain_width/ds.domain_dimensions +Dx = ds.domain_width / ds.domain_dimensions dr = Dx[0] rmin = ds.domain_left_edge[0] rmax = ds.domain_right_edge[0] nr = phi.shape[0] -r = np.linspace(rmin+dr/2.,rmax-dr/2.,nr) -B = 1.0/np.log(0.1/0.5) -A = -B*np.log(0.5) +r = np.linspace(rmin + dr / 2.0, rmax - dr / 2.0, nr) +B = 1.0 / np.log(0.1 / 0.5) +A = -B * np.log(0.5) err = 0.0 errmax_phi = 0.0 errmax_Er = 0.0 for i in range(len(r)): # outside EB and last cutcell - if r[i] > 0.1*m + dr: - phi_theory = A+B*np.log(r[i]) - Er_theory = -B/float(r[i]) - err = abs( phi_theory - phi[i,:] ).max() / phi_theory - if err>errmax_phi: + if r[i] > 0.1 * m + dr: + phi_theory = A + B * np.log(r[i]) + Er_theory = -B / float(r[i]) + err = abs(phi_theory - phi[i, :]).max() / phi_theory + if err > errmax_phi: errmax_phi = err - err = abs( Er_theory - Er[i,:] ).max() / Er_theory + err = abs(Er_theory - Er[i, :]).max() / Er_theory # Exclude the last inaccurate interpolation. - if err>errmax_Er and i errmax_Er and i < len(r) - 1: errmax_Er = err -print('max error of phi = ', errmax_phi) -print('max error of Er = ', errmax_Er) -print('tolerance = ', tolerance) -assert(errmax_phi= (0.1+dr))) + rmin = np.min(np.argwhere(r >= (0.1 + dr))) rmax = -1 r = r[rmin:rmax] - phi_sim = phi_sim[:,rmin:rmax] - Er_sim = Er_sim[:,rmin:rmax] + phi_sim = phi_sim[:, rmin:rmax] + Er_sim = Er_sim[:, rmin:rmax] - phi_theory = A + B*np.log(r) - phi_theory = np.tile(phi_theory, (phi_sim.shape[0],1)) - phi_error = np.max(np.abs(phi_theory-phi_sim) / np.abs(phi_theory)) + phi_theory = A + B * np.log(r) + phi_theory = np.tile(phi_theory, (phi_sim.shape[0], 1)) + phi_error = np.max(np.abs(phi_theory - phi_sim) / np.abs(phi_theory)) - Er_theory = -B/r - Er_theory = np.tile(Er_theory, (Er_sim.shape[0],1)) - Er_error = np.max(np.abs(Er_theory-Er_sim) / np.abs(Er_theory)) + Er_theory = -B / r + Er_theory = np.tile(Er_theory, (Er_sim.shape[0], 1)) + Er_error = np.max(np.abs(Er_theory - Er_sim) / np.abs(Er_theory)) + + print(f"max error of phi[lev={level}]: {phi_error}") + print(f"max error of Er[lev={level}]: {Er_error}") + assert phi_error < tolerance + assert Er_error < tolerance - print(f'max error of phi[lev={level}]: {phi_error}') - print(f'max error of Er[lev={level}]: {Er_error}') - assert(phi_error < tolerance) - assert(Er_error < tolerance) ts = OpenPMDTimeSeries(fn) -level_fields = [field for field in ts.avail_fields if 'lvl' in field] +level_fields = [field for field in ts.avail_fields if "lvl" in field] nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) -for level in range(nlevels+1): - get_error_per_lev(ts,level) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, output_format="openpmd") +for level in range(nlevels + 1): + get_error_per_lev(ts, level) diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_3d b/Examples/Tests/electrostatic_sphere_eb/inputs_3d deleted file mode 100644 index 13ad42da070..00000000000 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_3d +++ /dev/null @@ -1,37 +0,0 @@ -max_step = 1 -amr.n_cell = 64 64 64 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 128 -geometry.dims = 3 -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 -geometry.prob_lo = -0.5 -0.5 -0.5 -geometry.prob_hi = 0.5 0.5 0.5 -warpx.const_dt = 1e-6 - -warpx.do_electrostatic = labframe -warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.1**2)" -warpx.eb_potential(x,y,z,t) = "1." -warpx.self_fields_required_precision = 1.e-7 - -algo.field_gathering = momentum-conserving - -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez rho phi - -warpx.reduced_diags_names = eb_charge eb_charge_one_eighth -eb_charge.type = ChargeOnEB -eb_charge.intervals = 1 -eb_charge_one_eighth.type = ChargeOnEB -eb_charge_one_eighth.intervals = 1 -# Select only one eighth of the sphere -eb_charge_one_eighth.weighting_function(x,y,z) = (x>0)*(y<0)*(z>0) diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_3d_mixed_BCs b/Examples/Tests/electrostatic_sphere_eb/inputs_3d_mixed_BCs deleted file mode 100644 index 0c1b9130ded..00000000000 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_3d_mixed_BCs +++ /dev/null @@ -1,26 +0,0 @@ -max_step = 1 -amr.n_cell = 64 64 64 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 128 -geometry.dims = 3 -boundary.field_lo = pec pec neumann -boundary.field_hi = pec neumann neumann -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -geometry.prob_lo = -0.5 -0.5 -0.5 -geometry.prob_hi = 0.5 0.5 0.5 -warpx.const_dt = 1e-6 - -warpx.do_electrostatic = labframe -warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.3**2)" -warpx.eb_potential(x,y,z,t) = "1." -warpx.self_fields_required_precision = 1.e-7 - -algo.field_gathering = momentum-conserving - -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez rho phi diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_rz b/Examples/Tests/electrostatic_sphere_eb/inputs_rz deleted file mode 100644 index 28ebadb1cc7..00000000000 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_rz +++ /dev/null @@ -1,29 +0,0 @@ -max_step = 1 -amr.n_cell = 64 64 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 128 -boundary.field_lo = none periodic -boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 -geometry.dims = RZ -geometry.prob_lo = 0.0 -0.5 -geometry.prob_hi = 0.5 0.5 -warpx.const_dt = 1e-6 - -warpx.do_electrostatic = labframe -warpx.eb_implicit_function = "-(x**2-0.1**2)" -warpx.eb_potential(x,y,z,t) = "1." -warpx.self_fields_required_precision = 1.e-7 - -algo.field_gathering = momentum-conserving - -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Er phi diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr deleted file mode 100644 index 722fc916416..00000000000 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr +++ /dev/null @@ -1,33 +0,0 @@ -amr.max_level = 1 -warpx.fine_tag_lo = 0.0 -0.25 -warpx.fine_tag_hi = 0.25 0.25 - -max_step = 1 -amr.n_cell = 64 64 -amr.blocking_factor = 8 -amr.max_grid_size = 128 -boundary.field_lo = none periodic -boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 -geometry.dims = RZ -geometry.prob_lo = 0.0 -0.5 -geometry.prob_hi = 0.5 0.5 -warpx.const_dt = 1e-6 - -warpx.do_electrostatic = labframe -warpx.eb_implicit_function = "-(x**2-0.1**2)" -warpx.eb_potential(x,y,z,t) = "1." -warpx.self_fields_required_precision = 1.e-7 - -algo.field_gathering = momentum-conserving - -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Er phi -diag1.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb new file mode 100644 index 00000000000..f738e1c5d3a --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb @@ -0,0 +1,38 @@ +max_step = 1 +amr.n_cell = 64 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +geometry.dims = 3 +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +warpx.const_dt = 1e-6 + +warpx.do_electrostatic = labframe +warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.1**2)" +warpx.eb_potential(x,y,z,t) = "1." +warpx.self_fields_required_precision = 1.e-7 +warpx.abort_on_warning_threshold = medium + +algo.field_gathering = momentum-conserving + +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez rho phi + +warpx.reduced_diags_names = eb_charge eb_charge_one_eighth +eb_charge.type = ChargeOnEB +eb_charge.intervals = 1 +eb_charge_one_eighth.type = ChargeOnEB +eb_charge_one_eighth.intervals = 1 +# Select only one eighth of the sphere +eb_charge_one_eighth.weighting_function(x,y,z) = (x>0)*(y<0)*(z>0) diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_mixed_bc b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_mixed_bc new file mode 100644 index 00000000000..de2c0d0646c --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_mixed_bc @@ -0,0 +1,27 @@ +max_step = 1 +amr.n_cell = 64 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +geometry.dims = 3 +boundary.field_lo = pec pec neumann +boundary.field_hi = pec neumann neumann +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +warpx.const_dt = 1e-6 + +warpx.do_electrostatic = labframe +warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.3**2)" +warpx.eb_potential(x,y,z,t) = "1." +warpx.self_fields_required_precision = 1.e-7 +warpx.abort_on_warning_threshold = medium + +algo.field_gathering = momentum-conserving + +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez rho phi diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_picmi.py b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_picmi.py new file mode 100755 index 00000000000..37d280e77d2 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_test_3d_electrostatic_sphere_eb_picmi.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +########################## +# physics parameters +########################## + +V_domain_boundary = 0.0 +V_embedded_boundary = 1.0 + + +########################## +# numerics parameters +########################## + +dt = 1e-6 + +# --- Nb time steps + +max_steps = 2 + +# --- grid + +nx = 64 +ny = 64 +nz = 64 + +xmin = -0.5 +xmax = 0.5 +ymin = -0.5 +ymax = 0.5 +zmin = -0.5 +zmax = 0.5 + + +########################## +# numerics components +########################## + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + warpx_potential_lo_x=V_domain_boundary, + warpx_potential_hi_x=V_domain_boundary, + warpx_potential_lo_y=V_domain_boundary, + warpx_potential_hi_y=V_domain_boundary, + warpx_potential_lo_z=V_domain_boundary, + warpx_potential_hi_z=V_domain_boundary, + warpx_blocking_factor=8, + warpx_max_grid_size=128, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, method="Multigrid", required_precision=1e-7 +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-(x**2+y**2+z**2-radius**2)", + potential=V_embedded_boundary, + radius=0.1, +) + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=1, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=1, + data_list=["Ex", "Ey", "Ez", "phi", "rho"], +) + +reduced_diag = picmi.ReducedDiagnostic( + diag_type="ChargeOnEB", name="eb_charge", period=1 +) + +reduced_diag_one_eighth = picmi.ReducedDiagnostic( + diag_type="ChargeOnEB", + name="eb_charge_one_eighth", + weighting_function="(x>0)*(y>0)*(z>0)", + period=1, +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=dt, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + warpx_field_gathering_algo="momentum-conserving", +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(reduced_diag) +sim.add_diagnostic(reduced_diag_one_eighth) + +########################## +# simulation run +########################## + +sim.step(1) + +sim.extension.warpx.set_potential_on_eb("2.") + +sim.step(1) diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb b/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb new file mode 100644 index 00000000000..8ace9cd9b4a --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb @@ -0,0 +1,30 @@ +max_step = 1 +amr.n_cell = 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 +warpx.const_dt = 1e-6 + +warpx.do_electrostatic = labframe +warpx.eb_implicit_function = "-(x**2-0.1**2)" +warpx.eb_potential(x,y,z,t) = "1." +warpx.self_fields_required_precision = 1.e-7 +warpx.abort_on_warning_threshold = medium + +algo.field_gathering = momentum-conserving + +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er phi diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb_mr b/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb_mr new file mode 100644 index 00000000000..d984ba35b5d --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_test_rz_electrostatic_sphere_eb_mr @@ -0,0 +1,35 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.25 +warpx.fine_tag_hi = 0.25 0.25 + +max_step = 1 +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +amr.ref_ratio_vect = 2 2 2 +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 +warpx.const_dt = 1e-6 + +warpx.do_electrostatic = labframe +warpx.eb_implicit_function = "-(x**2-0.1**2)" +warpx.eb_potential(x,y,z,t) = "1." +warpx.self_fields_required_precision = 1.e-7 +warpx.abort_on_warning_threshold = medium + +algo.field_gathering = momentum-conserving + +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er phi +diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_cube/CMakeLists.txt b/Examples/Tests/embedded_boundary_cube/CMakeLists.txt new file mode 100644 index 00000000000..ac509955088 --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/CMakeLists.txt @@ -0,0 +1,38 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_cube # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_cube # inputs + "analysis_fields_2d.py diags/diag1000114" # analysis + "analysis_default_regression.py --path diags/diag1000114" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_cube # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_cube # inputs + "analysis_fields.py diags/diag1000208" # analysis + "analysis_default_regression.py --path diags/diag1000208" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_cube_macroscopic # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_cube_macroscopic # inputs + "analysis_fields.py diags/diag1000208" # analysis + "analysis_default_regression.py --path diags/diag1000208" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_cube/analysis_default_regression.py b/Examples/Tests/embedded_boundary_cube/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_cube/analysis_fields.py index dc6af9d57d2..4cb4a60f603 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields.py @@ -8,9 +8,6 @@ import yt from scipy.constants import c, mu_0, pi -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # This is a script that analyses the simulation results from # the script `inputs_3d`. This simulates a TMmnp mode in a PEC cubic resonator. # The magnetic field in the simulation is given (in theory) by: @@ -23,12 +20,14 @@ # $$ k_y = \frac{n\pi}{L}$$ # $$ k_z = \frac{p\pi}{L}$$ +test_name = os.path.split(os.getcwd())[1] + hi = [0.8, 0.8, 0.8] lo = [-0.8, -0.8, -0.8] ncells = [48, 48, 48] -dx = (hi[0] - lo[0])/ncells[0] -dy = (hi[1] - lo[1])/ncells[1] -dz = (hi[2] - lo[2])/ncells[2] +dx = (hi[0] - lo[0]) / ncells[0] +dy = (hi[1] - lo[1]) / ncells[1] +dz = (hi[2] - lo[2]) / ncells[2] m = 0 n = 1 p = 1 @@ -40,17 +39,19 @@ # Open the right plot file filename = sys.argv[1] ds = yt.load(filename) -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Parse test name and check whether this use the macroscopic solver # (i.e. solving the equation in a dielectric) -macroscopic = True if re.search( 'macroscopic', filename ) else False +macroscopic = True if re.search("macroscopic", test_name) else False # Calculate frequency of the mode oscillation -omega = np.sqrt( h_2 ) * c +omega = np.sqrt(h_2) * c if macroscopic: # Relative permittivity used in this test: epsilon_r = 1.5 - omega *= 1./np.sqrt(1.5) + omega *= 1.0 / np.sqrt(1.5) t = ds.current_time.to_value() @@ -61,40 +62,47 @@ for i in range(ncells[0]): for j in range(ncells[1]): for k in range(ncells[2]): - x = i*dx + lo[0] - y = (j+0.5)*dy + lo[1] - z = k*dz + lo[2] - - By_th[i, j, k] = -2/h_2*mu_0*(n * pi/Ly)*(p * pi/Lz) * (np.cos(m * pi/Lx * (x - Lx/2)) * - np.sin(n * pi/Ly * (y - Ly/2)) * - np.cos(p * pi/Lz * (z - Lz/2)) * - (-Lx/2 <= x < Lx/2) * - (-Ly/2 <= y < Ly/2) * - (-Lz/2 <= z < Lz/2) * - np.cos(omega * t)) - - x = i*dx + lo[0] - y = j*dy + lo[1] - z = (k+0.5)*dz + lo[2] - Bz_th[i, j, k] = mu_0*(np.cos(m * pi/Lx * (x - Lx/2)) * - np.cos(n * pi/Ly * (y - Ly/2)) * - np.sin(p * pi/Lz * (z - Lz/2)) * - (-Lx/2 <= x < Lx/2) * - (-Ly/2 <= y < Ly/2) * - (-Lz/2 <= z < Lz/2) * - np.cos(omega * t)) + x = i * dx + lo[0] + y = (j + 0.5) * dy + lo[1] + z = k * dz + lo[2] + + By_th[i, j, k] = ( + -2 + / h_2 + * mu_0 + * (n * pi / Ly) + * (p * pi / Lz) + * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.sin(n * pi / Ly * (y - Ly / 2)) + * np.cos(p * pi / Lz * (z - Lz / 2)) + * (-Lx / 2 <= x < Lx / 2) + * (-Ly / 2 <= y < Ly / 2) + * (-Lz / 2 <= z < Lz / 2) + * np.cos(omega * t) + ) + ) + + x = i * dx + lo[0] + y = j * dy + lo[1] + z = (k + 0.5) * dz + lo[2] + Bz_th[i, j, k] = mu_0 * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.cos(n * pi / Ly * (y - Ly / 2)) + * np.sin(p * pi / Lz * (z - Lz / 2)) + * (-Lx / 2 <= x < Lx / 2) + * (-Ly / 2 <= y < Ly / 2) + * (-Lz / 2 <= z < Lz / 2) + * np.cos(omega * t) + ) rel_tol_err = 1e-1 # Compute relative l^2 error on By -By_sim = data[('mesh','By')].to_ndarray() -rel_err_y = np.sqrt( np.sum(np.square(By_sim - By_th)) / np.sum(np.square(By_th))) -assert(rel_err_y < rel_tol_err) +By_sim = data[("mesh", "By")].to_ndarray() +rel_err_y = np.sqrt(np.sum(np.square(By_sim - By_th)) / np.sum(np.square(By_th))) +assert rel_err_y < rel_tol_err # Compute relative l^2 error on Bz -Bz_sim = data[('mesh','Bz')].to_ndarray() -rel_err_z = np.sqrt( np.sum(np.square(Bz_sim - Bz_th)) / np.sum(np.square(Bz_th))) -assert(rel_err_z < rel_tol_err) - -test_name = os.path.split(os.getcwd())[1] - -checksumAPI.evaluate_checksum(test_name, filename) +Bz_sim = data[("mesh", "Bz")].to_ndarray() +rel_err_z = np.sqrt(np.sum(np.square(Bz_sim - Bz_th)) / np.sum(np.square(Bz_th))) +assert rel_err_z < rel_tol_err diff --git a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py index 8faa299025e..bb35ad93cb8 100755 --- a/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py @@ -1,15 +1,11 @@ #!/usr/bin/env python3 -import os import sys import numpy as np import yt from scipy.constants import c, mu_0, pi -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # This is a script that analyses the simulation results from # the script `inputs_3d`. This simulates a TMmnp mode in a PEC cubic resonator. # The magnetic field in the simulation is given (in theory) by: @@ -32,7 +28,9 @@ # Open the right plot file filename = sys.argv[1] ds = yt.load(filename) -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) t = ds.current_time.to_value() @@ -40,25 +38,23 @@ By_th = np.zeros(ncells) for i in range(ncells[0]): for j in range(ncells[1]): - x = (i+0.5) * dx + lo[0] - z = (j+0.5) * dz + lo[1] - - By_th[i, j, 0] = mu_0 * (np.cos(m * pi / Lx * (x - Lx / 2)) * - np.cos(n * pi / Lz * (z - Lz / 2)) * - (-Lx / 2 <= x < Lx / 2) * - (-Lz / 2 <= z < Lz / 2) * - np.cos(np.pi / Lx * c * t)) + x = (i + 0.5) * dx + lo[0] + z = (j + 0.5) * dz + lo[1] + + By_th[i, j, 0] = mu_0 * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.cos(n * pi / Lz * (z - Lz / 2)) + * (-Lx / 2 <= x < Lx / 2) + * (-Lz / 2 <= z < Lz / 2) + * np.cos(np.pi / Lx * c * t) + ) rel_tol_err = 1e-3 # Compute relative l^2 error on By -By_sim = data['By'].to_ndarray() +By_sim = data["By"].to_ndarray() rel_err_y = np.sqrt(np.sum(np.square(By_sim - By_th)) / np.sum(np.square(By_th))) -assert (rel_err_y < rel_tol_err) +assert rel_err_y < rel_tol_err # Compute relative l^2 error on Ey -Ey_sim = data['Ey'].to_ndarray() -rel_err_y = np.sqrt(np.sum(np.square(Ey_sim/c - By_th)) / np.sum(np.square(By_th))) - -test_name = os.path.split(os.getcwd())[1] - -checksumAPI.evaluate_checksum(test_name, filename) +Ey_sim = data["Ey"].to_ndarray() +rel_err_y = np.sqrt(np.sum(np.square(Ey_sim / c - By_th)) / np.sum(np.square(By_th))) diff --git a/Examples/Tests/embedded_boundary_cube/inputs_2d b/Examples/Tests/embedded_boundary_cube/inputs_2d deleted file mode 100644 index 372e0dc0340..00000000000 --- a/Examples/Tests/embedded_boundary_cube/inputs_2d +++ /dev/null @@ -1,45 +0,0 @@ -stop_time = 1.3342563807926085e-08 -amr.n_cell = 32 32 -amr.max_grid_size = 128 -amr.max_level = 0 - -geometry.dims = 2 -geometry.prob_lo = -0.8 -0.8 -geometry.prob_hi = 0.8 0.8 -warpx.cfl = 1 - -boundary.field_lo = pec pec -boundary.field_hi = pec pec - -my_constants.xmin = -0.5 -my_constants.zmin = -0.5 -my_constants.xmax = 0.5 -my_constants.zmax = 0.5 -# Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. -warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" - -# To test the initial electrostatic solver, we also add uniform potential -# Since, the potential is uniform here, it should not modify the fields, so this mainly -# tests that calling the electrostatic solver does not raise any error/crash -warpx.eb_potential(x,y,z,t) = "1" - -warpx.B_ext_grid_init_style = parse_B_ext_grid_function -warpx.E_ext_grid_init_style = parse_E_ext_grid_function - -my_constants.m = 0 -my_constants.p = 1 -my_constants.Lx = 1 -my_constants.Lz = 1 - -warpx.Bz_external_grid_function(x,y,z) = 0 -warpx.Bx_external_grid_function(x,y,z) = 0 -warpx.By_external_grid_function(x,y,z) = cos(m * pi / Lx * (x - Lx / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0 - -warpx.Ez_external_grid_function(x,y,z) = 0 -warpx.Ex_external_grid_function(x,y,z) = 0 -warpx.Ey_external_grid_function(x,y,z) = cos(m * pi / Lx * (x - Lx / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0*clight - -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_cube/inputs_3d b/Examples/Tests/embedded_boundary_cube/inputs_3d deleted file mode 100644 index 61eb1192e04..00000000000 --- a/Examples/Tests/embedded_boundary_cube/inputs_3d +++ /dev/null @@ -1,43 +0,0 @@ -stop_time = 1.3342563807926085e-08 -amr.n_cell = 48 48 48 -amr.max_grid_size = 128 -amr.max_level = 0 - -geometry.dims = 3 -geometry.prob_lo = -0.8 -0.8 -0.8 -geometry.prob_hi = 0.8 0.8 0.8 -warpx.cfl = 1 - -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec - -eb2.geom_type = box -eb2.box_lo = -0.5 -0.5 -0.5 -eb2.box_hi = 0.5 0.5 0.5 -eb2.box_has_fluid_inside = true -# Alternatively one could use parser to build EB -# Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. -# warpx.eb_implicit_function = "max(max(max(x-0.5,-0.5-x), max(y-0.5,-0.5-y)), max(z-0.5,-0.5-z))" - -# To test the initial electrostatic solver, we also add uniform potential -# Since, the potential is uniform here, it should not modify the fields, so this mainly -# tests that calling the electrostatic solver does not raise any error/crash -warpx.eb_potential(x,y,z,t) = "1" - -warpx.B_ext_grid_init_style = parse_B_ext_grid_function -my_constants.m = 0 -my_constants.n = 1 -my_constants.p = 1 -my_constants.Lx = 1 -my_constants.Ly = 1 -my_constants.Lz = 1 -my_constants.h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 - -warpx.By_external_grid_function(x,y,z) = -2/h_2 * (n * pi / Ly) * (p * pi / Lz) * cos(m * pi / Lx * (x - Lx / 2)) * sin(n * pi / Ly * (y - Ly / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0*(x>-Lx/2)*(x-Ly/2)*(y-Lz/2)*(z-Lx/2)*(x-Ly/2)*(y-Lz/2)*(z-0.5)*(x<0.5)*(y>-0.5)*(y<0.5)*(z>-0.5)*(z<0.5) - -diagnostics.diags_names = diag1 -diag1.intervals = 1000 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_cube/inputs_base_3d b/Examples/Tests/embedded_boundary_cube/inputs_base_3d new file mode 100644 index 00000000000..70ddd8f8f64 --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/inputs_base_3d @@ -0,0 +1,44 @@ +stop_time = 1.3342563807926085e-08 +amr.n_cell = 48 48 48 +amr.max_grid_size = 128 +amr.max_level = 0 + +geometry.dims = 3 +geometry.prob_lo = -0.8 -0.8 -0.8 +geometry.prob_hi = 0.8 0.8 0.8 +warpx.cfl = 1 +warpx.abort_on_warning_threshold = medium + +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec + +eb2.geom_type = box +eb2.box_lo = -0.501 -0.501 -0.501 # Ensures that the stair-case EB is exactly at -0.5 +eb2.box_hi = 0.501 0.501 0.501 # Ensures that the stair-case EB is exactly at 0.5 +eb2.box_has_fluid_inside = true +# Alternatively one could use parser to build EB +# Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. +# warpx.eb_implicit_function = "max(max(max(x-0.5,-0.5-x), max(y-0.5,-0.5-y)), max(z-0.5,-0.5-z))" + +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function +my_constants.m = 0 +my_constants.n = 1 +my_constants.p = 1 +my_constants.Lx = 1 +my_constants.Ly = 1 +my_constants.Lz = 1 +my_constants.h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 + +warpx.By_external_grid_function(x,y,z) = -2/h_2 * (n * pi / Ly) * (p * pi / Lz) * cos(m * pi / Lx * (x - Lx / 2)) * sin(n * pi / Ly * (y - Ly / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0*(x>-Lx/2)*(x-Ly/2)*(y-Lz/2)*(z-Lx/2)*(x-Ly/2)*(y-Lz/2)*(z-0.5)*(x<0.5)*(y>-0.5)*(y<0.5)*(z>-0.5)*(z<0.5) + +diagnostics.diags_names = diag1 +diag1.intervals = 1000 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube new file mode 100644 index 00000000000..46272052c2c --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube @@ -0,0 +1,46 @@ +stop_time = 1.3342563807926085e-08 +amr.n_cell = 32 32 +amr.max_grid_size = 128 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.8 -0.8 +geometry.prob_hi = 0.8 0.8 +warpx.cfl = 1 +warpx.abort_on_warning_threshold = medium + +boundary.field_lo = pec pec +boundary.field_hi = pec pec + +my_constants.xmin = -0.501 # Ensures that the stair-case EB is exactly at -0.5 +my_constants.zmin = -0.501 # Ensures that the stair-case EB is exactly at -0.5 +my_constants.xmax = 0.501 # Ensures that the stair-case EB is exactly at 0.5 +my_constants.zmax = 0.501 # Ensures that the stair-case EB is exactly at 0.5 +# Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. +warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" + +# To test the initial electrostatic solver, we also add uniform potential +# Since, the potential is uniform here, it should not modify the fields, so this mainly +# tests that calling the electrostatic solver does not raise any error/crash +warpx.eb_potential(x,y,z,t) = "1" + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function +warpx.E_ext_grid_init_style = parse_E_ext_grid_function + +my_constants.m = 0 +my_constants.p = 1 +my_constants.Lx = 1 +my_constants.Lz = 1 + +warpx.Bz_external_grid_function(x,y,z) = 0 +warpx.Bx_external_grid_function(x,y,z) = 0 +warpx.By_external_grid_function(x,y,z) = cos(m * pi / Lx * (x - Lx / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0 + +warpx.Ez_external_grid_function(x,y,z) = 0 +warpx.Ex_external_grid_function(x,y,z) = 0 +warpx.Ey_external_grid_function(x,y,z) = cos(m * pi / Lx * (x - Lx / 2)) * cos(p * pi / Lz * (z - Lz / 2))*mu0*clight + +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube b/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube new file mode 100644 index 00000000000..9d612bd62da --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube @@ -0,0 +1,2 @@ +# base inpute parameters +FILE = inputs_base_3d diff --git a/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube_macroscopic b/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube_macroscopic new file mode 100644 index 00000000000..1bcb49dec54 --- /dev/null +++ b/Examples/Tests/embedded_boundary_cube/inputs_test_3d_embedded_boundary_cube_macroscopic @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.em_solver_medium = macroscopic +macroscopic.epsilon = 1.5*8.8541878128e-12 +macroscopic.mu = 1.25663706212e-06 +macroscopic.sigma = 0 diff --git a/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt b/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt new file mode 100644 index 00000000000..456e9f9b630 --- /dev/null +++ b/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_diffraction # name + RZ # dims + 2 # nprocs + inputs_test_rz_embedded_boundary_diffraction # inputs + "analysis_fields.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_diffraction/analysis_default_regression.py b/Examples/Tests/embedded_boundary_diffraction/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_diffraction/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_diffraction/analysis_fields.py b/Examples/Tests/embedded_boundary_diffraction/analysis_fields.py index da344f332a1..599bcea71f9 100755 --- a/Examples/Tests/embedded_boundary_diffraction/analysis_fields.py +++ b/Examples/Tests/embedded_boundary_diffraction/analysis_fields.py @@ -6,36 +6,34 @@ occurs along the angle given by the theoretical Airy pattern, i.e. theta_diffraction = 1.22 * lambda / d """ -import os + import sys import numpy as np from openpmd_viewer import OpenPMDTimeSeries from scipy.ndimage import gaussian_filter1d -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -ts = OpenPMDTimeSeries('./EmbeddedBoundaryDiffraction_plt/') +filename = sys.argv[1] +ts = OpenPMDTimeSeries(filename) # Extract the intensity as a function of r and z -Ex, info = ts.get_field('E', 'x', iteration=300) -I = gaussian_filter1d(Ex**2, sigma=5, axis=0) # Extract intensity by averaging E^2 over wavelength -irmax = np.argmax( I, axis=-1) +Ex, info = ts.get_field("E", "x", iteration=300) +In = gaussian_filter1d( + Ex**2, sigma=5, axis=0 +) # Extract intensity by averaging E^2 over wavelength +irmax = np.argmax(In, axis=-1) + # Find the radius of the first minimum, as a function of z def r_first_minimum(iz): - ir = len(info.r)//2 - while I[iz, ir+1] < I[iz, ir]: + ir = len(info.r) // 2 + while In[iz, ir + 1] < In[iz, ir]: ir += 1 return info.r[ir] -r = np.array([ r_first_minimum(iz) for iz in range(len(info.z)) ]) -# Check that this corresponds to the prediction from the Airy pattern -theta_diffraction = np.arcsin(1.22*0.1/0.4)/2 -assert np.all( abs(r[50:] - theta_diffraction*info.z[50:]) < 0.03 ) -# Open the right plot file -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') +r = np.array([r_first_minimum(iz) for iz in range(len(info.z))]) + +# Check that this corresponds to the prediction from the Airy pattern +theta_diffraction = np.arcsin(1.22 * 0.1 / 0.4) / 2 +assert np.all(abs(r[50:] - theta_diffraction * info.z[50:]) < 0.03) diff --git a/Examples/Tests/embedded_boundary_diffraction/inputs_rz b/Examples/Tests/embedded_boundary_diffraction/inputs_test_rz_embedded_boundary_diffraction similarity index 100% rename from Examples/Tests/embedded_boundary_diffraction/inputs_rz rename to Examples/Tests/embedded_boundary_diffraction/inputs_test_rz_embedded_boundary_diffraction diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt new file mode 100644 index 00000000000..0aa5b48b2b7 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt @@ -0,0 +1,110 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py new file mode 100755 index 00000000000..3ba44a8ac1b --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +""" +This analysis script checks that there is no spurious charge build-up when a particle is absorbed by an embedded boundary. + +More specifically, this test simulates two particles of oppposite charge that are initialized at +the same position and then move in opposite directions. The particles are surrounded by a cylindrical +embedded boundary, and are absorbed when their trajectory intersects this boundary. With an +electromagnetic solver, this can lead to spurious charge build-up (i.e., div(E)!= rho/epsion_0) +that remains at the position where particle was absorbed. + +Note that, in this test, there will also be a (non-spurious) component of div(E) that propagates +along the embedded boundary, due to electromagnetic waves reflecting on this boundary. +When checking for static, spurious charge build-up, we average div(E) in time to remove this component. + +The test is performed in 2D, 3D and RZ. +(In 2D, the cylindrical embedded boundary becomes two parallel plates) +""" + +from openpmd_viewer import OpenPMDTimeSeries + +ts = OpenPMDTimeSeries("./diags/diag1/") + +divE_stacked = ts.iterate( + lambda iteration: ts.get_field("divE", iteration=iteration)[0] +) +start_avg_iter = 25 +end_avg_iter = 100 +divE_avg = divE_stacked[start_avg_iter:end_avg_iter].mean(axis=0) + +# Adjust the tolerance so that the remaining error due to the propagating +# div(E) (after averaging) is below this tolerance, but so that any typical +# spurious charge build-up is above this tolerance. This is dimension-dependent. +dim = ts.fields_metadata["divE"]["geometry"] +if dim == "3dcartesian": + tolerance = 7e-11 +elif dim == "2dcartesian": + tolerance = 3.5e-10 +elif dim == "thetaMode": + # In RZ: there are issues with divE on axis + # Set the few cells around the axis to 0 for this test + divE_avg[:, 13:19] = 0 + tolerance = 4e-12 + + +def check_tolerance(array, tolerance): + assert abs(array).max() <= tolerance, ( + f"Test did not pass: the max error {abs(array).max()} exceeded the tolerance of {tolerance}." + ) + print("All elements of are within the tolerance.") + + +check_tolerance(divE_avg, tolerance) diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base new file mode 100644 index 00000000000..6c940d2298e --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base @@ -0,0 +1,35 @@ +max_step = 100 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.const_dt = 1.17957283598e-09 +warpx.use_filter = 0 + +my_constants.R = 6.35 +warpx.eb_implicit_function = "(x**2 + y**2 - R**2)" + +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 0.0 +electron.single_particle_u = 1.e20 0.0 0.4843221e20 # gamma*beta +electron.single_particle_weight = 1.0 + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 0.0 0.0 0.0 +positron.single_particle_u = -1.e20 0.0 -0.4843221e20 # gamma*beta +positron.single_particle_weight = 1.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = divE rho +diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..99110df1634 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..4cdbfc5dd5e --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..6113f0668fe --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..ea977877a2d --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..ea977877a2d --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..553e5e058e7 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..7faf7fd8934 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..7faf7fd8934 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..aad65594c96 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt b/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt new file mode 100644 index 00000000000..3e79e526218 --- /dev/null +++ b/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py deleted file mode 100755 index faec3ed4668..00000000000 --- a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -import numpy as np - -from pywarpx import fields, picmi - -max_steps = 1 -unit = 1e-3 - -################################## -# Define the mesh -################################## -# mesh cells per direction -nx = 64 -ny = 64 -nz = 64 - -# mesh bounds for domain -xmin = -32*unit -xmax = 32*unit -ymin = -32*unit -ymax = 32*unit -zmin = -32*unit -zmax = 32*unit - -########################## -# numerics components -########################## -lower_boundary_conditions = ['open', 'dirichlet', 'periodic'] -upper_boundary_conditions = ['open', 'dirichlet', 'periodic'] - -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = lower_boundary_conditions, - upper_boundary_conditions = upper_boundary_conditions, - lower_boundary_conditions_particles = ['absorbing', 'absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing', 'periodic'], - moving_window_velocity = None, - warpx_max_grid_size = 64 -) - - -flag_correct_div = False - -solver = picmi.ElectromagneticSolver(grid=grid, method='Yee', cfl=1.) - -n_cavity=30 -L_cavity = n_cavity*unit - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="max(max(max(x-L_cavity/2,-L_cavity/2-x),max(y-L_cavity/2,-L_cavity/2-y)),max(z-L_cavity/2,-L_cavity/2-z))", - L_cavity=L_cavity -) - - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 1, - write_dir = '.', - warpx_file_prefix = "embedded_boundary_python_API_plt" -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 1, - data_list = ['Ex'], - write_dir = '.', - warpx_file_prefix = "embedded_boundary_python_API_plt" -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - warpx_embedded_boundary=embedded_boundary, - verbose = 1 -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -sim.initialize_inputs() - -sim.step(1) - -edge_lengths_x = fields.EdgeLengthsxWrapper() -edge_lengths_y = fields.EdgeLengthsyWrapper() -edge_lengths_z = fields.EdgeLengthszWrapper() -face_areas_x = fields.FaceAreasxWrapper() -face_areas_y = fields.FaceAreasyWrapper() -face_areas_z = fields.FaceAreaszWrapper() - -print("======== Testing the wrappers of m_edge_lengths =========") - -ly_slice_x = edge_lengths_y[nx//2,:,:] -lz_slice_x = edge_lengths_z[nx//2,:,:] - -n_edge_y_lo = (ny - 30)//2 -n_edge_y_hi = ny - (ny - 30)//2 -n_edge_z_lo = (nz - 30)//2 -n_edge_z_hi = nz - (nz - 30)//2 - -perimeter_slice_x = (np.sum(ly_slice_x[n_edge_y_lo:n_edge_y_hi, n_edge_z_lo+1]) + - np.sum(ly_slice_x[n_edge_y_lo:n_edge_y_hi, n_edge_z_hi-1]) + - np.sum(lz_slice_x[n_edge_y_lo+1, n_edge_z_lo:n_edge_z_hi]) + - np.sum(lz_slice_x[n_edge_y_hi-1, n_edge_z_lo:n_edge_z_hi])) - -perimeter_slice_x_true = L_cavity*4 - -print("Perimeter of the middle x-slice:", perimeter_slice_x) -assert np.isclose(perimeter_slice_x, perimeter_slice_x_true, rtol=1e-05, atol=1e-08) - - -lx_slice_y = edge_lengths_x[:,ny//2,:] -lz_slice_y = edge_lengths_z[:,ny//2,:] - -n_edge_x_lo = (nx - 30)//2 -n_edge_x_hi = nx - (nx - 30)//2 -n_edge_z_lo = (nz - 30)//2 -n_edge_z_hi = nz - (nz - 30)//2 - -perimeter_slice_y = (np.sum(lx_slice_y[n_edge_x_lo:n_edge_x_hi, n_edge_z_lo+1]) + - np.sum(lx_slice_y[n_edge_x_lo:n_edge_x_hi, n_edge_z_hi-1]) + - np.sum(lz_slice_y[n_edge_x_lo+1, n_edge_z_lo:n_edge_z_hi]) + - np.sum(lz_slice_y[n_edge_x_hi-1, n_edge_z_lo:n_edge_z_hi])) - -perimeter_slice_y_true = L_cavity*4 - - -print("Perimeter of the middle y-slice:", perimeter_slice_y) -assert np.isclose(perimeter_slice_y, perimeter_slice_y_true, rtol=1e-05, atol=1e-08) - - -lx_slice_z = edge_lengths_x[:,:,nz//2] -ly_slice_z = edge_lengths_y[:,:,nz//2] - -n_edge_x_lo = (nx - 30)//2 -n_edge_x_hi = nx - (nx - 30)//2 -n_edge_y_lo = (ny - 30)//2 -n_edge_y_hi = ny - (ny - 30)//2 - -perimeter_slice_z = (np.sum(lx_slice_z[n_edge_x_lo:n_edge_x_hi, n_edge_y_lo+1]) + - np.sum(lx_slice_z[n_edge_x_lo:n_edge_x_hi, n_edge_y_hi-1]) + - np.sum(ly_slice_z[n_edge_x_lo+1, n_edge_y_lo:n_edge_y_hi]) + - np.sum(ly_slice_z[n_edge_x_hi-1, n_edge_y_lo:n_edge_y_hi])) - -perimeter_slice_z_true = L_cavity*4 - -print("Perimeter of the middle z-slice:", perimeter_slice_z) -assert np.isclose(perimeter_slice_z, perimeter_slice_z_true, rtol=1e-05, atol=1e-08) - -print("======== Testing the wrappers of m_face_areas =========") - -Sx_slice = np.sum(face_areas_x[nx//2,:,:]) -dx = (xmax-xmin)/nx -Ax = dx*dx -Sx_slice_true = L_cavity*L_cavity - 2*Ax -print("Area of the middle x-slice:", Sx_slice) -assert np.isclose(Sx_slice, Sx_slice_true, rtol=1e-05, atol=1e-08) - -Sy_slice = np.sum(face_areas_y[:,ny//2,:]) -dy = (ymax-ymin)/ny -Ay = dy*dy -Sy_slice_true = L_cavity*L_cavity - 2*Ay -print("Area of the middle y-slice:", Sx_slice) -assert np.isclose(Sy_slice, Sy_slice_true, rtol=1e-05, atol=1e-08) - -Sz_slice = np.sum(face_areas_z[:,:,nz//2]) -dz = (zmax-zmin)/nz -Az = dz*dz -Sz_slice_true = L_cavity*L_cavity - 2*Az -print("Area of the middle z-slice:", Sz_slice) -assert np.isclose(Sz_slice, Sz_slice_true, rtol=1e-05, atol=1e-08) - -sim.step(1) diff --git a/Examples/Tests/embedded_boundary_python_api/analysis.py b/Examples/Tests/embedded_boundary_python_api/analysis.py deleted file mode 100755 index 09cc2accfea..00000000000 --- a/Examples/Tests/embedded_boundary_python_api/analysis.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# This script just checks that the PICMI file executed successfully. -# If it did there will be a plotfile for the final step. - -import sys - -step = int(sys.argv[1][-5:]) - -assert step == 2 diff --git a/Examples/Tests/embedded_boundary_python_api/analysis_default_regression.py b/Examples/Tests/embedded_boundary_python_api/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_python_api/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py new file mode 100755 index 00000000000..7bd7cd68f25 --- /dev/null +++ b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import numpy as np + +from pywarpx import fields, picmi + +max_steps = 1 +unit = 1e-3 + +################################## +# Define the mesh +################################## +# mesh cells per direction +nx = 64 +ny = 64 +nz = 64 + +# mesh bounds for domain +xmin = -32 * unit +xmax = 32 * unit +ymin = -32 * unit +ymax = 32 * unit +zmin = -32 * unit +zmax = 32 * unit + +########################## +# numerics components +########################## +lower_boundary_conditions = ["open", "dirichlet", "periodic"] +upper_boundary_conditions = ["open", "dirichlet", "periodic"] + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=lower_boundary_conditions, + upper_boundary_conditions=upper_boundary_conditions, + lower_boundary_conditions_particles=["absorbing", "absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "periodic"], + moving_window_velocity=None, + warpx_max_grid_size=64, +) + + +flag_correct_div = False + +solver = picmi.ElectromagneticSolver(grid=grid, method="ECT", cfl=1.0) + +n_cavity = 30 +L_cavity = n_cavity * unit + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="max(max(max(x-L_cavity/2,-L_cavity/2-x),max(y-L_cavity/2,-L_cavity/2-y)),max(z-L_cavity/2,-L_cavity/2-z))", + L_cavity=L_cavity, +) + + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=1, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=1, + data_list=["Ex"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + verbose=1, +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +sim.initialize_inputs() + +sim.step(1) + +edge_lengths_x = fields.EdgeLengthsxWrapper() +edge_lengths_y = fields.EdgeLengthsyWrapper() +edge_lengths_z = fields.EdgeLengthszWrapper() +face_areas_x = fields.FaceAreasxWrapper() +face_areas_y = fields.FaceAreasyWrapper() +face_areas_z = fields.FaceAreaszWrapper() + +print("======== Testing the wrappers of edge_lengths =========") + +ly_slice_x = edge_lengths_y[nx // 2, :, :] +lz_slice_x = edge_lengths_z[nx // 2, :, :] + +n_edge_y_lo = (ny - 30) // 2 +n_edge_y_hi = ny - (ny - 30) // 2 +n_edge_z_lo = (nz - 30) // 2 +n_edge_z_hi = nz - (nz - 30) // 2 + +perimeter_slice_x = ( + np.sum(ly_slice_x[n_edge_y_lo:n_edge_y_hi, n_edge_z_lo + 1]) + + np.sum(ly_slice_x[n_edge_y_lo:n_edge_y_hi, n_edge_z_hi - 1]) + + np.sum(lz_slice_x[n_edge_y_lo + 1, n_edge_z_lo:n_edge_z_hi]) + + np.sum(lz_slice_x[n_edge_y_hi - 1, n_edge_z_lo:n_edge_z_hi]) +) + +perimeter_slice_x_true = L_cavity * 4 + +print("Perimeter of the middle x-slice:", perimeter_slice_x) +assert np.isclose(perimeter_slice_x, perimeter_slice_x_true, rtol=1e-05, atol=1e-08) + + +lx_slice_y = edge_lengths_x[:, ny // 2, :] +lz_slice_y = edge_lengths_z[:, ny // 2, :] + +n_edge_x_lo = (nx - 30) // 2 +n_edge_x_hi = nx - (nx - 30) // 2 +n_edge_z_lo = (nz - 30) // 2 +n_edge_z_hi = nz - (nz - 30) // 2 + +perimeter_slice_y = ( + np.sum(lx_slice_y[n_edge_x_lo:n_edge_x_hi, n_edge_z_lo + 1]) + + np.sum(lx_slice_y[n_edge_x_lo:n_edge_x_hi, n_edge_z_hi - 1]) + + np.sum(lz_slice_y[n_edge_x_lo + 1, n_edge_z_lo:n_edge_z_hi]) + + np.sum(lz_slice_y[n_edge_x_hi - 1, n_edge_z_lo:n_edge_z_hi]) +) + +perimeter_slice_y_true = L_cavity * 4 + + +print("Perimeter of the middle y-slice:", perimeter_slice_y) +assert np.isclose(perimeter_slice_y, perimeter_slice_y_true, rtol=1e-05, atol=1e-08) + + +lx_slice_z = edge_lengths_x[:, :, nz // 2] +ly_slice_z = edge_lengths_y[:, :, nz // 2] + +n_edge_x_lo = (nx - 30) // 2 +n_edge_x_hi = nx - (nx - 30) // 2 +n_edge_y_lo = (ny - 30) // 2 +n_edge_y_hi = ny - (ny - 30) // 2 + +perimeter_slice_z = ( + np.sum(lx_slice_z[n_edge_x_lo:n_edge_x_hi, n_edge_y_lo + 1]) + + np.sum(lx_slice_z[n_edge_x_lo:n_edge_x_hi, n_edge_y_hi - 1]) + + np.sum(ly_slice_z[n_edge_x_lo + 1, n_edge_y_lo:n_edge_y_hi]) + + np.sum(ly_slice_z[n_edge_x_hi - 1, n_edge_y_lo:n_edge_y_hi]) +) + +perimeter_slice_z_true = L_cavity * 4 + +print("Perimeter of the middle z-slice:", perimeter_slice_z) +assert np.isclose(perimeter_slice_z, perimeter_slice_z_true, rtol=1e-05, atol=1e-08) + +print("======== Testing the wrappers of face_areas =========") + +Sx_slice = np.sum(face_areas_x[nx // 2, :, :]) +dx = (xmax - xmin) / nx +Ax = dx * dx +Sx_slice_true = L_cavity * L_cavity - 2 * Ax +print("Area of the middle x-slice:", Sx_slice) +assert np.isclose(Sx_slice, Sx_slice_true, rtol=1e-05, atol=1e-08) + +Sy_slice = np.sum(face_areas_y[:, ny // 2, :]) +dy = (ymax - ymin) / ny +Ay = dy * dy +Sy_slice_true = L_cavity * L_cavity - 2 * Ay +print("Area of the middle y-slice:", Sx_slice) +assert np.isclose(Sy_slice, Sy_slice_true, rtol=1e-05, atol=1e-08) + +Sz_slice = np.sum(face_areas_z[:, :, nz // 2]) +dz = (zmax - zmin) / nz +Az = dz * dz +Sz_slice_true = L_cavity * L_cavity - 2 * Az +print("Area of the middle z-slice:", Sz_slice) +assert np.isclose(Sz_slice, Sz_slice_true, rtol=1e-05, atol=1e-08) + +sim.step(1) diff --git a/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt b/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt new file mode 100644 index 00000000000..cb7fa405210 --- /dev/null +++ b/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_rotated_cube # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_rotated_cube # inputs + "analysis_fields_2d.py diags/diag1000068" # analysis + "analysis_default_regression.py --path diags/diag1000068" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_rotated_cube # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_rotated_cube # inputs + "analysis_fields_3d.py diags/diag1000111" # analysis + "analysis_default_regression.py --path diags/diag1000111" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_default_regression.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py deleted file mode 100755 index e849958468f..00000000000 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 Lorenzo Giacomel -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -import os -import sys - -import numpy as np -import yt -from scipy.constants import c, mu_0, pi - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# This is a script that analyses the simulation results from -# the script `inputs_3d`. This simulates a TMmnp mode in a PEC cubic resonator rotated by pi/8. -# The magnetic field in the simulation is given (in theory) by: -# $$ B_x = \frac{-2\mu}{h^2}\, k_x k_z \sin(k_x x)\cos(k_y y)\cos(k_z z)\cos( \omega_p t)$$ -# $$ B_y = \frac{-2\mu}{h^2}\, k_y k_z \cos(k_x x)\sin(k_y y)\cos(k_z z)\cos( \omega_p t)$$ -# $$ B_z = \cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ -# with -# $$ h^2 = k_x^2 + k_y^2 + k_z^2$$ -# $$ k_x = \frac{m\pi}{L}$$ -# $$ k_y = \frac{n\pi}{L}$$ -# $$ k_z = \frac{p\pi}{L}$$ - -hi = [0.8, 0.8, 0.8] -lo = [-0.8, -0.8, -0.8] -m = 0 -n = 1 -p = 1 -Lx = 1 -Ly = 1 -Lz = 1 -h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 -theta = np.pi/6 - -# Open the right plot file -filename = sys.argv[1] -ds = yt.load(filename) -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - -t = ds.current_time.to_value() - -rel_tol_err = 1e-2 -my_grid = ds.index.grids[0] - -By_sim = my_grid['raw', 'By_fp'].squeeze().v -Bz_sim = my_grid['raw', 'Bz_fp'].squeeze().v - -ncells = np.array(np.shape(By_sim[:, :, :, 0])) -dx = (hi[0] - lo[0])/ncells[0] -dy = (hi[1] - lo[1])/ncells[1] -dz = (hi[2] - lo[2])/ncells[2] - -# Compute the analytic solution -Bx_th = np.zeros(ncells) -By_th = np.zeros(ncells) -Bz_th = np.zeros(ncells) -for i in range(ncells[0]): - for j in range(ncells[1]): - for k in range(ncells[2]): - x0 = (i+0.5)*dx + lo[0] - y0 = j*dy + lo[1] - z0 = (k+0.5)*dz + lo[2] - - x = x0 - y = y0*np.cos(-theta)-z0*np.sin(-theta) - z = y0*np.sin(-theta)+z0*np.cos(-theta) - By = -2/h_2*mu_0*(n * pi/Ly)*(p * pi/Lz) * (np.cos(m * pi/Lx * (x - Lx/2)) * - np.sin(n * pi/Ly * (y - Ly/2)) * - np.cos(p * pi/Lz * (z - Lz/2)) * - np.cos(np.sqrt(2) * - np.pi / Lx * c * t)) - - Bz = mu_0*(np.cos(m * pi/Lx * (x - Lx/2)) * - np.cos(n * pi/Ly * (y - Ly/2)) * - np.sin(p * pi/Lz * (z - Lz/2)) * - np.cos(np.sqrt(2) * np.pi / Lx * c * t)) - - By_th[i, j, k] = (By*np.cos(theta) - Bz*np.sin(theta))*(By_sim[i, j, k, 0] != 0) - - x0 = (i+0.5)*dx + lo[0] - y0 = (j+0.5)*dy + lo[1] - z0 = k*dz + lo[2] - - x = x0 - y = y0*np.cos(-theta)-z0*np.sin(-theta) - z = y0*np.sin(-theta)+z0*np.cos(-theta) - - By = -2/h_2*mu_0*(n * pi/Ly)*(p * pi/Lz) * (np.cos(m * pi/Lx * (x - Lx/2)) * - np.sin(n * pi/Ly * (y - Ly/2)) * - np.cos(p * pi/Lz * (z - Lz/2)) * - np.cos(np.sqrt(2) * - np.pi / Lx * c * t)) - - Bz = mu_0*(np.cos(m * pi/Lx * (x - Lx/2)) * - np.cos(n * pi/Ly * (y - Ly/2)) * - np.sin(p * pi/Lz * (z - Lz/2)) * - np.cos(np.sqrt(2) * np.pi / Lx * c * t)) - - Bz_th[i, j, k] = (By*np.sin(theta) + Bz*np.cos(theta))*(Bz_sim[i, j, k, 0] != 0) - - -# Compute relative l^2 error on By -rel_err_y = np.sqrt( np.sum(np.square(By_sim[:, :, :, 0] - By_th)) / np.sum(np.square(By_th))) -assert(rel_err_y < rel_tol_err) -# Compute relative l^2 error on Bz -rel_err_z = np.sqrt( np.sum(np.square(Bz_sim[:, :, :, 0] - Bz_th)) / np.sum(np.square(Bz_th))) -assert(rel_err_z < rel_tol_err) - -test_name = os.path.split(os.getcwd())[1] - -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py index dcdbc83a729..dbb74b174e7 100755 --- a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py @@ -1,15 +1,11 @@ #!/usr/bin/env python3 -import os import sys import numpy as np import yt from scipy.constants import c, mu_0, pi -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # This is a script that analyses the simulation results from # the script `inputs_3d`. This simulates a TMmnp mode in a PEC cubic resonator. # The magnetic field in the simulation is given (in theory) by: @@ -32,14 +28,16 @@ # Open the right plot file filename = sys.argv[1] ds = yt.load(filename) -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) my_grid = ds.index.grids[0] -By_sim = my_grid['By'].squeeze().v +By_sim = my_grid["By"].squeeze().v t = ds.current_time.to_value() -theta = np.pi/8 +theta = np.pi / 8 # Compute the analytic solution By_th = np.zeros(ncells) @@ -47,19 +45,21 @@ for j in range(ncells[1]): x = i * dx + lo[0] z = j * dz + lo[1] - xr = x*np.cos(-theta) + z*np.sin(-theta) - zr = -x*np.sin(-theta) + z*np.cos(-theta) + xr = x * np.cos(-theta) + z * np.sin(-theta) + zr = -x * np.sin(-theta) + z * np.cos(-theta) - By_th[i, j] = mu_0 * (np.cos(m * pi / Lx * (xr - Lx / 2)) * - np.cos(n * pi / Lz * (zr - Lz / 2)) * - np.cos(np.pi / Lx * c * t))*(By_sim[i, j] != 0) + By_th[i, j] = ( + mu_0 + * ( + np.cos(m * pi / Lx * (xr - Lx / 2)) + * np.cos(n * pi / Lz * (zr - Lz / 2)) + * np.cos(np.pi / Lx * c * t) + ) + * (By_sim[i, j] != 0) + ) rel_tol_err = 1e-1 # Compute relative l^2 error on By rel_err_y = np.sqrt(np.sum(np.square(By_sim - By_th)) / np.sum(np.square(By_th))) -assert (rel_err_y < rel_tol_err) - -test_name = os.path.split(os.getcwd())[1] - -checksumAPI.evaluate_checksum(test_name, filename) +assert rel_err_y < rel_tol_err diff --git a/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_3d.py b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_3d.py new file mode 100755 index 00000000000..00d1ba2280f --- /dev/null +++ b/Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_3d.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Lorenzo Giacomel +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import sys + +import numpy as np +import yt +from scipy.constants import c, mu_0, pi + +# This is a script that analyses the simulation results from +# the script `inputs_3d`. This simulates a TMmnp mode in a PEC cubic resonator rotated by pi/8. +# The magnetic field in the simulation is given (in theory) by: +# $$ B_x = \frac{-2\mu}{h^2}\, k_x k_z \sin(k_x x)\cos(k_y y)\cos(k_z z)\cos( \omega_p t)$$ +# $$ B_y = \frac{-2\mu}{h^2}\, k_y k_z \cos(k_x x)\sin(k_y y)\cos(k_z z)\cos( \omega_p t)$$ +# $$ B_z = \cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ +# with +# $$ h^2 = k_x^2 + k_y^2 + k_z^2$$ +# $$ k_x = \frac{m\pi}{L}$$ +# $$ k_y = \frac{n\pi}{L}$$ +# $$ k_z = \frac{p\pi}{L}$$ + +hi = [0.8, 0.8, 0.8] +lo = [-0.8, -0.8, -0.8] +m = 0 +n = 1 +p = 1 +Lx = 1 +Ly = 1 +Lz = 1 +h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 +theta = np.pi / 6 + +# Open the right plot file +filename = sys.argv[1] +ds = yt.load(filename) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) + +t = ds.current_time.to_value() + +rel_tol_err = 1e-2 +my_grid = ds.index.grids[0] + +By_sim = my_grid["raw", "By_fp"].squeeze().v +Bz_sim = my_grid["raw", "Bz_fp"].squeeze().v + +ncells = np.array(np.shape(By_sim[:, :, :, 0])) +dx = (hi[0] - lo[0]) / ncells[0] +dy = (hi[1] - lo[1]) / ncells[1] +dz = (hi[2] - lo[2]) / ncells[2] + +# Compute the analytic solution +Bx_th = np.zeros(ncells) +By_th = np.zeros(ncells) +Bz_th = np.zeros(ncells) +for i in range(ncells[0]): + for j in range(ncells[1]): + for k in range(ncells[2]): + x0 = (i + 0.5) * dx + lo[0] + y0 = j * dy + lo[1] + z0 = (k + 0.5) * dz + lo[2] + + x = x0 + y = y0 * np.cos(-theta) - z0 * np.sin(-theta) + z = y0 * np.sin(-theta) + z0 * np.cos(-theta) + By = ( + -2 + / h_2 + * mu_0 + * (n * pi / Ly) + * (p * pi / Lz) + * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.sin(n * pi / Ly * (y - Ly / 2)) + * np.cos(p * pi / Lz * (z - Lz / 2)) + * np.cos(np.sqrt(2) * np.pi / Lx * c * t) + ) + ) + + Bz = mu_0 * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.cos(n * pi / Ly * (y - Ly / 2)) + * np.sin(p * pi / Lz * (z - Lz / 2)) + * np.cos(np.sqrt(2) * np.pi / Lx * c * t) + ) + + By_th[i, j, k] = (By * np.cos(theta) - Bz * np.sin(theta)) * ( + By_sim[i, j, k, 0] != 0 + ) + + x0 = (i + 0.5) * dx + lo[0] + y0 = (j + 0.5) * dy + lo[1] + z0 = k * dz + lo[2] + + x = x0 + y = y0 * np.cos(-theta) - z0 * np.sin(-theta) + z = y0 * np.sin(-theta) + z0 * np.cos(-theta) + + By = ( + -2 + / h_2 + * mu_0 + * (n * pi / Ly) + * (p * pi / Lz) + * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.sin(n * pi / Ly * (y - Ly / 2)) + * np.cos(p * pi / Lz * (z - Lz / 2)) + * np.cos(np.sqrt(2) * np.pi / Lx * c * t) + ) + ) + + Bz = mu_0 * ( + np.cos(m * pi / Lx * (x - Lx / 2)) + * np.cos(n * pi / Ly * (y - Ly / 2)) + * np.sin(p * pi / Lz * (z - Lz / 2)) + * np.cos(np.sqrt(2) * np.pi / Lx * c * t) + ) + + Bz_th[i, j, k] = (By * np.sin(theta) + Bz * np.cos(theta)) * ( + Bz_sim[i, j, k, 0] != 0 + ) + + +# Compute relative l^2 error on By +rel_err_y = np.sqrt( + np.sum(np.square(By_sim[:, :, :, 0] - By_th)) / np.sum(np.square(By_th)) +) +assert rel_err_y < rel_tol_err +# Compute relative l^2 error on Bz +rel_err_z = np.sqrt( + np.sum(np.square(Bz_sim[:, :, :, 0] - Bz_th)) / np.sum(np.square(Bz_th)) +) +assert rel_err_z < rel_tol_err diff --git a/Examples/Tests/embedded_boundary_rotated_cube/inputs_2d b/Examples/Tests/embedded_boundary_rotated_cube/inputs_2d deleted file mode 100644 index e7e03168824..00000000000 --- a/Examples/Tests/embedded_boundary_rotated_cube/inputs_2d +++ /dev/null @@ -1,44 +0,0 @@ -stop_time = 8.019424744948937e-09 -amr.n_cell = 32 32 -amr.max_grid_size = 128 -amr.max_level = 0 - -geometry.dims = 2 -geometry.prob_lo = -0.8 -0.8 -geometry.prob_hi = 0.8 0.8 -warpx.cfl = 1 - -boundary.field_lo = pec pec -boundary.field_hi = pec pec - -algo.maxwell_solver = ect - -my_constants.xmin = -0.53 -my_constants.zmin = -0.53 -my_constants.xmax = 0.53 -my_constants.zmax = 0.53 -my_constants.pi = 3.141592653589793 -my_constants.theta = pi/8 - -warpx.eb_implicit_function = "xr=x*cos(-theta)+z*sin(-theta); zr=-x*sin(-theta)+z*cos(-theta); max(max(xr+xmin,-(xr+xmax)), max(zr+zmin,-(zr+zmax)))" - -my_constants.m = 0 -my_constants.p = 1 -my_constants.Lx = 1.06 -my_constants.Lz = 1.06 -my_constants.x_cent = 0. -my_constants.z_cent = 0. -my_constants.mu_0 = 1.25663706212e-06 - -warpx.B_ext_grid_init_style = parse_B_ext_grid_function - -warpx.Bx_external_grid_function(x,y,z) = 0 -warpx.By_external_grid_function(x,y,z) = "mu_0 * - cos(m * pi / Lx * (x*cos(-theta)+z*sin(-theta) - Lx / 2 - x_cent)) * - cos(p * pi / Lz * (-x*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent))" -warpx.Bz_external_grid_function(x,y,z) = 0 - -diagnostics.diags_names = diag1 -diag1.intervals = 1000 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_rotated_cube/inputs_3d b/Examples/Tests/embedded_boundary_rotated_cube/inputs_3d deleted file mode 100644 index 77e259e8975..00000000000 --- a/Examples/Tests/embedded_boundary_rotated_cube/inputs_3d +++ /dev/null @@ -1,72 +0,0 @@ -stop_time = 5.303669113650618e-09 -amr.n_cell = 64 64 64 -amr.max_grid_size = 128 -amr.max_level = 0 - -geometry.dims = 3 -geometry.prob_lo = -0.8 -0.8 -0.8 -geometry.prob_hi = 0.8 0.8 0.8 -warpx.cfl = 1 - -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec - -algo.maxwell_solver = ect - -my_constants.xmin = -0.5 -my_constants.ymin = -0.5 -my_constants.zmin = -0.5 -my_constants.xmax = 0.5 -my_constants.ymax = 0.5 -my_constants.zmax = 0.5 -my_constants.pi = 3.141592653589793 -my_constants.theta = pi/6 - -warpx.eb_implicit_function = "max(max(max(x+xmin,-(x+xmax)), max(y*cos(-theta)-z*sin(-theta)+ymin,-(y*cos(-theta)-z*sin(-theta)+ymax))), max(y*sin(-theta)+z*cos(-theta)+zmin,-(y*sin(-theta)+z*cos(-theta)+zmax)))" - -my_constants.m = 0 -my_constants.n = 1 -my_constants.p = 1 -my_constants.Lx = 1 -my_constants.Ly = 1 -my_constants.Lz = 1 -my_constants.x_cent = 0. -my_constants.y_cent = 0. -my_constants.z_cent = 0. -my_constants.h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 -my_constants.mu_0 = 1.25663706212e-06 - -warpx.B_ext_grid_init_style = parse_B_ext_grid_function -warpx.Bx_external_grid_function(x,y,z) = "-2/h_2 * mu_0 * (m * pi / Lx) * (p * pi / Lz) * - sin(m * pi / Lx * (x - Lx / 2 - x_cent)) * - cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * - cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent))" - -warpx.By_external_grid_function(x,y,z) = "-2/h_2 * mu_0 * (n * pi / Ly) * (p * pi / Lz) * - cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * - sin(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * - cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * - cos(theta) - - mu_0 * - cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * - cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * - sin(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * - sin(theta)" - -warpx.Bz_external_grid_function(x,y,z) = "mu_0 * - cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * - cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * - sin(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * - cos(theta) - - 2/h_2 * mu_0 * (n * pi / Ly) * (p * pi / Lz) * - cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * - sin(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * - cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * - sin(theta)" - - -diagnostics.diags_names = diag1 -diag1.intervals = 1000 -diag1.diag_type = Full -diag1.plot_raw_fields = 1 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_2d_embedded_boundary_rotated_cube b/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_2d_embedded_boundary_rotated_cube new file mode 100644 index 00000000000..24fb2938c2d --- /dev/null +++ b/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_2d_embedded_boundary_rotated_cube @@ -0,0 +1,45 @@ +stop_time = 8.019424744948937e-09 +amr.n_cell = 32 32 +amr.max_grid_size = 128 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.8 -0.8 +geometry.prob_hi = 0.8 0.8 +warpx.cfl = 1 +warpx.abort_on_warning_threshold = medium + +boundary.field_lo = pec pec +boundary.field_hi = pec pec + +algo.maxwell_solver = ect + +my_constants.xmin = -0.53 +my_constants.zmin = -0.53 +my_constants.xmax = 0.53 +my_constants.zmax = 0.53 +my_constants.pi = 3.141592653589793 +my_constants.theta = pi/8 + +warpx.eb_implicit_function = "xr=x*cos(-theta)+z*sin(-theta); zr=-x*sin(-theta)+z*cos(-theta); max(max(xr+xmin,-(xr+xmax)), max(zr+zmin,-(zr+zmax)))" + +my_constants.m = 0 +my_constants.p = 1 +my_constants.Lx = 1.06 +my_constants.Lz = 1.06 +my_constants.x_cent = 0. +my_constants.z_cent = 0. +my_constants.mu_0 = 1.25663706212e-06 + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function + +warpx.Bx_external_grid_function(x,y,z) = 0 +warpx.By_external_grid_function(x,y,z) = "mu_0 * + cos(m * pi / Lx * (x*cos(-theta)+z*sin(-theta) - Lx / 2 - x_cent)) * + cos(p * pi / Lz * (-x*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent))" +warpx.Bz_external_grid_function(x,y,z) = 0 + +diagnostics.diags_names = diag1 +diag1.intervals = 1000 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_3d_embedded_boundary_rotated_cube b/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_3d_embedded_boundary_rotated_cube new file mode 100644 index 00000000000..faefeec2206 --- /dev/null +++ b/Examples/Tests/embedded_boundary_rotated_cube/inputs_test_3d_embedded_boundary_rotated_cube @@ -0,0 +1,73 @@ +stop_time = 5.303669113650618e-09 +amr.n_cell = 64 64 64 +amr.max_grid_size = 128 +amr.max_level = 0 + +geometry.dims = 3 +geometry.prob_lo = -0.8 -0.8 -0.8 +geometry.prob_hi = 0.8 0.8 0.8 +warpx.cfl = 1 +warpx.abort_on_warning_threshold = medium + +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec + +algo.maxwell_solver = ect + +my_constants.xmin = -0.5 +my_constants.ymin = -0.5 +my_constants.zmin = -0.5 +my_constants.xmax = 0.5 +my_constants.ymax = 0.5 +my_constants.zmax = 0.5 +my_constants.pi = 3.141592653589793 +my_constants.theta = pi/6 + +warpx.eb_implicit_function = "max(max(max(x+xmin,-(x+xmax)), max(y*cos(-theta)-z*sin(-theta)+ymin,-(y*cos(-theta)-z*sin(-theta)+ymax))), max(y*sin(-theta)+z*cos(-theta)+zmin,-(y*sin(-theta)+z*cos(-theta)+zmax)))" + +my_constants.m = 0 +my_constants.n = 1 +my_constants.p = 1 +my_constants.Lx = 1 +my_constants.Ly = 1 +my_constants.Lz = 1 +my_constants.x_cent = 0. +my_constants.y_cent = 0. +my_constants.z_cent = 0. +my_constants.h_2 = (m * pi / Lx) ** 2 + (n * pi / Ly) ** 2 + (p * pi / Lz) ** 2 +my_constants.mu_0 = 1.25663706212e-06 + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function +warpx.Bx_external_grid_function(x,y,z) = "-2/h_2 * mu_0 * (m * pi / Lx) * (p * pi / Lz) * + sin(m * pi / Lx * (x - Lx / 2 - x_cent)) * + cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * + cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent))" + +warpx.By_external_grid_function(x,y,z) = "-2/h_2 * mu_0 * (n * pi / Ly) * (p * pi / Lz) * + cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * + sin(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * + cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * + cos(theta) - + mu_0 * + cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * + cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * + sin(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * + sin(theta)" + +warpx.Bz_external_grid_function(x,y,z) = "mu_0 * + cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * + cos(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * + sin(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * + cos(theta) - + 2/h_2 * mu_0 * (n * pi / Ly) * (p * pi / Lz) * + cos(m * pi / Lx * (x - Lx / 2 - x_cent)) * + sin(n * pi / Ly * (y*cos(-theta)-z*sin(-theta) - Ly / 2 - y_cent)) * + cos(p * pi / Lz * (y*sin(-theta)+z*cos(-theta) - Lz / 2 - z_cent)) * + sin(theta)" + + +diagnostics.diags_names = diag1 +diag1.intervals = 1000 +diag1.diag_type = Full +diag1.plot_raw_fields = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz diff --git a/Examples/Tests/embedded_circle/CMakeLists.txt b/Examples/Tests/embedded_circle/CMakeLists.txt new file mode 100644 index 00000000000..1a0577da82e --- /dev/null +++ b/Examples/Tests/embedded_circle/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_circle # name + 2 # dims + 2 # nprocs + inputs_test_2d_embedded_circle # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000011 --rtol 1e-2" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_circle/analysis.py b/Examples/Tests/embedded_circle/analysis.py deleted file mode 100755 index 6401b47bb90..00000000000 --- a/Examples/Tests/embedded_circle/analysis.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# Get name of the test -test_name = os.path.split(os.getcwd())[1] - -# Run checksum regression test -checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-2) diff --git a/Examples/Tests/embedded_circle/analysis_default_regression.py b/Examples/Tests/embedded_circle/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_circle/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_circle/inputs_2d b/Examples/Tests/embedded_circle/inputs_test_2d_embedded_circle similarity index 100% rename from Examples/Tests/embedded_circle/inputs_2d rename to Examples/Tests/embedded_circle/inputs_test_2d_embedded_circle diff --git a/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt b/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt new file mode 100644 index 00000000000..a925cc537f4 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_energy_conserving_thermal_plasma # name + 2 # dims + 2 # nprocs + inputs_test_2d_energy_conserving_thermal_plasma # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency +) diff --git a/Examples/Tests/energy_conserving_thermal_plasma/analysis.py b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py index 43e9b6d9822..5991d888e20 100755 --- a/Examples/Tests/energy_conserving_thermal_plasma/analysis.py +++ b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py @@ -12,27 +12,14 @@ # than other gathering scheme. This tests checks that the energy does not increase by # more than 0.3% over the duration of the simulatoin. -import os -import sys - import numpy as np -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - # Get energy as a function of time, from reduced diagnostics -EFdata = np.genfromtxt('./diags/reducedfiles/EF.txt') # Field energy -EPdata = np.genfromtxt('./diags/reducedfiles/EP.txt') # Particle energy -field_energy = EFdata[:,2] -particle_energy = EPdata[:,2] +EFdata = np.genfromtxt("./diags/reducedfiles/EF.txt") # Field energy +EPdata = np.genfromtxt("./diags/reducedfiles/EP.txt") # Particle energy +field_energy = EFdata[:, 2] +particle_energy = EPdata[:, 2] E = field_energy + particle_energy -print(abs(E-E[0])/E[0]) +print(abs(E - E[0]) / E[0]) # Check that the energy is conserved to 0.3% -assert np.all( abs(E-E[0])/E[0] < 0.003 ) - -# Checksum test -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +assert np.all(abs(E - E[0]) / E[0] < 0.003) diff --git a/Examples/Tests/energy_conserving_thermal_plasma/analysis_default_regression.py b/Examples/Tests/energy_conserving_thermal_plasma/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic b/Examples/Tests/energy_conserving_thermal_plasma/inputs_test_2d_energy_conserving_thermal_plasma similarity index 100% rename from Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic rename to Examples/Tests/energy_conserving_thermal_plasma/inputs_test_2d_energy_conserving_thermal_plasma diff --git a/Examples/Tests/field_ionization/CMakeLists.txt b/Examples/Tests/field_ionization/CMakeLists.txt new file mode 100644 index 00000000000..71e34dbc5fc --- /dev/null +++ b/Examples/Tests/field_ionization/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_ionization_boost # name + 2 # dims + 2 # nprocs + inputs_test_2d_ionization_boost # inputs + "analysis.py diags/diag1000420" # analysis + "analysis_default_regression.py --path diags/diag1000420" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_ionization_lab # name + 2 # dims + 2 # nprocs + inputs_test_2d_ionization_lab # inputs + "analysis.py diags/diag1001600" # analysis + "analysis_default_regression.py --path diags/diag1001600" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_ionization_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_ionization_picmi.py # inputs + "analysis.py diags/diag1001600" # analysis + "analysis_default_regression.py --path diags/diag1001600" # checksum + OFF # dependency +) diff --git a/Examples/Tests/field_ionization/README.rst b/Examples/Tests/field_ionization/README.rst new file mode 100644 index 00000000000..c2f11ed1a40 --- /dev/null +++ b/Examples/Tests/field_ionization/README.rst @@ -0,0 +1,59 @@ +.. _examples-tests-field_ionization: + +Field Ionization +================ + +Run Test +-------- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: lab frame + + This example can be run **either** as: + + * **Python** script: ``python3 inputs_test_2d_ionization_picmi.py`` or + * WarpX **executable** using an input file: ``warpx.2d inputs_test_2d_ionization_lab max_step=1600`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: inputs_test_2d_ionization_picmi.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/field_ionization/inputs_test_2d_ionization_picmi.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_test_2d_ionization_lab + :language: ini + :caption: You can copy this file from ``Examples/Tests/field_ionization/inputs_test_2d_ionization_lab``. + + .. tab-item:: boosted frame + + This example can be run as: + + * WarpX **executable** using an input file: ``warpx.2d inputs_test_2d_ionization_boost max_step=420`` + + .. literalinclude:: inputs_test_2d_ionization_boost + :language: ini + :caption: You can copy this file from ``Examples/Tests/field_ionization/inputs_test_2d_ionization_boost``. + +Analyze +------- + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/field_ionization/analysis.py``. + +Visualize +--------- + +.. figure:: https://gist.githubusercontent.com/johvandewetering/48d092c003915f1d1689b507caa2865b/raw/29f5d12ed77831047ca12f456a07dbf3b99770d5/image_ionization.png + :alt: Electric field of the laser pulse with (top) ions with ionization levels and (bottom) ionized electrons. + + Electric field of the laser pulse with (top) ions with ionization levels and (bottom) ionized electrons. diff --git a/Examples/Tests/field_ionization/analysis.py b/Examples/Tests/field_ionization/analysis.py new file mode 100755 index 00000000000..bafc47f2145 --- /dev/null +++ b/Examples/Tests/field_ionization/analysis.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +""" +This script tests the result of the ionization module in WarpX. + +Input files inputs.rt and inputs.bf.rt are used to reproduce the test from +Chen, JCP, 2013, figure 2 (in the lab frame and in a boosted frame, +respectively): a plane-wave laser pulse propagates through a +uniform N2+ neutral plasma and further ionizes the Nitrogen atoms. This test +checks that, after the laser went through the plasma, ~32% of Nitrogen +ions are N5+, in agreement with theory from Chen's article. +""" + +import sys + +import numpy as np +import yt + +yt.funcs.mylog.setLevel(0) + +# Open plotfile specified in command line, and get ion's ionization level. +filename = sys.argv[1] +ds = yt.load(filename) +ad = ds.all_data() +ilev = ad["ions", "particle_ionizationLevel"].v + +# Fraction of Nitrogen ions that are N5+. +N5_fraction = ilev[ilev == 5].size / float(ilev.size) + +print("Number of ions: " + str(ilev.size)) +print("Number of N5+ : " + str(ilev[ilev == 5].size)) +print("N5_fraction : " + str(N5_fraction)) + +do_plot = False +if do_plot: + import matplotlib.pyplot as plt + + all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions + ) + F = all_data_level_0["boxlib", "Ex"].v.squeeze() + extent = [ + ds.domain_left_edge[1], + ds.domain_right_edge[1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], + ] + ad = ds.all_data() + + # Plot ions with ionization levels + species = "ions" + xi = ad[species, "particle_position_x"].v + zi = ad[species, "particle_position_y"].v + ii = ad[species, "particle_ionizationLevel"].v + plt.figure(figsize=(10, 10)) + plt.subplot(211) + plt.imshow(np.abs(F), extent=extent, aspect="auto", cmap="magma", origin="default") + plt.colorbar() + for lev in range(int(np.max(ii) + 1)): + select = ii == lev + plt.scatter( + zi[select], xi[select], s=0.2, label="ionization level: " + str(lev) + ) + plt.legend() + plt.title("abs(Ex) (V/m) and ions") + plt.xlabel("z (m)") + plt.ylabel("x (m)") + plt.subplot(212) + plt.imshow(np.abs(F), extent=extent, aspect="auto", cmap="magma", origin="default") + plt.colorbar() + + # Plot electrons + species = "electrons" + if species in [x[0] for x in ds.field_list]: + xe = ad[species, "particle_position_x"].v + ze = ad[species, "particle_position_y"].v + plt.scatter(ze, xe, s=0.1, c="r", label="electrons") + plt.title("abs(Ex) (V/m) and electrons") + plt.xlabel("z (m)") + plt.ylabel("x (m)") + plt.savefig("image_ionization.pdf", bbox_inches="tight") + +error_rel = abs(N5_fraction - 0.32) / 0.32 +tolerance_rel = 0.07 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert error_rel < tolerance_rel + +# Check that the user runtime component (if it exists) worked as expected +try: + orig_z = ad["electrons", "particle_orig_z"].v + print(f"orig_z: min = {np.min(orig_z)}, max = {np.max(orig_z)}") + assert np.all((orig_z > 0.0) & (orig_z < 1.5e-5)) + print("particle_orig_z has reasonable values") +except yt.utilities.exceptions.YTFieldNotFound: + pass # The backtransformed diagnostic version of the test does not have orig_z diff --git a/Examples/Tests/field_ionization/analysis_default_regression.py b/Examples/Tests/field_ionization/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/field_ionization/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/field_ionization/catalyst_pipeline.py b/Examples/Tests/field_ionization/catalyst_pipeline.py new file mode 100644 index 00000000000..321a23f4053 --- /dev/null +++ b/Examples/Tests/field_ionization/catalyst_pipeline.py @@ -0,0 +1,403 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.11.1-1332-ga0a402a54e +# and validated with paraview 5.12,1 +import paraview + +paraview.compatibility.major = 5 +paraview.compatibility.minor = 12 + +#### import the simple module from the paraview +from paraview.simple import * # noqa: F403 + +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# ---------------------------------------------------------------- +# setup views used in the visualization +# ---------------------------------------------------------------- + +# Create a new 'Render View' +renderView1 = CreateView("RenderView") +renderView1.ViewSize = [889, 820] +renderView1.InteractionMode = "2D" +renderView1.AxesGrid = "Grid Axes 3D Actor" +renderView1.CenterOfRotation = [5.125578491129211e-06, 5.125219398642561e-06, 0.0] +renderView1.StereoType = "Crystal Eyes" +renderView1.CameraPosition = [ + -3.2986377960746266e-08, + 1.2519774889107966e-05, + 6.655318906751408e-05, +] +renderView1.CameraFocalPoint = [-3.2986377960746266e-08, 1.2519774889107966e-05, 0.0] +renderView1.CameraFocalDisk = 1.0 +renderView1.CameraParallelScale = 9.246257677588838e-06 +renderView1.LegendGrid = "Legend Grid Actor" +renderView1.UseColorPaletteForBackground = 0 +renderView1.Background = [0.13725490196078433, 0.1450980392156863, 0.17647058823529413] + +SetActiveView(None) + +# ---------------------------------------------------------------- +# setup view layouts +# ---------------------------------------------------------------- + +# create new layout object 'Layout #1' +layout1 = CreateLayout(name="Layout #1") +layout1.AssignView(0, renderView1) +layout1.SetSize(889, 820) + +# ---------------------------------------------------------------- +# restore active view +# ---------------------------------------------------------------- +SetActiveView(renderView1) + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create a new 'XML Partitioned Dataset Collection Reader' +particles = PVTrivialProducer(registrationName="particles") + +# create a new 'Clip' +clip1 = Clip(registrationName="Clip1", Input=particles) +clip1.ClipType = "Plane" +clip1.HyperTreeGridClipper = "Plane" +clip1.Scalars = ["CELLS", "particle_electrons_cpu"] +clip1.Value = -396581.0 +clip1.Invert = 0 + +# init the 'Plane' selected for 'ClipType' +clip1.ClipType.Origin = [5.12557849112921e-06, 5.02521939864256e-06, 0.0] +clip1.ClipType.Normal = [0.0, 1.0, 0.0] + +# init the 'Plane' selected for 'HyperTreeGridClipper' +clip1.HyperTreeGridClipper.Origin = [5.12557849112921e-06, 5.12521939864256e-06, 0.0] + +# ---------------------------------------------------------------- +# setup the visualization in view 'renderView1' +# ---------------------------------------------------------------- + +# show data from clip1 +clip1Display = Show(clip1, renderView1, "UnstructuredGridRepresentation") + +# get 2D transfer function for 'particle_electrons_ux' +particle_electrons_uxTF2D = GetTransferFunction2D("particle_electrons_ux") +particle_electrons_uxTF2D.ScalarRangeInitialized = 1 +particle_electrons_uxTF2D.Range = [-719020005.8608257, 713207105.8017796, 0.0, 1.0] + +# get color transfer function/color map for 'particle_electrons_ux' +particle_electrons_uxLUT = GetColorTransferFunction("particle_electrons_ux") +particle_electrons_uxLUT.TransferFunction2D = particle_electrons_uxTF2D +particle_electrons_uxLUT.RGBPoints = [ + -813777010.3574873, + 0.0862745098039216, + 0.00392156862745098, + 0.298039215686275, + -764496926.60325, + 0.113725, + 0.0235294, + 0.45098, + -723569500.3656244, + 0.105882, + 0.0509804, + 0.509804, + -695170821.8676349, + 0.0392157, + 0.0392157, + 0.560784, + -667607367.8918439, + 0.0313725, + 0.0980392, + 0.6, + -640879233.4148993, + 0.0431373, + 0.164706, + 0.639216, + -602457575.7205275, + 0.054902, + 0.243137, + 0.678431, + -551506980.3332543, + 0.054902, + 0.317647, + 0.709804, + -488862937.3087116, + 0.0509804, + 0.396078, + 0.741176, + -448248678.177614, + 0.0392157, + 0.466667, + 0.768627, + -407634419.0465164, + 0.0313725, + 0.537255, + 0.788235, + -365245168.54020303, + 0.0313725, + 0.615686, + 0.811765, + -321811973.7593288, + 0.0235294, + 0.709804, + 0.831373, + -278378684.00181174, + 0.0509804, + 0.8, + 0.85098, + -242462794.85069358, + 0.0705882, + 0.854902, + 0.870588, + -209052579.2661966, + 0.262745, + 0.901961, + 0.862745, + -179818676.24599826, + 0.423529, + 0.941176, + 0.87451, + -134714937.4440689, + 0.572549, + 0.964706, + 0.835294, + -104645714.92502928, + 0.658824, + 0.980392, + 0.843137, + -82929117.53459203, + 0.764706, + 0.980392, + 0.866667, + -59542023.61142123, + 0.827451, + 0.980392, + 0.886275, + -13603012.798971772, + 0.913725, + 0.988235, + 0.937255, + 596326.4500254393, + 1.0, + 1.0, + 0.972549019607843, + 14795665.69902289, + 0.988235, + 0.980392, + 0.870588, + 35677038.567251325, + 0.992156862745098, + 0.972549019607843, + 0.803921568627451, + 51546874.348982334, + 0.992157, + 0.964706, + 0.713725, + 78275008.82592702, + 0.988235, + 0.956863, + 0.643137, + 120872979.08459747, + 0.980392, + 0.917647, + 0.509804, + 156788963.21235335, + 0.968627, + 0.87451, + 0.407843, + 193540171.8623129, + 0.94902, + 0.823529, + 0.321569, + 220268306.3392564, + 0.929412, + 0.776471, + 0.278431, + 259525283.53246915, + 0.909804, + 0.717647, + 0.235294, + 294605948.1613816, + 0.890196, + 0.658824, + 0.196078, + 323422245.31323636, + 0.878431, + 0.619608, + 0.168627, + 364036504.4443337, + 0.870588, + 0.54902, + 0.156863, + 404650763.575431, + 0.85098, + 0.47451, + 0.145098, + 445265022.7065283, + 0.831373, + 0.411765, + 0.133333, + 485879281.83762586, + 0.811765, + 0.345098, + 0.113725, + 526493540.9687232, + 0.788235, + 0.266667, + 0.0941176, + 567107800.0998205, + 0.741176, + 0.184314, + 0.0745098, + 607722059.2309178, + 0.690196, + 0.12549, + 0.0627451, + 648336318.3620149, + 0.619608, + 0.0627451, + 0.0431373, + 686340409.4987272, + 0.54902, + 0.027451, + 0.0705882, + 719750596.8870386, + 0.470588, + 0.0156863, + 0.0901961, + 757337055.2873727, + 0.4, + 0.00392157, + 0.101961, + 810793354.8864042, + 0.188235294117647, + 0.0, + 0.0705882352941176, +] +particle_electrons_uxLUT.ColorSpace = "Lab" +particle_electrons_uxLUT.NanOpacity = 0.0 +particle_electrons_uxLUT.ScalarRangeInitialized = 1.0 + +# get opacity transfer function/opacity map for 'particle_electrons_ux' +particle_electrons_uxPWF = GetOpacityTransferFunction("particle_electrons_ux") +particle_electrons_uxPWF.Points = [ + -813777010.3574873, + 0.0, + 0.5, + 0.0, + 810793354.8864042, + 1.0, + 0.5, + 0.0, +] +particle_electrons_uxPWF.ScalarRangeInitialized = 1 + +# trace defaults for the display properties. +clip1Display.Representation = "Surface" +clip1Display.ColorArrayName = ["CELLS", "particle_electrons_ux"] +clip1Display.LookupTable = particle_electrons_uxLUT +clip1Display.SelectTCoordArray = "None" +clip1Display.SelectNormalArray = "None" +clip1Display.SelectTangentArray = "None" +clip1Display.OSPRayScaleFunction = "Piecewise Function" +clip1Display.Assembly = "Hierarchy" +clip1Display.SelectOrientationVectors = "None" +clip1Display.ScaleFactor = 2.0093934867816264e-06 +clip1Display.SelectScaleArray = "None" +clip1Display.GlyphType = "Arrow" +clip1Display.GlyphTableIndexArray = "None" +clip1Display.GaussianRadius = 1.0046967433908131e-07 +clip1Display.SetScaleArray = [None, ""] +clip1Display.ScaleTransferFunction = "Piecewise Function" +clip1Display.OpacityArray = [None, ""] +clip1Display.OpacityTransferFunction = "Piecewise Function" +clip1Display.DataAxesGrid = "Grid Axes Representation" +clip1Display.PolarAxes = "Polar Axes Representation" +clip1Display.ScalarOpacityFunction = particle_electrons_uxPWF +clip1Display.ScalarOpacityUnitDistance = 6.795294978934044e-07 +clip1Display.OpacityArrayName = ["CELLS", "particle_electrons_cpu"] +clip1Display.SelectInputVectors = [None, ""] +clip1Display.WriteLog = "" + +# setup the color legend parameters for each legend in this view + +# get color legend/bar for particle_electrons_uxLUT in view renderView1 +particle_electrons_uxLUTColorBar = GetScalarBar(particle_electrons_uxLUT, renderView1) +particle_electrons_uxLUTColorBar.WindowLocation = "Any Location" +particle_electrons_uxLUTColorBar.Position = [0.7817772778402697, 0.5853658536585366] +particle_electrons_uxLUTColorBar.Title = "particle_electrons_ux" +particle_electrons_uxLUTColorBar.ComponentTitle = "" +particle_electrons_uxLUTColorBar.ScalarBarLength = 0.32999999999999985 + +# set color bar visibility +particle_electrons_uxLUTColorBar.Visibility = 1 + +# show color legend +clip1Display.SetScalarBarVisibility(renderView1, True) + +# ---------------------------------------------------------------- +# setup color maps and opacity maps used in the visualization +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# ---------------------------------------------------------------- +# setup animation scene, tracks and keyframes +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get time animation track +timeAnimationCue1 = GetTimeTrack() + +# initialize the animation scene + +# get the time-keeper +timeKeeper1 = GetTimeKeeper() + +# initialize the timekeeper + +# initialize the animation track + +# get animation scene +animationScene1 = GetAnimationScene() + +# initialize the animation scene +animationScene1.ViewModules = renderView1 +animationScene1.Cues = timeAnimationCue1 +animationScene1.AnimationTime = 1.3329221244118056e-13 +animationScene1.EndTime = 1.3329221244118056e-13 +animationScene1.PlayMode = "Snap To TimeSteps" + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +pNG1 = CreateExtractor("PNG", renderView1, registrationName="PNG1") +# trace defaults for the extractor. +pNG1.Trigger = "Time Step" + +# init the 'PNG' selected for 'Writer' +pNG1.Writer.FileName = "RenderView1_{timestep:06d}{camera}.png" +pNG1.Writer.ImageResolution = [889, 820] +pNG1.Writer.Format = "PNG" + +# ---------------------------------------------------------------- +# restore active source +# ---------------------------------------------------------------- +SetActiveSource(pNG1) + +# ------------------------------------------------------------------------------ +# Catalyst options +# ------------------------------------------------------------------------------ +from paraview import catalyst + +options = catalyst.Options() +options.GlobalTrigger = "Time Step" +options.CatalystLiveTrigger = "Time Step" + +if __name__ == "__main__": + from paraview.simple import SaveExtractsUsingCatalystOptions + + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options) diff --git a/Examples/Tests/ionization/inputs_2d_bf_rt b/Examples/Tests/field_ionization/inputs_test_2d_ionization_boost similarity index 100% rename from Examples/Tests/ionization/inputs_2d_bf_rt rename to Examples/Tests/field_ionization/inputs_test_2d_ionization_boost diff --git a/Examples/Tests/ionization/inputs_2d_rt b/Examples/Tests/field_ionization/inputs_test_2d_ionization_lab similarity index 100% rename from Examples/Tests/ionization/inputs_2d_rt rename to Examples/Tests/field_ionization/inputs_test_2d_ionization_lab diff --git a/Examples/Tests/field_ionization/inputs_test_2d_ionization_picmi.py b/Examples/Tests/field_ionization/inputs_test_2d_ionization_picmi.py new file mode 100644 index 00000000000..6d1d485cc8c --- /dev/null +++ b/Examples/Tests/field_ionization/inputs_test_2d_ionization_picmi.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c + +# Number of time steps +max_steps = 1600 + +# Number of cells +nx = 16 +nz = 800 + +# Physical domain +xmin = -5e-06 +xmax = 5e-06 +zmin = 0e-06 +zmax = 20e-06 + +# Domain decomposition +max_grid_size = 64 +blocking_factor = 16 + +# Create grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=["periodic", "open"], + upper_boundary_conditions=["periodic", "open"], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor, +) + +# Particles: electrons and ions +ions_density = 1 +ions_xmin = None +ions_ymin = None +ions_zmin = 5e-06 +ions_xmax = None +ions_ymax = None +ions_zmax = 15e-06 +uniform_distribution = picmi.UniformDistribution( + density=ions_density, + lower_bound=[ions_xmin, ions_ymin, ions_zmin], + upper_bound=[ions_xmax, ions_ymax, ions_zmax], + fill_in=True, +) +electrons = picmi.Species( + particle_type="electron", + name="electrons", + warpx_add_real_attributes={"orig_z": "z"}, +) +ions = picmi.Species( + particle_type="N", + name="ions", + charge_state=2, + initial_distribution=uniform_distribution, + warpx_add_real_attributes={"orig_z": "z"}, +) + +# Field ionization +nitrogen_ionization = picmi.FieldIonization( + model="ADK", # Ammosov-Delone-Krainov model + ionized_species=ions, + product_species=electrons, +) + +# Laser +position_z = 3e-06 +profile_t_peak = 60.0e-15 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=1e10, + duration=26.685e-15, + focal_position=[0, 0, position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[1, 0, 0], + a0=1.8, + fill_in=False, +) +laser_antenna = picmi.LaserAntenna( + position=[0.0, 0.0, position_z], normal_vector=[0, 0, 1] +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="CKC", cfl=0.999) + +# Diagnostics +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10000, + species=[electrons, ions], + data_list=["ux", "uy", "uz", "x", "z", "weighting", "orig_z"], +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10000, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, max_steps=max_steps, particle_shape="linear", warpx_use_filter=0 +) + +# Add electrons and ions +sim.add_species( + electrons, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[0, 0, 0]) +) +sim.add_species( + ions, layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2, 1, 1]) +) + +# Add field ionization +sim.add_interaction(nitrogen_ionization) + +# Add laser +sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name="inputs_2d_picmi") + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Tests/field_probe/CMakeLists.txt b/Examples/Tests/field_probe/CMakeLists.txt new file mode 100644 index 00000000000..8b052dc3b66 --- /dev/null +++ b/Examples/Tests/field_probe/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_2d_field_probe # name + 2 # dims + 2 # nprocs + inputs_test_2d_field_probe # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000544" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/field_probe/analysis.py b/Examples/Tests/field_probe/analysis.py new file mode 100755 index 00000000000..57085fb7cdc --- /dev/null +++ b/Examples/Tests/field_probe/analysis.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# +# Copyright 2021-2022 Elisa Rheaume +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the accuracy of the FieldProbe diagnostic by observing a plane +wave undergoing single slit diffraction. The input file inputs_2d is used. This +file defines the simulation box, laser pulse, embedded boundary with single slit, +and line of detector points. The plane wave initializes near the negative Z end +of the simulation box. The wave interacts with the embedded boundary at Z=0. The +wave undergoes diffraction at the slit. The electromagnetic flux is calculated +at the line detector which is placed perpendicular to Z beyond the slit. This +test will check if the detected EM flux matches expected values, +which can be solved analytically. +""" + +import numpy as np +import pandas as pd + +filename = "diags/reducedfiles/FP_line.txt" + +# Open data file +df = pd.read_csv(filename, sep=" ") +df = df.sort_values(by=["[2]part_x_lev0-(m)"]) + +# Select position and Intensity of timestep 500 +x = df.query("`[0]step()` == 500")["[2]part_x_lev0-(m)"] +S = df.query("`[0]step()` == 500")["[11]part_S_lev0-(W*s/m^2)"] +xvals = x.to_numpy() +svals = S.to_numpy() + +# Default intensity is highest measured value for plane +# wave interacting with single slit +I_0 = np.max(S) + + +def I_envelope(x, lam=0.2e-6, a=0.3e-6, D=1.7e-6): + arg = np.pi * a / lam * np.sin(np.arctan(x / D)) + return np.sinc(arg / np.pi) ** 2 + + +# Count non-outlier values away from simulation boundaries +counter = np.arange(60, 140, 2) + +# Count average error from expected values +error = 0 +for a in counter: + b = I_0 * I_envelope(xvals[a]) + c = svals[a] + error += abs((c - b) / b) * 100.0 +averror = error / (len(counter) - 1) + +# average error range set at 2.5% +if averror > 2.5: + print("Average error greater than 2.5%") + +assert averror < 2.5 diff --git a/Examples/Tests/field_probe/analysis_default_regression.py b/Examples/Tests/field_probe/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/field_probe/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/field_probe/analysis_field_probe.py b/Examples/Tests/field_probe/analysis_field_probe.py deleted file mode 100755 index e167942d77c..00000000000 --- a/Examples/Tests/field_probe/analysis_field_probe.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2021-2022 Elisa Rheaume -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -""" -This script tests the accuracy of the FieldProbe diagnostic by observing a plane -wave undergoing single slit diffraction. The input file inputs_2d is used. This -file defines the simulation box, laser pulse, embedded boundary with single slit, -and line of detector points. The plane wave initializes near the negative Z end -of the simulation box. The wave interacts with the embedded boundary at Z=0. The -wave undergoes diffraction at the slit. The electromagnetic flux is calculated -at the line detector which is placed perpendicular to Z beyond the slit. This -test will check if the detected EM flux matches expected values, -which can be solved analytically. -""" -import numpy as np -import pandas as pd - -filename = "diags/reducedfiles/FP_line.txt" - -# Open data file -df = pd.read_csv(filename, sep=' ') -df = df.sort_values(by=['[2]part_x_lev0-(m)']) - -# Select position and Intensity of timestep 500 -x = df.query('`[0]step()` == 500')['[2]part_x_lev0-(m)'] -S = df.query('`[0]step()` == 500')['[11]part_S_lev0-(W*s/m^2)'] -xvals = x.to_numpy() -svals = S.to_numpy() - -# Default intensity is highest measured value for plane -# wave interacting with single slit -I_0 = np.max(S) -def I_envelope (x, lam = 0.2e-6, a = 0.3e-6, D = 1.7e-6): - arg = np.pi * a / lam * np.sin(np.arctan(x / D)) - return np.sinc( arg / np.pi )**2 - -# Count non-outlier values away from simulation boundaries -counter = np.arange(60, 140, 2) - -# Count average error from expected values -error = 0 -for a in counter: - b = I_0 * I_envelope(xvals[a]) - c = svals[a] - error += abs((c-b)/b) * 100.0 -averror = error / (len(counter) - 1) - -# average error range set at 2.5% -if averror > 2.5: - print('Average error greater than 2.5%') - -assert averror < 2.5 diff --git a/Examples/Tests/field_probe/inputs_2d b/Examples/Tests/field_probe/inputs_test_2d_field_probe similarity index 100% rename from Examples/Tests/field_probe/inputs_2d rename to Examples/Tests/field_probe/inputs_test_2d_field_probe diff --git a/Examples/Tests/flux_injection/CMakeLists.txt b/Examples/Tests/flux_injection/CMakeLists.txt new file mode 100644 index 00000000000..390c76ec58e --- /dev/null +++ b/Examples/Tests/flux_injection/CMakeLists.txt @@ -0,0 +1,52 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_flux_injection # name + 3 # dims + 2 # nprocs + inputs_test_3d_flux_injection # inputs + "analysis_flux_injection_3d.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_flux_injection # name + RZ # dims + 2 # nprocs + inputs_test_rz_flux_injection # inputs + "analysis_flux_injection_rz.py diags/diag1000120" # analysis + "analysis_default_regression.py --path diags/diag1000120" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_flux_injection_from_eb # name + 3 # dims + 2 # nprocs + inputs_test_3d_flux_injection_from_eb # inputs + "analysis_flux_injection_from_eb.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_flux_injection_from_eb # name + RZ # dims + 2 # nprocs + inputs_test_rz_flux_injection_from_eb # inputs + "analysis_flux_injection_from_eb.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_flux_injection_from_eb # name + 2 # dims + 2 # nprocs + inputs_test_2d_flux_injection_from_eb # inputs + "analysis_flux_injection_from_eb.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) diff --git a/Examples/Tests/flux_injection/analysis_default_regression.py b/Examples/Tests/flux_injection/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/flux_injection/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index d0271f6aa94..53baf9511f4 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -20,8 +20,7 @@ velocity distribution (Gaussian or Gaussian-flux depending on the direction of space) """ -import os -import re + import sys import matplotlib.pyplot as plt @@ -30,21 +29,18 @@ from scipy.constants import c, m_e, m_p from scipy.special import erf -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - yt.funcs.mylog.setLevel(0) # Open plotfile specified in command line fn = sys.argv[1] -ds = yt.load( fn ) +ds = yt.load(fn) ad = ds.all_data() t_max = ds.current_time.item() # time of simulation # Total number of electrons expected: # Simulation parameters determine the total number of particles emitted (Ntot) -flux = 1. # in m^-2.s^-1, from the input script -emission_surface = 8*8 # in m^2 +flux = 1.0 # in m^-2.s^-1, from the input script +emission_surface = 8 * 8 # in m^2 Ntot = flux * emission_surface * t_max # Parameters of the histogram @@ -53,92 +49,96 @@ # Define function that histogram and check the data + def gaussian_dist(u, u_th): - return 1./((2*np.pi)**.5*u_th) * np.exp(-u**2/(2*u_th**2) ) + return 1.0 / ((2 * np.pi) ** 0.5 * u_th) * np.exp(-(u**2) / (2 * u_th**2)) + def gaussian_flux_dist(u, u_th, u_m): - normalization_factor = u_th**2 * np.exp(-u_m**2/(2*u_th**2)) + (np.pi/2)**.5*u_m*u_th * (1 + erf(u_m/(2**.5*u_th))) - result = 1./normalization_factor * np.where( u>0, u * np.exp(-(u-u_m)**2/(2*u_th**2)), 0 ) + normalization_factor = u_th**2 * np.exp(-(u_m**2) / (2 * u_th**2)) + ( + np.pi / 2 + ) ** 0.5 * u_m * u_th * (1 + erf(u_m / (2**0.5 * u_th))) + result = ( + 1.0 + / normalization_factor + * np.where(u > 0, u * np.exp(-((u - u_m) ** 2) / (2 * u_th**2)), 0) + ) return result -def compare_gaussian(u, w, u_th, label=''): - du = (hist_range[1]-hist_range[0])/hist_bins - w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w/du, range=hist_range) - u_hist = 0.5*(u_hist[1:]+u_hist[:-1]) - w_th = Ntot*gaussian_dist(u_hist, u_th) - plt.plot( u_hist, w_hist, label=label+': simulation' ) - plt.plot( u_hist, w_th, '--', label=label+': theory' ) - assert np.allclose( w_hist, w_th, atol=0.07*w_th.max() ) - -def compare_gaussian_flux(u, w, u_th, u_m, label=''): - du = (hist_range[1]-hist_range[0])/hist_bins - w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w/du, range=hist_range) - u_hist = 0.5*(u_hist[1:]+u_hist[:-1]) - w_th = Ntot*gaussian_flux_dist(u_hist, u_th, u_m) - plt.plot( u_hist, w_hist, label=label+': simulation' ) - plt.plot( u_hist, w_th, '--', label=label+': theory' ) - assert np.allclose( w_hist, w_th, atol=0.05*w_th.max() ) + +def compare_gaussian(u, w, u_th, label=""): + du = (hist_range[1] - hist_range[0]) / hist_bins + w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w / du, range=hist_range) + u_hist = 0.5 * (u_hist[1:] + u_hist[:-1]) + w_th = Ntot * gaussian_dist(u_hist, u_th) + plt.plot(u_hist, w_hist, label=label + ": simulation") + plt.plot(u_hist, w_th, "--", label=label + ": theory") + assert np.allclose(w_hist, w_th, atol=0.07 * w_th.max()) + + +def compare_gaussian_flux(u, w, u_th, u_m, label=""): + du = (hist_range[1] - hist_range[0]) / hist_bins + w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w / du, range=hist_range) + u_hist = 0.5 * (u_hist[1:] + u_hist[:-1]) + w_th = Ntot * gaussian_flux_dist(u_hist, u_th, u_m) + plt.plot(u_hist, w_hist, label=label + ": simulation") + plt.plot(u_hist, w_th, "--", label=label + ": theory") + assert np.allclose(w_hist, w_th, atol=0.05 * w_th.max()) + # Load data and perform check -plt.figure(figsize=(8,7)) +plt.figure(figsize=(8, 7)) plt.subplot(221) -plt.title('Electrons u_m=0.07') +plt.title("Electrons u_m=0.07") -ux = ad['electron','particle_momentum_x'].to_ndarray()/(m_e*c) -uy = ad['electron','particle_momentum_y'].to_ndarray()/(m_e*c) -uz = ad['electron','particle_momentum_z'].to_ndarray()/(m_e*c) -w = ad['electron', 'particle_weight'].to_ndarray() +ux = ad["electron", "particle_momentum_x"].to_ndarray() / (m_e * c) +uy = ad["electron", "particle_momentum_y"].to_ndarray() / (m_e * c) +uz = ad["electron", "particle_momentum_z"].to_ndarray() / (m_e * c) +w = ad["electron", "particle_weight"].to_ndarray() -compare_gaussian(ux, w, u_th=0.1, label='u_x') -compare_gaussian_flux(uy, w, u_th=0.1, u_m=0.07, label='u_y') -compare_gaussian(uz, w, u_th=0.1, label='u_z') +compare_gaussian(ux, w, u_th=0.1, label="u_x") +compare_gaussian_flux(uy, w, u_th=0.1, u_m=0.07, label="u_y") +compare_gaussian(uz, w, u_th=0.1, label="u_z") plt.subplot(223) -plt.title('Protons u_m=0.05') +plt.title("Protons u_m=0.05") -ux = ad['proton','particle_momentum_x'].to_ndarray()/(m_p*c) -uy = ad['proton','particle_momentum_y'].to_ndarray()/(m_p*c) -uz = ad['proton','particle_momentum_z'].to_ndarray()/(m_p*c) -w = ad['proton', 'particle_weight'].to_ndarray() +ux = ad["proton", "particle_momentum_x"].to_ndarray() / (m_p * c) +uy = ad["proton", "particle_momentum_y"].to_ndarray() / (m_p * c) +uz = ad["proton", "particle_momentum_z"].to_ndarray() / (m_p * c) +w = ad["proton", "particle_weight"].to_ndarray() -compare_gaussian_flux(-ux, w, u_th=0.1, u_m=0.05, label='u_x') -compare_gaussian(uy, w, u_th=0.1, label='u_y') -compare_gaussian(uz, w, u_th=0.1, label='u_z') +compare_gaussian_flux(-ux, w, u_th=0.1, u_m=0.05, label="u_x") +compare_gaussian(uy, w, u_th=0.1, label="u_y") +compare_gaussian(uz, w, u_th=0.1, label="u_z") plt.subplot(222) -plt.title('Electrons u_m=-0.07') +plt.title("Electrons u_m=-0.07") -ux = ad['electron_negative','particle_momentum_x'].to_ndarray()/(m_e*c) -uy = ad['electron_negative','particle_momentum_y'].to_ndarray()/(m_e*c) -uz = ad['electron_negative','particle_momentum_z'].to_ndarray()/(m_e*c) -w = ad['electron_negative', 'particle_weight'].to_ndarray() +ux = ad["electron_negative", "particle_momentum_x"].to_ndarray() / (m_e * c) +uy = ad["electron_negative", "particle_momentum_y"].to_ndarray() / (m_e * c) +uz = ad["electron_negative", "particle_momentum_z"].to_ndarray() / (m_e * c) +w = ad["electron_negative", "particle_weight"].to_ndarray() -compare_gaussian(ux, w, u_th=0.1, label='u_x') -compare_gaussian(uy, w, u_th=0.1, label='u_y') -compare_gaussian_flux(uz, w, u_th=0.1, u_m=-0.07, label='u_z') +compare_gaussian(ux, w, u_th=0.1, label="u_x") +compare_gaussian(uy, w, u_th=0.1, label="u_y") +compare_gaussian_flux(uz, w, u_th=0.1, u_m=-0.07, label="u_z") plt.legend(loc=(1.02, 0.5)) plt.subplot(224) -plt.title('Protons u_m=-0.05') +plt.title("Protons u_m=-0.05") -ux = ad['proton_negative','particle_momentum_x'].to_ndarray()/(m_p*c) -uy = ad['proton_negative','particle_momentum_y'].to_ndarray()/(m_p*c) -uz = ad['proton_negative','particle_momentum_z'].to_ndarray()/(m_p*c) -w = ad['proton_negative', 'particle_weight'].to_ndarray() +ux = ad["proton_negative", "particle_momentum_x"].to_ndarray() / (m_p * c) +uy = ad["proton_negative", "particle_momentum_y"].to_ndarray() / (m_p * c) +uz = ad["proton_negative", "particle_momentum_z"].to_ndarray() / (m_p * c) +w = ad["proton_negative", "particle_weight"].to_ndarray() -compare_gaussian(ux, w, u_th=0.1, label='u_x') -compare_gaussian(uy, w, u_th=0.1, label='u_y') -compare_gaussian_flux(-uz, w, u_th=0.1, u_m=-0.05, label='u_z') -#plt.legend(loc=0) +compare_gaussian(ux, w, u_th=0.1, label="u_x") +compare_gaussian(uy, w, u_th=0.1, label="u_y") +compare_gaussian_flux(-uz, w, u_th=0.1, u_m=-0.05, label="u_z") +# plt.legend(loc=0) plt.tight_layout() -plt.savefig('Distribution.png') - -# Verify checksum -test_name = os.path.split(os.getcwd())[1] -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +plt.savefig("Distribution.png") diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py b/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py new file mode 100755 index 00000000000..96488fd7e71 --- /dev/null +++ b/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the emission of particles from the embedded boundary. +(In this case, the embedded boundary is a sphere in 3D and RZ, a cylinder in 2D.) +We check that the embedded boundary emits the correct number of particles, and that +the particle distributions are consistent with the expected distributions. +""" + +import re +import sys + +import matplotlib.pyplot as plt +import numpy as np +import yt +from scipy.constants import c, m_e +from scipy.special import erf + +yt.funcs.mylog.setLevel(0) + +# Open plotfile specified in command line +fn = sys.argv[1] +ds = yt.load(fn) +ad = ds.all_data() +t_inj = 0.5e-8 # duration for which the flux injection was active + +# Extract the dimensionality of the simulation +with open("./warpx_used_inputs", "r") as f: + warpx_used_inputs = f.read() +if re.search("geometry.dims = 2", warpx_used_inputs): + dims = "2D" +elif re.search("geometry.dims = RZ", warpx_used_inputs): + dims = "RZ" +elif re.search("geometry.dims = 3", warpx_used_inputs): + dims = "3D" + +# Total number of electrons expected: +# Simulation parameters determine the total number of particles emitted (Ntot) +flux = 1.0 # in m^-2.s^-1, from the input script +R = 2.0 # in m, radius of the sphere +if dims == "3D" or dims == "RZ": + emission_surface = 4 * np.pi * R**2 # in m^2 +elif dims == "2D": + emission_surface = 2 * np.pi * R # in m +Ntot = flux * emission_surface * t_inj + +# Parameters of the histogram +hist_bins = 50 +hist_range = [-0.5, 0.5] + + +# Define function that histograms and checks the data +def gaussian_dist(u, u_th): + return 1.0 / ((2 * np.pi) ** 0.5 * u_th) * np.exp(-(u**2) / (2 * u_th**2)) + + +def gaussian_flux_dist(u, u_th, u_m): + normalization_factor = u_th**2 * np.exp(-(u_m**2) / (2 * u_th**2)) + ( + np.pi / 2 + ) ** 0.5 * u_m * u_th * (1 + erf(u_m / (2**0.5 * u_th))) + result = ( + 1.0 + / normalization_factor + * np.where(u > 0, u * np.exp(-((u - u_m) ** 2) / (2 * u_th**2)), 0) + ) + return result + + +def compare_gaussian(u, w, u_th, label=""): + du = (hist_range[1] - hist_range[0]) / hist_bins + w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w / du, range=hist_range) + u_hist = 0.5 * (u_hist[1:] + u_hist[:-1]) + w_th = Ntot * gaussian_dist(u_hist, u_th) + plt.plot(u_hist, w_hist, label=label + ": simulation") + plt.plot(u_hist, w_th, "--", label=label + ": theory") + assert np.allclose(w_hist, w_th, atol=0.07 * w_th.max()) + + +def compare_gaussian_flux(u, w, u_th, u_m, label=""): + du = (hist_range[1] - hist_range[0]) / hist_bins + w_hist, u_hist = np.histogram(u, bins=hist_bins, weights=w / du, range=hist_range) + u_hist = 0.5 * (u_hist[1:] + u_hist[:-1]) + w_th = Ntot * gaussian_flux_dist(u_hist, u_th, u_m) + plt.plot(u_hist, w_hist, label=label + ": simulation") + plt.plot(u_hist, w_th, "--", label=label + ": theory") + assert np.allclose(w_hist, w_th, atol=0.05 * w_th.max()) + + +# Load data and perform check + +plt.figure() + +if dims == "3D": + x = ad["electron", "particle_position_x"].to_ndarray() + y = ad["electron", "particle_position_y"].to_ndarray() + z = ad["electron", "particle_position_z"].to_ndarray() +elif dims == "2D": + x = ad["electron", "particle_position_x"].to_ndarray() + y = np.zeros_like(x) + z = ad["electron", "particle_position_y"].to_ndarray() +elif dims == "RZ": + theta = ad["electron", "particle_theta"].to_ndarray() + r = ad["electron", "particle_position_x"].to_ndarray() + x = r * np.cos(theta) + y = r * np.sin(theta) + z = ad["electron", "particle_position_y"].to_ndarray() +ux = ad["electron", "particle_momentum_x"].to_ndarray() / (m_e * c) +uy = ad["electron", "particle_momentum_y"].to_ndarray() / (m_e * c) +uz = ad["electron", "particle_momentum_z"].to_ndarray() / (m_e * c) +w = ad["electron", "particle_weight"].to_ndarray() + +# Check that the total number of particles emitted is correct +Ntot_sim = np.sum(w) +print("Ntot_sim = ", Ntot_sim) +print("Ntot = ", Ntot) +assert np.isclose(Ntot_sim, Ntot, rtol=0.01) + +# Check that none of the particles are inside the EB +# A factor 0.98 is applied to accomodate +# the cut-cell approximation of the sphere +assert np.all(x**2 + y**2 + z**2 > (0.98 * R) ** 2) + +# Check that the normal component of the velocity is consistent with the expected distribution +r = np.sqrt(x**2 + y**2 + z**2) +nx = x / r +ny = y / r +nz = z / r +u_n = ux * nx + uy * ny + uz * nz # normal component +compare_gaussian_flux(u_n, w, u_th=0.1, u_m=0.07, label="u_n") + +# Pick a direction that is orthogonal to the normal direction, and check the distribution +vx = ny / np.sqrt(nx**2 + ny**2) +vy = -nx / np.sqrt(nx**2 + ny**2) +vz = 0 +u_perp = ux * vx + uy * vy + uz * vz +compare_gaussian(u_perp, w, u_th=0.01, label="u_perp") + +# Pick the other perpendicular direction, and check the distribution +# The third direction is obtained by the cross product (n x v) +wx = ny * vz - nz * vy +wy = nz * vx - nx * vz +wz = nx * vy - ny * vx +u_perp2 = ux * wx + uy * wy + uz * wz +compare_gaussian(u_perp2, w, u_th=0.01, label="u_perp2") + +plt.legend() +plt.tight_layout() +plt.savefig("Distribution.png") diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_rz.py b/Examples/Tests/flux_injection/analysis_flux_injection_rz.py index 8ec944c715d..170fb08128d 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_rz.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_rz.py @@ -24,42 +24,31 @@ velocity was along the azimuthal direction.) - The total number of electrons corresponds to the expected flux. """ -import os -import re + import sys import numpy as np import yt -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - yt.funcs.mylog.setLevel(0) # Open plotfile specified in command line fn = sys.argv[1] -ds = yt.load( fn ) +ds = yt.load(fn) t_max = ds.current_time.item() # time of simulation # Total number of electrons expected: -flux = 1. # in m^-2.s^-1, from the input script -emission_surface = 0.8 # in m^2, +flux = 1.0 # in m^-2.s^-1, from the input script +emission_surface = 0.8 # in m^2, # given that xmin = 1.5, xmax = 1.9, zmin = -1.0, zmax = 1. n_tot = flux * emission_surface * t_max # Read particle data ad = ds.all_data() -r = ad['particle_position_x'].to_ndarray() # Corresponds to the radial coordinate in RZ -w = ad['particle_weight'].to_ndarray() +r = ad["particle_position_x"].to_ndarray() # Corresponds to the radial coordinate in RZ +w = ad["particle_weight"].to_ndarray() # Check that the number of particles matches the expected one -assert np.allclose( w.sum(), n_tot, rtol=0.05 ) +assert np.allclose(w.sum(), n_tot, rtol=0.05) # Check that the particles are at the right radius -assert np.all( (r >= 1.48) & (r <=1.92) ) - -test_name = os.path.split(os.getcwd())[1] - -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +assert np.all((r >= 1.48) & (r <= 1.92)) diff --git a/Examples/Tests/flux_injection/inputs_base_from_eb b/Examples/Tests/flux_injection/inputs_base_from_eb new file mode 100644 index 00000000000..618fd1c941a --- /dev/null +++ b/Examples/Tests/flux_injection/inputs_base_from_eb @@ -0,0 +1,44 @@ +# Maximum number of time steps +max_step = 20 + +# The lo and hi ends of grids are multipliers of blocking factor +amr.blocking_factor = 8 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 8 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Deactivate Maxwell solver +algo.maxwell_solver = none +warpx.const_dt = 0.5e-9 + +# Embedded boundary +warpx.eb_implicit_function = "-(x**2+y**2+z**2-2**2)" + +# particles +particles.species_names = electron +algo.particle_shape = 3 + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = NFluxPerCell +electron.inject_from_embedded_boundary = 1 +electron.num_particles_per_cell = 100 +electron.flux_profile = parse_flux_function +electron.flux_function(x,y,z,t) = "1." +electron.flux_tmin = 0.25e-8 +electron.flux_tmax = 0.75e-8 +electron.momentum_distribution_type = gaussianflux +electron.ux_th = 0.01 +electron.uy_th = 0.01 +electron.uz_th = 0.1 +electron.uz_m = 0.07 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 10 +diag1.diag_type = Full +diag1.fields_to_plot = none diff --git a/Examples/Tests/flux_injection/inputs_test_2d_flux_injection_from_eb b/Examples/Tests/flux_injection/inputs_test_2d_flux_injection_from_eb new file mode 100644 index 00000000000..291ef329ad6 --- /dev/null +++ b/Examples/Tests/flux_injection/inputs_test_2d_flux_injection_from_eb @@ -0,0 +1,13 @@ +FILE = inputs_base_from_eb + +# number of grid points +amr.n_cell = 32 32 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -4 -4 +geometry.prob_hi = 4 4 + +# Boundary condition +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic diff --git a/Examples/Tests/flux_injection/inputs_3d b/Examples/Tests/flux_injection/inputs_test_3d_flux_injection similarity index 100% rename from Examples/Tests/flux_injection/inputs_3d rename to Examples/Tests/flux_injection/inputs_test_3d_flux_injection diff --git a/Examples/Tests/flux_injection/inputs_test_3d_flux_injection_from_eb b/Examples/Tests/flux_injection/inputs_test_3d_flux_injection_from_eb new file mode 100644 index 00000000000..59db133e484 --- /dev/null +++ b/Examples/Tests/flux_injection/inputs_test_3d_flux_injection_from_eb @@ -0,0 +1,13 @@ +FILE = inputs_base_from_eb + +# number of grid points +amr.n_cell = 32 32 32 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -4 -4 -4 +geometry.prob_hi = 4 4 4 + +# Boundary condition +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic diff --git a/Examples/Tests/flux_injection/inputs_rz b/Examples/Tests/flux_injection/inputs_test_rz_flux_injection similarity index 100% rename from Examples/Tests/flux_injection/inputs_rz rename to Examples/Tests/flux_injection/inputs_test_rz_flux_injection diff --git a/Examples/Tests/flux_injection/inputs_test_rz_flux_injection_from_eb b/Examples/Tests/flux_injection/inputs_test_rz_flux_injection_from_eb new file mode 100644 index 00000000000..c206a154646 --- /dev/null +++ b/Examples/Tests/flux_injection/inputs_test_rz_flux_injection_from_eb @@ -0,0 +1,15 @@ +FILE = inputs_base_from_eb + +# number of grid points +amr.n_cell = 16 32 + +# Geometry +geometry.dims = RZ +geometry.prob_lo = 0 -4 +geometry.prob_hi = 4 4 + +# Boundary condition +boundary.field_lo = none periodic +boundary.field_hi = pec periodic + +electron.num_particles_per_cell = 300 diff --git a/Examples/Tests/gaussian_beam/CMakeLists.txt b/Examples/Tests/gaussian_beam/CMakeLists.txt new file mode 100644 index 00000000000..2a1f4918458 --- /dev/null +++ b/Examples/Tests/gaussian_beam/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_focusing_gaussian_beam # name + 3 # dims + 2 # nprocs + inputs_test_3d_focusing_gaussian_beam # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000000" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_gaussian_beam_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_gaussian_beam_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) diff --git a/Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py b/Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py deleted file mode 100755 index b9d06034394..00000000000 --- a/Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 - -#from warp import picmi -import argparse - -from pywarpx import picmi - -parser = argparse.ArgumentParser(description="Gaussian beam PICMI example") - -parser.add_argument('--diagformat', type=str, - help='Format of the full diagnostics (plotfile, openpmd, ascent, sensei, ...)', - default='plotfile') -parser.add_argument('--fields_to_plot', type=str, - help='List of fields to write to diagnostics', - default=['E', 'B', 'J', 'part_per_cell'], - nargs = '*') - -args = parser.parse_args() - -constants = picmi.constants - -nx = 32 -ny = 32 -nz = 32 - -xmin = -2. -xmax = +2. -ymin = -2. -ymax = +2. -zmin = -2. -zmax = +2. - -number_sim_particles = 32768 -total_charge = 8.010883097437485e-07 - -beam_rms_size = 0.25 -electron_beam_divergence = -0.04*constants.c - -em_order = 3 - -grid = picmi.Cartesian3DGrid(number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['periodic', 'periodic', 'open'], - upper_boundary_conditions = ['periodic', 'periodic', 'open'], - lower_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - upper_boundary_conditions_particles = ['periodic', 'periodic', 'absorbing'], - warpx_max_grid_size=16) - -solver = picmi.ElectromagneticSolver(grid = grid, - cfl = 1., - stencil_order=[em_order,em_order,em_order]) - -electron_beam = picmi.GaussianBunchDistribution(n_physical_particles = total_charge/constants.q_e, - rms_bunch_size = [beam_rms_size, beam_rms_size, beam_rms_size], - velocity_divergence = [electron_beam_divergence, electron_beam_divergence, electron_beam_divergence]) - -proton_beam = picmi.GaussianBunchDistribution(n_physical_particles = total_charge/constants.q_e, - rms_bunch_size = [beam_rms_size, beam_rms_size, beam_rms_size]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=electron_beam) -protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=proton_beam) - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = 10, - data_list = args.fields_to_plot, - warpx_format = args.diagformat, - write_dir = '.', - warpx_file_prefix = 'Python_gaussian_beam_plt') - -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = 10, - species = [electrons, protons], - data_list = ['weighting', 'momentum'], - warpx_format = args.diagformat) - -sim = picmi.Simulation(solver = solver, - max_steps = 10, - verbose = 1, - warpx_current_deposition_algo = 'direct', - warpx_use_filter = 0) - -sim.add_species(electrons, layout=picmi.PseudoRandomLayout(n_macroparticles=number_sim_particles)) -sim.add_species(protons, layout=picmi.PseudoRandomLayout(n_macroparticles=number_sim_particles)) - -sim.add_diagnostic(field_diag1) -sim.add_diagnostic(part_diag1) - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Tests/gaussian_beam/README.rst b/Examples/Tests/gaussian_beam/README.rst index bfca2bb2398..e3baf9842c2 100644 --- a/Examples/Tests/gaussian_beam/README.rst +++ b/Examples/Tests/gaussian_beam/README.rst @@ -11,7 +11,7 @@ Run This example can be run **either** as: -* **Python** script: ``python3 PICMI_inputs_gaussian_beam.py`` or +* **Python** script: ``python3 inputs_test_3d_gaussian_beam_picmi.py`` or * WarpX **executable** using an input file: (*TODO*) For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. @@ -20,15 +20,15 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. tab-item:: Python: Script - .. literalinclude:: PICMI_inputs_gaussian_beam.py + .. literalinclude:: inputs_test_3d_gaussian_beam_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py``. + :caption: You can copy this file from ``Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py``. .. tab-item:: Executable: Input File .. note:: - TODO: This input file should be created following the ``PICMI_inputs_gaussian_beam.py`` file. + TODO: This input file should be created following the ``inputs_test_3d_gaussian_beam_picmi.py`` file. Analyze diff --git a/Examples/Tests/gaussian_beam/analysis.py b/Examples/Tests/gaussian_beam/analysis.py new file mode 100755 index 00000000000..a5a6caf8e42 --- /dev/null +++ b/Examples/Tests/gaussian_beam/analysis.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Arianna Formenti +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, eV, m_e, micro, nano + +GeV = 1e9 * eV +energy = 125.0 * GeV +gamma = energy / (m_e * c**2) +sigmax = 516.0 * nano +sigmay = 7.7 * nano +sigmaz = 300.0 * micro +nz = 256 +Lz = 20 * sigmaz +gridz = np.linspace(-0.5 * Lz, 0.5 * Lz, nz) +tol = gridz[1] - gridz[0] +emitx = 50 * micro +emity = 20 * nano +focal_distance = 4 * sigmaz + + +def s(z, sigma0, emit): + """The theoretical size of a focusing beam (in the absence of space charge), + at position z, given its emittance and size at focus.""" + return np.sqrt(sigma0**2 + emit**2 * (z - focal_distance) ** 2 / sigma0**2) + + +ts = OpenPMDTimeSeries("./diags/openpmd/") + +( + x, + y, + z, + w, +) = ts.get_particle(["x", "y", "z", "w"], species="beam1", iteration=0, plot=False) + +imin = np.argmin(np.sqrt((gridz + 0.8 * focal_distance) ** 2)) +imax = np.argmin(np.sqrt((gridz - 0.8 * focal_distance) ** 2)) + +sx, sy = [], [] +# Compute the size of the beam in each z slice +subgrid = gridz[imin:imax] +for d in subgrid: + i = np.sqrt((z - d) ** 2) < tol + if np.sum(i) != 0: + mux = np.average(x[i], weights=w[i]) + muy = np.average(y[i], weights=w[i]) + sx.append(np.sqrt(np.average((x[i] - mux) ** 2, weights=w[i]))) + sy.append(np.sqrt(np.average((y[i] - muy) ** 2, weights=w[i]))) + +# Theoretical prediction for the size of the beam in each z slice +sx_theory = s(subgrid, sigmax, emitx / gamma) +sy_theory = s(subgrid, sigmay, emity / gamma) + +assert np.allclose(sx, sx_theory, rtol=0.051, atol=0) +assert np.allclose(sy, sy_theory, rtol=0.038, atol=0) diff --git a/Examples/Tests/gaussian_beam/analysis_default_regression.py b/Examples/Tests/gaussian_beam/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/gaussian_beam/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/gaussian_beam/analysis_focusing_beam.py b/Examples/Tests/gaussian_beam/analysis_focusing_beam.py deleted file mode 100755 index 4a5fa3b927b..00000000000 --- a/Examples/Tests/gaussian_beam/analysis_focusing_beam.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2024 Arianna Formenti -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -import os -import sys - -import numpy as np -from scipy.constants import c, eV, m_e, micro, nano - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - -import checksumAPI -from openpmd_viewer import OpenPMDTimeSeries - -GeV=1e9*eV -energy = 125.*GeV -gamma = energy/(m_e*c**2) -sigmax = 516.0*nano -sigmay = 7.7*nano -sigmaz = 300.*micro -nz = 256 -Lz = 20*sigmaz -gridz = np.linspace(-0.5*Lz, 0.5*Lz, nz) -tol = gridz[1] - gridz[0] -emitx = 50*micro -emity = 20*nano -focal_distance = 4*sigmaz - -def s(z, sigma0, emit): - '''The theoretical size of a focusing beam (in the absence of space charge), - at position z, given its emittance and size at focus.''' - return np.sqrt(sigma0**2 + emit**2 * (z - focal_distance)**2 / sigma0**2) - -filename = sys.argv[1] - -ts = OpenPMDTimeSeries('./diags/openpmd/') - -x, y, z, w, = ts.get_particle( ['x', 'y', 'z', 'w'], species='beam1', iteration=0, plot=False) - -imin = np.argmin(np.sqrt((gridz+0.8*focal_distance)**2)) -imax = np.argmin(np.sqrt((gridz-0.8*focal_distance)**2)) - -sx, sy = [], [] -# Compute the size of the beam in each z slice -subgrid = gridz[imin:imax] -for d in subgrid: - i = np.sqrt((z - d)**2) < tol - if (np.sum(i)!=0): - mux = np.average(x[i], weights=w[i]) - muy = np.average(y[i], weights=w[i]) - sx.append(np.sqrt(np.average((x[i]-mux)**2, weights=w[i]))) - sy.append(np.sqrt(np.average((y[i]-muy)**2, weights=w[i]))) - -# Theoretical prediction for the size of the beam in each z slice -sx_theory = s(subgrid, sigmax, emitx/gamma) -sy_theory = s(subgrid, sigmay, emity/gamma) - -assert(np.allclose(sx, sx_theory, rtol=0.051, atol=0)) -assert(np.allclose(sy, sy_theory, rtol=0.038, atol=0)) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/gaussian_beam/inputs_focusing_beam b/Examples/Tests/gaussian_beam/inputs_test_3d_focusing_gaussian_beam similarity index 100% rename from Examples/Tests/gaussian_beam/inputs_focusing_beam rename to Examples/Tests/gaussian_beam/inputs_test_3d_focusing_gaussian_beam diff --git a/Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py b/Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py new file mode 100755 index 00000000000..cd169110f8a --- /dev/null +++ b/Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +# from warp import picmi +import argparse + +from pywarpx import picmi + +parser = argparse.ArgumentParser(description="Gaussian beam PICMI example") + +parser.add_argument( + "--diagformat", + type=str, + help="Format of the full diagnostics (plotfile, openpmd, ascent, sensei, ...)", + default="plotfile", +) +parser.add_argument( + "--fields_to_plot", + type=str, + help="List of fields to write to diagnostics", + default=["E", "B", "J", "part_per_cell"], + nargs="*", +) + +args = parser.parse_args() + +constants = picmi.constants + +nx = 32 +ny = 32 +nz = 32 + +xmin = -2.0 +xmax = +2.0 +ymin = -2.0 +ymax = +2.0 +zmin = -2.0 +zmax = +2.0 + +number_sim_particles = 32768 +total_charge = 8.010883097437485e-07 + +beam_rms_size = 0.25 +electron_beam_divergence = -0.04 * constants.c + +em_order = 3 + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "open"], + upper_boundary_conditions=["periodic", "periodic", "open"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + warpx_max_grid_size=16, +) + +solver = picmi.ElectromagneticSolver( + grid=grid, cfl=1.0, stencil_order=[em_order, em_order, em_order] +) + +electron_beam = picmi.GaussianBunchDistribution( + n_physical_particles=total_charge / constants.q_e, + rms_bunch_size=[beam_rms_size, beam_rms_size, beam_rms_size], + velocity_divergence=[ + electron_beam_divergence, + electron_beam_divergence, + electron_beam_divergence, + ], +) + +proton_beam = picmi.GaussianBunchDistribution( + n_physical_particles=total_charge / constants.q_e, + rms_bunch_size=[beam_rms_size, beam_rms_size, beam_rms_size], +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=electron_beam +) +protons = picmi.Species( + particle_type="proton", name="protons", initial_distribution=proton_beam +) + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=args.fields_to_plot, + warpx_format=args.diagformat, +) + +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=10, + species=[electrons, protons], + data_list=["weighting", "momentum"], + warpx_format=args.diagformat, +) + +sim = picmi.Simulation( + solver=solver, + max_steps=10, + verbose=1, + warpx_current_deposition_algo="direct", + warpx_use_filter=0, +) + +sim.add_species( + electrons, layout=picmi.PseudoRandomLayout(n_macroparticles=number_sim_particles) +) +sim.add_species( + protons, layout=picmi.PseudoRandomLayout(n_macroparticles=number_sim_particles) +) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt new file mode 100644 index 00000000000..e4bde9bbeaf --- /dev/null +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -0,0 +1,64 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_semi_implicit_picard # name + 1 # dims + 2 # nprocs + inputs_test_1d_semi_implicit_picard # inputs + "analysis_1d.py" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_theta_implicit_picard # name + 1 # dims + 2 # nprocs + inputs_test_1d_theta_implicit_picard # inputs + "analysis_1d.py" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_theta_implicit_jfnk_vandb # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_jfnk_vandb # inputs + "analysis_vandb_jfnk_2d.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_theta_implicit_jfnk_vandb_filtered # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_jfnk_vandb_filtered # inputs + "analysis_vandb_jfnk_2d.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_theta_implicit_jfnk_vandb_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_jfnk_vandb_picmi.py # inputs + "analysis_vandb_jfnk_2d.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_2d_theta_implicit_strang_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_strang_psatd # inputs + "analysis_2d_psatd.py" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/implicit/analysis_1d.py b/Examples/Tests/implicit/analysis_1d.py new file mode 100755 index 00000000000..aa54cd279ce --- /dev/null +++ b/Examples/Tests/implicit/analysis_1d.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# Copyright 2023 David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. +import os +import re + +import numpy as np + +field_energy = np.loadtxt("diags/reducedfiles/field_energy.txt", skiprows=1) +particle_energy = np.loadtxt("diags/reducedfiles/particle_energy.txt", skiprows=1) + +total_energy = field_energy[:, 2] + particle_energy[:, 2] + +delta_E = (total_energy - total_energy[0]) / total_energy[0] +max_delta_E = np.abs(delta_E).max() + +test_name = os.path.split(os.getcwd())[1] +if re.match("test_1d_semi_implicit_picard", test_name): + tolerance_rel = 2.5e-5 +elif re.match("test_1d_theta_implicit_picard", test_name): + # This case should have near machine precision conservation of energy + tolerance_rel = 1.0e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel}") + +assert max_delta_E < tolerance_rel diff --git a/Examples/Tests/implicit/analysis_2d_psatd.py b/Examples/Tests/implicit/analysis_2d_psatd.py new file mode 100755 index 00000000000..507fc6f2c4a --- /dev/null +++ b/Examples/Tests/implicit/analysis_2d_psatd.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus, David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. + +import numpy as np + +field_energy = np.loadtxt("diags/reducedfiles/field_energy.txt", skiprows=1) +particle_energy = np.loadtxt("diags/reducedfiles/particle_energy.txt", skiprows=1) + +total_energy = field_energy[:, 2] + particle_energy[:, 2] + +delta_E = (total_energy - total_energy[0]) / total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.1e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert max_delta_E < tolerance_rel_energy diff --git a/Examples/Tests/implicit/analysis_default_regression.py b/Examples/Tests/implicit/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/implicit/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/implicit/analysis_vandb_jfnk_2d.py b/Examples/Tests/implicit/analysis_vandb_jfnk_2d.py new file mode 100755 index 00000000000..dcbacdfde1f --- /dev/null +++ b/Examples/Tests/implicit/analysis_vandb_jfnk_2d.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. +import sys + +import numpy as np +import yt +from scipy.constants import e, epsilon_0 + +field_energy = np.loadtxt("diags/reducedfiles/field_energy.txt", skiprows=1) +particle_energy = np.loadtxt("diags/reducedfiles/particle_energy.txt", skiprows=1) + +total_energy = field_energy[:, 2] + particle_energy[:, 2] + +delta_E = (total_energy - total_energy[0]) / total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.0e-14 +tolerance_rel_charge = 2.0e-15 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert max_delta_E < tolerance_rel_energy + +# check for machine precision conservation of charge density +n0 = 1.0e30 + +pltdir = sys.argv[1] +ds = yt.load(pltdir) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) + +divE = data["boxlib", "divE"].value +rho = data["boxlib", "rho"].value + +# compute local error in Gauss's law +drho = (rho - epsilon_0 * divE) / e / n0 + +# compute RMS on in error on the grid +nX = drho.shape[0] +nZ = drho.shape[1] +drho2_avg = (drho**2).sum() / (nX * nZ) +drho_rms = np.sqrt(drho2_avg) + +print(f"rms error in charge conservation: {drho_rms}") +print(f"tolerance: {tolerance_rel_charge}") + +assert drho_rms < tolerance_rel_charge diff --git a/Examples/Tests/implicit/inputs_test_1d_semi_implicit_picard b/Examples/Tests/implicit/inputs_test_1d_semi_implicit_picard new file mode 100644 index 00000000000..39df05ff72c --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_1d_semi_implicit_picard @@ -0,0 +1,90 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.abort_on_warning_threshold = high +warpx.verbose = 1 +warpx.const_dt = dt +algo.evolve_scheme = semi_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 5 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +reduced_diags.intervals = 1 +particle_energy.type = ParticleEnergy +field_energy.type = FieldEnergy diff --git a/Examples/Tests/implicit/inputs_test_1d_theta_implicit_picard b/Examples/Tests/implicit/inputs_test_1d_theta_implicit_picard new file mode 100644 index 00000000000..80e4e7033fc --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_1d_theta_implicit_picard @@ -0,0 +1,90 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.abort_on_warning_threshold = high +warpx.verbose = 1 +warpx.const_dt = dt +algo.evolve_scheme = theta_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 31 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +reduced_diags.intervals = 1 +particle_energy.type = ParticleEnergy +field_energy.type = FieldEnergy diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb new file mode 100644 index 00000000000..bab9a03878c --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb @@ -0,0 +1,114 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.abort_on_warning_threshold = high +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +reduced_diags.intervals = 1 +particle_energy.type = ParticleEnergy +field_energy.type = FieldEnergy diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered new file mode 100644 index 00000000000..c7457e02af8 --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered @@ -0,0 +1,114 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.abort_on_warning_threshold = high +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 1 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +reduced_diags.intervals = 1 +particle_energy.type = ParticleEnergy +field_energy.type = FieldEnergy diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_picmi.py b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_picmi.py new file mode 100755 index 00000000000..8fa29127a7f --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_picmi.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# +# --- Tests the python interface to the Implicit solver + +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +n0 = 1.0e30 # m^-3 +Ti = 100.0 # eV +Te = 100.0 # eV +wpe = constants.q_e * np.sqrt(n0 / (constants.m_e * constants.ep0)) +de0 = constants.c / wpe +nppcz = 10 # number of particles/cell in z +dt = 0.1 / wpe # s + +vthe = np.sqrt(Te * constants.q_e / constants.m_e) +vthi = np.sqrt(Ti * constants.q_e / constants.m_p) + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 20 +diagnostic_intervals = "::20" + +# --- Grid +nx = 40 +ny = 40 + +xmin = 0.0 +ymin = 0.0 +xmax = 10.0 * de0 +ymax = 10.0 * de0 + +number_per_cell_each_dim = [nppcz, nppcz] + +########################## +# physics components +########################## + +electrons_uniform_plasma = picmi.UniformDistribution( + density=n0, rms_velocity=[vthe, vthe, vthe] +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=electrons_uniform_plasma, +) + +protons_uniform_plasma = picmi.UniformDistribution( + density=n0, rms_velocity=[vthi, vthi, vthi] +) + +protons = picmi.Species( + particle_type="proton", name="protons", initial_distribution=protons_uniform_plasma +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic"], + warpx_max_grid_size=8, + warpx_blocking_factor=8, +) + +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee") + +GMRES_solver = picmi.GMRESLinearSolver( + verbose_int=2, + max_iterations=1000, + relative_tolerance=1.0e-8, + absolute_tolerance=0.0, +) + +newton_solver = picmi.NewtonNonlinearSolver( + verbose=True, + max_iterations=20, + relative_tolerance=1.0e-12, + absolute_tolerance=0.0, + require_convergence=False, + linear_solver=GMRES_solver, + max_particle_iterations=21, + particle_tolerance=1.0e-12, +) + +evolve_scheme = picmi.ThetaImplicitEMEvolveScheme( + theta=0.5, nonlinear_solver=newton_solver +) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["Ex", "Ey", "Ez", "Bx", "By", "Bz", "Jx", "Jy", "Jz", "rho", "divE"], +) + +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, + species=[electrons, protons], + data_list=["weighting", "position", "momentum"], +) + +particle_energy_diag = picmi.ReducedDiagnostic( + diag_type="ParticleEnergy", name="particle_energy", period=1 +) + +field_energy_diag = picmi.ReducedDiagnostic( + diag_type="FieldEnergy", name="field_energy", period=1 +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + particle_shape=2, + time_step_size=dt, + max_steps=max_steps, + verbose=1, + warpx_evolve_scheme=evolve_scheme, + warpx_current_deposition_algo="villasenor", + warpx_particle_pusher_algo="boris", + warpx_serialize_initial_conditions=1, + warpx_use_filter=0, +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) +sim.add_species( + protons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) +sim.add_diagnostic(particle_energy_diag) +sim.add_diagnostic(field_energy_diag) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +sim.write_input_file(file_name="inputs2d_from_PICMI") + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd new file mode 100644 index 00000000000..46bc6b3d301 --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd @@ -0,0 +1,97 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.nz = 40 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = nz nz +amr.max_grid_size = nz +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = psatd +algo.evolve_scheme = strang_implicit_spectral_em +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 9 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +algo.particle_pusher = "boris" + +algo.particle_shape = 2 +algo.current_deposition = direct +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +interpolation.galerkin_scheme = 0 + +psatd.periodic_single_box_fft = 1 +psatd.update_with_rho = 0 + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +reduced_diags.intervals = 1 +particle_energy.type = ParticleEnergy +field_energy.type = FieldEnergy diff --git a/Examples/Tests/initial_distribution/CMakeLists.txt b/Examples/Tests/initial_distribution/CMakeLists.txt new file mode 100644 index 00000000000..06fce4dddcb --- /dev/null +++ b/Examples/Tests/initial_distribution/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_initial_distribution # name + 3 # dims + 1 # nprocs + inputs_test_3d_initial_distribution # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/initial_distribution/analysis.py b/Examples/Tests/initial_distribution/analysis.py new file mode 100755 index 00000000000..8b2c8ca74e2 --- /dev/null +++ b/Examples/Tests/initial_distribution/analysis.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Yinjian Zhao +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests initial distributions. +# 1 denotes gaussian distribution. +# 2 denotes maxwell-boltzmann distribution. +# 3 denotes maxwell-juttner distribution. +# 4 denotes gaussian position distribution. +# 5 denotes maxwell-juttner distribution w/ spatially varying temperature +# 6 denotes maxwell-boltzmann distribution w/ constant velocity +# 7 denotes maxwell-boltzmann distribution w/ spatially-varying velocity +# 8 denotes uniform distribution +# 9 denotes gaussian_parser distribution w/ spatially-varying mean and thermal velocity +# The distribution is obtained through reduced diagnostic ParticleHistogram. + +import numpy as np +import scipy.constants as scc +import scipy.special as scs +from read_raw_data import read_reduced_diags, read_reduced_diags_histogram + +# print tolerance +tolerance = 0.02 +print("Tolerance:", tolerance) + +# =============================== +# gaussian and maxwell-boltzmann +# =============================== + +# load data +bin_value, h1x = read_reduced_diags_histogram("h1x.txt")[2:] +h1y = read_reduced_diags_histogram("h1y.txt")[3] +h1z = read_reduced_diags_histogram("h1z.txt")[3] +h2x = read_reduced_diags_histogram("h2x.txt")[3] +h2y = read_reduced_diags_histogram("h2y.txt")[3] +h2z = read_reduced_diags_histogram("h2z.txt")[3] + +# parameters of theory +u_rms = 0.01 +gamma = np.sqrt(1.0 + u_rms * u_rms) +v_rms = u_rms / gamma * scc.c +n = 1.0e21 +V = 8.0 +db = 0.0016 + +# compute the analytical solution +f = ( + n + * V + * scc.c + * db + * np.exp(-0.5 * (bin_value * scc.c / v_rms) ** 2) + / (v_rms * np.sqrt(2.0 * scc.pi)) +) +f_peak = np.amax(f) + +# compute error +# note that parameters are chosen such that gaussian and +# maxwell-boltzmann distributions are identical +f1_error = ( + np.sum(np.abs(f - h1x) + np.abs(f - h1y) + np.abs(f - h1z)) + / bin_value.size + / f_peak +) +f2_error = ( + np.sum(np.abs(f - h2x) + np.abs(f - h2y) + np.abs(f - h2z)) + / bin_value.size + / f_peak +) + +print("Gaussian distribution difference:", f1_error) +print("Maxwell-Boltzmann distribution difference:", f2_error) + +assert f1_error < tolerance +assert f2_error < tolerance + +# ================ +# maxwell-juttner +# ================ + +# load data +bin_value, bin_data = read_reduced_diags_histogram("h3.txt")[2:] +bin_data_filtered = read_reduced_diags_histogram("h3_filtered.txt")[3] + +# parameters of theory +theta = 1.0 +K2 = scs.kn(2, 1.0 / theta) +n = 1.0e21 +V = 8.0 +db = 0.22 + +# compute the analytical solution +f = ( + n + * V + * db + * bin_value**2 + * np.sqrt(1.0 - 1.0 / bin_value**2) + / (theta * K2) + * np.exp(-bin_value / theta) +) +f_peak = np.amax(f) + +# analytical solution for the filtered histogram: we just filter out gamma values < 5.5 +f_filtered = f * (bin_value > 5.5) + +# compute error +f3_error = ( + np.sum(np.abs(f - bin_data) + np.abs(f_filtered - bin_data_filtered)) + / bin_value.size + / f_peak +) + +print("Maxwell-Juttner distribution difference:", f3_error) + +assert f3_error < tolerance + +# ============== +# gaussian beam +# ============== + +# load data +bin_value, h4x = read_reduced_diags_histogram("h4x.txt")[2:] +h4y = read_reduced_diags_histogram("h4y.txt")[3] +h4z = read_reduced_diags_histogram("h4z.txt")[3] +_, bmmntr = read_reduced_diags("bmmntr.txt") +charge = bmmntr["charge"][0] + +# parameters of theory +x_rms = 0.25 +z_cut = 2.0 +q_tot = -1.0e-20 +q_e = -1.602176634e-19 +npart = q_tot / q_e +db = bin_value[1] - bin_value[0] + +# compute the analytical solution +f_xy = ( + npart + * db + * np.exp(-0.5 * (bin_value / x_rms) ** 2) + / (x_rms * np.sqrt(2.0 * scc.pi)) + * scs.erf(z_cut / np.sqrt(2.0)) +) +f_z = ( + npart + * db + * np.exp(-0.5 * (bin_value / x_rms) ** 2) + / (x_rms * np.sqrt(2.0 * scc.pi)) +) +f_z[np.absolute(bin_value) > z_cut * x_rms] = 0.0 +f_peak = np.amax(f_z) +q_tot_cut = q_tot * scs.erf(z_cut / np.sqrt(2.0)) + +# compute error +f4_error = ( + np.sum(np.abs(f_xy - h4x) + np.abs(f_xy - h4y) + np.abs(f_z - h4z)) + / bin_value.size + / f_peak +) +charge_error = np.abs((q_tot_cut - charge) / q_tot) + +do_plot = False +if do_plot: + import matplotlib.pyplot as plt + + plt.figure() + plt.subplot(121) + plt.plot(bin_value, f_xy, "+-", label="ref") + plt.plot(bin_value, h4x, "+--", label="sim") + plt.legend() + plt.subplot(122) + plt.plot(bin_value, f_z, "+-", label="ref") + plt.plot(bin_value, h4z, "+--", label="sim") + plt.legend() + plt.savefig("toto.pdf", bbox_inches="tight") + +print("Gaussian position distribution difference:", f4_error) +assert f4_error < tolerance + +print("Relative beam charge difference:", charge_error) +assert charge_error < tolerance + +# ============================================= +# maxwell-juttner with temperature from parser +# ============================================= + +# load data +bin_value, bin_data_neg = read_reduced_diags_histogram("h5_neg.txt")[2:] +bin_data_pos = read_reduced_diags_histogram("h5_pos.txt")[3] + +# parameters of theory +# _neg denotes where x<0, _pos where x>0 +theta_neg = 1.0 +theta_pos = 2.0 +K2_neg = scs.kn(2, 1.0 / theta_neg) +K2_pos = scs.kn(2, 1.0 / theta_pos) +n = 1.0e21 +V = 8.0 / 2 # because each of these are for half the domain +db = 0.22 + +# compute the analytical solution for each half of the domain +f_neg = ( + n + * V + * db + * bin_value**2 + * np.sqrt(1.0 - 1.0 / bin_value**2) + / (theta_neg * K2_neg) + * np.exp(-bin_value / theta_neg) +) +f_neg_peak = np.amax(f_neg) +f_pos = ( + n + * V + * db + * bin_value**2 + * np.sqrt(1.0 - 1.0 / bin_value**2) + / (theta_pos * K2_pos) + * np.exp(-bin_value / theta_pos) +) +f_pos_peak = np.amax(f_pos) +f_peak = max(f_neg_peak, f_pos_peak) + +# compute error +f5_error = ( + np.sum(np.abs(f_neg - bin_data_neg) + np.abs(f_pos - bin_data_pos)) + / bin_value.size + / f_peak +) + +print("Maxwell-Juttner parser temperature difference:", f5_error) + +assert f5_error < tolerance + +# ============================================== +# maxwell-boltzmann with constant bulk velocity +# ============================================== + +# load data +bin_value_g, bin_data_g = read_reduced_diags_histogram("h6.txt")[2:] +bin_value_uy, bin_data_uy = read_reduced_diags_histogram("h6uy.txt")[2:] + +# Expected values for beta and u = beta*gamma +beta_const = 0.2 +g_const = 1.0 / np.sqrt(1.0 - beta_const * beta_const) +uy_const = beta_const * g_const +g_bin_size = 0.004 +g_bin_min = 1.0 +uy_bin_size = 0.04 +uy_bin_min = -1.0 +V = 8.0 # volume in m^3 +n = 1.0e21 # number density in 1/m^3 + +f_g = np.zeros_like(bin_value_g) +i_g = int(np.floor((g_const - g_bin_min) / g_bin_size)) +f_g[i_g] = n * V +f_peak = np.amax(f_g) + +f_uy = np.zeros_like(bin_value_uy) +i_uy = int(np.floor((-uy_const - uy_bin_min) / uy_bin_size)) +f_uy[i_uy] = n * V + +f6_error = ( + np.sum(np.abs(f_g - bin_data_g) + np.abs(f_uy - bin_data_uy)) + / bin_value_g.size + / f_peak +) + +print("Maxwell-Boltzmann constant velocity difference:", f6_error) + +assert f6_error < tolerance + +# ============================================ +# maxwell-boltzmann with parser bulk velocity +# ============================================ + +# load data +bin_value_g, bin_data_g = read_reduced_diags_histogram("h7.txt")[2:] +bin_value_uy, bin_data_uy_neg = read_reduced_diags_histogram("h7uy_neg.txt")[2:] +bin_data_uy_pos = read_reduced_diags_histogram("h7uy_pos.txt")[3] + +# Expected values for beta and u = beta*gamma +beta_const = 0.2 +g_const = 1.0 / np.sqrt(1.0 - beta_const * beta_const) +uy_const = beta_const * g_const +g_bin_size = 0.004 +g_bin_min = 1.0 +uy_bin_size = 0.04 +uy_bin_min = -1.0 +V = 8.0 # volume in m^3 +n = 1.0e21 # number density in 1/m^3 + +f_g = np.zeros_like(bin_value_g) +i_g = int(np.floor((g_const - g_bin_min) / g_bin_size)) +f_g[i_g] = n * V +f_peak = np.amax(f_g) + +f_uy_neg = np.zeros_like(bin_value_uy) +i_uy_neg = int(np.floor((uy_const - uy_bin_min) / uy_bin_size)) +f_uy_neg[i_uy_neg] = n * V / 2.0 + +f_uy_pos = np.zeros_like(bin_value_uy) +i_uy_pos = int(np.floor((-uy_const - uy_bin_min) / uy_bin_size)) +f_uy_pos[i_uy_pos] = n * V / 2.0 + +f7_error = ( + np.sum( + np.abs(f_g - bin_data_g) + + np.abs(f_uy_pos - bin_data_uy_pos) + + np.abs(f_uy_neg - bin_data_uy_neg) + ) + / bin_value_g.size + / f_peak +) + +print("Maxwell-Boltzmann parser velocity difference:", f7_error) + +assert f7_error < tolerance + + +# ============================================ +# Cuboid distribution in momentum space +# ============================================ + +bin_value_x, h8x = read_reduced_diags_histogram("h8x.txt")[2:] +bin_value_y, h8y = read_reduced_diags_histogram("h8y.txt")[2:] +bin_value_z, h8z = read_reduced_diags_histogram("h8z.txt")[2:] + +# Analytical distribution +ux_min = -0.2 +ux_max = 0.3 +uy_min = -0.1 +uy_max = 0.1 +uz_min = 10 +uz_max = 11.2 + +N0 = n * V + +# Distributions along the three momentum axes are independent: +# we can test them separately + + +# This counts the number of bins where we expect the distribution to be nonzero +def nonzero_bins(bins, low, high): + # Bin with nonzero distribution is defined when b_{i+1} > u_min & b_i < u_max + # `bins` contains the bin centers + + db = bins[1] - bins[0] + loweredges = bins - 0.5 * db + upperedges = bins + 0.5 * db + return (upperedges > low) & (loweredges < high) + + +# Function that checks the validity of the histogram. +# We have to call it for each of the axis +def check_validity_uniform(bins, histogram, u_min, u_max, Ntrials=1000): + """ + - `bins` contains the bin centers + - `histogram` contains the normalized histogram (i.e. np.sum(histogram) = 1) + - `u_min` is the minimum of the histogram domain + - `u_max` is the maximum of the histogram domain + """ + nzbins = nonzero_bins(bins, u_min, u_max) + Nbins = np.count_nonzero(nzbins) + db = bins[1] - bins[0] + loweredges = bins - 0.5 * db + upperedges = bins + 0.5 * db + + # First we check if Nbins = 1 because this covers the case + # u_max = u_min (i.e. a delta distribution) + if Nbins == 1: + # In this case the result should be exact + assert (histogram[nzbins].item() - 1) < 1e-8 + + return + + # The probability of filling a given bin is proportional to the bin width. + # We normalize it to the "full bin" value (i.e. every bin except from the edges + # is expected to have the same p in a uniform distribution). + # The fill ratio is therefore 1 for a bin fully included in the domain and < 1 else. + # Filling a given bin is a binomial process, so we basically test each histogram with the + # expected average value to be (x - mu) < 3 sigma + + probability = ( + np.clip(upperedges, u_min, u_max) - np.clip(loweredges, u_min, u_max) + ) / (u_max - u_min) + variance = probability * (1 - probability) + nzprob = probability[nzbins] + nzhist = histogram[nzbins] + nzvar = variance[nzbins] + samplesigma = 1 / np.sqrt(Ntrials) + + normalizedvariable = np.abs(nzhist - nzprob) / np.sqrt(nzvar) + + assert np.all(normalizedvariable < 3 * samplesigma) + + +# Test the distribution at every time step +# (this assumes that no interaction is happening) +for timestep in range(len(h8x)): + check_validity_uniform(bin_value_x, h8x[timestep] / N0, ux_min, ux_max) + check_validity_uniform(bin_value_y, h8y[timestep] / N0, uy_min, uy_max) + check_validity_uniform(bin_value_z, h8z[timestep] / N0, uz_min, uz_max) + +# ================================================= +# Gaussian with parser mean and standard deviation +# ================================================= + +# load data +bin_value_ux, bin_data_ux = read_reduced_diags_histogram("h9x.txt")[2:] +bin_value_uy, bin_data_uy = read_reduced_diags_histogram("h9y.txt")[2:] +bin_value_uz, bin_data_uz = read_reduced_diags_histogram("h9z.txt")[2:] + + +def Gaussian(mean, sigma, u): + V = 8.0 # volume in m^3 + n = 1.0e21 # number density in 1/m^3 + return (n * V / (sigma * np.sqrt(2.0 * np.pi))) * np.exp( + -((u - mean) ** 2) / (2.0 * sigma**2) + ) + + +du = 2.0 / 50 +f_ux = Gaussian(0.1, 0.2, bin_value_ux) * du +f_uy = Gaussian(0.12, 0.21, bin_value_uy) * du +f_uz = Gaussian(0.14, 0.22, bin_value_uz) * du + +f9_error = ( + np.sum( + np.abs(f_ux - bin_data_ux) / f_ux.max() + + np.abs(f_uy - bin_data_uy) / f_ux.max() + + np.abs(f_uz - bin_data_uz) / f_uz.max() + ) + / bin_value_ux.size +) + +print("gaussian_parse_momentum_function velocity difference:", f9_error) + +assert f9_error < tolerance diff --git a/Examples/Tests/initial_distribution/analysis_default_regression.py b/Examples/Tests/initial_distribution/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/initial_distribution/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/initial_distribution/analysis_distribution.py b/Examples/Tests/initial_distribution/analysis_distribution.py deleted file mode 100755 index 5a3774133db..00000000000 --- a/Examples/Tests/initial_distribution/analysis_distribution.py +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Yinjian Zhao -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This script tests initial distributions. -# 1 denotes gaussian distribution. -# 2 denotes maxwell-boltzmann distribution. -# 3 denotes maxwell-juttner distribution. -# 4 denotes gaussian position distribution. -# 5 denotes maxwell-juttner distribution w/ spatially varying temperature -# 6 denotes maxwell-boltzmann distribution w/ constant velocity -# 7 denotes maxwell-boltzmann distribution w/ spatially-varying velocity -# 8 denotes uniform distribution -# 9 denotes gaussian_parser distribution w/ spatially-varying mean and thermal velocity -# The distribution is obtained through reduced diagnostic ParticleHistogram. - -import os -import sys - -import numpy as np -import scipy.constants as scc -import scipy.special as scs -from read_raw_data import read_reduced_diags, read_reduced_diags_histogram - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] - -# print tolerance -tolerance = 0.02 -print('Tolerance:', tolerance) - -#=============================== -# gaussian and maxwell-boltzmann -#=============================== - -# load data -bin_value, h1x = read_reduced_diags_histogram("h1x.txt")[2:] -h1y = read_reduced_diags_histogram("h1y.txt")[3] -h1z = read_reduced_diags_histogram("h1z.txt")[3] -h2x = read_reduced_diags_histogram("h2x.txt")[3] -h2y = read_reduced_diags_histogram("h2y.txt")[3] -h2z = read_reduced_diags_histogram("h2z.txt")[3] - -# parameters of theory -u_rms = 0.01 -gamma = np.sqrt(1.0+u_rms*u_rms) -v_rms = u_rms / gamma * scc.c -n = 1.0e21 -V = 8.0 -db = 0.0016 - -# compute the analytical solution -f = n*V*scc.c*db*np.exp(-0.5*(bin_value*scc.c/v_rms)**2)/(v_rms*np.sqrt(2.0*scc.pi)) -f_peak = np.amax(f) - -# compute error -# note that parameters are chosen such that gaussian and -# maxwell-boltzmann distributions are identical -f1_error = np.sum(np.abs(f-h1x)+np.abs(f-h1y)+np.abs(f-h1z))/bin_value.size / f_peak -f2_error = np.sum(np.abs(f-h2x)+np.abs(f-h2y)+np.abs(f-h2z))/bin_value.size / f_peak - -print('Gaussian distribution difference:', f1_error) -print('Maxwell-Boltzmann distribution difference:', f2_error) - -assert(f1_error < tolerance) -assert(f2_error < tolerance) - -#================ -# maxwell-juttner -#================ - -# load data -bin_value, bin_data = read_reduced_diags_histogram("h3.txt")[2:] -bin_data_filtered = read_reduced_diags_histogram("h3_filtered.txt")[3] - -# parameters of theory -theta = 1.0 -K2 = scs.kn(2,1.0/theta) -n = 1.0e21 -V = 8.0 -db = 0.22 - -# compute the analytical solution -f = n*V*db * bin_value**2 * np.sqrt(1.0-1.0/bin_value**2) / \ - (theta*K2) * np.exp(-bin_value/theta) -f_peak = np.amax(f) - -# analytical solution for the filtered histogram: we just filter out gamma values < 5.5 -f_filtered = f*(bin_value > 5.5) - -# compute error -f3_error = np.sum( np.abs(f-bin_data) + np.abs(f_filtered-bin_data_filtered) ) \ - / bin_value.size / f_peak - -print('Maxwell-Juttner distribution difference:', f3_error) - -assert(f3_error < tolerance) - -#============== -# gaussian beam -#============== - -# load data -bin_value, h4x = read_reduced_diags_histogram("h4x.txt")[2:] -h4y = read_reduced_diags_histogram("h4y.txt")[3] -h4z = read_reduced_diags_histogram("h4z.txt")[3] -_, bmmntr = read_reduced_diags("bmmntr.txt") -charge = bmmntr['charge'][0] - -# parameters of theory -x_rms = 0.25 -z_cut = 2. -q_tot = -1.0e-20 -q_e = -1.602176634e-19 -npart = q_tot/q_e -db = bin_value[1]-bin_value[0] - -# compute the analytical solution -f_xy = npart*db * np.exp(-0.5*(bin_value/x_rms)**2)/(x_rms*np.sqrt(2.0*scc.pi)) * scs.erf(z_cut/np.sqrt(2.)) -f_z = npart*db * np.exp(-0.5*(bin_value/x_rms)**2)/(x_rms*np.sqrt(2.0*scc.pi)) -f_z[ np.absolute(bin_value) > z_cut * x_rms ] = 0. -f_peak = np.amax(f_z) -q_tot_cut = q_tot * scs.erf(z_cut/np.sqrt(2.)) - -# compute error -f4_error = np.sum(np.abs(f_xy-h4x)+np.abs(f_xy-h4y)+np.abs(f_z-h4z))/bin_value.size / f_peak -charge_error = np.abs((q_tot_cut - charge) / q_tot) - -do_plot = False -if do_plot: - import matplotlib.pyplot as plt - plt.figure() - plt.subplot(121) - plt.plot(bin_value, f_xy, '+-', label='ref') - plt.plot(bin_value, h4x, '+--', label='sim') - plt.legend() - plt.subplot(122) - plt.plot(bin_value, f_z, '+-', label='ref') - plt.plot(bin_value, h4z, '+--', label='sim') - plt.legend() - plt.savefig('toto.pdf', bbox_inches='tight') - -print('Gaussian position distribution difference:', f4_error) -assert(f4_error < tolerance) - -print('Relative beam charge difference:', charge_error) -assert(charge_error < tolerance) - -#============================================= -# maxwell-juttner with temperature from parser -#============================================= - -# load data -bin_value, bin_data_neg = read_reduced_diags_histogram("h5_neg.txt")[2:] -bin_data_pos = read_reduced_diags_histogram("h5_pos.txt")[3] - -# parameters of theory -# _neg denotes where x<0, _pos where x>0 -theta_neg = 1.0 -theta_pos = 2.0 -K2_neg = scs.kn(2,1.0/theta_neg) -K2_pos = scs.kn(2,1.0/theta_pos) -n = 1.0e21 -V = 8.0 / 2 # because each of these are for half the domain -db = 0.22 - -# compute the analytical solution for each half of the domain -f_neg = n*V*db * bin_value**2 * np.sqrt(1.0-1.0/bin_value**2) / \ - (theta_neg*K2_neg) * np.exp(-bin_value/theta_neg) -f_neg_peak = np.amax(f_neg) -f_pos = n*V*db * bin_value**2 * np.sqrt(1.0-1.0/bin_value**2) / \ - (theta_pos*K2_pos) * np.exp(-bin_value/theta_pos) -f_pos_peak = np.amax(f_pos) -f_peak = max(f_neg_peak, f_pos_peak) - -# compute error -f5_error = np.sum( np.abs(f_neg-bin_data_neg) + np.abs(f_pos-bin_data_pos) ) \ - / bin_value.size / f_peak - -print('Maxwell-Juttner parser temperature difference:', f5_error) - -assert(f5_error < tolerance) - -#============================================== -# maxwell-boltzmann with constant bulk velocity -#============================================== - -# load data -bin_value_g, bin_data_g = read_reduced_diags_histogram("h6.txt")[2:] -bin_value_uy, bin_data_uy = read_reduced_diags_histogram("h6uy.txt")[2:] - -# Expected values for beta and u = beta*gamma -beta_const = 0.2 -g_const = 1. / np.sqrt(1. - beta_const * beta_const) -uy_const = beta_const * g_const -g_bin_size = 0.004 -g_bin_min = 1. -uy_bin_size = 0.04 -uy_bin_min = -1. -V = 8.0 # volume in m^3 -n = 1.0e21 # number density in 1/m^3 - -f_g = np.zeros_like(bin_value_g) -i_g = int(np.floor((g_const - g_bin_min) / g_bin_size)) -f_g[i_g] = n * V -f_peak = np.amax(f_g) - -f_uy = np.zeros_like(bin_value_uy) -i_uy = int(np.floor((-uy_const - uy_bin_min) / uy_bin_size)) -f_uy[i_uy] = n * V - -f6_error = np.sum(np.abs(f_g - bin_data_g) + np.abs(f_uy - bin_data_uy)) \ - / bin_value_g.size / f_peak - -print('Maxwell-Boltzmann constant velocity difference:', f6_error) - -assert(f6_error < tolerance) - -#============================================ -# maxwell-boltzmann with parser bulk velocity -#============================================ - -# load data -bin_value_g, bin_data_g = read_reduced_diags_histogram("h7.txt")[2:] -bin_value_uy, bin_data_uy_neg = read_reduced_diags_histogram("h7uy_neg.txt")[2:] -bin_data_uy_pos = read_reduced_diags_histogram("h7uy_pos.txt")[3] - -# Expected values for beta and u = beta*gamma -beta_const = 0.2 -g_const = 1. / np.sqrt(1. - beta_const * beta_const) -uy_const = beta_const * g_const -g_bin_size = 0.004 -g_bin_min = 1. -uy_bin_size = 0.04 -uy_bin_min = -1. -V = 8.0 # volume in m^3 -n = 1.0e21 # number density in 1/m^3 - -f_g = np.zeros_like(bin_value_g) -i_g = int(np.floor((g_const - g_bin_min) / g_bin_size)) -f_g[i_g] = n * V -f_peak = np.amax(f_g) - -f_uy_neg = np.zeros_like(bin_value_uy) -i_uy_neg = int(np.floor((uy_const - uy_bin_min) / uy_bin_size)) -f_uy_neg[i_uy_neg] = n * V / 2. - -f_uy_pos = np.zeros_like(bin_value_uy) -i_uy_pos = int(np.floor((-uy_const - uy_bin_min) / uy_bin_size)) -f_uy_pos[i_uy_pos] = n * V / 2. - -f7_error = np.sum(np.abs(f_g - bin_data_g) + np.abs(f_uy_pos - bin_data_uy_pos) \ - + np.abs(f_uy_neg - bin_data_uy_neg)) / bin_value_g.size / f_peak - -print('Maxwell-Boltzmann parser velocity difference:', f7_error) - -assert(f7_error < tolerance) - - -#============================================ -# Cuboid distribution in momentum space -#============================================ - -bin_value_x, h8x = read_reduced_diags_histogram("h8x.txt")[2:] -bin_value_y, h8y = read_reduced_diags_histogram("h8y.txt")[2:] -bin_value_z, h8z = read_reduced_diags_histogram("h8z.txt")[2:] - -# Analytical distribution -ux_min = -0.2 -ux_max = 0.3 -uy_min = -0.1 -uy_max = 0.1 -uz_min = 10 -uz_max = 11.2 - -N0 = n * V - -# Distributions along the three momentum axes are independent: -# we can test them separately - -# This counts the number of bins where we expect the distribution to be nonzero -def nonzero_bins(bins, low, high): - # Bin with nonzero distribution is defined when b_{i+1} > u_min & b_i < u_max - # `bins` contains the bin centers - - db = bins[1] - bins[0] - loweredges = bins - 0.5 * db - upperedges = bins + 0.5 * db - return ((upperedges > low) & (loweredges < high)) - -# Function that checks the validity of the histogram. -# We have to call it for each of the axis -def check_validity_uniform(bins, histogram, u_min, u_max, Ntrials=1000): - """ - - `bins` contains the bin centers - - `histogram` contains the normalized histogram (i.e. np.sum(histogram) = 1) - - `u_min` is the minimum of the histogram domain - - `u_max` is the maximum of the histogram domain - """ - nzbins = nonzero_bins(bins, u_min, u_max) - Nbins = np.count_nonzero(nzbins) - db = bins[1] - bins[0] - loweredges = bins - 0.5 * db - upperedges = bins + 0.5 * db - - # First we check if Nbins = 1 because this covers the case - # u_max = u_min (i.e. a delta distribution) - if Nbins == 1: - # In this case the result should be exact - assert( (histogram[nzbins].item() - 1) < 1e-8 ) - - return - - # The probability of filling a given bin is proportional to the bin width. - # We normalize it to the "full bin" value (i.e. every bin except from the edges - # is expected to have the same p in a uniform distribution). - # The fill ratio is therefore 1 for a bin fully included in the domain and < 1 else. - # Filling a given bin is a binomial process, so we basically test each histogram with the - # expected average value to be (x - mu) < 3 sigma - - probability = (np.clip(upperedges, u_min, u_max) - np.clip(loweredges, u_min, u_max)) / (u_max - u_min) - variance = probability * (1 - probability) - nzprob = probability[nzbins] - nzhist = histogram[nzbins] - nzvar = variance[nzbins] - samplesigma = 1 / np.sqrt(Ntrials) - - normalizedvariable = np.abs(nzhist - nzprob) / np.sqrt(nzvar) - - assert np.all(normalizedvariable < 3 * samplesigma) - -# Test the distribution at every time step -# (this assumes that no interaction is happening) -for timestep in range(len(h8x)): - check_validity_uniform(bin_value_x, h8x[timestep] / N0, ux_min, ux_max) - check_validity_uniform(bin_value_y, h8y[timestep] / N0, uy_min, uy_max) - check_validity_uniform(bin_value_z, h8z[timestep] / N0, uz_min, uz_max) - -#================================================= -# Gaussian with parser mean and standard deviation -#================================================= - -# load data -bin_value_ux, bin_data_ux = read_reduced_diags_histogram("h9x.txt")[2:] -bin_value_uy, bin_data_uy = read_reduced_diags_histogram("h9y.txt")[2:] -bin_value_uz, bin_data_uz = read_reduced_diags_histogram("h9z.txt")[2:] - -def Gaussian(mean, sigma, u): - V = 8.0 # volume in m^3 - n = 1.0e21 # number density in 1/m^3 - return (n*V/(sigma*np.sqrt(2.*np.pi)))*np.exp(-(u - mean)**2/(2.*sigma**2)) - -du = 2./50 -f_ux = Gaussian(0.1 , 0.2 , bin_value_ux)*du -f_uy = Gaussian(0.12, 0.21, bin_value_uy)*du -f_uz = Gaussian(0.14, 0.22, bin_value_uz)*du - -f9_error = np.sum(np.abs(f_ux - bin_data_ux)/f_ux.max() - +np.abs(f_uy - bin_data_uy)/f_ux.max() - +np.abs(f_uz - bin_data_uz)/f_uz.max()) / bin_value_ux.size - -print('gaussian_parse_momentum_function velocity difference:', f9_error) - -assert(f9_error < tolerance) - - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/initial_distribution/inputs b/Examples/Tests/initial_distribution/inputs_test_3d_initial_distribution similarity index 100% rename from Examples/Tests/initial_distribution/inputs rename to Examples/Tests/initial_distribution/inputs_test_3d_initial_distribution diff --git a/Examples/Tests/initial_plasma_profile/CMakeLists.txt b/Examples/Tests/initial_plasma_profile/CMakeLists.txt new file mode 100644 index 00000000000..064bbc29907 --- /dev/null +++ b/Examples/Tests/initial_plasma_profile/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_parabolic_channel_initialization # name + 2 # dims + 2 # nprocs + inputs_test_2d_parabolic_channel_initialization # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles --rtol 1e-4" # checksum + OFF # dependency +) diff --git a/Examples/Tests/initial_plasma_profile/analysis.py b/Examples/Tests/initial_plasma_profile/analysis.py deleted file mode 100755 index b8cfaad1048..00000000000 --- a/Examples/Tests/initial_plasma_profile/analysis.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Michael Rowan -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import os -import sys - -import yt - -yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Name of the plotfile -fn = sys.argv[1] - -test_name = os.path.split(os.getcwd())[1] - -checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-4, do_particles=False) diff --git a/Examples/Tests/initial_plasma_profile/analysis_default_regression.py b/Examples/Tests/initial_plasma_profile/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/initial_plasma_profile/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/initial_plasma_profile/inputs b/Examples/Tests/initial_plasma_profile/inputs_test_2d_parabolic_channel_initialization similarity index 100% rename from Examples/Tests/initial_plasma_profile/inputs rename to Examples/Tests/initial_plasma_profile/inputs_test_2d_parabolic_channel_initialization diff --git a/Examples/Tests/ion_stopping/CMakeLists.txt b/Examples/Tests/ion_stopping/CMakeLists.txt new file mode 100644 index 00000000000..666b28244dd --- /dev/null +++ b/Examples/Tests/ion_stopping/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_ion_stopping # name + 3 # dims + 1 # nprocs + inputs_test_3d_ion_stopping # inputs + "analysis.py diags/diag1000010" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) diff --git a/Examples/Tests/ion_stopping/analysis.py b/Examples/Tests/ion_stopping/analysis.py new file mode 100755 index 00000000000..6b92bb304a5 --- /dev/null +++ b/Examples/Tests/ion_stopping/analysis.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +# Copyright 2022 David Grote +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the ion stopping model. +# It uses the same stopping power formula that +# is used in the C++ to check the resulting +# particle energies. + +import sys + +import numpy as np +import yt +from scipy.constants import e, epsilon_0, k, m_e, m_p + +# Define constants using the WarpX names for the evals below +q_e = e +kb = k + +# Tolerance on the error in the final energy (in eV) +tolerance = 1.0e-7 + +last_filename = sys.argv[1] +last_it = 10 + + +def stopping_from_electrons(ne, Te, Zb, ion_mass): + """Calculate the coefficient in equation 14.13 from + "Introduction to Plasma Physics", Goldston and Rutherford. + ne: electron density + Te: electron temperature (eV) + Zb: ion charge state + ion_mass: (kg) + """ + vthe = np.sqrt(3.0 * Te * e / m_e) + wpe = np.sqrt(ne * e**2 / (epsilon_0 * m_e)) + lambdadb = vthe / wpe + loglambda = np.log((12.0 * np.pi / Zb) * (ne * lambdadb**3)) + # Goldston's equation 14.13 + dEdt = ( + -np.sqrt(2.0) + * ne + * Zb**2 + * e**4 + * np.sqrt(m_e) + * loglambda + / (6.0 * np.pi**1.5 * epsilon_0**2 * ion_mass * (Te * e) ** 1.5) + ) + return dEdt + + +def stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass, ion_energy): + """ + ni: background ion density + Ti: background ion temperature (eV) + mi: background ion mass + Zi: background ion charge state + Zb: ion charge state + ion_mass: (kg) + ion_energy: (eV) + """ + vthi = np.sqrt(3.0 * Ti * e / mi) + wpi = np.sqrt(ni * e**2 / (epsilon_0 * mi)) + lambdadb = vthi / wpi + loglambda = np.log((12.0 * np.pi / Zb) * (ni * lambdadb**3)) + alpha = ( + np.sqrt(2.0) + * ni + * Zi**2 + * Zb**2 + * e**4 + * np.sqrt(ion_mass) + * loglambda + / (8.0 * np.pi * epsilon_0**2 * mi) + ) + f1 = np.clip((ion_energy * e) ** (3.0 / 2.0) - 3.0 / 2.0 * alpha * dt, 0.0, None) + ion_energy = (f1) ** (2.0 / 3.0) / e + return ion_energy + + +# Fetch background parameters and initial particle data +ds0 = yt.load("diags/diag1000000") +ad0 = ds0.all_data() + +Zb = 1.0 # Ion charge state + +ne = float(ds0.parameters["stopping_on_electrons_constant.background_density"]) +Te = ( + eval(ds0.parameters["stopping_on_electrons_constant.background_temperature"]) + * kb + / e +) +ion_mass12 = m_p + +mi = eval(ds0.parameters["stopping_on_ions_constant.background_mass"]) +Zi = float(ds0.parameters["stopping_on_ions_constant.background_charge_state"]) +ni = float(ds0.parameters["stopping_on_ions_constant.background_density"]) +Ti = eval(ds0.parameters["stopping_on_ions_constant.background_temperature"]) * kb / e +ion_mass34 = 4.0 * m_p + +# For ions1, the background parameters are constants +vx = ad0[("ions1", "particle_momentum_x")].to_ndarray() / ion_mass12 +vy = ad0[("ions1", "particle_momentum_y")].to_ndarray() / ion_mass12 +vz = ad0[("ions1", "particle_momentum_z")].to_ndarray() / ion_mass12 +EE1 = 0.5 * ion_mass12 * (vx**2 + vy**2 + vz**2) / e + +# For ions2, the background parameters are parsed +xx = ad0[("ions2", "particle_position_x")].to_ndarray() / ion_mass12 +yy = ad0[("ions2", "particle_position_y")].to_ndarray() / ion_mass12 +ne2 = np.where(xx > 0.0, 1.0e20, 1.0e21) +Te2 = np.where(yy > 0.0, 1.0, 2.0) + +vx = ad0[("ions2", "particle_momentum_x")].to_ndarray() / ion_mass12 +vy = ad0[("ions2", "particle_momentum_y")].to_ndarray() / ion_mass12 +vz = ad0[("ions2", "particle_momentum_z")].to_ndarray() / ion_mass12 +EE2 = 0.5 * ion_mass12 * (vx**2 + vy**2 + vz**2) / e + +# For ions3, the background parameters are constants +vx = ad0[("ions3", "particle_momentum_x")].to_ndarray() / ion_mass34 +vy = ad0[("ions3", "particle_momentum_y")].to_ndarray() / ion_mass34 +vz = ad0[("ions3", "particle_momentum_z")].to_ndarray() / ion_mass34 +EE3 = 0.5 * ion_mass34 * (vx**2 + vy**2 + vz**2) / e + +# For ions4, the background parameters are parsed +xx = ad0[("ions4", "particle_position_x")].to_ndarray() / ion_mass34 +yy = ad0[("ions4", "particle_position_y")].to_ndarray() / ion_mass34 +ni4 = np.where(xx > 0.0, 1.0e20, 1.0e21) +Ti4 = np.where(yy > 0.0, 0.05, 0.10) + +vx = ad0[("ions4", "particle_momentum_x")].to_ndarray() / ion_mass34 +vy = ad0[("ions4", "particle_momentum_y")].to_ndarray() / ion_mass34 +vz = ad0[("ions4", "particle_momentum_z")].to_ndarray() / ion_mass34 +EE4 = 0.5 * ion_mass34 * (vx**2 + vy**2 + vz**2) / e + + +ds = yt.load(last_filename) +ad = ds.all_data() +dt = ds.current_time.to_value() / last_it + +# Step through the same number of time steps +a_EE1 = EE1 +a_EE2 = EE2 +a_EE3 = EE3 +a_EE4 = EE4 +for it in range(last_it): + dEdt1 = stopping_from_electrons(ne, Te, Zb, ion_mass12) + a_EE1 *= np.exp(dEdt1 * dt) + dEdt2 = stopping_from_electrons(ne2, Te2, Zb, ion_mass12) + a_EE2 *= np.exp(dEdt2 * dt) + a_EE3 = stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass34, a_EE3) + a_EE4 = stopping_from_ions(dt, ni4, Ti4, mi, Zi, Zb, ion_mass34, a_EE4) + +# Fetch the final particle data +vx = ad[("ions1", "particle_momentum_x")].to_ndarray() / ion_mass12 +vy = ad[("ions1", "particle_momentum_y")].to_ndarray() / ion_mass12 +vz = ad[("ions1", "particle_momentum_z")].to_ndarray() / ion_mass12 +EE1 = 0.5 * ion_mass12 * (vx**2 + vy**2 + vz**2) / e + +vx = ad[("ions2", "particle_momentum_x")].to_ndarray() / ion_mass12 +vy = ad[("ions2", "particle_momentum_y")].to_ndarray() / ion_mass12 +vz = ad[("ions2", "particle_momentum_z")].to_ndarray() / ion_mass12 +EE2 = 0.5 * ion_mass12 * (vx**2 + vy**2 + vz**2) / e + +vx = ad[("ions3", "particle_momentum_x")].to_ndarray() / ion_mass34 +vy = ad[("ions3", "particle_momentum_y")].to_ndarray() / ion_mass34 +vz = ad[("ions3", "particle_momentum_z")].to_ndarray() / ion_mass34 +EE3 = 0.5 * ion_mass34 * (vx**2 + vy**2 + vz**2) / e + +vx = ad[("ions4", "particle_momentum_x")].to_ndarray() / ion_mass34 +vy = ad[("ions4", "particle_momentum_y")].to_ndarray() / ion_mass34 +vz = ad[("ions4", "particle_momentum_z")].to_ndarray() / ion_mass34 +EE4 = 0.5 * ion_mass34 * (vx**2 + vy**2 + vz**2) / e + +error1 = np.abs(EE1 - a_EE1) +error2 = np.abs(EE2 - a_EE2) +error3 = np.abs(EE3 - a_EE3) +error4 = np.abs(EE4 - a_EE4) +print("stopping on electrons error with constant = ", error1) +print("stopping on electrons error with parsed = ", error2) +print("stopping on ions error with constant = ", error3) +print("stopping on ions error with parsed = ", error4) +print("tolerance = ", tolerance) + +assert np.all(error1 < tolerance) +assert np.all(error2 < tolerance) +assert np.all(error3 < tolerance) +assert np.all(error4 < tolerance) diff --git a/Examples/Tests/ion_stopping/analysis_default_regression.py b/Examples/Tests/ion_stopping/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ion_stopping/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py deleted file mode 100755 index d7774c14d6b..00000000000 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 David Grote -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This script tests the ion stopping model. -# It uses the same stopping power formula that -# is used in the C++ to check the resulting -# particle energies. - -import os -import re -import sys - -import numpy as np -import yt -from scipy.constants import e, epsilon_0, k, m_e, m_p - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Define constants using the WarpX names for the evals below -q_e = e -kb = k - -# Tolerance on the error in the final energy (in eV) -tolerance = 1.e-7 - -last_filename = sys.argv[1] - -# Remove trailing '/' from file name, if necessary -last_filename.rstrip('/') -# Find last iteration in file name, such as 'test_name_plt000001' (last_it = '000001') -last_it = re.search('\d+$', last_filename).group() -# Find output prefix in file name, such as 'test_name_plt000001' (prefix = 'test_name_plt') -prefix = last_filename[:-len(last_it)] - -def stopping_from_electrons(ne, Te, Zb, ion_mass): - """Calculate the coefficient in equation 14.13 from - "Introduction to Plasma Physics", Goldston and Rutherford. - ne: electron density - Te: electron temperature (eV) - Zb: ion charge state - ion_mass: (kg) - """ - vthe = np.sqrt(3.*Te*e/m_e) - wpe = np.sqrt(ne*e**2/(epsilon_0*m_e)) - lambdadb = vthe/wpe - loglambda = np.log((12.*np.pi/Zb)*(ne*lambdadb**3)) - # Goldston's equation 14.13 - dEdt = - np.sqrt(2.)*ne*Zb**2*e**4*np.sqrt(m_e)*loglambda/(6.*np.pi**1.5*epsilon_0**2*ion_mass*(Te*e)**1.5) - return dEdt - -def stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass, ion_energy): - """ - ni: background ion density - Ti: background ion temperature (eV) - mi: background ion mass - Zi: background ion charge state - Zb: ion charge state - ion_mass: (kg) - ion_energy: (eV) - """ - vthi = np.sqrt(3.*Ti*e/mi) - wpi = np.sqrt(ni*e**2/(epsilon_0*mi)) - lambdadb = vthi/wpi - loglambda = np.log((12.*np.pi/Zb)*(ni*lambdadb**3)) - alpha = np.sqrt(2.)*ni*Zi**2*Zb**2*e**4*np.sqrt(ion_mass)*loglambda/(8.*np.pi*epsilon_0**2*mi) - f1 = np.clip((ion_energy*e)**(3./2.) - 3./2.*alpha*dt, 0., None) - ion_energy = (f1)**(2./3.)/e - return ion_energy - -# Fetch background parameters and initial particle data -ds0 = yt.load(f'{prefix}{len(last_it)*"0"}') -ad0 = ds0.all_data() - -Zb = 1. # Ion charge state - -ne = float(ds0.parameters['stopping_on_electrons_constant.background_density']) -Te = eval(ds0.parameters['stopping_on_electrons_constant.background_temperature'])*kb/e -ion_mass12 = m_p - -mi = eval(ds0.parameters['stopping_on_ions_constant.background_mass']) -Zi = float(ds0.parameters['stopping_on_ions_constant.background_charge_state']) -ni = float(ds0.parameters['stopping_on_ions_constant.background_density']) -Ti = eval(ds0.parameters['stopping_on_ions_constant.background_temperature'])*kb/e -ion_mass34 = 4.*m_p - -# For ions1, the background parameters are constants -vx = ad0[('ions1', 'particle_momentum_x')].to_ndarray()/ion_mass12 -vy = ad0[('ions1', 'particle_momentum_y')].to_ndarray()/ion_mass12 -vz = ad0[('ions1', 'particle_momentum_z')].to_ndarray()/ion_mass12 -EE1 = 0.5*ion_mass12*(vx**2 + vy**2 + vz**2)/e - -# For ions2, the background parameters are parsed -xx = ad0[('ions2', 'particle_position_x')].to_ndarray()/ion_mass12 -yy = ad0[('ions2', 'particle_position_y')].to_ndarray()/ion_mass12 -ne2 = np.where(xx > 0., 1.e20, 1.e21) -Te2 = np.where(yy > 0., 1., 2.) - -vx = ad0[('ions2', 'particle_momentum_x')].to_ndarray()/ion_mass12 -vy = ad0[('ions2', 'particle_momentum_y')].to_ndarray()/ion_mass12 -vz = ad0[('ions2', 'particle_momentum_z')].to_ndarray()/ion_mass12 -EE2 = 0.5*ion_mass12*(vx**2 + vy**2 + vz**2)/e - -# For ions3, the background parameters are constants -vx = ad0[('ions3', 'particle_momentum_x')].to_ndarray()/ion_mass34 -vy = ad0[('ions3', 'particle_momentum_y')].to_ndarray()/ion_mass34 -vz = ad0[('ions3', 'particle_momentum_z')].to_ndarray()/ion_mass34 -EE3 = 0.5*ion_mass34*(vx**2 + vy**2 + vz**2)/e - -# For ions4, the background parameters are parsed -xx = ad0[('ions4', 'particle_position_x')].to_ndarray()/ion_mass34 -yy = ad0[('ions4', 'particle_position_y')].to_ndarray()/ion_mass34 -ni4 = np.where(xx > 0., 1.e20, 1.e21) -Ti4 = np.where(yy > 0., 0.05, 0.10) - -vx = ad0[('ions4', 'particle_momentum_x')].to_ndarray()/ion_mass34 -vy = ad0[('ions4', 'particle_momentum_y')].to_ndarray()/ion_mass34 -vz = ad0[('ions4', 'particle_momentum_z')].to_ndarray()/ion_mass34 -EE4 = 0.5*ion_mass34*(vx**2 + vy**2 + vz**2)/e - - -ds = yt.load(last_filename) -ad = ds.all_data() -dt = ds.current_time.to_value()/int(last_it) - -# Step through the same number of time steps -a_EE1 = EE1 -a_EE2 = EE2 -a_EE3 = EE3 -a_EE4 = EE4 -for it in range(int(last_it)): - dEdt1 = stopping_from_electrons(ne, Te, Zb, ion_mass12) - a_EE1 *= np.exp(dEdt1*dt) - dEdt2 = stopping_from_electrons(ne2, Te2, Zb, ion_mass12) - a_EE2 *= np.exp(dEdt2*dt) - a_EE3 = stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass34, a_EE3) - a_EE4 = stopping_from_ions(dt, ni4, Ti4, mi, Zi, Zb, ion_mass34, a_EE4) - -# Fetch the final particle data -vx = ad[('ions1', 'particle_momentum_x')].to_ndarray()/ion_mass12 -vy = ad[('ions1', 'particle_momentum_y')].to_ndarray()/ion_mass12 -vz = ad[('ions1', 'particle_momentum_z')].to_ndarray()/ion_mass12 -EE1 = 0.5*ion_mass12*(vx**2 + vy**2 + vz**2)/e - -vx = ad[('ions2', 'particle_momentum_x')].to_ndarray()/ion_mass12 -vy = ad[('ions2', 'particle_momentum_y')].to_ndarray()/ion_mass12 -vz = ad[('ions2', 'particle_momentum_z')].to_ndarray()/ion_mass12 -EE2 = 0.5*ion_mass12*(vx**2 + vy**2 + vz**2)/e - -vx = ad[('ions3', 'particle_momentum_x')].to_ndarray()/ion_mass34 -vy = ad[('ions3', 'particle_momentum_y')].to_ndarray()/ion_mass34 -vz = ad[('ions3', 'particle_momentum_z')].to_ndarray()/ion_mass34 -EE3 = 0.5*ion_mass34*(vx**2 + vy**2 + vz**2)/e - -vx = ad[('ions4', 'particle_momentum_x')].to_ndarray()/ion_mass34 -vy = ad[('ions4', 'particle_momentum_y')].to_ndarray()/ion_mass34 -vz = ad[('ions4', 'particle_momentum_z')].to_ndarray()/ion_mass34 -EE4 = 0.5*ion_mass34*(vx**2 + vy**2 + vz**2)/e - -error1 = np.abs(EE1 - a_EE1) -error2 = np.abs(EE2 - a_EE2) -error3 = np.abs(EE3 - a_EE3) -error4 = np.abs(EE4 - a_EE4) -print('stopping on electrons error with constant = ', error1) -print('stopping on electrons error with parsed = ', error2) -print('stopping on ions error with constant = ', error3) -print('stopping on ions error with parsed = ', error4) -print('tolerance = ', tolerance) - -assert np.all(error1 < tolerance) -assert np.all(error2 < tolerance) -assert np.all(error3 < tolerance) -assert np.all(error4 < tolerance) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, last_filename) diff --git a/Examples/Tests/ion_stopping/inputs_3d b/Examples/Tests/ion_stopping/inputs_3d deleted file mode 100644 index 291e1ca0a9e..00000000000 --- a/Examples/Tests/ion_stopping/inputs_3d +++ /dev/null @@ -1,100 +0,0 @@ -my_constants.max_ion_energy = 100. # eV -my_constants.max_ion_velocity12 = sqrt(2.*max_ion_energy*q_e/m_p)/clight -my_constants.max_ion_velocity34 = sqrt(2.*max_ion_energy*q_e/(4.*m_p))/clight - -max_step = 10 - -amr.n_cell = 8 8 8 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -1. -1. -1. -geometry.prob_hi = +1. +1. +1. -boundary.field_lo = periodic periodic periodic -boundary.field_hi = periodic periodic periodic -boundary.particle_lo = periodic periodic periodic -boundary.particle_hi = periodic periodic periodic -algo.particle_shape = 1 - -particles.species_names = ions1 ions2 ions3 ions4 - -ions1.species_type = proton -ions1.injection_style = "MultipleParticles" -ions1.multiple_particles_pos_x = 0. 0. 0. 0. -ions1.multiple_particles_pos_y = 0. 0. 0. 0. -ions1.multiple_particles_pos_z = 0. 0. 0. 0. -ions1.multiple_particles_ux = max_ion_velocity12 0. 0. max_ion_velocity12/2. -ions1.multiple_particles_uy = 0. max_ion_velocity12 0. max_ion_velocity12/2. -ions1.multiple_particles_uz = 0. 0. -max_ion_velocity12 -max_ion_velocity12/2. -ions1.multiple_particles_weight = 1. 1. 1. 1. -ions1.do_not_deposit = 1 - -ions2.species_type = proton -ions2.injection_style = "MultipleParticles" -ions2.multiple_particles_pos_x = -0.1 +0.1 -0.1 +0.1 -ions2.multiple_particles_pos_y = -0.1 -0.1 +0.1 +0.1 -ions2.multiple_particles_pos_z = 0. 0. 0. 0. -ions2.multiple_particles_ux = max_ion_velocity12 max_ion_velocity12 max_ion_velocity12 max_ion_velocity12 -ions2.multiple_particles_uy = 0. 0. 0. 0. -ions2.multiple_particles_uz = 0. 0. 0. 0. -ions2.multiple_particles_weight = 1. 1. 1. 1. -ions2.do_not_deposit = 1 - -ions3.charge = q_e -ions3.mass = 4*m_p -ions3.injection_style = "MultipleParticles" -ions3.multiple_particles_pos_x = 0. 0. 0. 0. -ions3.multiple_particles_pos_y = 0. 0. 0. 0. -ions3.multiple_particles_pos_z = 0. 0. 0. 0. -ions3.multiple_particles_ux = max_ion_velocity34 0. 0. max_ion_velocity34/2. -ions3.multiple_particles_uy = 0. max_ion_velocity34 0. max_ion_velocity34/2. -ions3.multiple_particles_uz = 0. 0. -max_ion_velocity34 -max_ion_velocity34/2. -ions3.multiple_particles_weight = 1. 1. 1. 1. -ions3.do_not_deposit = 1 - -ions4.charge = q_e -ions4.mass = 4*m_p -ions4.injection_style = "MultipleParticles" -ions4.multiple_particles_pos_x = -0.1 +0.1 -0.1 +0.1 -ions4.multiple_particles_pos_y = -0.1 -0.1 +0.1 +0.1 -ions4.multiple_particles_pos_z = 0. 0. 0. 0. -ions4.multiple_particles_ux = max_ion_velocity34 max_ion_velocity34 max_ion_velocity34 max_ion_velocity34 -ions4.multiple_particles_uy = 0. 0. 0. 0. -ions4.multiple_particles_uz = 0. 0. 0. 0. -ions4.multiple_particles_weight = 1. 1. 1. 1. -ions4.do_not_deposit = 1 - -collisions.collision_names = stopping_on_electrons_constant stopping_on_electrons_parsed stopping_on_ions_constant stopping_on_ions_parsed - -stopping_on_electrons_constant.type = background_stopping -stopping_on_electrons_constant.species = ions1 -stopping_on_electrons_constant.background_type = electrons -stopping_on_electrons_constant.background_mass = m_e -stopping_on_electrons_constant.background_density = 1.e20 -stopping_on_electrons_constant.background_temperature = 1.*q_e/kb # Kelvin - -stopping_on_electrons_parsed.type = background_stopping -stopping_on_electrons_parsed.species = ions2 -stopping_on_electrons_parsed.background_type = electrons -stopping_on_electrons_parsed.background_density(x,y,z,t) = if(x>0,1.e20,1.e21) -stopping_on_electrons_parsed.background_temperature(x,y,z,t) = if(y>0,1.,2.)*q_e/kb - -stopping_on_ions_constant.type = background_stopping -stopping_on_ions_constant.species = ions3 -stopping_on_ions_constant.background_type = ions -stopping_on_ions_constant.background_mass = m_p -stopping_on_ions_constant.background_charge_state = 1. -stopping_on_ions_constant.background_density = 1.e20 -stopping_on_ions_constant.background_temperature = 0.05*q_e/kb # Kelvin - -stopping_on_ions_parsed.type = background_stopping -stopping_on_ions_parsed.species = ions4 -stopping_on_ions_parsed.background_type = ions -stopping_on_ions_parsed.background_mass = m_p -stopping_on_ions_parsed.background_charge_state = 1. -stopping_on_ions_parsed.background_density(x,y,z,t) = if(x>0,1.e20,1.e21) -stopping_on_ions_parsed.background_temperature(x,y,z,t) = if(y>0,0.05,0.10)*q_e/kb - -diagnostics.diags_names = diag1 -diag1.diag_type = Full -diag1.format = plotfile -diag1.intervals = 1 diff --git a/Examples/Tests/ion_stopping/inputs_test_3d_ion_stopping b/Examples/Tests/ion_stopping/inputs_test_3d_ion_stopping new file mode 100644 index 00000000000..93b59bbde4a --- /dev/null +++ b/Examples/Tests/ion_stopping/inputs_test_3d_ion_stopping @@ -0,0 +1,101 @@ +my_constants.max_ion_energy = 100. # eV +my_constants.max_ion_velocity12 = sqrt(2.*max_ion_energy*q_e/m_p)/clight +my_constants.max_ion_velocity34 = sqrt(2.*max_ion_energy*q_e/(4.*m_p))/clight + +max_step = 10 + +amr.n_cell = 8 8 8 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -1. -1. -1. +geometry.prob_hi = +1. +1. +1. +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic +boundary.particle_lo = periodic periodic periodic +boundary.particle_hi = periodic periodic periodic +algo.particle_shape = 1 +warpx.cfl = 0.7 + +particles.species_names = ions1 ions2 ions3 ions4 + +ions1.species_type = proton +ions1.injection_style = "MultipleParticles" +ions1.multiple_particles_pos_x = 0. 0. 0. 0. +ions1.multiple_particles_pos_y = 0. 0. 0. 0. +ions1.multiple_particles_pos_z = 0. 0. 0. 0. +ions1.multiple_particles_ux = max_ion_velocity12 0. 0. max_ion_velocity12/2. +ions1.multiple_particles_uy = 0. max_ion_velocity12 0. max_ion_velocity12/2. +ions1.multiple_particles_uz = 0. 0. -max_ion_velocity12 -max_ion_velocity12/2. +ions1.multiple_particles_weight = 1. 1. 1. 1. +ions1.do_not_deposit = 1 + +ions2.species_type = proton +ions2.injection_style = "MultipleParticles" +ions2.multiple_particles_pos_x = -0.1 +0.1 -0.1 +0.1 +ions2.multiple_particles_pos_y = -0.1 -0.1 +0.1 +0.1 +ions2.multiple_particles_pos_z = 0. 0. 0. 0. +ions2.multiple_particles_ux = max_ion_velocity12 max_ion_velocity12 max_ion_velocity12 max_ion_velocity12 +ions2.multiple_particles_uy = 0. 0. 0. 0. +ions2.multiple_particles_uz = 0. 0. 0. 0. +ions2.multiple_particles_weight = 1. 1. 1. 1. +ions2.do_not_deposit = 1 + +ions3.charge = q_e +ions3.mass = 4*m_p +ions3.injection_style = "MultipleParticles" +ions3.multiple_particles_pos_x = 0. 0. 0. 0. +ions3.multiple_particles_pos_y = 0. 0. 0. 0. +ions3.multiple_particles_pos_z = 0. 0. 0. 0. +ions3.multiple_particles_ux = max_ion_velocity34 0. 0. max_ion_velocity34/2. +ions3.multiple_particles_uy = 0. max_ion_velocity34 0. max_ion_velocity34/2. +ions3.multiple_particles_uz = 0. 0. -max_ion_velocity34 -max_ion_velocity34/2. +ions3.multiple_particles_weight = 1. 1. 1. 1. +ions3.do_not_deposit = 1 + +ions4.charge = q_e +ions4.mass = 4*m_p +ions4.injection_style = "MultipleParticles" +ions4.multiple_particles_pos_x = -0.1 +0.1 -0.1 +0.1 +ions4.multiple_particles_pos_y = -0.1 -0.1 +0.1 +0.1 +ions4.multiple_particles_pos_z = 0. 0. 0. 0. +ions4.multiple_particles_ux = max_ion_velocity34 max_ion_velocity34 max_ion_velocity34 max_ion_velocity34 +ions4.multiple_particles_uy = 0. 0. 0. 0. +ions4.multiple_particles_uz = 0. 0. 0. 0. +ions4.multiple_particles_weight = 1. 1. 1. 1. +ions4.do_not_deposit = 1 + +collisions.collision_names = stopping_on_electrons_constant stopping_on_electrons_parsed stopping_on_ions_constant stopping_on_ions_parsed + +stopping_on_electrons_constant.type = background_stopping +stopping_on_electrons_constant.species = ions1 +stopping_on_electrons_constant.background_type = electrons +stopping_on_electrons_constant.background_mass = m_e +stopping_on_electrons_constant.background_density = 1.e20 +stopping_on_electrons_constant.background_temperature = 1.*q_e/kb # Kelvin + +stopping_on_electrons_parsed.type = background_stopping +stopping_on_electrons_parsed.species = ions2 +stopping_on_electrons_parsed.background_type = electrons +stopping_on_electrons_parsed.background_density(x,y,z,t) = if(x>0,1.e20,1.e21) +stopping_on_electrons_parsed.background_temperature(x,y,z,t) = if(y>0,1.,2.)*q_e/kb + +stopping_on_ions_constant.type = background_stopping +stopping_on_ions_constant.species = ions3 +stopping_on_ions_constant.background_type = ions +stopping_on_ions_constant.background_mass = m_p +stopping_on_ions_constant.background_charge_state = 1. +stopping_on_ions_constant.background_density = 1.e20 +stopping_on_ions_constant.background_temperature = 0.05*q_e/kb # Kelvin + +stopping_on_ions_parsed.type = background_stopping +stopping_on_ions_parsed.species = ions4 +stopping_on_ions_parsed.background_type = ions +stopping_on_ions_parsed.background_mass = m_p +stopping_on_ions_parsed.background_charge_state = 1. +stopping_on_ions_parsed.background_density(x,y,z,t) = if(x>0,1.e20,1.e21) +stopping_on_ions_parsed.background_temperature(x,y,z,t) = if(y>0,0.05,0.10)*q_e/kb + +diagnostics.diags_names = diag1 +diag1.diag_type = Full +diag1.format = plotfile +diag1.intervals = 1 diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py deleted file mode 100644 index 802bf5435ac..00000000000 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c - -# Number of time steps -max_steps = 1600 - -# Number of cells -nx = 16 -nz = 800 - -# Physical domain -xmin = -5e-06 -xmax = 5e-06 -zmin = 0e-06 -zmax = 20e-06 - -# Domain decomposition -max_grid_size = 64 -blocking_factor = 16 - -# Create grid -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, nz], - lower_bound = [xmin, zmin], - upper_bound = [xmax, zmax], - lower_boundary_conditions = ['periodic', 'open'], - upper_boundary_conditions = ['periodic', 'open'], - warpx_max_grid_size = max_grid_size, - warpx_blocking_factor = blocking_factor) - -# Particles: electrons and ions -ions_density = 1 -ions_xmin = None -ions_ymin = None -ions_zmin = 5e-06 -ions_xmax = None -ions_ymax = None -ions_zmax = 15e-06 -uniform_distribution = picmi.UniformDistribution( - density = ions_density, - lower_bound = [ions_xmin, ions_ymin, ions_zmin], - upper_bound = [ions_xmax, ions_ymax, ions_zmax], - fill_in = True) -electrons = picmi.Species( - particle_type = 'electron', - name = 'electrons', - warpx_add_real_attributes = {'orig_z': 'z'}) -ions = picmi.Species( - particle_type = 'N', - name = 'ions', - charge_state = 2, - initial_distribution = uniform_distribution, - warpx_add_real_attributes = {'orig_z': 'z'}) - -# Field ionization -nitrogen_ionization = picmi.FieldIonization( - model = "ADK", # Ammosov-Delone-Krainov model - ionized_species = ions, - product_species = electrons) - -# Laser -position_z = 3e-06 -profile_t_peak = 60.e-15 -laser = picmi.GaussianLaser( - wavelength = 0.8e-06, - waist = 1e10, - duration = 26.685e-15, - focal_position = [0, 0, position_z], - centroid_position = [0, 0, position_z - c*profile_t_peak], - propagation_direction = [0, 0, 1], - polarization_direction = [1, 0, 0], - a0 = 1.8, - fill_in = False) -laser_antenna = picmi.LaserAntenna( - position = [0., 0., position_z], - normal_vector = [0, 0, 1]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid = grid, - method = 'CKC', - cfl = 0.999) - -# Diagnostics -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10000, - species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting', 'orig_z'], - write_dir = '.', - warpx_file_prefix = 'Python_ionization_plt') -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 10000, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir = '.', - warpx_file_prefix = 'Python_ionization_plt') - -# Set up simulation -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - particle_shape = 'linear', - warpx_use_filter = 0) - -# Add electrons and ions -sim.add_species( - electrons, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [0, 0, 0])) -sim.add_species( - ions, - layout = picmi.GriddedLayout(grid = grid, n_macroparticle_per_cell = [2, 1, 1])) - -# Add field ionization -sim.add_interaction(nitrogen_ionization) - -# Add laser -sim.add_laser( - laser, - injection_method = laser_antenna) - -# Add diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Write input file that can be used to run with the compiled version -sim.write_input_file(file_name = 'inputs_2d_picmi') - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py deleted file mode 100755 index 90657915b50..00000000000 --- a/Examples/Tests/ionization/analysis_ionization.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -""" -This script tests the result of the ionization module in WarpX. - -Input files inputs.rt and inputs.bf.rt are used to reproduce the test from -Chen, JCP, 2013, figure 2 (in the lab frame and in a boosted frame, -respectively): a plane-wave laser pulse propagates through a -uniform N2+ neutral plasma and further ionizes the Nitrogen atoms. This test -checks that, after the laser went through the plasma, ~32% of Nitrogen -ions are N5+, in agreement with theory from Chen's article. -""" - -import os -import sys - -import numpy as np -import yt - -yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Open plotfile specified in command line, and get ion's ionization level. -filename = sys.argv[1] -ds = yt.load( filename ) -ad = ds.all_data() -ilev = ad['ions', 'particle_ionizationLevel'].v - -# Fraction of Nitrogen ions that are N5+. -N5_fraction = ilev[ilev == 5].size/float(ilev.size) - -print("Number of ions: " + str(ilev.size)) -print("Number of N5+ : " + str(ilev[ilev == 5].size)) -print("N5_fraction : " + str(N5_fraction)) - -do_plot = False -if do_plot: - import matplotlib.pyplot as plt - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) - F = all_data_level_0['boxlib', 'Ex'].v.squeeze() - extent = [ ds.domain_left_edge[1], ds.domain_right_edge[1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - ad = ds.all_data() - - # Plot ions with ionization levels - species = 'ions' - xi = ad[species, 'particle_position_x'].v - zi = ad[species, 'particle_position_y'].v - ii = ad[species, 'particle_ionizationLevel'].v - plt.figure(figsize=(10,10)) - plt.subplot(211) - plt.imshow(np.abs(F), extent=extent, aspect='auto', - cmap='magma', origin='default') - plt.colorbar() - for lev in range(int(np.max(ii)+1)): - select = (ii == lev) - plt.scatter(zi[select],xi[select],s=.2, - label='ionization level: ' + str(lev)) - plt.legend() - plt.title("abs(Ex) (V/m) and ions") - plt.xlabel("z (m)") - plt.ylabel("x (m)") - plt.subplot(212) - plt.imshow(np.abs(F), extent=extent, aspect='auto', - cmap='magma', origin='default') - plt.colorbar() - - # Plot electrons - species = 'electrons' - if species in [x[0] for x in ds.field_list]: - xe = ad[species, 'particle_position_x'].v - ze = ad[species, 'particle_position_y'].v - plt.scatter(ze,xe,s=.1,c='r',label='electrons') - plt.title("abs(Ex) (V/m) and electrons") - plt.xlabel("z (m)") - plt.ylabel("x (m)") - plt.savefig("image_ionization.pdf", bbox_inches='tight') - -error_rel = abs(N5_fraction-0.32) / 0.32 -tolerance_rel = 0.07 - -print("error_rel : " + str(error_rel)) -print("tolerance_rel: " + str(tolerance_rel)) - -assert( error_rel < tolerance_rel ) - -# Check that the user runtime component (if it exists) worked as expected -try: - orig_z = ad['electrons', 'particle_orig_z'].v - print(f"orig_z: min = {np.min(orig_z)}, max = {np.max(orig_z)}") - assert np.all( (orig_z > 0.0) & (orig_z < 1.5e-5) ) - print('particle_orig_z has reasonable values') -except yt.utilities.exceptions.YTFieldNotFound: - pass # The backtransformed diagnostic version of the test does not have orig_z - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/langmuir/CMakeLists.txt b/Examples/Tests/langmuir/CMakeLists.txt new file mode 100644 index 00000000000..c01fed9125a --- /dev/null +++ b/Examples/Tests/langmuir/CMakeLists.txt @@ -0,0 +1,411 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_langmuir_multi # name + 1 # dims + 2 # nprocs + inputs_test_1d_langmuir_multi # inputs + "analysis_1d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_multi # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_multi_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_mr # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_multi_mr_anisotropic # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_mr_anisotropic # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_multi_mr_momentum_conserving # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_mr_momentum_conserving # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_mr_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_mr_psatd # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +add_warpx_test( + test_2d_langmuir_multi_nodal # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_nodal # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_multi_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_psatd # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_current_correction # name + 2 # dims + 1 # nprocs + inputs_test_2d_langmuir_multi_psatd_current_correction # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_current_correction_nodal # name + 2 # dims + 1 # nprocs + inputs_test_2d_langmuir_multi_psatd_current_correction_nodal # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_momentum_conserving # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_psatd_momentum_conserving # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_multiJ # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_psatd_multiJ # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_multiJ_nodal # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_psatd_multiJ_nodal # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_nodal # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi_psatd_nodal # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_vay_deposition # name + 2 # dims + 1 # nprocs + inputs_test_2d_langmuir_multi_psatd_vay_deposition # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_vay_deposition_nodal # name + 2 # dims + 1 # nprocs + inputs_test_2d_langmuir_multi_psatd_vay_deposition_nodal # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 # name + 2 # dims + 1 # nprocs + inputs_test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +add_warpx_test( + test_3d_langmuir_multi # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_langmuir_multi_nodal # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_nodal # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_langmuir_multi_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_langmuir_multi_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_current_correction # name + 3 # dims + 1 # nprocs + inputs_test_3d_langmuir_multi_psatd_current_correction # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_current_correction_nodal # name + 3 # dims + 1 # nprocs + inputs_test_3d_langmuir_multi_psatd_current_correction_nodal # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_div_cleaning # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd_div_cleaning # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_momentum_conserving # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd_momentum_conserving # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_multiJ # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd_multiJ # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_multiJ_nodal # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd_multiJ_nodal # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_nodal # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_multi_psatd_nodal # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_vay_deposition # name + 3 # dims + 1 # nprocs + inputs_test_3d_langmuir_multi_psatd_vay_deposition # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_langmuir_multi_psatd_vay_deposition_nodal # name + 3 # dims + 1 # nprocs + inputs_test_3d_langmuir_multi_psatd_vay_deposition_nodal # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency + ) + label_warpx_test(test_3d_langmuir_multi_psatd_vay_deposition_nodal slow) +endif() + +add_warpx_test( + test_rz_langmuir_multi # name + RZ # dims + 2 # nprocs + inputs_test_rz_langmuir_multi # inputs + "analysis_rz.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_langmuir_multi_picmi # name + RZ # dims + 2 # nprocs + inputs_test_rz_langmuir_multi_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_rz_langmuir_multi_psatd # name + RZ # dims + 2 # nprocs + inputs_test_rz_langmuir_multi_psatd # inputs + "analysis_rz.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_langmuir_multi_psatd_current_correction # name + RZ # dims + 1 # nprocs + inputs_test_rz_langmuir_multi_psatd_current_correction # inputs + "analysis_rz.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_langmuir_multi_psatd_multiJ # name + RZ # dims + 2 # nprocs + inputs_test_rz_langmuir_multi_psatd_multiJ # inputs + "analysis_rz.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/langmuir/PICMI_inputs_2d.py b/Examples/Tests/langmuir/PICMI_inputs_2d.py deleted file mode 100755 index 4b9c3ac300f..00000000000 --- a/Examples/Tests/langmuir/PICMI_inputs_2d.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Simple example of Langmuir oscillations in a uniform plasma -# --- in two dimensions - -from pywarpx import picmi - -constants = picmi.constants - -########################## -# physics parameters -########################## - -plasma_density = 1.e25 -plasma_xmin = 0. -plasma_x_velocity = 0.1*constants.c - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 40 -diagnostic_intervals = "::10" - -# --- Grid -nx = 64 -ny = 64 - -xmin = -20.e-6 -ymin = -20.e-6 -xmax = +20.e-6 -ymax = +20.e-6 - -number_per_cell_each_dim = [2,2] - -########################## -# physics components -########################## - -uniform_plasma = picmi.UniformDistribution(density = 1.e25, - upper_bound = [0., None, None], - directed_velocity = [0.1*constants.c, 0., 0.]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=uniform_plasma) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['periodic', 'periodic'], - upper_boundary_conditions = ['periodic', 'periodic'], - moving_window_velocity = [0., 0., 0.], - warpx_max_grid_size = 32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.) - -########################## -# diagnostics -########################## - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['Ex', 'Jx'], - write_dir = '.', - warpx_file_prefix = 'Python_Langmuir_2d_plt') - -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = diagnostic_intervals, - species = [electrons], - data_list = ['weighting', 'ux']) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - warpx_current_deposition_algo = 'direct', - warpx_use_filter = 0) - -sim.add_species(electrons, - layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) - -sim.add_diagnostic(field_diag1) -sim.add_diagnostic(part_diag1) - -########################## -# simulation run -########################## - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -sim.write_input_file(file_name = 'inputs2d_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Tests/langmuir/PICMI_inputs_3d.py b/Examples/Tests/langmuir/PICMI_inputs_3d.py deleted file mode 100755 index 180180f5f45..00000000000 --- a/Examples/Tests/langmuir/PICMI_inputs_3d.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Simple example of Langmuir oscillations in a uniform plasma - -from pywarpx import picmi - -constants = picmi.constants - -########################## -# physics parameters -########################## - -plasma_density = 1.e25 -plasma_xmin = 0. -plasma_x_velocity = 0.1*constants.c - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 40 -diagnostic_interval = 10 - -# --- Grid -nx = 64 -ny = 64 -nz = 64 - -xmin = -20.e-6 -ymin = -20.e-6 -zmin = -20.e-6 -xmax = +20.e-6 -ymax = +20.e-6 -zmax = +20.e-6 - -number_per_cell_each_dim = [2,2,2] - -########################## -# physics components -########################## - -uniform_plasma = picmi.UniformDistribution(density = 1.e25, - upper_bound = [0., None, None], - directed_velocity = [0.1*constants.c, 0., 0.]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=uniform_plasma) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian3DGrid(number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['periodic', 'periodic', 'periodic'], - upper_boundary_conditions = ['periodic', 'periodic', 'periodic'], - moving_window_velocity = [0., 0., 0.], - warpx_max_grid_size = 32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.) - -########################## -# diagnostics -########################## - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = diagnostic_interval, - data_list = ['Ex', 'Jx'], - write_dir = '.', - warpx_file_prefix = 'Python_Langmuir_plt') - -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = diagnostic_interval, - species = [electrons], - data_list = ['weighting', 'ux']) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - warpx_current_deposition_algo = 'direct') - -sim.add_species(electrons, - layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) - -sim.add_diagnostic(field_diag1) -sim.add_diagnostic(part_diag1) - -########################## -# simulation run -########################## - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name = 'inputs_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py deleted file mode 100755 index 8da03b00469..00000000000 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env python3 -# -# This is a script that analyses the multimode simulation results. -# This simulates a RZ multimode periodic plasma wave. -# The electric field from the simulation is compared to the analytic value - -import matplotlib - -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import numpy as np - -from pywarpx import fields, picmi - -constants = picmi.constants - -########################## -# physics parameters -########################## - -density = 2.e24 -epsilon0 = 0.001*constants.c -epsilon1 = 0.001*constants.c -epsilon2 = 0.001*constants.c -w0 = 5.e-6 -n_osc_z = 3 - -# Plasma frequency -wp = np.sqrt((density*constants.q_e**2)/(constants.m_e*constants.ep0)) -kp = wp/constants.c - -########################## -# numerics parameters -########################## - -nr = 64 -nz = 200 - -rmin = 0.e0 -zmin = 0.e0 -rmax = +20.e-6 -zmax = +40.e-6 - -# Wave vector of the wave -k0 = 2.*np.pi*n_osc_z/(zmax - zmin) - -diagnostic_intervals = 40 - -########################## -# physics components -########################## - -uniform_plasma = picmi.UniformDistribution(density = density, - upper_bound = [+18e-6, +18e-6, None], - directed_velocity = [0., 0., 0.]) - -momentum_expressions = ["""+ epsilon0/kp*2*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - - epsilon1/kp*2/w0*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - + epsilon1/kp*4*x**2/w0**3*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - - epsilon2/kp*8*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - + epsilon2/kp*8*x*(x**2-y**2)/w0**4*exp(-(x**2+y**2)/w0**2)*sin(k0*z)""", - """+ epsilon0/kp*2*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - + epsilon1/kp*4*x*y/w0**3*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - + epsilon2/kp*8*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) - + epsilon2/kp*8*y*(x**2-y**2)/w0**4*exp(-(x**2+y**2)/w0**2)*sin(k0*z)""", - """- epsilon0/kp*k0*exp(-(x**2+y**2)/w0**2)*cos(k0*z) - - epsilon1/kp*k0*2*x/w0*exp(-(x**2+y**2)/w0**2)*cos(k0*z) - - epsilon2/kp*k0*4*(x**2-y**2)/w0**2*exp(-(x**2+y**2)/w0**2)*cos(k0*z)"""] - -analytic_plasma = picmi.AnalyticDistribution(density_expression = density, - upper_bound = [+18e-6, +18e-6, None], - epsilon0 = epsilon0, - epsilon1 = epsilon1, - epsilon2 = epsilon2, - kp = kp, - k0 = k0, - w0 = w0, - momentum_expressions = momentum_expressions) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=analytic_plasma) -protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=uniform_plasma) - -########################## -# numerics components -########################## - -grid = picmi.CylindricalGrid(number_of_cells = [nr, nz], - n_azimuthal_modes = 3, - lower_bound = [rmin, zmin], - upper_bound = [rmax, zmax], - lower_boundary_conditions = ['none', 'periodic'], - upper_boundary_conditions = ['none', 'periodic'], - lower_boundary_conditions_particles = ['none', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - moving_window_velocity = [0.,0.], - warpx_max_grid_size=64) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.) - -########################## -# diagnostics -########################## - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['Er', 'Ez', 'Bt', 'Jr', 'Jz', 'part_per_cell'], - write_dir = '.', - warpx_file_prefix = 'Python_Langmuir_rz_multimode_plt') - -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = diagnostic_intervals, - species = [electrons], - data_list = ['weighting', 'momentum']) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation(solver = solver, - max_steps = 40, - verbose = 1, - warpx_current_deposition_algo = 'esirkepov', - warpx_field_gathering_algo = 'energy-conserving', - warpx_particle_pusher_algo = 'boris', - warpx_use_filter = 0) - -sim.add_species(electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,16,2], grid=grid)) -sim.add_species(protons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,16,2], grid=grid)) - -sim.add_diagnostic(field_diag1) -sim.add_diagnostic(part_diag1) - -########################## -# simulation run -########################## - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -#sim.write_input_file(file_name='inputsrz_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -sim.step() - - -# Below is WarpX specific code to check the results. - -def calcEr( z, r, k0, w0, wp, t, epsilons) : - """ - Return the radial electric field as an array - of the same length as z and r, in the half-plane theta=0 - """ - Er_array = ( - epsilons[0] * constants.m_e*constants.c/constants.q_e * 2*r/w0**2 * - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - - epsilons[1] * constants.m_e*constants.c/constants.q_e * 2/w0 * - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - + epsilons[1] * constants.m_e*constants.c/constants.q_e * 4*r**2/w0**3 * - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - - epsilons[2] * constants.m_e*constants.c/constants.q_e * 8*r/w0**2 * - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - + epsilons[2] * constants.m_e*constants.c/constants.q_e * 8*r**3/w0**4 * - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t )) - return( Er_array ) - -def calcEz( z, r, k0, w0, wp, t, epsilons) : - """ - Return the longitudinal electric field as an array - of the same length as z and r, in the half-plane theta=0 - """ - Ez_array = ( - - epsilons[0] * constants.m_e*constants.c/constants.q_e * k0 * - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - - epsilons[1] * constants.m_e*constants.c/constants.q_e * k0 * 2*r/w0 * - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - - epsilons[2] * constants.m_e*constants.c/constants.q_e * k0 * 4*r**2/w0**2 * - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t )) - return( Ez_array ) - -# Current time of the simulation -t0 = sim.extension.warpx.gett_new(0) - -# Get the raw field data. Note that these are the real and imaginary -# parts of the fields for each azimuthal mode. -Ex_sim_wrap = fields.ExWrapper() -Ez_sim_wrap = fields.EzWrapper() -Ex_sim_modes = Ex_sim_wrap[...] -Ez_sim_modes = Ez_sim_wrap[...] - -rr_Er = Ex_sim_wrap.mesh('r') -zz_Er = Ex_sim_wrap.mesh('z') -rr_Ez = Ez_sim_wrap.mesh('r') -zz_Ez = Ez_sim_wrap.mesh('z') - -rr_Er = rr_Er[:,np.newaxis]*np.ones(zz_Er.shape[0])[np.newaxis,:] -zz_Er = zz_Er[np.newaxis,:]*np.ones(rr_Er.shape[0])[:,np.newaxis] -rr_Ez = rr_Ez[:,np.newaxis]*np.ones(zz_Ez.shape[0])[np.newaxis,:] -zz_Ez = zz_Ez[np.newaxis,:]*np.ones(rr_Ez.shape[0])[:,np.newaxis] - -# Sum the real components to get the field along x-axis (theta = 0) -Er_sim = Ex_sim_modes[:,:,0] + np.sum(Ex_sim_modes[:,:,1::2], axis=2) -Ez_sim = Ez_sim_modes[:,:,0] + np.sum(Ez_sim_modes[:,:,1::2], axis=2) - -# The analytical solutions -Er_th = calcEr(zz_Er, rr_Er, k0, w0, wp, t0, [epsilon0, epsilon1, epsilon2]) -Ez_th = calcEz(zz_Ez, rr_Ez, k0, w0, wp, t0, [epsilon0, epsilon1, epsilon2]) - -max_error_Er = abs(Er_sim - Er_th).max()/abs(Er_th).max() -max_error_Ez = abs(Ez_sim - Ez_th).max()/abs(Ez_th).max() -print("Max error Er %e"%max_error_Er) -print("Max error Ez %e"%max_error_Ez) - -# Plot the last field from the loop (Er at iteration 40) -fig, ax = plt.subplots(3) -im = ax[0].imshow( Er_sim, aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[0], orientation='vertical') -ax[0].set_title('Er, last iteration (simulation)') -ax[1].imshow( Er_th, aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[1], orientation='vertical') -ax[1].set_title('Er, last iteration (theory)') -im = ax[2].imshow( (Er_sim - Er_th)/abs(Er_th).max(), aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[2], orientation='vertical') -ax[2].set_title('Er, last iteration (difference)') -plt.savefig('langmuir_multi_rz_multimode_analysis_Er.png') - -fig, ax = plt.subplots(3) -im = ax[0].imshow( Ez_sim, aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[0], orientation='vertical') -ax[0].set_title('Ez, last iteration (simulation)') -ax[1].imshow( Ez_th, aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[1], orientation='vertical') -ax[1].set_title('Ez, last iteration (theory)') -im = ax[2].imshow( (Ez_sim - Ez_th)/abs(Ez_th).max(), aspect='auto', origin='lower' ) -fig.colorbar(im, ax=ax[2], orientation='vertical') -ax[2].set_title('Ez, last iteration (difference)') -plt.savefig('langmuir_multi_rz_multimode_analysis_Ez.png') - -assert max(max_error_Er, max_error_Ez) < 0.02 diff --git a/Examples/Tests/langmuir/README.rst b/Examples/Tests/langmuir/README.rst index 60c0018744c..4748f428dc1 100644 --- a/Examples/Tests/langmuir/README.rst +++ b/Examples/Tests/langmuir/README.rst @@ -23,19 +23,19 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. tab-item:: Python: Script - This example can be run as a **Python** script: ``python3 PICMI_inputs_3d.py``. + This example can be run as a **Python** script: ``python3 inputs_test_3d_langmuir_multi_picmi.py``. - .. literalinclude:: PICMI_inputs_3d.py + .. literalinclude:: inputs_test_3d_langmuir_multi_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_3d.py``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_picmi.py``. .. tab-item:: Executable: Input File - This example can be run as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + This example can be run as WarpX **executable** using an input file: ``warpx.3d inputs_test_3d_langmuir_multi`` - .. literalinclude:: inputs_3d + .. literalinclude:: inputs_test_3d_langmuir_multi :language: ini - :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_3d``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_3d_langmuir_multi``. .. tab-item:: 2D @@ -43,19 +43,19 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. tab-item:: Python: Script - This example can be run as a **Python** script: ``python3 PICMI_inputs_2d.py``. + This example can be run as a **Python** script: ``python3 inputs_test_2d_langmuir_multi_picmi.py``. - .. literalinclude:: PICMI_inputs_2d.py + .. literalinclude:: inputs_test_2d_langmuir_multi_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_2d.py``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_picmi.py``. .. tab-item:: Executable: Input File - This example can be run as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + This example can be run as WarpX **executable** using an input file: ``warpx.2d inputs_test_2d_langmuir_multi`` - .. literalinclude:: inputs_2d + .. literalinclude:: inputs_test_2d_langmuir_multi :language: ini - :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_2d``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_2d_langmuir_multi``. .. tab-item:: RZ @@ -64,19 +64,19 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. tab-item:: Python: Script - This example can be run as a **Python** script: ``python3 PICMI_inputs_rz.py``. + This example can be run as a **Python** script: ``python3 inputs_test_rz_langmuir_multi_picmi.py``. - .. literalinclude:: PICMI_inputs_rz.py + .. literalinclude:: inputs_test_rz_langmuir_multi_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_rz.py``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_picmi.py``. .. tab-item:: Executable: Input File - This example can be run as WarpX **executable** using an input file: ``warpx.rz inputs_rz`` + This example can be run as WarpX **executable** using an input file: ``warpx.rz inputs_test_rz_langmuir_multi`` - .. literalinclude:: inputs_rz + .. literalinclude:: inputs_test_rz_langmuir_multi :language: ini - :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_rz``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_rz_langmuir_multi``. .. tab-item:: 1D @@ -87,15 +87,15 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. note:: - TODO: This input file should be created, like the ``inputs_1d`` file. + TODO: This input file should be created, like the ``inputs_test_1d_langmuir_multi`` file. .. tab-item:: Executable: Input File - This example can be run as WarpX **executable** using an input file: ``warpx.1d inputs_1d`` + This example can be run as WarpX **executable** using an input file: ``warpx.1d inputs_test_1d_langmuir_multi`` - .. literalinclude:: inputs_1d + .. literalinclude:: inputs_test_1d_langmuir_multi :language: ini - :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_1d``. + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_test_1d_langmuir_multi``. Analyze diff --git a/Examples/Tests/langmuir/analysis_1d.py b/Examples/Tests/langmuir/analysis_1d.py index 7b05fac6643..60a088d1309 100755 --- a/Examples/Tests/langmuir/analysis_1d.py +++ b/Examples/Tests/langmuir/analysis_1d.py @@ -17,7 +17,7 @@ import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -26,94 +26,99 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +# test name +test_name = os.path.split(os.getcwd())[1] # this will be the name of the plot file fn = sys.argv[1] # Parse test name and check if current correction (psatd.current_correction=1) is applied -current_correction = True if re.search( 'current_correction', fn ) else False +current_correction = True if re.search("current_correction", test_name) else False # Parse test name and check if Vay current deposition (algo.current_deposition=vay) is used -vay_deposition = True if re.search( 'Vay_deposition', fn ) else False +vay_deposition = True if re.search("vay_deposition", test_name) else False # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_z = 2 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -kz = 2.*np.pi*n_osc_z/(zmax-zmin) +kz = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ez": kz} +cos = {"Ez": (1, 1, 0)} -k = {'Ez':kz} -cos = {'Ez':(1,1,0)} -def get_contribution( is_cos, k ): - du = (zmax-zmin)/Nz - u = zmin + du*( 0.5 + np.arange(Nz) ) +def get_contribution(is_cos, k): + du = (zmax - zmin) / Nz + u = zmin + du * (0.5 + np.arange(Nz)) if is_cos == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) + -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - z_contribution = get_contribution( cos_flag[2], kz ) + z_contribution = get_contribution(cos_flag[2], kz) E = amplitude * z_contribution - return( E ) + return E + # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Check the validity of the fields error_rel = 0 -for field in ['Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,0,0] +for field in ["Ez"]: + E_sim = data[("mesh", field)].to_ndarray()[:, 0, 0] E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 80) -plt.subplot2grid( (1,2), (0,0) ) -plt.plot( E_sim ) -#plt.colorbar() -plt.title('Ez, last iteration\n(simulation)') -plt.subplot2grid( (1,2), (0,1) ) -plt.plot( E_th ) -#plt.colorbar() -plt.title('Ez, last iteration\n(theory)') +plt.subplot2grid((1, 2), (0, 0)) +plt.plot(E_sim) +# plt.colorbar() +plt.title("Ez, last iteration\n(simulation)") +plt.subplot2grid((1, 2), (0, 1)) +plt.plot(E_th) +# plt.colorbar() +plt.title("Ez, last iteration\n(theory)") plt.tight_layout() -plt.savefig('langmuir_multi_1d_analysis.png') +plt.savefig("langmuir_multi_1d_analysis.png") tolerance_rel = 0.05 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) +assert error_rel < tolerance_rel # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) when # current correction (psatd.do_current_correction=1) is applied or when # Vay current deposition (algo.current_deposition=vay) is used if current_correction or vay_deposition: - rho = data[('boxlib','rho')].to_ndarray() - divE = data[('boxlib','divE')].to_ndarray() - error_rel = np.amax( np.abs( divE - rho/epsilon_0 ) ) / np.amax( np.abs( rho/epsilon_0 ) ) - tolerance = 1.e-9 + rho = data[("boxlib", "rho")].to_ndarray() + divE = data[("boxlib", "divE")].to_ndarray() + error_rel = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax( + np.abs(rho / epsilon_0) + ) + tolerance = 1.0e-9 print("Check charge conservation:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) - assert( error_rel < tolerance ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) + assert error_rel < tolerance diff --git a/Examples/Tests/langmuir/analysis_2d.py b/Examples/Tests/langmuir/analysis_2d.py index 94f97ca6de8..3aa246008a6 100755 --- a/Examples/Tests/langmuir/analysis_2d.py +++ b/Examples/Tests/langmuir/analysis_2d.py @@ -26,99 +26,114 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +# test name +test_name = os.path.split(os.getcwd())[1] # this will be the name of the plot file fn = sys.argv[1] # Parse test name and check if current correction (psatd.current_correction=1) is applied -current_correction = True if re.search( 'current_correction', fn ) else False +current_correction = True if re.search("current_correction", test_name) else False # Parse test name and check if Vay current deposition (algo.current_deposition=vay) is used -vay_deposition = True if re.search( 'Vay_deposition', fn ) else False +vay_deposition = True if re.search("vay_deposition", test_name) else False # Parse test name and check if particle_shape = 4 is used -particle_shape_4 = True if re.search('particle_shape_4', fn) else False +particle_shape_4 = True if re.search("particle_shape_4", test_name) else False -# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +# Parameters (must match the parameters in the inputs) +# FIXME read these parameters from warpx_used_inputs epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_x = 2 n_osc_z = 2 -xmin = -20e-6; xmax = 20.e-6; Nx = 128 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +xmin = -20e-6 +xmax = 20.0e-6 +Nx = 128 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -kx = 2.*np.pi*n_osc_x/(xmax-xmin) -kz = 2.*np.pi*n_osc_z/(zmax-zmin) +kx = 2.0 * np.pi * n_osc_x / (xmax - xmin) +kz = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ex": kx, "Ez": kz} +cos = {"Ex": (0, 1, 1), "Ez": (1, 1, 0)} -k = {'Ex':kx, 'Ez':kz} -cos = {'Ex': (0,1,1), 'Ez':(1,1,0)} -def get_contribution( is_cos, k ): - du = (xmax-xmin)/Nx - u = xmin + du*( 0.5 + np.arange(Nx) ) +def get_contribution(is_cos, k): + du = (xmax - xmin) / Nx + u = xmin + du * (0.5 + np.arange(Nx)) if is_cos == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) + -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag[0], kx ) - z_contribution = get_contribution( cos_flag[2], kz ) + x_contribution = get_contribution(cos_flag[0], kx) + z_contribution = get_contribution(cos_flag[2], kz) - E = amplitude * x_contribution[:, np.newaxis ] \ - * z_contribution[np.newaxis, :] + E = amplitude * x_contribution[:, np.newaxis] * z_contribution[np.newaxis, :] + + return E - return( E ) # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) -edge = np.array([(ds.domain_left_edge[1]).item(), (ds.domain_right_edge[1]).item(), \ - (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +edge = np.array( + [ + (ds.domain_left_edge[1]).item(), + (ds.domain_right_edge[1]).item(), + (ds.domain_left_edge[0]).item(), + (ds.domain_right_edge[0]).item(), + ] +) # Check the validity of the fields error_rel = 0 -for field in ['Ex', 'Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,:,0] +for field in ["Ex", "Ez"]: + E_sim = data[("mesh", field)].to_ndarray()[:, :, 0] E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 40) -fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi=100) # First plot vmin = E_sim.min() vmax = E_sim.max() -cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') -im1 = ax1.imshow(E_sim, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb1 = fig.colorbar(im1, cax = cax1) -ax1.set_xlabel(r'$z$') -ax1.set_ylabel(r'$x$') -ax1.set_title(r'$E_z$ (sim)') +cax1 = make_axes_locatable(ax1).append_axes("right", size="5%", pad="5%") +im1 = ax1.imshow(E_sim, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb1 = fig.colorbar(im1, cax=cax1) +ax1.set_xlabel(r"$z$") +ax1.set_ylabel(r"$x$") +ax1.set_title(r"$E_z$ (sim)") # Second plot vmin = E_th.min() vmax = E_th.max() -cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') -im2 = ax2.imshow(E_th, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb2 = fig.colorbar(im2, cax = cax2) -ax2.set_xlabel(r'$z$') -ax2.set_ylabel(r'$x$') -ax2.set_title(r'$E_z$ (theory)') +cax2 = make_axes_locatable(ax2).append_axes("right", size="5%", pad="5%") +im2 = ax2.imshow(E_th, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb2 = fig.colorbar(im2, cax=cax2) +ax2.set_xlabel(r"$z$") +ax2.set_ylabel(r"$x$") +ax2.set_title(r"$E_z$ (theory)") # Save figure fig.tight_layout() -fig.savefig('Langmuir_multi_2d_analysis.png', dpi = 200) +fig.savefig("Langmuir_multi_2d_analysis.png", dpi=200) if particle_shape_4: -# lower fidelity, due to smoothing + # lower fidelity, due to smoothing tolerance_rel = 0.07 else: tolerance_rel = 0.05 @@ -126,7 +141,7 @@ def get_theoretical_field( field, t ): print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) +assert error_rel < tolerance_rel # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) # with current correction (and periodic single box option) or with Vay current deposition @@ -135,13 +150,12 @@ def get_theoretical_field( field, t ): elif vay_deposition: tolerance = 1e-3 if current_correction or vay_deposition: - rho = data[('boxlib','rho')].to_ndarray() - divE = data[('boxlib','divE')].to_ndarray() - error_rel = np.amax( np.abs( divE - rho/epsilon_0 ) ) / np.amax( np.abs( rho/epsilon_0 ) ) + rho = data[("boxlib", "rho")].to_ndarray() + divE = data[("boxlib", "divE")].to_ndarray() + error_rel = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax( + np.abs(rho / epsilon_0) + ) print("Check charge conservation:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) - assert( error_rel < tolerance ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) + assert error_rel < tolerance diff --git a/Examples/Tests/langmuir/analysis_3d.py b/Examples/Tests/langmuir/analysis_3d.py index 68334f506ff..75b9c5ba71c 100755 --- a/Examples/Tests/langmuir/analysis_3d.py +++ b/Examples/Tests/langmuir/analysis_3d.py @@ -26,128 +26,139 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +# test name +test_name = os.path.split(os.getcwd())[1] # this will be the name of the plot file fn = sys.argv[1] # Parse test name and check if current correction (psatd.current_correction=1) is applied -current_correction = True if re.search( 'current_correction', fn ) else False +current_correction = True if re.search("current_correction", test_name) else False # Parse test name and check if Vay current deposition (algo.current_deposition=vay) is used -vay_deposition = True if re.search( 'Vay_deposition', fn ) else False +vay_deposition = True if re.search("vay_deposition", test_name) else False # Parse test name and check if div(E)/div(B) cleaning (warpx.do_div_cleaning=1) is used -div_cleaning = True if re.search('div_cleaning', fn) else False +div_cleaning = True if re.search("div_cleaning", test_name) else False # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_x = 2 n_osc_y = 2 n_osc_z = 2 -lo = [-20.e-6, -20.e-6, -20.e-6] -hi = [ 20.e-6, 20.e-6, 20.e-6] +lo = [-20.0e-6, -20.0e-6, -20.0e-6] +hi = [20.0e-6, 20.0e-6, 20.0e-6] Ncell = [64, 64, 64] # Wave vector of the wave -kx = 2.*np.pi*n_osc_x/(hi[0]-lo[0]) -ky = 2.*np.pi*n_osc_y/(hi[1]-lo[1]) -kz = 2.*np.pi*n_osc_z/(hi[2]-lo[2]) +kx = 2.0 * np.pi * n_osc_x / (hi[0] - lo[0]) +ky = 2.0 * np.pi * n_osc_y / (hi[1] - lo[1]) +kz = 2.0 * np.pi * n_osc_z / (hi[2] - lo[2]) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ex": kx, "Ey": ky, "Ez": kz} +cos = {"Ex": (0, 1, 1), "Ey": (1, 0, 1), "Ez": (1, 1, 0)} -k = {'Ex':kx, 'Ey':ky, 'Ez':kz} -cos = {'Ex': (0,1,1), 'Ey':(1,0,1), 'Ez':(1,1,0)} -def get_contribution( is_cos, k, idim ): - du = (hi[idim]-lo[idim])/Ncell[idim] - u = lo[idim] + du*( 0.5 + np.arange(Ncell[idim]) ) +def get_contribution(is_cos, k, idim): + du = (hi[idim] - lo[idim]) / Ncell[idim] + u = lo[idim] + du * (0.5 + np.arange(Ncell[idim])) if is_cos[idim] == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) + -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag, kx, 0 ) - y_contribution = get_contribution( cos_flag, ky, 1 ) - z_contribution = get_contribution( cos_flag, kz, 2 ) + x_contribution = get_contribution(cos_flag, kx, 0) + y_contribution = get_contribution(cos_flag, ky, 1) + z_contribution = get_contribution(cos_flag, kz, 2) - E = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ - * y_contribution[np.newaxis, :, np.newaxis] \ - * z_contribution[np.newaxis, np.newaxis, :] + E = ( + amplitude + * x_contribution[:, np.newaxis, np.newaxis] + * y_contribution[np.newaxis, :, np.newaxis] + * z_contribution[np.newaxis, np.newaxis, :] + ) + + return E - return( E ) # Read the file ds = yt.load(fn) # Check that the particle selective output worked: -species = 'electrons' -print('ds.field_list', ds.field_list) -for field in ['particle_weight', - 'particle_momentum_x']: - print('assert that this is in ds.field_list', (species, field)) +species = "electrons" +print("ds.field_list", ds.field_list) +for field in ["particle_weight", "particle_momentum_x"]: + print("assert that this is in ds.field_list", (species, field)) assert (species, field) in ds.field_list -for field in ['particle_momentum_y', - 'particle_momentum_z']: - print('assert that this is NOT in ds.field_list', (species, field)) +for field in ["particle_momentum_y", "particle_momentum_z"]: + print("assert that this is NOT in ds.field_list", (species, field)) assert (species, field) not in ds.field_list -species = 'positrons' -for field in ['particle_momentum_x', - 'particle_momentum_y']: - print('assert that this is NOT in ds.field_list', (species, field)) +species = "positrons" +for field in ["particle_momentum_x", "particle_momentum_y"]: + print("assert that this is NOT in ds.field_list", (species, field)) assert (species, field) not in ds.field_list t0 = ds.current_time.to_value() -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) -edge = np.array([(ds.domain_left_edge[2]).item(), (ds.domain_right_edge[2]).item(), \ - (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +edge = np.array( + [ + (ds.domain_left_edge[2]).item(), + (ds.domain_right_edge[2]).item(), + (ds.domain_left_edge[0]).item(), + (ds.domain_right_edge[0]).item(), + ] +) # Check the validity of the fields error_rel = 0 -for field in ['Ex', 'Ey', 'Ez']: - E_sim = data[('mesh',field)].to_ndarray() +for field in ["Ex", "Ey", "Ez"]: + E_sim = data[("mesh", field)].to_ndarray() E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 40) -fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi=100) # First plot (slice at y=0) -E_plot = E_sim[:,Ncell[1]//2+1,:] +E_plot = E_sim[:, Ncell[1] // 2 + 1, :] vmin = E_plot.min() vmax = E_plot.max() -cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') -im1 = ax1.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb1 = fig.colorbar(im1, cax = cax1) -ax1.set_xlabel(r'$z$') -ax1.set_ylabel(r'$x$') -ax1.set_title(r'$E_z$ (sim)') +cax1 = make_axes_locatable(ax1).append_axes("right", size="5%", pad="5%") +im1 = ax1.imshow(E_plot, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb1 = fig.colorbar(im1, cax=cax1) +ax1.set_xlabel(r"$z$") +ax1.set_ylabel(r"$x$") +ax1.set_title(r"$E_z$ (sim)") # Second plot (slice at y=0) -E_plot = E_th[:,Ncell[1]//2+1,:] +E_plot = E_th[:, Ncell[1] // 2 + 1, :] vmin = E_plot.min() vmax = E_plot.max() -cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') -im2 = ax2.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb2 = fig.colorbar(im2, cax = cax2) -ax2.set_xlabel(r'$z$') -ax2.set_ylabel(r'$x$') -ax2.set_title(r'$E_z$ (theory)') +cax2 = make_axes_locatable(ax2).append_axes("right", size="5%", pad="5%") +im2 = ax2.imshow(E_plot, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb2 = fig.colorbar(im2, cax=cax2) +ax2.set_xlabel(r"$z$") +ax2.set_ylabel(r"$x$") +ax2.set_title(r"$E_z$ (theory)") # Save figure fig.tight_layout() -fig.savefig('Langmuir_multi_analysis.png', dpi = 200) +fig.savefig("Langmuir_multi_analysis.png", dpi=200) tolerance_rel = 5e-2 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) +assert error_rel < tolerance_rel # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) # with current correction (and periodic single box option) or with Vay current deposition @@ -156,43 +167,44 @@ def get_theoretical_field( field, t ): elif vay_deposition: tolerance = 1e-3 if current_correction or vay_deposition: - rho = data[('boxlib','rho')].to_ndarray() - divE = data[('boxlib','divE')].to_ndarray() - error_rel = np.amax( np.abs( divE - rho/epsilon_0 ) ) / np.amax( np.abs( rho/epsilon_0 ) ) + rho = data[("boxlib", "rho")].to_ndarray() + divE = data[("boxlib", "divE")].to_ndarray() + error_rel = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax( + np.abs(rho / epsilon_0) + ) print("Check charge conservation:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) - assert( error_rel < tolerance ) + assert error_rel < tolerance if div_cleaning: - ds_old = yt.load('Langmuir_multi_psatd_div_cleaning_plt000038') - ds_mid = yt.load('Langmuir_multi_psatd_div_cleaning_plt000039') - ds_new = yt.load(fn) # this is the last plotfile - - ad_old = ds_old.covering_grid(level = 0, left_edge = ds_old.domain_left_edge, dims = ds_old.domain_dimensions) - ad_mid = ds_mid.covering_grid(level = 0, left_edge = ds_mid.domain_left_edge, dims = ds_mid.domain_dimensions) - ad_new = ds_new.covering_grid(level = 0, left_edge = ds_new.domain_left_edge, dims = ds_new.domain_dimensions) - - rho = ad_mid['rho'].v.squeeze() - divE = ad_mid['divE'].v.squeeze() - F_old = ad_old['F'].v.squeeze() - F_new = ad_new['F'].v.squeeze() + ds_old = yt.load("diags/diag1000038") + ds_mid = yt.load("diags/diag1000039") + ds_new = yt.load(fn) # this is the last plotfile + + ad_old = ds_old.covering_grid( + level=0, left_edge=ds_old.domain_left_edge, dims=ds_old.domain_dimensions + ) + ad_mid = ds_mid.covering_grid( + level=0, left_edge=ds_mid.domain_left_edge, dims=ds_mid.domain_dimensions + ) + ad_new = ds_new.covering_grid( + level=0, left_edge=ds_new.domain_left_edge, dims=ds_new.domain_dimensions + ) + + rho = ad_mid["rho"].v.squeeze() + divE = ad_mid["divE"].v.squeeze() + F_old = ad_old["F"].v.squeeze() + F_new = ad_new["F"].v.squeeze() # Check max norm of error on dF/dt = div(E) - rho/epsilon_0 # (the time interval between the old and new data is 2*dt) dt = 1.203645751e-15 x = F_new - F_old - y = (divE - rho/epsilon_0) * 2 * dt + y = (divE - rho / epsilon_0) * 2 * dt error_rel = np.amax(np.abs(x - y)) / np.amax(np.abs(y)) tolerance = 1e-2 print("Check div(E) cleaning:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) - assert(error_rel < tolerance) - -test_name = os.path.split(os.getcwd())[1] - -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) + assert error_rel < tolerance diff --git a/Examples/Tests/langmuir/analysis_default_regression.py b/Examples/Tests/langmuir/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/langmuir/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/langmuir/analysis_rz.py b/Examples/Tests/langmuir/analysis_rz.py index e5ae2194123..a0697b93ab9 100755 --- a/Examples/Tests/langmuir/analysis_rz.py +++ b/Examples/Tests/langmuir/analysis_rz.py @@ -19,7 +19,7 @@ import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -29,89 +29,111 @@ import post_processing_utils from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] +# test name test_name = os.path.split(os.getcwd())[1] # Parse test name and check if current correction (psatd.current_correction) is applied -current_correction = True if re.search('current_correction', fn) else False +current_correction = True if re.search("current_correction", test_name) else False # Parameters (these parameters must match the parameters in `inputs.multi.rz.rt`) epsilon = 0.01 -n = 2.e24 -w0 = 5.e-6 +n = 2.0e24 +w0 = 5.0e-6 n_osc_z = 2 -rmin = 0e-6; rmax = 20.e-6; Nr = 64 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +rmin = 0e-6 +rmax = 20.0e-6 +Nr = 64 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -k0 = 2.*np.pi*n_osc_z/(zmax-zmin) +k0 = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) -kp = wp/c +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) +kp = wp / c + -def Er( z, r, epsilon, k0, w0, wp, t) : +def Er(z, r, epsilon, k0, w0, wp, t): """ Return the radial electric field as an array of the same length as z and r, in the half-plane theta=0 """ - Er_array = \ - epsilon * m_e*c**2/e * 2*r/w0**2 * \ - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - return( Er_array ) - -def Ez( z, r, epsilon, k0, w0, wp, t) : + Er_array = ( + epsilon + * m_e + * c**2 + / e + * 2 + * r + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + ) + return Er_array + + +def Ez(z, r, epsilon, k0, w0, wp, t): """ Return the longitudinal electric field as an array of the same length as z and r, in the half-plane theta=0 """ - Ez_array = \ - - epsilon * m_e*c**2/e * k0 * \ - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - return( Ez_array ) + Ez_array = ( + -epsilon + * m_e + * c**2 + / e + * k0 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.sin(wp * t) + ) + return Ez_array + # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Get cell centered coordinates -dr = (rmax - rmin)/Nr -dz = (zmax - zmin)/Nz -coords = np.indices([Nr, Nz],'d') -rr = rmin + (coords[0] + 0.5)*dr -zz = zmin + (coords[1] + 0.5)*dz +dr = (rmax - rmin) / Nr +dz = (zmax - zmin) / Nz +coords = np.indices([Nr, Nz], "d") +rr = rmin + (coords[0] + 0.5) * dr +zz = zmin + (coords[1] + 0.5) * dz # Check the validity of the fields overall_max_error = 0 -Er_sim = data[('boxlib','Er')].to_ndarray()[:,:,0] +Er_sim = data[("boxlib", "Er")].to_ndarray()[:, :, 0] Er_th = Er(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Er_sim-Er_th).max()/abs(Er_th).max() -print('Er: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Er_sim - Er_th).max() / abs(Er_th).max() +print("Er: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) -Ez_sim = data[('boxlib','Ez')].to_ndarray()[:,:,0] +Ez_sim = data[("boxlib", "Ez")].to_ndarray()[:, :, 0] Ez_th = Ez(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Ez_sim-Ez_th).max()/abs(Ez_th).max() -print('Ez: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Ez_sim - Ez_th).max() / abs(Ez_th).max() +print("Ez: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) # Plot the last field from the loop (Ez at iteration 40) -plt.subplot2grid( (1,2), (0,0) ) -plt.imshow( Ez_sim ) +plt.subplot2grid((1, 2), (0, 0)) +plt.imshow(Ez_sim) plt.colorbar() -plt.title('Ez, last iteration\n(simulation)') -plt.subplot2grid( (1,2), (0,1) ) -plt.imshow( Ez_th ) +plt.title("Ez, last iteration\n(simulation)") +plt.subplot2grid((1, 2), (0, 1)) +plt.imshow(Ez_th) plt.colorbar() -plt.title('Ez, last iteration\n(theory)') +plt.title("Ez, last iteration\n(theory)") plt.tight_layout() -plt.savefig(test_name+'_analysis.png') +plt.savefig(test_name + "_analysis.png") error_rel = overall_max_error @@ -120,18 +142,18 @@ def Ez( z, r, epsilon, k0, w0, wp, t) : print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) +assert error_rel < tolerance_rel # Check charge conservation (relative L-infinity norm of error) with current correction if current_correction: - divE = data[('boxlib','divE')].to_ndarray() - rho = data[('boxlib','rho')].to_ndarray() / epsilon_0 + divE = data[("boxlib", "divE")].to_ndarray() + rho = data[("boxlib", "rho")].to_ndarray() / epsilon_0 error_rel = np.amax(np.abs(divE - rho)) / max(np.amax(divE), np.amax(rho)) - tolerance = 1.e-9 + tolerance = 1.0e-9 print("Check charge conservation:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) - assert( error_rel < tolerance ) + assert error_rel < tolerance ## In the final past of the test, we verify that the diagnostic particle filter function works as @@ -142,17 +164,18 @@ def Ez( z, r, epsilon, k0, w0, wp, t) : parser_filter_fn = "diags/diag_parser_filter000080" parser_filter_expression = "(py-pz < 0) * (r<10e-6) * (z > 0)" -post_processing_utils.check_particle_filter(fn, parser_filter_fn, parser_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + fn, parser_filter_fn, parser_filter_expression, dim, species_name +) uniform_filter_fn = "diags/diag_uniform_filter000080" uniform_filter_expression = "ids%3 == 0" -post_processing_utils.check_particle_filter(fn, uniform_filter_fn, uniform_filter_expression, - dim, species_name) +post_processing_utils.check_particle_filter( + fn, uniform_filter_fn, uniform_filter_expression, dim, species_name +) random_filter_fn = "diags/diag_random_filter000080" random_fraction = 0.66 -post_processing_utils.check_random_filter(fn, random_filter_fn, random_fraction, - dim, species_name) - -checksumAPI.evaluate_checksum(test_name, fn) +post_processing_utils.check_random_filter( + fn, random_filter_fn, random_fraction, dim, species_name +) diff --git a/Examples/Tests/langmuir/inputs_1d b/Examples/Tests/langmuir/inputs_1d deleted file mode 100644 index af1cf367553..00000000000 --- a/Examples/Tests/langmuir/inputs_1d +++ /dev/null @@ -1,85 +0,0 @@ -# Maximum number of time steps -max_step = 80 - -# number of grid points -amr.n_cell = 128 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 1 -geometry.prob_lo = -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 - -# Boundary condition -boundary.field_lo = periodic -boundary.field_hi = periodic - -warpx.serialize_initial_conditions = 1 - -# Verbosity -warpx.verbose = 1 - -# Algorithms -algo.field_gathering = energy-conserving -warpx.use_filter = 0 - -# Order of particle shape factors -algo.particle_shape = 1 - -# CFL -warpx.cfl = 0.8 - -# Parameters for the plasma wave -my_constants.epsilon = 0.01 -my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 -my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency -my_constants.kp = wp/clight # plasma wavenumber -my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber -# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) -# k is calculated so as to have 2 periods within the 40e-6 wide box. - -# Particles -particles.species_names = electrons positrons - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 2 -electrons.zmin = -20.e-6 -electrons.zmax = 20.e-6 - -electrons.profile = constant -electrons.density = n0 # number of electrons per m^3 -electrons.momentum_distribution_type = parse_momentum_function -electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" -electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" -electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" - -positrons.charge = q_e -positrons.mass = m_e -positrons.injection_style = "NUniformPerCell" -positrons.num_particles_per_cell_each_dim = 2 -positrons.zmin = -20.e-6 -positrons.zmax = 20.e-6 - -positrons.profile = constant -positrons.density = n0 # number of positrons per m^3 -positrons.momentum_distribution_type = parse_momentum_function -positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" -positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" -positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" - -# Diagnostics -diagnostics.diags_names = diag1 openpmd -diag1.intervals = 40 -diag1.diag_type = Full - -openpmd.intervals = 40 -openpmd.diag_type = Full -openpmd.format = openpmd diff --git a/Examples/Tests/langmuir/inputs_2d b/Examples/Tests/langmuir/inputs_base_2d similarity index 100% rename from Examples/Tests/langmuir/inputs_2d rename to Examples/Tests/langmuir/inputs_base_2d diff --git a/Examples/Tests/langmuir/inputs_3d b/Examples/Tests/langmuir/inputs_base_3d similarity index 100% rename from Examples/Tests/langmuir/inputs_3d rename to Examples/Tests/langmuir/inputs_base_3d diff --git a/Examples/Tests/langmuir/inputs_rz b/Examples/Tests/langmuir/inputs_base_rz similarity index 100% rename from Examples/Tests/langmuir/inputs_rz rename to Examples/Tests/langmuir/inputs_base_rz diff --git a/Examples/Tests/langmuir/inputs_test_1d_langmuir_multi b/Examples/Tests/langmuir/inputs_test_1d_langmuir_multi new file mode 100644 index 00000000000..e2fd1da4b94 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_1d_langmuir_multi @@ -0,0 +1,88 @@ +# Maximum number of time steps +max_step = 80 + +# number of grid points +amr.n_cell = 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 1 +geometry.prob_lo = -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 + +# Boundary condition +boundary.field_lo = periodic +boundary.field_hi = periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.field_gathering = energy-conserving +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +# CFL +warpx.cfl = 0.8 + +# Parameters for the plasma wave +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Particles +particles.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 2 +electrons.zmin = -20.e-6 +electrons.zmax = 20.e-6 + +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.injection_style = "NUniformPerCell" +positrons.num_particles_per_cell_each_dim = 2 +positrons.zmin = -20.e-6 +positrons.zmax = 20.e-6 + +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 openpmd +diag1.intervals = 40 +diag1.diag_type = Full +diag1.electrons.variables = z w ux uy uz +diag1.positrons.variables = z w ux uy uz + +openpmd.intervals = 40 +openpmd.diag_type = Full +openpmd.format = openpmd diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi new file mode 100644 index 00000000000..df0189acd91 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = direct +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr new file mode 100644 index 00000000000..8adf73023be --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = ckc +amr.max_level = 1 +amr.ref_ratio = 4 +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.fine_tag_hi = 10.e-6 10.e-6 +warpx.fine_tag_lo = -10.e-6 -10.e-6 +warpx.use_filter = 1 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_anisotropic b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_anisotropic new file mode 100644 index 00000000000..047943373c0 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_anisotropic @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = ckc +amr.max_level = 1 +amr.ref_ratio_vect = 4 2 +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.fine_tag_hi = 10.e-6 10.e-6 +warpx.fine_tag_lo = -10.e-6 -10.e-6 +warpx.use_filter = 1 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_momentum_conserving b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_momentum_conserving new file mode 100644 index 00000000000..201f19f32c2 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_momentum_conserving @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.field_gathering = momentum-conserving +algo.maxwell_solver = ckc +amr.max_level = 1 +amr.ref_ratio = 4 +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.use_filter = 1 +warpx.fine_tag_lo = -10.e-6 -10.e-6 +warpx.fine_tag_hi = 10.e-6 10.e-6 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_psatd b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_psatd new file mode 100644 index 00000000000..cf95a07e2fc --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_mr_psatd @@ -0,0 +1,14 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +amr.max_level = 1 +amr.ref_ratio = 4 +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.fine_tag_hi = 10.e-6 10.e-6 +warpx.fine_tag_lo = -10.e-6 -10.e-6 +warpx.use_filter = 1 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_nodal b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_nodal new file mode 100644 index 00000000000..99d952d79d9 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_nodal @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = direct +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_picmi.py b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_picmi.py new file mode 100755 index 00000000000..dc7fa3a2ba7 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_picmi.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# --- Simple example of Langmuir oscillations in a uniform plasma +# --- in two dimensions + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +plasma_density = 1.0e25 +plasma_xmin = 0.0 +plasma_x_velocity = 0.1 * constants.c + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 40 +diagnostic_intervals = "::10" + +# --- Grid +nx = 64 +ny = 64 + +xmin = -20.0e-6 +ymin = -20.0e-6 +xmax = +20.0e-6 +ymax = +20.0e-6 + +number_per_cell_each_dim = [2, 2] + +########################## +# physics components +########################## + +uniform_plasma = picmi.UniformDistribution( + density=1.0e25, + upper_bound=[0.0, None, None], + directed_velocity=[0.1 * constants.c, 0.0, 0.0], +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=uniform_plasma +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic"], + moving_window_velocity=[0.0, 0.0, 0.0], + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.0) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["Ex", "Jx"], +) + +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, + species=[electrons], + data_list=["weighting", "ux"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + warpx_current_deposition_algo="direct", + warpx_use_filter=0, +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +sim.write_input_file(file_name="inputs2d_from_PICMI") + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd new file mode 100644 index 00000000000..2386f9e462f --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction new file mode 100644 index 00000000000..c56572ac957 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = esirkepov +algo.maxwell_solver = psatd +amr.max_grid_size = 128 +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +warpx.cfl = 0.7071067811865475 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction_nodal b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction_nodal new file mode 100644 index 00000000000..5359d8703f3 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_current_correction_nodal @@ -0,0 +1,14 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +amr.max_grid_size = 128 +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +warpx.cfl = 0.7071067811865475 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_momentum_conserving b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_momentum_conserving new file mode 100644 index 00000000000..694f65fe233 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_momentum_conserving @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.field_gathering = momentum-conserving +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ new file mode 100644 index 00000000000..793f077b0f7 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +psatd.J_in_time = linear +psatd.solution_type = first-order +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 2 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ_nodal b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ_nodal new file mode 100644 index 00000000000..573337abb76 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_multiJ_nodal @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +psatd.J_in_time = linear +psatd.solution_type = first-order +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 2 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_nodal b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_nodal new file mode 100644 index 00000000000..8f02d4f8aae --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_nodal @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition new file mode 100644 index 00000000000..209e48e10e6 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = vay +algo.maxwell_solver = psatd +amr.max_grid_size = 128 +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.cfl = 0.7071067811865475 diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_nodal b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_nodal new file mode 100644 index 00000000000..d327c1b37b2 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_nodal @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = vay +algo.maxwell_solver = psatd +amr.max_grid_size = 128 +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.cfl = 0.7071067811865475 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 new file mode 100644 index 00000000000..fc5d780cef1 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = vay +algo.maxwell_solver = psatd +algo.particle_shape = 4 +amr.max_grid_size = 128 +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz +warpx.cfl = 0.7071067811865475 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi new file mode 100644 index 00000000000..7665a846eef --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_3d diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_nodal b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_nodal new file mode 100644 index 00000000000..9620cd97f33 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_nodal @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_picmi.py b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_picmi.py new file mode 100755 index 00000000000..11ea1843e27 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_picmi.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# +# --- Simple example of Langmuir oscillations in a uniform plasma + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +plasma_density = 1.0e25 +plasma_xmin = 0.0 +plasma_x_velocity = 0.1 * constants.c + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 40 +diagnostic_interval = 10 + +# --- Grid +nx = 64 +ny = 64 +nz = 64 + +xmin = -20.0e-6 +ymin = -20.0e-6 +zmin = -20.0e-6 +xmax = +20.0e-6 +ymax = +20.0e-6 +zmax = +20.0e-6 + +number_per_cell_each_dim = [2, 2, 2] + +########################## +# physics components +########################## + +uniform_plasma = picmi.UniformDistribution( + density=1.0e25, + upper_bound=[0.0, None, None], + directed_velocity=[0.1 * constants.c, 0.0, 0.0], +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=uniform_plasma +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic", "periodic"], + moving_window_velocity=[0.0, 0.0, 0.0], + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.0) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_interval, + data_list=["Ex", "Jx"], +) + +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_interval, + species=[electrons], + data_list=["weighting", "ux"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + warpx_current_deposition_algo="direct", +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name = 'inputs_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd new file mode 100644 index 00000000000..427de2993b1 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.maxwell_solver = psatd +warpx.cfl = 0.5773502691896258 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction new file mode 100644 index 00000000000..86f33d131ce --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = esirkepov +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +warpx.cfl = 0.5773502691896258 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction_nodal b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction_nodal new file mode 100644 index 00000000000..7f67b9100b2 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_current_correction_nodal @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +warpx.cfl = 0.5773502691896258 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_div_cleaning b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_div_cleaning new file mode 100644 index 00000000000..d372b789336 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_div_cleaning @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +diag1.intervals = 0, 38:40:1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE F +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.5773502691896258 +warpx.do_dive_cleaning = 1 +warpx.do_divb_cleaning = 1 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_momentum_conserving b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_momentum_conserving new file mode 100644 index 00000000000..15a4c7d6985 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_momentum_conserving @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.field_gathering = momentum-conserving +algo.maxwell_solver = psatd +warpx.cfl = 0.5773502691896258 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ new file mode 100644 index 00000000000..e1cd25cd93d --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +warpx.cfl = 0.5773502691896258 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 2 +psatd.J_in_time = linear +psatd.solution_type = first-order +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ_nodal b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ_nodal new file mode 100644 index 00000000000..4a828d2e8b5 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_multiJ_nodal @@ -0,0 +1,14 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +psatd.J_in_time = linear +psatd.solution_type = first-order +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.5773502691896258 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 2 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_nodal b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_nodal new file mode 100644 index 00000000000..fd03e00968a --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_nodal @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.5773502691896258 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition new file mode 100644 index 00000000000..5e2ffa9d407 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = vay +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +warpx.cfl = 0.5773502691896258 diff --git a/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition_nodal b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition_nodal new file mode 100644 index 00000000000..df311b0fb3c --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_3d_langmuir_multi_psatd_vay_deposition_nodal @@ -0,0 +1,9 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.current_deposition = vay +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +warpx.cfl = 0.5773502691896258 +warpx.grid_type = collocated diff --git a/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi new file mode 100644 index 00000000000..45665b67266 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +diag1.dump_rz_modes = 0 +diag1.electrons.variables = x y z w ux uy uz +diag1.ions.variables = x y z w ux uy uz diff --git a/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_picmi.py b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_picmi.py new file mode 100755 index 00000000000..24eedd703d2 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_picmi.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +# +# This is a script that analyses the multimode simulation results. +# This simulates a RZ multimode periodic plasma wave. +# The electric field from the simulation is compared to the analytic value + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +from pywarpx import fields, picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +density = 2.0e24 +epsilon0 = 0.001 * constants.c +epsilon1 = 0.001 * constants.c +epsilon2 = 0.001 * constants.c +w0 = 5.0e-6 +n_osc_z = 3 + +# Plasma frequency +wp = np.sqrt((density * constants.q_e**2) / (constants.m_e * constants.ep0)) +kp = wp / constants.c + +########################## +# numerics parameters +########################## + +nr = 64 +nz = 200 + +rmin = 0.0e0 +zmin = 0.0e0 +rmax = +20.0e-6 +zmax = +40.0e-6 + +# Wave vector of the wave +k0 = 2.0 * np.pi * n_osc_z / (zmax - zmin) + +diagnostic_intervals = 40 + +########################## +# physics components +########################## + +uniform_plasma = picmi.UniformDistribution( + density=density, + upper_bound=[+18e-6, +18e-6, None], + directed_velocity=[0.0, 0.0, 0.0], +) + +momentum_expressions = [ + """+ epsilon0/kp*2*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + - epsilon1/kp*2/w0*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + + epsilon1/kp*4*x**2/w0**3*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + - epsilon2/kp*8*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + + epsilon2/kp*8*x*(x**2-y**2)/w0**4*exp(-(x**2+y**2)/w0**2)*sin(k0*z)""", + """+ epsilon0/kp*2*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + + epsilon1/kp*4*x*y/w0**3*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + + epsilon2/kp*8*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z) + + epsilon2/kp*8*y*(x**2-y**2)/w0**4*exp(-(x**2+y**2)/w0**2)*sin(k0*z)""", + """- epsilon0/kp*k0*exp(-(x**2+y**2)/w0**2)*cos(k0*z) + - epsilon1/kp*k0*2*x/w0*exp(-(x**2+y**2)/w0**2)*cos(k0*z) + - epsilon2/kp*k0*4*(x**2-y**2)/w0**2*exp(-(x**2+y**2)/w0**2)*cos(k0*z)""", +] + +analytic_plasma = picmi.AnalyticDistribution( + density_expression=density, + upper_bound=[+18e-6, +18e-6, None], + epsilon0=epsilon0, + epsilon1=epsilon1, + epsilon2=epsilon2, + kp=kp, + k0=k0, + w0=w0, + momentum_expressions=momentum_expressions, +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=analytic_plasma +) +protons = picmi.Species( + particle_type="proton", name="protons", initial_distribution=uniform_plasma +) + +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + n_azimuthal_modes=3, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=["none", "periodic"], + upper_boundary_conditions=["none", "periodic"], + lower_boundary_conditions_particles=["none", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + moving_window_velocity=[0.0, 0.0], + warpx_max_grid_size=64, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.0) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["Er", "Ez", "Bt", "Jr", "Jz", "part_per_cell"], +) + +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, + species=[electrons], + data_list=["weighting", "momentum"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=40, + verbose=1, + warpx_current_deposition_algo="esirkepov", + warpx_field_gathering_algo="energy-conserving", + warpx_particle_pusher_algo="boris", + warpx_use_filter=0, +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2, 16, 2], grid=grid), +) +sim.add_species( + protons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2, 16, 2], grid=grid) +) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +# sim.write_input_file(file_name='inputsrz_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() + + +# Below is WarpX specific code to check the results. + + +def calcEr(z, r, k0, w0, wp, t, epsilons): + """ + Return the radial electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Er_array = ( + epsilons[0] + * constants.m_e + * constants.c + / constants.q_e + * 2 + * r + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + - epsilons[1] + * constants.m_e + * constants.c + / constants.q_e + * 2 + / w0 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + + epsilons[1] + * constants.m_e + * constants.c + / constants.q_e + * 4 + * r**2 + / w0**3 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + - epsilons[2] + * constants.m_e + * constants.c + / constants.q_e + * 8 + * r + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + + epsilons[2] + * constants.m_e + * constants.c + / constants.q_e + * 8 + * r**3 + / w0**4 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + ) + return Er_array + + +def calcEz(z, r, k0, w0, wp, t, epsilons): + """ + Return the longitudinal electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Ez_array = ( + -epsilons[0] + * constants.m_e + * constants.c + / constants.q_e + * k0 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.sin(wp * t) + - epsilons[1] + * constants.m_e + * constants.c + / constants.q_e + * k0 + * 2 + * r + / w0 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.sin(wp * t) + - epsilons[2] + * constants.m_e + * constants.c + / constants.q_e + * k0 + * 4 + * r**2 + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.sin(wp * t) + ) + return Ez_array + + +# Current time of the simulation +t0 = sim.extension.warpx.gett_new(0) + +# Get the raw field data. Note that these are the real and imaginary +# parts of the fields for each azimuthal mode. +Ex_sim_wrap = fields.ExWrapper() +Ez_sim_wrap = fields.EzWrapper() +Ex_sim_modes = Ex_sim_wrap[...] +Ez_sim_modes = Ez_sim_wrap[...] + +rr_Er = Ex_sim_wrap.mesh("r") +zz_Er = Ex_sim_wrap.mesh("z") +rr_Ez = Ez_sim_wrap.mesh("r") +zz_Ez = Ez_sim_wrap.mesh("z") + +rr_Er = rr_Er[:, np.newaxis] * np.ones(zz_Er.shape[0])[np.newaxis, :] +zz_Er = zz_Er[np.newaxis, :] * np.ones(rr_Er.shape[0])[:, np.newaxis] +rr_Ez = rr_Ez[:, np.newaxis] * np.ones(zz_Ez.shape[0])[np.newaxis, :] +zz_Ez = zz_Ez[np.newaxis, :] * np.ones(rr_Ez.shape[0])[:, np.newaxis] + +# Sum the real components to get the field along x-axis (theta = 0) +Er_sim = Ex_sim_modes[:, :, 0] + np.sum(Ex_sim_modes[:, :, 1::2], axis=2) +Ez_sim = Ez_sim_modes[:, :, 0] + np.sum(Ez_sim_modes[:, :, 1::2], axis=2) + +# The analytical solutions +Er_th = calcEr(zz_Er, rr_Er, k0, w0, wp, t0, [epsilon0, epsilon1, epsilon2]) +Ez_th = calcEz(zz_Ez, rr_Ez, k0, w0, wp, t0, [epsilon0, epsilon1, epsilon2]) + +max_error_Er = abs(Er_sim - Er_th).max() / abs(Er_th).max() +max_error_Ez = abs(Ez_sim - Ez_th).max() / abs(Ez_th).max() +print("Max error Er %e" % max_error_Er) +print("Max error Ez %e" % max_error_Ez) + +# Plot the last field from the loop (Er at iteration 40) +fig, ax = plt.subplots(3) +im = ax[0].imshow(Er_sim, aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[0], orientation="vertical") +ax[0].set_title("Er, last iteration (simulation)") +ax[1].imshow(Er_th, aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[1], orientation="vertical") +ax[1].set_title("Er, last iteration (theory)") +im = ax[2].imshow((Er_sim - Er_th) / abs(Er_th).max(), aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[2], orientation="vertical") +ax[2].set_title("Er, last iteration (difference)") +plt.savefig("langmuir_multi_rz_multimode_analysis_Er.png") + +fig, ax = plt.subplots(3) +im = ax[0].imshow(Ez_sim, aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[0], orientation="vertical") +ax[0].set_title("Ez, last iteration (simulation)") +ax[1].imshow(Ez_th, aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[1], orientation="vertical") +ax[1].set_title("Ez, last iteration (theory)") +im = ax[2].imshow((Ez_sim - Ez_th) / abs(Ez_th).max(), aspect="auto", origin="lower") +fig.colorbar(im, ax=ax[2], orientation="vertical") +ax[2].set_title("Ez, last iteration (difference)") +plt.savefig("langmuir_multi_rz_multimode_analysis_Ez.png") + +assert max(max_error_Er, max_error_Ez) < 0.02 diff --git a/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd new file mode 100644 index 00000000000..5537335629d --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd @@ -0,0 +1,15 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +diag1.dump_rz_modes = 0 +diag1.electrons.variables = x y z w ux uy uz +diag1.ions.variables = x y z w ux uy uz +electrons.random_theta = 0 +ions.random_theta = 0 +psatd.current_correction = 0 +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.do_dive_cleaning = 0 diff --git a/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_current_correction b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_current_correction new file mode 100644 index 00000000000..fac41cea4cd --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_current_correction @@ -0,0 +1,16 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +amr.max_grid_size = 128 +diag1.dump_rz_modes = 0 +diag1.electrons.variables = x y z w ux uy uz +diag1.fields_to_plot = jr jz Er Ez Bt rho divE +diag1.ions.variables = x y z w ux uy uz +electrons.random_theta = 0 +ions.random_theta = 0 +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +warpx.do_dive_cleaning = 0 diff --git a/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_multiJ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_multiJ new file mode 100644 index 00000000000..0ff617af8eb --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_rz_langmuir_multi_psatd_multiJ @@ -0,0 +1,22 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +algo.current_deposition = direct +algo.maxwell_solver = psatd +amr.max_grid_size = 32 +diag1.dump_rz_modes = 0 +diag1.electrons.variables = x y z w ux uy uz +diag1.ions.variables = x y z w ux uy uz +electrons.num_particles_per_cell_each_dim = 2 4 2 +electrons.random_theta = 0 +ions.num_particles_per_cell_each_dim = 2 4 2 +ions.random_theta = 0 +psatd.current_correction = 0 +psatd.update_with_rho = 1 +warpx.abort_on_warning_threshold = medium +warpx.do_dive_cleaning = 0 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 4 +warpx.n_rz_azimuthal_modes = 2 +warpx.use_filter = 1 diff --git a/Examples/Tests/langmuir_fluids/CMakeLists.txt b/Examples/Tests/langmuir_fluids/CMakeLists.txt new file mode 100644 index 00000000000..df6732200c6 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/CMakeLists.txt @@ -0,0 +1,42 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_langmuir_fluid # name + 1 # dims + 2 # nprocs + inputs_test_1d_langmuir_fluid # inputs + "analysis_1d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_langmuir_fluid # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_fluid # inputs + "analysis_2d.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_langmuir_fluid # name + 3 # dims + 2 # nprocs + inputs_test_3d_langmuir_fluid # inputs + "analysis_3d.py diags/diag1000040" # analysis + "analysis_default_regression.py --path diags/diag1000040" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_langmuir_fluid # name + RZ # dims + 2 # nprocs + inputs_test_rz_langmuir_fluid # inputs + "analysis_rz.py diags/diag1000080" # analysis + "analysis_default_regression.py --path diags/diag1000080" # checksum + OFF # dependency +) diff --git a/Examples/Tests/langmuir_fluids/analysis_1d.py b/Examples/Tests/langmuir_fluids/analysis_1d.py index 2d1a8f69d1d..f60c76660b5 100755 --- a/Examples/Tests/langmuir_fluids/analysis_1d.py +++ b/Examples/Tests/langmuir_fluids/analysis_1d.py @@ -11,12 +11,11 @@ # the script `inputs.multi.rt`. This simulates a 1D periodic plasma wave. # The electric field in the simulation is given (in theory) by: # $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\sin(k_z z)\sin( \omega_p t)$$ -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -25,113 +24,122 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_z = 2 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -kz = 2.*np.pi*n_osc_z/(zmax-zmin) +kz = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ez": kz, "Jz": kz} +cos = {"Ez": (1, 1, 0), "Jz": (1, 1, 0)} +cos_rho = {"rho": (1, 1, 1)} -k = {'Ez':kz,'Jz':kz} -cos = {'Ez':(1,1,0), 'Jz':(1,1,0)} -cos_rho = {'rho': (1,1,1)} -def get_contribution( is_cos, k ): - du = (zmax-zmin)/Nz - u = zmin + du*( 0.5 + np.arange(Nz) ) +def get_contribution(is_cos, k): + du = (zmax - zmin) / Nz + u = zmin + du * (0.5 + np.arange(Nz)) if is_cos == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - z_contribution = get_contribution( cos_flag[2], kz ) + z_contribution = get_contribution(cos_flag[2], kz) E = amplitude * z_contribution - return( E ) + return E + -def get_theoretical_J_field( field, t ): +def get_theoretical_J_field(field, t): # wpdt/2 accounts for the Yee halfstep offset of the current - dt = t / 80 # SPECIFIC to config parameters! - amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + dt = t / 80 # SPECIFIC to config parameters! + amplitude = ( + -epsilon_0 + * wp + * epsilon + * (m_e * c**2 * k[field]) + / e + * np.cos(wp * t - wp * dt / 2) + ) cos_flag = cos[field] - z_contribution = get_contribution( cos_flag[2], kz ) + z_contribution = get_contribution(cos_flag[2], kz) - J = amplitude * z_contribution + J = amplitude * z_contribution - return( J ) + return J -def get_theoretical_rho_field( field, t ): - amplitude = epsilon_0 * epsilon * (m_e*c**2*(kz*kz))/e * np.sin(wp*t) + +def get_theoretical_rho_field(field, t): + amplitude = epsilon_0 * epsilon * (m_e * c**2 * (kz * kz)) / e * np.sin(wp * t) cos_flag = cos_rho[field] - z_contribution = get_contribution( cos_flag[2], kz) + z_contribution = get_contribution(cos_flag[2], kz) rho = amplitude * z_contribution - return( rho ) + return rho + # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Check the validity of the fields error_rel = 0 -for field in ['Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,0,0] +for field in ["Ez"]: + E_sim = data[("mesh", field)].to_ndarray()[:, 0, 0] E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the currents -for field in ['Jz']: - J_sim = data[('mesh',field)].to_ndarray()[:,0,0] +for field in ["Jz"]: + J_sim = data[("mesh", field)].to_ndarray()[:, 0, 0] J_th = get_theoretical_J_field(field, t0) - max_error = abs(J_sim-J_th).max()/abs(J_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(J_sim - J_th).max() / abs(J_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the charge -for field in ['rho']: - rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] +for field in ["rho"]: + rho_sim = data[("boxlib", field)].to_ndarray()[:, 0, 0] rho_th = get_theoretical_rho_field(field, t0) - max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(rho_sim - rho_th).max() / abs(rho_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 80) -plt.subplot2grid( (1,2), (0,0) ) -plt.plot( E_sim ) -#plt.colorbar() -plt.title('Ez, last iteration\n(simulation)') -plt.subplot2grid( (1,2), (0,1) ) -plt.plot( E_th ) -#plt.colorbar() -plt.title('Ez, last iteration\n(theory)') +plt.subplot2grid((1, 2), (0, 0)) +plt.plot(E_sim) +# plt.colorbar() +plt.title("Ez, last iteration\n(simulation)") +plt.subplot2grid((1, 2), (0, 1)) +plt.plot(E_th) +# plt.colorbar() +plt.title("Ez, last iteration\n(theory)") plt.tight_layout() -plt.savefig('langmuir_fluid_multi_1d_analysis.png') +plt.savefig("langmuir_fluid_multi_1d_analysis.png") tolerance_rel = 0.05 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/langmuir_fluids/analysis_2d.py b/Examples/Tests/langmuir_fluids/analysis_2d.py index f7244f87137..46b9948884c 100755 --- a/Examples/Tests/langmuir_fluids/analysis_2d.py +++ b/Examples/Tests/langmuir_fluids/analysis_2d.py @@ -13,7 +13,6 @@ # $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ # $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ # $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ -import os import sys import matplotlib.pyplot as plt @@ -25,135 +24,152 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_x = 2 n_osc_z = 2 -xmin = -20e-6; xmax = 20.e-6; Nx = 128 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +xmin = -20e-6 +xmax = 20.0e-6 +Nx = 128 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -kx = 2.*np.pi*n_osc_x/(xmax-xmin) -kz = 2.*np.pi*n_osc_z/(zmax-zmin) +kx = 2.0 * np.pi * n_osc_x / (xmax - xmin) +kz = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ex": kx, "Ez": kz, "Jx": kx, "Jz": kz} +cos = {"Ex": (0, 1, 1), "Ez": (1, 1, 0), "Jx": (0, 1, 1), "Jz": (1, 1, 0)} +cos_rho = {"rho": (1, 1, 1)} -k = {'Ex':kx, 'Ez':kz, 'Jx':kx, 'Jz':kz} -cos = {'Ex': (0,1,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jz':(1,1,0)} -cos_rho = {'rho': (1,1,1)} -def get_contribution( is_cos, k ): - du = (xmax-xmin)/Nx - u = xmin + du*( 0.5 + np.arange(Nx) ) +def get_contribution(is_cos, k): + du = (xmax - xmin) / Nx + u = xmin + du * (0.5 + np.arange(Nx)) if is_cos == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag[0], kx ) - z_contribution = get_contribution( cos_flag[2], kz ) + x_contribution = get_contribution(cos_flag[0], kx) + z_contribution = get_contribution(cos_flag[2], kz) + + E = amplitude * x_contribution[:, np.newaxis] * z_contribution[np.newaxis, :] - E = amplitude * x_contribution[:, np.newaxis ] \ - * z_contribution[np.newaxis, :] + return E - return( E ) -def get_theoretical_J_field( field, t ): +def get_theoretical_J_field(field, t): # wpdt/2 accounts for the Yee halfstep offset of the current - dt = t / 40 # SPECIFIC to config parameters! - amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + dt = t / 40 # SPECIFIC to config parameters! + amplitude = ( + -epsilon_0 + * wp + * epsilon + * (m_e * c**2 * k[field]) + / e + * np.cos(wp * t - wp * dt / 2) + ) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag[0], kx ) - z_contribution = get_contribution( cos_flag[2], kz ) + x_contribution = get_contribution(cos_flag[0], kx) + z_contribution = get_contribution(cos_flag[2], kz) - J = amplitude * x_contribution[:, np.newaxis] \ - * z_contribution[np.newaxis, :] + J = amplitude * x_contribution[:, np.newaxis] * z_contribution[np.newaxis, :] - return( J ) + return J -def get_theoretical_rho_field( field, t ): - amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+kz*kz))/e * np.sin(wp*t) + +def get_theoretical_rho_field(field, t): + amplitude = ( + epsilon_0 * epsilon * (m_e * c**2 * (kx * kx + kz * kz)) / e * np.sin(wp * t) + ) cos_flag = cos_rho[field] - x_contribution = get_contribution( cos_flag[0], kx ) - z_contribution = get_contribution( cos_flag[2], kz ) + x_contribution = get_contribution(cos_flag[0], kx) + z_contribution = get_contribution(cos_flag[2], kz) + + rho = amplitude * x_contribution[:, np.newaxis] * z_contribution[np.newaxis, :] - rho = amplitude * x_contribution[:, np.newaxis] \ - * z_contribution[ np.newaxis, :] + return rho - return( rho ) # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) -edge = np.array([(ds.domain_left_edge[1]).item(), (ds.domain_right_edge[1]).item(), \ - (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +edge = np.array( + [ + (ds.domain_left_edge[1]).item(), + (ds.domain_right_edge[1]).item(), + (ds.domain_left_edge[0]).item(), + (ds.domain_right_edge[0]).item(), + ] +) # Check the validity of the fields error_rel = 0 -for field in ['Ex', 'Ez']: - E_sim = data[('mesh',field)].to_ndarray()[:,:,0] +for field in ["Ex", "Ez"]: + E_sim = data[("mesh", field)].to_ndarray()[:, :, 0] E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the currents -for field in ['Jx', 'Jz']: - J_sim = data[('mesh',field)].to_ndarray()[:,:,0] +for field in ["Jx", "Jz"]: + J_sim = data[("mesh", field)].to_ndarray()[:, :, 0] J_th = get_theoretical_J_field(field, t0) - max_error = abs(J_sim-J_th).max()/abs(J_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(J_sim - J_th).max() / abs(J_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the charge -for field in ['rho']: - rho_sim = data[('boxlib',field)].to_ndarray()[:,:,0] +for field in ["rho"]: + rho_sim = data[("boxlib", field)].to_ndarray()[:, :, 0] rho_th = get_theoretical_rho_field(field, t0) - max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(rho_sim - rho_th).max() / abs(rho_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 40) -fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi=100) # First plot vmin = E_sim.min() vmax = E_sim.max() -cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') -im1 = ax1.imshow(E_sim, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb1 = fig.colorbar(im1, cax = cax1) -ax1.set_xlabel(r'$z$') -ax1.set_ylabel(r'$x$') -ax1.set_title(r'$E_z$ (sim)') +cax1 = make_axes_locatable(ax1).append_axes("right", size="5%", pad="5%") +im1 = ax1.imshow(E_sim, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb1 = fig.colorbar(im1, cax=cax1) +ax1.set_xlabel(r"$z$") +ax1.set_ylabel(r"$x$") +ax1.set_title(r"$E_z$ (sim)") # Second plot vmin = E_th.min() vmax = E_th.max() -cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') -im2 = ax2.imshow(E_th, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb2 = fig.colorbar(im2, cax = cax2) -ax2.set_xlabel(r'$z$') -ax2.set_ylabel(r'$x$') -ax2.set_title(r'$E_z$ (theory)') +cax2 = make_axes_locatable(ax2).append_axes("right", size="5%", pad="5%") +im2 = ax2.imshow(E_th, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb2 = fig.colorbar(im2, cax=cax2) +ax2.set_xlabel(r"$z$") +ax2.set_ylabel(r"$x$") +ax2.set_title(r"$E_z$ (theory)") # Save figure fig.tight_layout() -fig.savefig('Langmuir_fluid_multi_2d_analysis.png', dpi = 200) +fig.savefig("Langmuir_fluid_multi_2d_analysis.png", dpi=200) tolerance_rel = 0.05 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/langmuir_fluids/analysis_3d.py b/Examples/Tests/langmuir_fluids/analysis_3d.py index 686907f103a..6a15c175843 100755 --- a/Examples/Tests/langmuir_fluids/analysis_3d.py +++ b/Examples/Tests/langmuir_fluids/analysis_3d.py @@ -13,8 +13,6 @@ # $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ # $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ # $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ -import os -import re import sys import matplotlib.pyplot as plt @@ -26,154 +24,186 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 -n = 4.e24 +n = 4.0e24 n_osc_x = 2 n_osc_y = 2 n_osc_z = 2 -lo = [-20.e-6, -20.e-6, -20.e-6] -hi = [ 20.e-6, 20.e-6, 20.e-6] +lo = [-20.0e-6, -20.0e-6, -20.0e-6] +hi = [20.0e-6, 20.0e-6, 20.0e-6] Ncell = [64, 64, 64] # Wave vector of the wave -kx = 2.*np.pi*n_osc_x/(hi[0]-lo[0]) -ky = 2.*np.pi*n_osc_y/(hi[1]-lo[1]) -kz = 2.*np.pi*n_osc_z/(hi[2]-lo[2]) +kx = 2.0 * np.pi * n_osc_x / (hi[0] - lo[0]) +ky = 2.0 * np.pi * n_osc_y / (hi[1] - lo[1]) +kz = 2.0 * np.pi * n_osc_z / (hi[2] - lo[2]) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) - -k = {'Ex':kx, 'Ey':ky, 'Ez':kz, 'Jx':kx, 'Jy':ky, 'Jz':kz} -cos = {'Ex': (0,1,1), 'Ey':(1,0,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jy':(1,0,1), 'Jz':(1,1,0)} -cos_rho = {'rho': (1,1,1)} - -def get_contribution( is_cos, k, idim ): - du = (hi[idim]-lo[idim])/Ncell[idim] - u = lo[idim] + du*( 0.5 + np.arange(Ncell[idim]) ) +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) + +k = {"Ex": kx, "Ey": ky, "Ez": kz, "Jx": kx, "Jy": ky, "Jz": kz} +cos = { + "Ex": (0, 1, 1), + "Ey": (1, 0, 1), + "Ez": (1, 1, 0), + "Jx": (0, 1, 1), + "Jy": (1, 0, 1), + "Jz": (1, 1, 0), +} +cos_rho = {"rho": (1, 1, 1)} + + +def get_contribution(is_cos, k, idim): + du = (hi[idim] - lo[idim]) / Ncell[idim] + u = lo[idim] + du * (0.5 + np.arange(Ncell[idim])) if is_cos[idim] == 1: - return( np.cos(k*u) ) + return np.cos(k * u) else: - return( np.sin(k*u) ) + return np.sin(k * u) -def get_theoretical_field( field, t ): - amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + +def get_theoretical_field(field, t): + amplitude = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag, kx, 0 ) - y_contribution = get_contribution( cos_flag, ky, 1 ) - z_contribution = get_contribution( cos_flag, kz, 2 ) + x_contribution = get_contribution(cos_flag, kx, 0) + y_contribution = get_contribution(cos_flag, ky, 1) + z_contribution = get_contribution(cos_flag, kz, 2) + + E = ( + amplitude + * x_contribution[:, np.newaxis, np.newaxis] + * y_contribution[np.newaxis, :, np.newaxis] + * z_contribution[np.newaxis, np.newaxis, :] + ) - E = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ - * y_contribution[np.newaxis, :, np.newaxis] \ - * z_contribution[np.newaxis, np.newaxis, :] + return E - return( E ) -def get_theoretical_J_field( field, t ): +def get_theoretical_J_field(field, t): # wpdt/2 accounts for the Yee halfstep offset of the current - dt = t / 40 # SPECIFIC to config parameters! - amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + dt = t / 40 # SPECIFIC to config parameters! + amplitude = ( + -epsilon_0 + * wp + * epsilon + * (m_e * c**2 * k[field]) + / e + * np.cos(wp * t - wp * dt / 2) + ) cos_flag = cos[field] - x_contribution = get_contribution( cos_flag, kx, 0 ) - y_contribution = get_contribution( cos_flag, ky, 1 ) - z_contribution = get_contribution( cos_flag, kz, 2 ) - - J = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ - * y_contribution[np.newaxis, :, np.newaxis] \ - * z_contribution[np.newaxis, np.newaxis, :] - - return( J ) - -def get_theoretical_rho_field( field, t ): - amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+ky*ky+kz*kz))/e * np.sin(wp*t) + x_contribution = get_contribution(cos_flag, kx, 0) + y_contribution = get_contribution(cos_flag, ky, 1) + z_contribution = get_contribution(cos_flag, kz, 2) + + J = ( + amplitude + * x_contribution[:, np.newaxis, np.newaxis] + * y_contribution[np.newaxis, :, np.newaxis] + * z_contribution[np.newaxis, np.newaxis, :] + ) + + return J + + +def get_theoretical_rho_field(field, t): + amplitude = ( + epsilon_0 + * epsilon + * (m_e * c**2 * (kx * kx + ky * ky + kz * kz)) + / e + * np.sin(wp * t) + ) cos_flag = cos_rho[field] - x_contribution = get_contribution( cos_flag, kx, 0 ) - y_contribution = get_contribution( cos_flag, ky, 1 ) - z_contribution = get_contribution( cos_flag, kz, 2 ) + x_contribution = get_contribution(cos_flag, kx, 0) + y_contribution = get_contribution(cos_flag, ky, 1) + z_contribution = get_contribution(cos_flag, kz, 2) + + rho = ( + amplitude + * x_contribution[:, np.newaxis, np.newaxis] + * y_contribution[np.newaxis, :, np.newaxis] + * z_contribution[np.newaxis, np.newaxis, :] + ) - rho = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ - * y_contribution[np.newaxis, :, np.newaxis] \ - * z_contribution[np.newaxis, np.newaxis, :] + return rho - return( rho ) # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) -edge = np.array([(ds.domain_left_edge[2]).item(), (ds.domain_right_edge[2]).item(), \ - (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +edge = np.array( + [ + (ds.domain_left_edge[2]).item(), + (ds.domain_right_edge[2]).item(), + (ds.domain_left_edge[0]).item(), + (ds.domain_right_edge[0]).item(), + ] +) # Check the validity of the fields error_rel = 0 -for field in ['Ex', 'Ey', 'Ez']: - E_sim = data[('mesh',field)].to_ndarray() +for field in ["Ex", "Ey", "Ez"]: + E_sim = data[("mesh", field)].to_ndarray() E_th = get_theoretical_field(field, t0) - max_error = abs(E_sim-E_th).max()/abs(E_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(E_sim - E_th).max() / abs(E_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the currents -for field in ['Jx', 'Jy', 'Jz']: - J_sim = data[('mesh',field)].to_ndarray() +for field in ["Jx", "Jy", "Jz"]: + J_sim = data[("mesh", field)].to_ndarray() J_th = get_theoretical_J_field(field, t0) - max_error = abs(J_sim-J_th).max()/abs(J_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(J_sim - J_th).max() / abs(J_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Check the validity of the charge -for field in ['rho']: - rho_sim = data[('boxlib',field)].to_ndarray() +for field in ["rho"]: + rho_sim = data[("boxlib", field)].to_ndarray() rho_th = get_theoretical_rho_field(field, t0) - max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() - print('%s: Max error: %.2e' %(field,max_error)) - error_rel = max( error_rel, max_error ) + max_error = abs(rho_sim - rho_th).max() / abs(rho_th).max() + print("%s: Max error: %.2e" % (field, max_error)) + error_rel = max(error_rel, max_error) # Plot the last field from the loop (Ez at iteration 40) -fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi=100) # First plot (slice at y=0) -E_plot = E_sim[:,Ncell[1]//2+1,:] +E_plot = E_sim[:, Ncell[1] // 2 + 1, :] vmin = E_plot.min() vmax = E_plot.max() -cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') -im1 = ax1.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb1 = fig.colorbar(im1, cax = cax1) -ax1.set_xlabel(r'$z$') -ax1.set_ylabel(r'$x$') -ax1.set_title(r'$E_z$ (sim)') +cax1 = make_axes_locatable(ax1).append_axes("right", size="5%", pad="5%") +im1 = ax1.imshow(E_plot, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb1 = fig.colorbar(im1, cax=cax1) +ax1.set_xlabel(r"$z$") +ax1.set_ylabel(r"$x$") +ax1.set_title(r"$E_z$ (sim)") # Second plot (slice at y=0) -E_plot = E_th[:,Ncell[1]//2+1,:] +E_plot = E_th[:, Ncell[1] // 2 + 1, :] vmin = E_plot.min() vmax = E_plot.max() -cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') -im2 = ax2.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) -cb2 = fig.colorbar(im2, cax = cax2) -ax2.set_xlabel(r'$z$') -ax2.set_ylabel(r'$x$') -ax2.set_title(r'$E_z$ (theory)') +cax2 = make_axes_locatable(ax2).append_axes("right", size="5%", pad="5%") +im2 = ax2.imshow(E_plot, origin="lower", extent=edge, vmin=vmin, vmax=vmax) +cb2 = fig.colorbar(im2, cax=cax2) +ax2.set_xlabel(r"$z$") +ax2.set_ylabel(r"$x$") +ax2.set_title(r"$E_z$ (theory)") # Save figure fig.tight_layout() -fig.savefig('Langmuir_fluid_multi_analysis.png', dpi = 200) +fig.savefig("Langmuir_fluid_multi_analysis.png", dpi=200) tolerance_rel = 5e-2 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] - -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/langmuir_fluids/analysis_default_regression.py b/Examples/Tests/langmuir_fluids/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/langmuir_fluids/analysis_rz.py b/Examples/Tests/langmuir_fluids/analysis_rz.py index 108d054e75a..de6853db556 100755 --- a/Examples/Tests/langmuir_fluids/analysis_rz.py +++ b/Examples/Tests/langmuir_fluids/analysis_rz.py @@ -14,12 +14,11 @@ # Unrelated to the Langmuir waves, we also test the plotfile particle filter function in this # analysis script. import os -import re import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -28,139 +27,188 @@ import numpy as np from scipy.constants import c, e, epsilon_0, m_e -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] test_name = os.path.split(os.getcwd())[1] -# Parse test name and check if current correction (psatd.current_correction) is applied -current_correction = True if re.search('current_correction', fn) else False - # Parameters (these parameters must match the parameters in `inputs.multi.rz.rt`) epsilon = 0.01 -n = 2.e24 -w0 = 5.e-6 +n = 2.0e24 +w0 = 5.0e-6 n_osc_z = 2 -rmin = 0e-6; rmax = 20.e-6; Nr = 64 -zmin = -20e-6; zmax = 20.e-6; Nz = 128 +rmin = 0e-6 +rmax = 20.0e-6 +Nr = 64 +zmin = -20e-6 +zmax = 20.0e-6 +Nz = 128 # Wave vector of the wave -k0 = 2.*np.pi*n_osc_z/(zmax-zmin) +k0 = 2.0 * np.pi * n_osc_z / (zmax - zmin) # Plasma frequency -wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) -kp = wp/c +wp = np.sqrt((n * e**2) / (m_e * epsilon_0)) +kp = wp / c + -def Er( z, r, epsilon, k0, w0, wp, t) : +def Er(z, r, epsilon, k0, w0, wp, t): """ Return the radial electric field as an array of the same length as z and r, in the half-plane theta=0 """ - Er_array = \ - epsilon * m_e*c**2/e * 2*r/w0**2 * \ - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - return( Er_array ) - -def Ez( z, r, epsilon, k0, w0, wp, t) : + Er_array = ( + epsilon + * m_e + * c**2 + / e + * 2 + * r + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.sin(wp * t) + ) + return Er_array + + +def Ez(z, r, epsilon, k0, w0, wp, t): """ Return the longitudinal electric field as an array of the same length as z and r, in the half-plane theta=0 """ - Ez_array = \ - - epsilon * m_e*c**2/e * k0 * \ - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - return( Ez_array ) - -def Jr( z, r, epsilon, k0, w0, wp, t) : + Ez_array = ( + -epsilon + * m_e + * c**2 + / e + * k0 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.sin(wp * t) + ) + return Ez_array + + +def Jr(z, r, epsilon, k0, w0, wp, t): """ Return the radial current density as an array of the same length as z and r, in the half-plane theta=0 """ - dt = t / 80 # SPECIFIC to config parameters! - Jr_array = \ - - epsilon_0 * epsilon * m_e*c**2/e * 2*r/w0**2 * \ - np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 - return( Jr_array ) - -def Jz( z, r, epsilon, k0, w0, wp, t) : + dt = t / 80 # SPECIFIC to config parameters! + Jr_array = ( + -epsilon_0 + * epsilon + * m_e + * c**2 + / e + * 2 + * r + / w0**2 + * np.exp(-(r**2) / w0**2) + * np.sin(k0 * z) + * np.cos(wp * t - wp * dt / 2) + * wp + ) # phase_error = wp*dt/2 + return Jr_array + + +def Jz(z, r, epsilon, k0, w0, wp, t): """ Return the longitudinal current density as an array of the same length as z and r, in the half-plane theta=0 """ - dt = t / 80 # SPECIFIC to config parameters! - Jz_array = \ - epsilon_0 * epsilon * m_e*c**2/e * k0 * \ - np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 - return( Jz_array ) - -def rho( z, r, epsilon, k0, w0, wp, t) : + dt = t / 80 # SPECIFIC to config parameters! + Jz_array = ( + epsilon_0 + * epsilon + * m_e + * c**2 + / e + * k0 + * np.exp(-(r**2) / w0**2) + * np.cos(k0 * z) + * np.cos(wp * t - wp * dt / 2) + * wp + ) # phase_error = wp*dt/2 + return Jz_array + + +def rho(z, r, epsilon, k0, w0, wp, t): """ Return the charge density as an array of the same length as z and r, in the half-plane theta=0 """ - rho_array = \ - epsilon_0 * epsilon * m_e*c**2/e * np.sin( wp*t ) * np.sin( k0*z ) * np.exp( -r**2/w0**2 ) * \ - ((4.0/(w0**2))*(1 - (r**2)/(w0**2)) + k0**2) - return( rho_array ) + rho_array = ( + epsilon_0 + * epsilon + * m_e + * c**2 + / e + * np.sin(wp * t) + * np.sin(k0 * z) + * np.exp(-(r**2) / w0**2) + * ((4.0 / (w0**2)) * (1 - (r**2) / (w0**2)) + k0**2) + ) + return rho_array + # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) # Get cell centered coordinates -dr = (rmax - rmin)/Nr -dz = (zmax - zmin)/Nz -coords = np.indices([Nr, Nz],'d') -rr = rmin + (coords[0] + 0.5)*dr -zz = zmin + (coords[1] + 0.5)*dz +dr = (rmax - rmin) / Nr +dz = (zmax - zmin) / Nz +coords = np.indices([Nr, Nz], "d") +rr = rmin + (coords[0] + 0.5) * dr +zz = zmin + (coords[1] + 0.5) * dz # Check the validity of the fields overall_max_error = 0 -Er_sim = data[('boxlib','Er')].to_ndarray()[:,:,0] +Er_sim = data[("boxlib", "Er")].to_ndarray()[:, :, 0] Er_th = Er(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Er_sim-Er_th).max()/abs(Er_th).max() -print('Er: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Er_sim - Er_th).max() / abs(Er_th).max() +print("Er: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) -Ez_sim = data[('boxlib','Ez')].to_ndarray()[:,:,0] +Ez_sim = data[("boxlib", "Ez")].to_ndarray()[:, :, 0] Ez_th = Ez(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Ez_sim-Ez_th).max()/abs(Ez_th).max() -print('Ez: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Ez_sim - Ez_th).max() / abs(Ez_th).max() +print("Ez: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) -Jr_sim = data[('boxlib','jr')].to_ndarray()[:,:,0] +Jr_sim = data[("boxlib", "jr")].to_ndarray()[:, :, 0] Jr_th = Jr(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Jr_sim-Jr_th).max()/abs(Jr_th).max() -print('Jr: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Jr_sim - Jr_th).max() / abs(Jr_th).max() +print("Jr: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) -Jz_sim = data[('boxlib','jz')].to_ndarray()[:,:,0] +Jz_sim = data[("boxlib", "jz")].to_ndarray()[:, :, 0] Jz_th = Jz(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(Jz_sim-Jz_th).max()/abs(Jz_th).max() -print('Jz: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(Jz_sim - Jz_th).max() / abs(Jz_th).max() +print("Jz: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) -rho_sim = data[('boxlib','rho')].to_ndarray()[:,:,0] +rho_sim = data[("boxlib", "rho")].to_ndarray()[:, :, 0] rho_th = rho(zz, rr, epsilon, k0, w0, wp, t0) -max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() -print('rho: Max error: %.2e' %(max_error)) -overall_max_error = max( overall_max_error, max_error ) +max_error = abs(rho_sim - rho_th).max() / abs(rho_th).max() +print("rho: Max error: %.2e" % (max_error)) +overall_max_error = max(overall_max_error, max_error) # Plot the last field from the loop (Ez at iteration 40) -plt.subplot2grid( (1,2), (0,0) ) -plt.imshow( Ez_sim ) +plt.subplot2grid((1, 2), (0, 0)) +plt.imshow(Ez_sim) plt.colorbar() -plt.title('Ez, last iteration\n(simulation)') -plt.subplot2grid( (1,2), (0,1) ) -plt.imshow( Ez_th ) +plt.title("Ez, last iteration\n(simulation)") +plt.subplot2grid((1, 2), (0, 1)) +plt.imshow(Ez_th) plt.colorbar() -plt.title('Ez, last iteration\n(theory)') +plt.title("Ez, last iteration\n(theory)") plt.tight_layout() -plt.savefig(test_name+'_analysis.png') +plt.savefig(test_name + "_analysis.png") error_rel = overall_max_error @@ -169,6 +217,4 @@ def rho( z, r, epsilon, k0, w0, wp, t) : print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -checksumAPI.evaluate_checksum(test_name, fn) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/langmuir_fluids/inputs_1d b/Examples/Tests/langmuir_fluids/inputs_test_1d_langmuir_fluid similarity index 100% rename from Examples/Tests/langmuir_fluids/inputs_1d rename to Examples/Tests/langmuir_fluids/inputs_test_1d_langmuir_fluid diff --git a/Examples/Tests/langmuir_fluids/inputs_2d b/Examples/Tests/langmuir_fluids/inputs_test_2d_langmuir_fluid similarity index 100% rename from Examples/Tests/langmuir_fluids/inputs_2d rename to Examples/Tests/langmuir_fluids/inputs_test_2d_langmuir_fluid diff --git a/Examples/Tests/langmuir_fluids/inputs_3d b/Examples/Tests/langmuir_fluids/inputs_test_3d_langmuir_fluid similarity index 100% rename from Examples/Tests/langmuir_fluids/inputs_3d rename to Examples/Tests/langmuir_fluids/inputs_test_3d_langmuir_fluid diff --git a/Examples/Tests/langmuir_fluids/inputs_rz b/Examples/Tests/langmuir_fluids/inputs_test_rz_langmuir_fluid similarity index 100% rename from Examples/Tests/langmuir_fluids/inputs_rz rename to Examples/Tests/langmuir_fluids/inputs_test_rz_langmuir_fluid diff --git a/Examples/Tests/larmor/CMakeLists.txt b/Examples/Tests/larmor/CMakeLists.txt new file mode 100644 index 00000000000..f089b4dc958 --- /dev/null +++ b/Examples/Tests/larmor/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_larmor # name + 2 # dims + 2 # nprocs + inputs_test_2d_larmor # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) diff --git a/Examples/Tests/larmor/analysis_default_regression.py b/Examples/Tests/larmor/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/larmor/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/larmor/inputs_2d_mr b/Examples/Tests/larmor/inputs_2d_mr deleted file mode 100644 index 5d7af3d67a4..00000000000 --- a/Examples/Tests/larmor/inputs_2d_mr +++ /dev/null @@ -1,79 +0,0 @@ -# Maximum number of time steps -max_step = 400 - -# number of grid points -amr.n_cell = 64 64 - -# The lo and hi ends of grids are multipliers of blocking factor -amr.blocking_factor = 16 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 1 - -warpx.fine_tag_lo = -0.8 -0.8 -warpx.fine_tag_hi = 0.8 0.8 - -# Geometry -geometry.dims = 2 -geometry.prob_lo = -2.0 -2.0 # physical domain -geometry.prob_hi = 2.0 2.0 - -# Boundary condition -boundary.field_lo = pml pml pml -boundary.field_hi = pml pml pml - -# PML -warpx.pml_ncell = 10 - -particles.B_ext_particle_init_style = "constant" -particles.B_external_particle = 0.0 0.00078110417851950768 0.0 - -# Verbosity -warpx.verbose = 1 - -# Algorithms -warpx.use_filter = 0 - -# CFL -warpx.cfl = 1.0 - -# particles -particles.species_names = electron positron - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 -1.25 -electron.single_particle_u = -0.45825756949558416 0.0 0.0 # gamma*beta - -positron.charge = q_e -positron.mass = m_e -positron.injection_style = "SingleParticle" -positron.single_particle_pos = 0.0 0.0 -1.25 -positron.single_particle_u = 0.45825756949558416 0.0 0.0 # gamma*beta - -electron.single_particle_weight = 1.0e12 -positron.single_particle_weight = 1.0e12 - -# Order of particle shape factors -algo.particle_shape = 3 - -# Moving window -warpx.do_moving_window = 0 - -warpx.do_dive_cleaning = 1 - -# Diagnostics -diagnostics.diags_names = diag1 diagraw -diag1.intervals = 2 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divE divB - -diagraw.intervals = 2 -diagraw.plot_raw_fields=1 -diagraw.diag_type = Full -diagraw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divE divB diff --git a/Examples/Tests/larmor/inputs_test_2d_larmor b/Examples/Tests/larmor/inputs_test_2d_larmor new file mode 100644 index 00000000000..76e4f76ee22 --- /dev/null +++ b/Examples/Tests/larmor/inputs_test_2d_larmor @@ -0,0 +1,79 @@ +# Maximum number of time steps +max_step = 10 + +# number of grid points +amr.n_cell = 64 64 + +# The lo and hi ends of grids are multipliers of blocking factor +amr.blocking_factor = 16 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 1 + +warpx.fine_tag_lo = -0.8 -0.8 +warpx.fine_tag_hi = 0.8 0.8 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -2.0 -2.0 # physical domain +geometry.prob_hi = 2.0 2.0 + +# Boundary condition +boundary.field_lo = pml pml pml +boundary.field_hi = pml pml pml + +# PML +warpx.pml_ncell = 10 + +particles.B_ext_particle_init_style = "constant" +particles.B_external_particle = 0.0 0.00078110417851950768 0.0 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# particles +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 -1.25 +electron.single_particle_u = -0.45825756949558416 0.0 0.0 # gamma*beta + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 0.0 0.0 -1.25 +positron.single_particle_u = 0.45825756949558416 0.0 0.0 # gamma*beta + +electron.single_particle_weight = 1.0e12 +positron.single_particle_weight = 1.0e12 + +# Order of particle shape factors +algo.particle_shape = 3 + +# Moving window +warpx.do_moving_window = 0 + +warpx.do_dive_cleaning = 1 + +# Diagnostics +diagnostics.diags_names = diag1 diagraw +diag1.intervals = 2 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divE divB + +diagraw.intervals = 2 +diagraw.plot_raw_fields=1 +diagraw.diag_type = Full +diagraw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divE divB diff --git a/Examples/Tests/laser_injection/CMakeLists.txt b/Examples/Tests/laser_injection/CMakeLists.txt new file mode 100644 index 00000000000..30d18c6d063 --- /dev/null +++ b/Examples/Tests/laser_injection/CMakeLists.txt @@ -0,0 +1,52 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_laser_injection # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_injection # inputs + "analysis_1d.py diags/diag1000240" # analysis + "analysis_default_regression.py --path diags/diag1000240" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_injection # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_injection # inputs + "analysis_2d.py diags/diag1000240" # analysis + "analysis_default_regression.py --path diags/diag1000240" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_laser_injection # name + 3 # dims + 2 # nprocs + inputs_test_3d_laser_injection # inputs + "analysis_3d.py" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_laser_injection_implicit # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_injection_implicit # inputs + "analysis_1d.py diags/diag1000240" # analysis + "analysis_default_regression.py --path diags/diag1000240" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_injection_implicit # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_injection_implicit # inputs + "analysis_2d.py diags/diag1000240" # analysis + "analysis_default_regression.py --path diags/diag1000240" # checksum + OFF # dependency +) diff --git a/Examples/Tests/laser_injection/analysis_1d.py b/Examples/Tests/laser_injection/analysis_1d.py index 5aca707ff3a..98ce6ca47c5 100755 --- a/Examples/Tests/laser_injection/analysis_1d.py +++ b/Examples/Tests/laser_injection/analysis_1d.py @@ -12,20 +12,16 @@ # the simulation and it compares it with theory. It also checks that the # central frequency of the Fourier transform is the expected one. -import os import sys import matplotlib import yt -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np from scipy.signal import hilbert -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # Maximum acceptable error for this test relative_error_threshold = 0.05 @@ -33,102 +29,114 @@ small_num = 1.0e-8 # Physical parameters -um = 1.e-6 -fs = 1.e-15 +um = 1.0e-6 +fs = 1.0e-15 c = 299792458 # Parameters of the gaussian beam -wavelength = 1.*um -w0 = 5.*um -tt = 10.*fs -t_c = 24.*fs +wavelength = 1.0 * um +w0 = 5.0 * um +tt = 10.0 * fs +t_c = 24.0 * fs E_max = 4e12 # laser direction -dir_vector = np.array([0,0,1.0]) +dir_vector = np.array([0, 0, 1.0]) dir_vector /= np.linalg.norm(dir_vector) # polarization vector -pol_vector = np.array([1.0,1.0,0.0]) +pol_vector = np.array([1.0, 1.0, 0.0]) pol_vector /= np.linalg.norm(pol_vector) + # Calculates the envelope of a Gaussian beam -def gauss_env(T,Z): - '''Function to compute the theory for the envelope - ''' - inv_tau2 = 1./tt/tt - exp_arg = - inv_tau2 / c/c * (Z-T*c)*(Z-T*c) +def gauss_env(T, Z): + """Function to compute the theory for the envelope""" + inv_tau2 = 1.0 / tt / tt + exp_arg = -inv_tau2 / c / c * (Z - T * c) * (Z - T * c) return E_max * np.real(np.exp(exp_arg)) + # Checks envelope and central frequency for a given laser component -def check_component(data, component, t_env_theory, coeff,Z,dz): +def check_component(data, component, t_env_theory, coeff, Z, dz): print("*** Checking " + component + " ***") - field = data['boxlib', component].v.squeeze() + field = data["boxlib", component].v.squeeze() env = abs(hilbert(field)) - env_theory = t_env_theory*np.abs(coeff) + env_theory = t_env_theory * np.abs(coeff) # Plot results - fig = plt.figure(figsize=(12,6)) + fig = plt.figure(figsize=(12, 6)) ax1 = fig.add_subplot(221) - ax1.set_title('PIC field') - ax1.plot(Z,field) + ax1.set_title("PIC field") + ax1.plot(Z, field) ax2 = fig.add_subplot(222) - ax2.set_title('PIC envelope') - ax2.plot(Z,env) + ax2.set_title("PIC envelope") + ax2.plot(Z, env) ax3 = fig.add_subplot(223) - ax3.set_title('Theory envelope') - ax3.plot(Z,env_theory, label="theory") - ax3.plot(Z,env, label="simulation") + ax3.set_title("Theory envelope") + ax3.plot(Z, env_theory, label="theory") + ax3.plot(Z, env, label="simulation") ax3.legend(loc="upper right") ax4 = fig.add_subplot(224) - ax4.set_title('Difference') - ax4.plot(Z,env-env_theory) + ax4.set_title("Difference") + ax4.plot(Z, env - env_theory) plt.tight_layout() - plt.savefig("plt_" + component + ".png", bbox_inches='tight') + plt.savefig("plt_" + component + ".png", bbox_inches="tight") - if(np.abs(coeff) < small_num): + if np.abs(coeff) < small_num: is_field_zero = np.sum(np.abs(env)) < small_num - if is_field_zero : + if is_field_zero: print("[OK] Field component expected to be 0 is ~ 0") - else : + else: print("[FAIL] Field component expected to be 0 is NOT ~ 0") - assert(is_field_zero) + assert is_field_zero print("******\n") return fft_field = np.fft.fft(field) - freq_cols = np.fft.fftfreq(fft_field.shape[0],dz/c) + freq_cols = np.fft.fftfreq(fft_field.shape[0], dz / c) pos_max = np.unravel_index(np.abs(fft_field).argmax(), fft_field.shape) freq = np.abs(freq_cols[pos_max[0]]) - exp_freq = c/wavelength + exp_freq = c / wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq + relative_error_freq = np.abs(freq - exp_freq) / exp_freq is_freq_ok = relative_error_freq < relative_error_threshold - if is_freq_ok : - print("[OK] Relative error frequency: {:6.3f} %".format(relative_error_freq*100)) - else : - print("[FAIL] Relative error frequency: {:6.3f} %".format(relative_error_freq*100)) - assert(is_freq_ok) + if is_freq_ok: + print( + "[OK] Relative error frequency: {:6.3f} %".format(relative_error_freq * 100) + ) + else: + print( + "[FAIL] Relative error frequency: {:6.3f} %".format( + relative_error_freq * 100 + ) + ) + assert is_freq_ok print("******\n") - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env_theory)) + relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env_theory)) is_env_ok = relative_error_env < relative_error_threshold - if is_env_ok : - print("[OK] Relative error envelope: {:6.3f} %".format(relative_error_env*100)) - else : - print("[FAIL] Relative error envelope: {:6.3f} %".format(relative_error_env*100)) - assert(is_env_ok) + if is_env_ok: + print( + "[OK] Relative error envelope: {:6.3f} %".format(relative_error_env * 100) + ) + else: + print( + "[FAIL] Relative error envelope: {:6.3f} %".format(relative_error_env * 100) + ) + assert is_env_ok + def check_laser(filename): ds = yt.load(filename) @@ -136,20 +144,26 @@ def check_laser(filename): # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() + if "force_periodicity" in dir(ds): + ds.force_periodicity() z = np.linspace( - ds.domain_left_edge[0].v, - ds.domain_right_edge[0].v, - ds.domain_dimensions[0]) + ds.domain_left_edge[0].v, ds.domain_right_edge[0].v, ds.domain_dimensions[0] + ) - dz = (ds.domain_right_edge[0].v-ds.domain_left_edge[0].v)/(ds.domain_dimensions[0]-1) + dz = (ds.domain_right_edge[0].v - ds.domain_left_edge[0].v) / ( + ds.domain_dimensions[0] - 1 + ) # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(),z)+gauss_env(-t_c+ds.current_time.to_value(),z) + env_theory = gauss_env(+t_c - ds.current_time.to_value(), z) + gauss_env( + -t_c + ds.current_time.to_value(), z + ) # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) + all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions + ) b_vector = np.cross(dir_vector, pol_vector) @@ -160,20 +174,20 @@ def check_laser(filename): pol_vector[2], b_vector[0], b_vector[1], - b_vector[2]] + b_vector[2], + ] - field_facts = [1, 1, 1, 1/c, 1/c, 1/c] + field_facts = [1, 1, 1, 1 / c, 1 / c, 1 / c] for comp, coeff, field_fact in zip(components, coeffs, field_facts): - check_component(all_data_level_0, comp, field_fact*env_theory, coeff, z, dz) + check_component(all_data_level_0, comp, field_fact * env_theory, coeff, z, dz) + def main(): filename_end = sys.argv[1] check_laser(filename_end) - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) if __name__ == "__main__": main() diff --git a/Examples/Tests/laser_injection/analysis_2d.py b/Examples/Tests/laser_injection/analysis_2d.py index 4424fe134bc..33b87823ebd 100755 --- a/Examples/Tests/laser_injection/analysis_2d.py +++ b/Examples/Tests/laser_injection/analysis_2d.py @@ -17,21 +17,17 @@ # the simulation and it compares it with theory. It also checks that the # central frequency of the Fourier transform is the expected one. -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import yt from mpl_toolkits.axes_grid1 import make_axes_locatable from scipy.signal import hilbert -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # Maximum acceptable error for this test relative_error_threshold = 0.05 @@ -39,147 +35,166 @@ small_num = 1.0e-8 # Physical parameters -um = 1.e-6 -fs = 1.e-15 +um = 1.0e-6 +fs = 1.0e-15 c = 299792458 # Parameters of the gaussian beam -wavelength = 1.*um -w0 = 5.*um -tt = 10.*fs -x_c = 10.*um -t_c = 24.*fs +wavelength = 1.0 * um +w0 = 5.0 * um +tt = 10.0 * fs +x_c = 10.0 * um +t_c = 24.0 * fs # foc_dist = 13.109*um (not actually used) E_max = 4e12 # laser direction -dir_vector = np.array([2.,0,1.0]) +dir_vector = np.array([2.0, 0, 1.0]) dir_vector /= np.linalg.norm(dir_vector) -rot_angle = np.arctan(dir_vector[2]/dir_vector[0]) +rot_angle = np.arctan(dir_vector[2] / dir_vector[0]) # polarization vector -pol_vector = np.array([1.0,1.0,-2.0]) +pol_vector = np.array([1.0, 1.0, -2.0]) pol_vector /= np.linalg.norm(pol_vector) + # Calculates the envelope of a Gaussian beam -def gauss_env(T,XX,ZZ): - '''Function to compute the theory for the envelope - ''' +def gauss_env(T, XX, ZZ): + """Function to compute the theory for the envelope""" - Z = np.cos(rot_angle)*(XX-x_c) + np.sin(rot_angle)*ZZ - X = -np.sin(rot_angle)*(XX-x_c) + np.cos(rot_angle)*ZZ + Z = np.cos(rot_angle) * (XX - x_c) + np.sin(rot_angle) * ZZ + X = -np.sin(rot_angle) * (XX - x_c) + np.cos(rot_angle) * ZZ - inv_tau2 = 1./tt/tt - inv_w_2 = 1.0/(w0*w0) - exp_arg = - (X*X)*inv_w_2 - inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + inv_w_2 = 1.0 / (w0 * w0) + exp_arg = -(X * X) * inv_w_2 - inv_tau2 / c / c * (Z - T * c) * (Z - T * c) return E_max * np.real(np.exp(exp_arg)) + # Checks envelope and central frequency for a given laser component -def check_component(data, component, t_env_theory, coeff, X,Z,dx,dz): +def check_component(data, component, t_env_theory, coeff, X, Z, dx, dz): print("*** Checking " + component + " ***") - field = data['boxlib', component].v.squeeze() + field = data["boxlib", component].v.squeeze() env = abs(hilbert(field)) - env_theory = t_env_theory*np.abs(coeff) + env_theory = t_env_theory * np.abs(coeff) # Plot results - fig = plt.figure(figsize=(12,6)) - - ax1 = fig.add_subplot(221, aspect='equal') - ax1.set_title('PIC field') - p1 = ax1.pcolormesh(X,Z,field) - cax1 = make_axes_locatable(ax1).append_axes('right', size='5%', pad=0.05) - fig.colorbar(p1, cax=cax1, orientation='vertical') - - ax2 = fig.add_subplot(222, aspect='equal') - ax2.set_title('PIC envelope') - p2 = ax2.pcolormesh(X,Z,env) - cax2 = make_axes_locatable(ax2).append_axes('right', size='5%', pad=0.05) - fig.colorbar(p2, cax=cax2, orientation='vertical') - - ax3 = fig.add_subplot(223, aspect='equal') - ax3.set_title('Theory envelope') - p3 = ax3.pcolormesh(X,Z,env_theory) - cax3 = make_axes_locatable(ax3).append_axes('right', size='5%', pad=0.05) - fig.colorbar(p3, cax=cax3, orientation='vertical') - - ax4 = fig.add_subplot(224, aspect='equal') - ax4.set_title('Difference') - p4 = ax4.pcolormesh(X,Z,env-env_theory) - cax4 = make_axes_locatable(ax4).append_axes('right', size='5%', pad=0.05) - fig.colorbar(p4, cax=cax4, orientation='vertical') + fig = plt.figure(figsize=(12, 6)) + + ax1 = fig.add_subplot(221, aspect="equal") + ax1.set_title("PIC field") + p1 = ax1.pcolormesh(X, Z, field) + cax1 = make_axes_locatable(ax1).append_axes("right", size="5%", pad=0.05) + fig.colorbar(p1, cax=cax1, orientation="vertical") + + ax2 = fig.add_subplot(222, aspect="equal") + ax2.set_title("PIC envelope") + p2 = ax2.pcolormesh(X, Z, env) + cax2 = make_axes_locatable(ax2).append_axes("right", size="5%", pad=0.05) + fig.colorbar(p2, cax=cax2, orientation="vertical") + + ax3 = fig.add_subplot(223, aspect="equal") + ax3.set_title("Theory envelope") + p3 = ax3.pcolormesh(X, Z, env_theory) + cax3 = make_axes_locatable(ax3).append_axes("right", size="5%", pad=0.05) + fig.colorbar(p3, cax=cax3, orientation="vertical") + + ax4 = fig.add_subplot(224, aspect="equal") + ax4.set_title("Difference") + p4 = ax4.pcolormesh(X, Z, env - env_theory) + cax4 = make_axes_locatable(ax4).append_axes("right", size="5%", pad=0.05) + fig.colorbar(p4, cax=cax4, orientation="vertical") plt.tight_layout() - plt.savefig("plt_" + component + ".png", bbox_inches='tight') + plt.savefig("plt_" + component + ".png", bbox_inches="tight") - if(np.abs(coeff) < small_num): + if np.abs(coeff) < small_num: is_field_zero = np.sum(np.abs(env)) < small_num - if is_field_zero : + if is_field_zero: print("[OK] Field component expected to be 0 is ~ 0") - else : + else: print("[FAIL] Field component expected to be 0 is NOT ~ 0") - assert(is_field_zero) + assert is_field_zero print("******\n") return - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env_theory)) + relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env_theory)) is_env_ok = relative_error_env < relative_error_threshold - if is_env_ok : - print("[OK] Relative error envelope: {:6.3f} %".format(relative_error_env*100)) - else : - print("[FAIL] Relative error envelope: {:6.3f} %".format(relative_error_env*100)) - assert(is_env_ok) + if is_env_ok: + print( + "[OK] Relative error envelope: {:6.3f} %".format(relative_error_env * 100) + ) + else: + print( + "[FAIL] Relative error envelope: {:6.3f} %".format(relative_error_env * 100) + ) + assert is_env_ok fft_field = np.fft.fft2(field) - freq_rows = np.fft.fftfreq(fft_field.shape[0],dx/c) - freq_cols = np.fft.fftfreq(fft_field.shape[1],dz/c) + freq_rows = np.fft.fftfreq(fft_field.shape[0], dx / c) + freq_cols = np.fft.fftfreq(fft_field.shape[1], dz / c) pos_max = np.unravel_index(np.abs(fft_field).argmax(), fft_field.shape) - freq = np.sqrt((freq_rows[pos_max[0]])**2 + (freq_cols[pos_max[1]]**2)) - exp_freq = c/wavelength + freq = np.sqrt((freq_rows[pos_max[0]]) ** 2 + (freq_cols[pos_max[1]] ** 2)) + exp_freq = c / wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq + relative_error_freq = np.abs(freq - exp_freq) / exp_freq is_freq_ok = relative_error_freq < relative_error_threshold - if is_freq_ok : - print("[OK] Relative error frequency: {:6.3f} %".format(relative_error_freq*100)) - else : - print("[FAIL] Relative error frequency: {:6.3f} %".format(relative_error_freq*100)) - assert(is_freq_ok) + if is_freq_ok: + print( + "[OK] Relative error frequency: {:6.3f} %".format(relative_error_freq * 100) + ) + else: + print( + "[FAIL] Relative error frequency: {:6.3f} %".format( + relative_error_freq * 100 + ) + ) + assert is_freq_ok print("******\n") + def check_laser(filename): ds = yt.load(filename) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() + if "force_periodicity" in dir(ds): + ds.force_periodicity() x = np.linspace( - ds.domain_left_edge[0].v, - ds.domain_right_edge[0].v, - ds.domain_dimensions[0]) + ds.domain_left_edge[0].v, ds.domain_right_edge[0].v, ds.domain_dimensions[0] + ) - dx = (ds.domain_right_edge[0].v-ds.domain_left_edge[0].v)/(ds.domain_dimensions[0]-1) + dx = (ds.domain_right_edge[0].v - ds.domain_left_edge[0].v) / ( + ds.domain_dimensions[0] - 1 + ) z = np.linspace( - ds.domain_left_edge[1].v, - ds.domain_right_edge[1].v, - ds.domain_dimensions[1]) + ds.domain_left_edge[1].v, ds.domain_right_edge[1].v, ds.domain_dimensions[1] + ) - dz = (ds.domain_right_edge[1].v-ds.domain_left_edge[1].v)/(ds.domain_dimensions[1]-1) + dz = (ds.domain_right_edge[1].v - ds.domain_left_edge[1].v) / ( + ds.domain_dimensions[1] - 1 + ) - X, Z = np.meshgrid(x, z, indexing='ij') + X, Z = np.meshgrid(x, z, indexing="ij") # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(),X,Z)+gauss_env(-t_c+ds.current_time.to_value(),X,Z) + env_theory = gauss_env(+t_c - ds.current_time.to_value(), X, Z) + gauss_env( + -t_c + ds.current_time.to_value(), X, Z + ) # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) + all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions + ) b_vector = np.cross(dir_vector, pol_vector) @@ -190,20 +205,22 @@ def check_laser(filename): pol_vector[2], b_vector[0], b_vector[1], - b_vector[2]] + b_vector[2], + ] - field_facts = [1, 1, 1, 1/c, 1/c, 1/c] + field_facts = [1, 1, 1, 1 / c, 1 / c, 1 / c] for comp, coeff, field_fact in zip(components, coeffs, field_facts): - check_component(all_data_level_0, comp, field_fact*env_theory, coeff, X, Z, dx, dz) + check_component( + all_data_level_0, comp, field_fact * env_theory, coeff, X, Z, dx, dz + ) + def main(): filename_end = sys.argv[1] check_laser(filename_end) - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) if __name__ == "__main__": main() diff --git a/Examples/Tests/laser_injection/analysis_3d.py b/Examples/Tests/laser_injection/analysis_3d.py new file mode 100755 index 00000000000..2ce123169d5 --- /dev/null +++ b/Examples/Tests/laser_injection/analysis_3d.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Andrew Myers, Jean-Luc Vay, Maxence Thevenet +# Remi Lehe, Weiqun Zhang +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +# you can save an image to be displayed on the website +t = np.arange(0.0, 2.0, 0.01) +s = 1 + np.sin(2 * np.pi * t) +plt.plot(t, s) +plt.savefig("laser_analysis.png") diff --git a/Examples/Tests/laser_injection/analysis_default_regression.py b/Examples/Tests/laser_injection/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/laser_injection/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/laser_injection/analysis_laser.py b/Examples/Tests/laser_injection/analysis_laser.py deleted file mode 100755 index 8dde8d6e96d..00000000000 --- a/Examples/Tests/laser_injection/analysis_laser.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Andrew Myers, Jean-Luc Vay, Maxence Thevenet -# Remi Lehe, Weiqun Zhang -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -import os -import sys - -import matplotlib - -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import numpy as np - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# you can save an image to be displayed on the website -t = np.arange(0.0, 2.0, 0.01) -s = 1 + np.sin(2*np.pi*t) -plt.plot(t, s) -plt.savefig("laser_analysis.png") - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/laser_injection/inputs_3d_rt b/Examples/Tests/laser_injection/inputs_3d_rt deleted file mode 100644 index 72464f86aaf..00000000000 --- a/Examples/Tests/laser_injection/inputs_3d_rt +++ /dev/null @@ -1,59 +0,0 @@ -# Maximum number of time steps -max_step = 1000 - -# number of grid points -amr.n_cell = 32 32 240 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 16 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -12.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 12.e-6 - -# Boundary condition -boundary.field_lo = periodic periodic pec -boundary.field_hi = periodic periodic pec - -warpx.serialize_initial_conditions = 1 - -# Verbosity -warpx.verbose = 1 - -# Algorithms -warpx.use_filter = 0 - -# CFL -warpx.cfl = 1.0 - -# Order of particle shape factors -algo.particle_shape = 1 - -# Laser -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. 9.e-6 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 15.e-15 # The duration of the laser (in seconds) -laser1.profile_t_peak = 30.e-15 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 20 -diag1.diag_type = Full -diag1.fields_to_plot = jx jy jz Ex Ey Ez Bx Bz - -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light diff --git a/Examples/Tests/laser_injection/inputs_1d_rt b/Examples/Tests/laser_injection/inputs_test_1d_laser_injection similarity index 100% rename from Examples/Tests/laser_injection/inputs_1d_rt rename to Examples/Tests/laser_injection/inputs_test_1d_laser_injection diff --git a/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit b/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit new file mode 100644 index 00000000000..758e2cebaa1 --- /dev/null +++ b/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit @@ -0,0 +1,79 @@ +# Maximum number of time steps +max_step = 240 + +# number of grid points +amr.n_cell = 352 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 32 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 1 +geometry.prob_lo = -15.e-6 # physical domain +geometry.prob_hi = 15.e-6 + +boundary.field_lo = pec +boundary.field_hi = pec + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# implicit evolve scheme +algo.evolve_scheme = "semi_implicit_em" +# +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 21 +newton.relative_tolerance = 1.0e-8 +newton.require_convergence = true +# +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-4 + +# CFL +warpx.cfl = 0.9 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0.e-6 0.e-6 0.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 1. 1. 0. # The main polarization vector +laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) +laser1.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 24.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 13.109e-6 # Focal distance from the antenna (in meters) + # With this focal distance the laser is at focus + # at the end of the simulation. + +# Diagnostics +diagnostics.diags_names = diag1 openpmd +diag1.intervals = 20 +diag1.diag_type = Full + +openpmd.intervals = 20 +openpmd.diag_type = Full +openpmd.format = openpmd + +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.start_moving_window_step = 20 +warpx.end_moving_window_step = 200 diff --git a/Examples/Tests/laser_injection/inputs_2d_rt b/Examples/Tests/laser_injection/inputs_test_2d_laser_injection similarity index 100% rename from Examples/Tests/laser_injection/inputs_2d_rt rename to Examples/Tests/laser_injection/inputs_test_2d_laser_injection diff --git a/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit b/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit new file mode 100644 index 00000000000..be6a704b171 --- /dev/null +++ b/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit @@ -0,0 +1,75 @@ +# Maximum number of time steps +max_step = 240 + +# number of grid points +amr.n_cell = 480 352 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 32 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -20.e-6 -15.e-6 # physical domain +geometry.prob_hi = 20.e-6 15.e-6 + +boundary.field_lo = pec periodic +boundary.field_hi = pec periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# implicit evolve scheme +algo.evolve_scheme = "semi_implicit_em" +# +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 21 +newton.relative_tolerance = 1.0e-8 +newton.require_convergence = true +# +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-4 + +# CFL +warpx.cfl = 1.0 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 10.e-6 0.e-6 0.e-6 # This point is on the laser plane +laser1.direction = 2. 0. 1. # The plane normal direction +laser1.polarization = 1. 1. -2. # The main polarization vector +laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) +laser1.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 24.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 13.109e-6 # Focal distance from the antenna (in meters) + # With this focal distance the laser is at focus + # at the end of the simulation. + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 240 +diag1.diag_type = Full + +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = x +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.start_moving_window_step = 20 +warpx.end_moving_window_step = 200 diff --git a/Examples/Tests/laser_injection/inputs_test_3d_laser_injection b/Examples/Tests/laser_injection/inputs_test_3d_laser_injection new file mode 100644 index 00000000000..250a0160881 --- /dev/null +++ b/Examples/Tests/laser_injection/inputs_test_3d_laser_injection @@ -0,0 +1,59 @@ +# Maximum number of time steps +max_step = 20 + +# number of grid points +amr.n_cell = 32 32 240 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 16 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -20.e-6 -20.e-6 -12.e-6 # physical domain +geometry.prob_hi = 20.e-6 20.e-6 12.e-6 + +# Boundary condition +boundary.field_lo = periodic periodic pec +boundary.field_hi = periodic periodic pec + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. 9.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 15.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 30.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in meters) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = jx jy jz Ex Ey Ez Bx Bz + +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # in units of the speed of light diff --git a/Examples/Tests/laser_injection_from_file/CMakeLists.txt b/Examples/Tests/laser_injection_from_file/CMakeLists.txt new file mode 100644 index 00000000000..d585160bc8f --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/CMakeLists.txt @@ -0,0 +1,142 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_laser_injection_from_lasy_file_prepare # name + 1 # dims + 1 # nprocs + inputs_test_1d_laser_injection_from_lasy_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_laser_injection_from_lasy_file # name + 1 # dims + 1 # nprocs + inputs_test_1d_laser_injection_from_lasy_file # inputs + "analysis_1d.py diags/diag1000251" # analysis + "analysis_default_regression.py --path diags/diag1000251" # checksum + test_1d_laser_injection_from_lasy_file_prepare # dependency +) + +add_warpx_test( + test_1d_laser_injection_from_lasy_file_boost_prepare # name + 1 # dims + 1 # nprocs + inputs_test_1d_laser_injection_from_lasy_file_boost_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_laser_injection_from_lasy_file_boost # name + 1 # dims + 1 # nprocs + inputs_test_1d_laser_injection_from_lasy_file_boost # inputs + "analysis_1d_boost.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + test_1d_laser_injection_from_lasy_file_boost_prepare # dependency +) + +add_warpx_test( + test_2d_laser_injection_from_binary_file_prepare # name + 2 # dims + 1 # nprocs + inputs_test_2d_laser_injection_from_binary_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_injection_from_binary_file # name + 2 # dims + 1 # nprocs + inputs_test_2d_laser_injection_from_binary_file # inputs + "analysis_2d_binary.py diags/diag1000250" # analysis + "analysis_default_regression.py --path diags/diag1000250" # checksum + test_2d_laser_injection_from_binary_file_prepare # dependency +) + +add_warpx_test( + test_2d_laser_injection_from_lasy_file_prepare # name + 2 # dims + 1 # nprocs + inputs_test_2d_laser_injection_from_lasy_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_laser_injection_from_lasy_file # name + 2 # dims + 1 # nprocs + inputs_test_2d_laser_injection_from_lasy_file # inputs + "analysis_2d.py diags/diag1000251" # analysis + "analysis_default_regression.py --path diags/diag1000251" # checksum + test_2d_laser_injection_from_lasy_file_prepare # dependency +) + +add_warpx_test( + test_3d_laser_injection_from_lasy_file_prepare # name + 3 # dims + 1 # nprocs + inputs_test_3d_laser_injection_from_lasy_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_laser_injection_from_lasy_file # name + 3 # dims + 1 # nprocs + inputs_test_3d_laser_injection_from_lasy_file # inputs + "analysis_3d.py diags/diag1000251" # analysis + "analysis_default_regression.py --path diags/diag1000251" # checksum + test_3d_laser_injection_from_lasy_file_prepare # dependency +) + +add_warpx_test( + test_rz_laser_injection_from_lasy_file_prepare # name + RZ # dims + 1 # nprocs + inputs_test_rz_laser_injection_from_lasy_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_laser_injection_from_lasy_file # name + RZ # dims + 1 # nprocs + inputs_test_rz_laser_injection_from_lasy_file # inputs + "analysis_rz.py diags/diag1000252" # analysis + "analysis_default_regression.py --path diags/diag1000252" # checksum + test_rz_laser_injection_from_lasy_file_prepare # dependency +) + +add_warpx_test( + test_rz_laser_injection_from_RZ_lasy_file_prepare # name + RZ # dims + 1 # nprocs + inputs_test_rz_laser_injection_from_RZ_lasy_file_prepare.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_laser_injection_from_RZ_lasy_file # name + RZ # dims + 1 # nprocs + inputs_test_rz_laser_injection_from_RZ_lasy_file # inputs + "analysis_from_RZ_file.py diags/diag1000612" # analysis + "analysis_default_regression.py --path diags/diag1000612" # checksum + test_rz_laser_injection_from_RZ_lasy_file_prepare # dependency +) diff --git a/Examples/Tests/laser_injection_from_file/analysis_1d.py b/Examples/Tests/laser_injection_from_file/analysis_1d.py index a595a912858..bd6a78f5949 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_1d.py +++ b/Examples/Tests/laser_injection_from_file/analysis_1d.py @@ -9,138 +9,102 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external lasy file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in 1D, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, epsilon_0 from scipy.signal import hilbert -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + # Function for the envelope -def gauss_env(T,Z): +def gauss_env(T, Z): # Function to compute the theory for the envelope - inv_tau2 = 1./tt/tt - exp_arg = - inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + exp_arg = -inv_tau2 / c / c * (Z - T * c) * (Z - T * c) return E_max * np.real(np.exp(exp_arg)) -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - dt = ds.current_time.to_value()/steps - - z = np.linspace( - ds.domain_left_edge[0].v, - ds.domain_right_edge[0].v, - ds.domain_dimensions[0]) - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), z)+gauss_env(-t_c+ds.current_time.to_value(),z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Ey'].v.squeeze() - env = abs(hilbert(F_laser)) - - # Plot results - plt.figure(figsize=(8,8)) - plt.subplot(221) - plt.title('PIC field') - plt.plot(z, F_laser) - plt.subplot(222) - plt.title('PIC envelope') - plt.plot(z, env) - plt.subplot(223) - plt.title('Theory envelope') - plt.plot(z,env_theory) - plt.subplot(224) - plt.title('Difference') - plt.plot(z,env-env_theory) - - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_z = np.fft.fftfreq(F_laser.shape[0],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.abs(freq_z[pos_max[0]]) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.1d_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000251/", "comp_unf.pdf", 251) - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import GaussianProfile - - # Create a laser using lasy - pol = (1, 0) - profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) - dim = "xyt" - lo = (-25e-6, -25e-6, -20e-15) - hi = (+25e-6, +25e-6, +20e-15) - npoints = (100, 100, 100) - laser = Laser(dim, lo, hi, npoints, profile) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("gaussianlaser3d") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000251/" - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 251 +ds = yt.load(filename) +dt = ds.current_time.to_value() / steps + +z = np.linspace( + ds.domain_left_edge[0].v, ds.domain_right_edge[0].v, ds.domain_dimensions[0] +) + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), z) + gauss_env( + -t_c + ds.current_time.to_value(), z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Ey"].v.squeeze() +env = abs(hilbert(F_laser)) + +# Plot results +plt.figure(figsize=(8, 8)) +plt.subplot(221) +plt.title("PIC field") +plt.plot(z, F_laser) +plt.subplot(222) +plt.title("PIC envelope") +plt.plot(z, env) +plt.subplot(223) +plt.title("Theory envelope") +plt.plot(z, env_theory) +plt.subplot(224) +plt.title("Difference") +plt.plot(z, env - env_theory) + +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_z = np.fft.fftfreq(F_laser.shape[0], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.abs(freq_z[pos_max[0]]) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_1d_boost.py b/Examples/Tests/laser_injection_from_file/analysis_1d_boost.py index 1fcc46d54d3..b51b32714de 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_1d_boost.py +++ b/Examples/Tests/laser_injection_from_file/analysis_1d_boost.py @@ -9,139 +9,102 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external lasy file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in 1D, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, epsilon_0 from scipy.signal import hilbert -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + # Function for the envelope -def gauss_env(T,Z): +def gauss_env(T, Z): # Function to compute the theory for the envelope - inv_tau2 = 1./tt/tt - exp_arg = - inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + exp_arg = -inv_tau2 / c / c * (Z - T * c) * (Z - T * c) return E_max * np.real(np.exp(exp_arg)) -def do_analysis(fname, compname): - ds = yt.load(fname) - dz = (ds.domain_right_edge[0].v-ds.domain_left_edge[0].v)/ds.domain_dimensions[0] - dt = dz/c - - z = np.linspace( - ds.domain_left_edge[0].v, - ds.domain_right_edge[0].v, - ds.domain_dimensions[0]) - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), z)+gauss_env(-t_c+ds.current_time.to_value(),z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Ey'].v.squeeze() - env = abs(hilbert(F_laser)) - - # Plot results - plt.figure(figsize=(8,8)) - plt.subplot(221) - plt.title('PIC field') - plt.plot(z, F_laser) - plt.subplot(222) - plt.title('PIC envelope') - plt.plot(z, env) - plt.subplot(223) - plt.title('Theory envelope') - plt.plot(z,env_theory) - plt.subplot(224) - plt.title('Difference') - plt.plot(z,env-env_theory) - - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_z = np.fft.fftfreq(F_laser.shape[0],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.abs(freq_z[pos_max[0]]) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.1d_boost_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000001/", "comp_unf.pdf") - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import GaussianProfile - - # Create a laser using lasy - pol = (1, 0) - profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) - dim = "xyt" - lo = (-25e-6, -25e-6, -20e-15) - hi = (+25e-6, +25e-6, +20e-15) - npoints = (100, 100, 100) - laser = Laser(dim, lo, hi, npoints, profile) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("gaussianlaser3d") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000001/" - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +ds = yt.load(filename) +dz = (ds.domain_right_edge[0].v - ds.domain_left_edge[0].v) / ds.domain_dimensions[0] +dt = dz / c + +z = np.linspace( + ds.domain_left_edge[0].v, ds.domain_right_edge[0].v, ds.domain_dimensions[0] +) + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), z) + gauss_env( + -t_c + ds.current_time.to_value(), z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Ey"].v.squeeze() +env = abs(hilbert(F_laser)) + +# Plot results +plt.figure(figsize=(8, 8)) +plt.subplot(221) +plt.title("PIC field") +plt.plot(z, F_laser) +plt.subplot(222) +plt.title("PIC envelope") +plt.plot(z, env) +plt.subplot(223) +plt.title("Theory envelope") +plt.plot(z, env_theory) +plt.subplot(224) +plt.title("Difference") +plt.plot(z, env - env_theory) + +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_z = np.fft.fftfreq(F_laser.shape[0], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.abs(freq_z[pos_max[0]]) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_2d.py b/Examples/Tests/laser_injection_from_file/analysis_2d.py index 23b7e12dbcd..21f5c186b7a 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_2d.py +++ b/Examples/Tests/laser_injection_from_file/analysis_2d.py @@ -9,156 +9,128 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external lasy file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in 2D, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, epsilon_0 from scipy.signal import hilbert -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + # Function for the envelope def gauss_env(T, X, Y, Z): # Function to compute the theory for the envelope - inv_tau2 = 1./tt/tt - inv_w_2 = 1.0/(w0*w0) - exp_arg = - (X*X)*inv_w_2 - (Y*Y)*inv_w_2- inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + inv_w_2 = 1.0 / (w0 * w0) + exp_arg = ( + -(X * X) * inv_w_2 + - (Y * Y) * inv_w_2 + - inv_tau2 / c / c * (Z - T * c) * (Z - T * c) + ) return E_max * np.real(np.exp(exp_arg)) -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - dt = ds.current_time.to_value()/steps - - # Define 3D meshes - x = np.linspace( - ds.domain_left_edge[0], - ds.domain_right_edge[0], - ds.domain_dimensions[0]).v - y = np.linspace( - ds.domain_left_edge[1], - ds.domain_right_edge[1], - ds.domain_dimensions[1]).v - z = np.linspace( - ds.domain_left_edge[ds.dimensionality-1], - ds.domain_right_edge[ds.dimensionality-1], - ds.domain_dimensions[ds.dimensionality-1]).v - X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing='ij') - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), X,Y,Z)+gauss_env(-t_c+ds.current_time.to_value(), X,Y,Z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Ey'].v.squeeze() - env = abs(hilbert(F_laser)) - extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - env_theory_slice= env_theory[:,env_theory.shape[1]//2, :] - - # Plot results - plt.figure(figsize=(8,6)) - plt.subplot(221) - plt.title('PIC field') - plt.imshow(F_laser, extent=extent) - plt.colorbar() - plt.subplot(222) - plt.title('PIC envelope') - plt.imshow(env, extent=extent) - plt.colorbar() - plt.subplot(223) - plt.title('Theory envelope') - plt.imshow(env_theory_slice, extent=extent) - plt.colorbar() - plt.subplot(224) - plt.title('Difference') - plt.imshow(env-env_theory_slice, extent=extent) - plt.colorbar() - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory_slice)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_x = np.fft.fftfreq(F_laser.shape[0],dt) - freq_z = np.fft.fftfreq(F_laser.shape[1],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.sqrt((freq_x[pos_max[0]])**2 + (freq_z[pos_max[1]])**2) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.2d_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000251/", "comp_unf.pdf", 251) - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import GaussianProfile - - # Create a laser using lasy - pol = (1, 0) - profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) - dim = "xyt" - lo = (-25e-6, -25e-6, -20e-15) - hi = (+25e-6, +25e-6, +20e-15) - npoints = (100, 100, 100) - laser = Laser(dim, lo, hi, npoints, profile) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("gaussianlaser3d") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000251/" - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 251 +ds = yt.load(filename) +dt = ds.current_time.to_value() / steps + +# Define 3D meshes +x = np.linspace( + ds.domain_left_edge[0], ds.domain_right_edge[0], ds.domain_dimensions[0] +).v +y = np.linspace( + ds.domain_left_edge[1], ds.domain_right_edge[1], ds.domain_dimensions[1] +).v +z = np.linspace( + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_dimensions[ds.dimensionality - 1], +).v +X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing="ij") + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), X, Y, Z) + gauss_env( + -t_c + ds.current_time.to_value(), X, Y, Z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Ey"].v.squeeze() +env = abs(hilbert(F_laser)) +extent = [ + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], +] +env_theory_slice = env_theory[:, env_theory.shape[1] // 2, :] + +# Plot results +plt.figure(figsize=(8, 6)) +plt.subplot(221) +plt.title("PIC field") +plt.imshow(F_laser, extent=extent) +plt.colorbar() +plt.subplot(222) +plt.title("PIC envelope") +plt.imshow(env, extent=extent) +plt.colorbar() +plt.subplot(223) +plt.title("Theory envelope") +plt.imshow(env_theory_slice, extent=extent) +plt.colorbar() +plt.subplot(224) +plt.title("Difference") +plt.imshow(env - env_theory_slice, extent=extent) +plt.colorbar() +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory_slice)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_x = np.fft.fftfreq(F_laser.shape[0], dt) +freq_z = np.fft.fftfreq(F_laser.shape[1], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.sqrt((freq_x[pos_max[0]]) ** 2 + (freq_z[pos_max[1]]) ** 2) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py b/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py index c5bdd84d023..d223026a073 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py +++ b/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py @@ -9,207 +9,135 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external binary file. -# -# - Generate an input binary file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external binary file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in 2D, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.signal import hilbert -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 c = 299792458 -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 6.*um -tt = 10.*fs -x_c = 0.*um -t_c = 20.*fs -foc_dist = 10*um +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 6.0 * um +tt = 10.0 * fs +x_c = 0.0 * um +t_c = 20.0 * fs +foc_dist = 10 * um E_max = 1e12 -rot_angle = -np.pi/4.0 +rot_angle = -np.pi / 4.0 -#Parameters of the tx grid -x_l = -12.0*um -x_r = 12.0*um +# Parameters of the tx grid +x_l = -12.0 * um +x_r = 12.0 * um x_points = 480 -t_l = 0.0*fs -t_r = 40.0*fs +t_l = 0.0 * fs +t_r = 40.0 * fs t_points = 400 tcoords = np.linspace(t_l, t_r, t_points) xcoords = np.linspace(x_l, x_r, x_points) -def gauss(T,X,Y,opt): - """Compute the electric field for a Gaussian laser pulse. - This is used to write the binary input file. - """ - - k0 = 2.0*np.pi/wavelength - inv_tau2 = 1./tt/tt - osc_phase = k0*c*(T-t_c) - - diff_factor = 1.0 + 1.0j* foc_dist * 2/(k0*w0*w0) - inv_w_2 = 1.0/(w0*w0*diff_factor) - - pre_fact = np.exp(1.0j * osc_phase) - - if opt == '3d': - pre_fact = pre_fact/diff_factor - else: - pre_fact = pre_fact/np.sqrt(diff_factor) - - exp_arg = - (X*X + Y*Y)*inv_w_2 - inv_tau2 * (T-t_c)*(T-t_c) - - return np.real(pre_fact * np.exp(exp_arg)) # Function for the envelope -def gauss_env(T,XX,ZZ): - '''Function to compute the theory for the envelope - ''' +def gauss_env(T, XX, ZZ): + """Function to compute the theory for the envelope""" - X = np.cos(rot_angle)*XX + np.sin(rot_angle)*ZZ - Z = -np.sin(rot_angle)*XX + np.cos(rot_angle)*ZZ + X = np.cos(rot_angle) * XX + np.sin(rot_angle) * ZZ + Z = -np.sin(rot_angle) * XX + np.cos(rot_angle) * ZZ - inv_tau2 = 1./tt/tt - inv_w_2 = 1.0/(w0*w0) - exp_arg = - (X*X)*inv_w_2 - inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + inv_w_2 = 1.0 / (w0 * w0) + exp_arg = -(X * X) * inv_w_2 - inv_tau2 / c / c * (Z - T * c) * (Z - T * c) return E_max * np.real(np.exp(exp_arg)) -def write_file(fname, x, y, t, E): - """ For a given filename fname, space coordinates x and y, time coordinate t - and field E, write a WarpX-compatible input binary file containing the - profile of the laser pulse. This function should be used in the case - of a uniform spatio-temporal mesh - """ - - with open(fname, 'wb') as file: - flag_unif = 1 - file.write(flag_unif.to_bytes(1, byteorder='little')) - file.write((len(t)).to_bytes(4, byteorder='little', signed=False)) - file.write((len(x)).to_bytes(4, byteorder='little', signed=False)) - file.write((len(y)).to_bytes(4, byteorder='little', signed=False)) - file.write(t[0].tobytes()) - file.write(t[-1].tobytes()) - file.write(x[0].tobytes()) - file.write(x[-1].tobytes()) - if len(y) == 1 : - file.write(y[0].tobytes()) - else : - file.write(y[0].tobytes()) - file.write(y[-1].tobytes()) - file.write(E.tobytes()) - -def create_gaussian_2d(): - T, X, Y = np.meshgrid(tcoords, xcoords, np.array([0.0]), indexing='ij') - E_t = gauss(T,X,Y,'2d') - write_file("gauss_2d", xcoords, np.array([0.0]), tcoords, E_t) - -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - - dt = ds.current_time.to_value()/steps - - # Define 2D meshes - x = np.linspace( - ds.domain_left_edge[0], - ds.domain_right_edge[0], - ds.domain_dimensions[0]).v - z = np.linspace( - ds.domain_left_edge[ds.dimensionality-1], - ds.domain_right_edge[ds.dimensionality-1], - ds.domain_dimensions[ds.dimensionality-1]).v - X, Z = np.meshgrid(x, z, sparse=False, indexing='ij') - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), X,Z)+gauss_env(-t_c+ds.current_time.to_value(), X,Z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Ey'].v.squeeze() - env = abs(hilbert(F_laser)) - extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - - # Plot results - plt.figure(figsize=(8,6)) - plt.subplot(221) - plt.title('PIC field') - plt.imshow(F_laser, extent=extent) - plt.colorbar() - plt.subplot(222) - plt.title('PIC envelope') - plt.imshow(env, extent=extent) - plt.colorbar() - plt.subplot(223) - plt.title('Theory envelope') - plt.imshow(env_theory, extent=extent) - plt.colorbar() - plt.subplot(224) - plt.title('Difference') - plt.imshow(env-env_theory, extent=extent) - plt.colorbar() - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fft2(F_laser) - - freq_rows = np.fft.fftfreq(F_laser.shape[0],dt) - freq_cols = np.fft.fftfreq(F_laser.shape[1],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.sqrt((freq_rows[pos_max[0]])**2 + (freq_cols[pos_max[1]]**2)) - exp_freq = c/wavelength - - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - create_gaussian_2d() - os.system("./" + executable + " inputs.2d_test_binary diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000250/", "comp_unf.pdf", 250) - - -def main() : - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000250/" - test_name = "LaserInjectionFromBINARYFile" - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 250 +ds = yt.load(filename) + +dt = ds.current_time.to_value() / steps + +# Define 2D meshes +x = np.linspace( + ds.domain_left_edge[0], ds.domain_right_edge[0], ds.domain_dimensions[0] +).v +z = np.linspace( + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_dimensions[ds.dimensionality - 1], +).v +X, Z = np.meshgrid(x, z, sparse=False, indexing="ij") + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), X, Z) + gauss_env( + -t_c + ds.current_time.to_value(), X, Z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Ey"].v.squeeze() +env = abs(hilbert(F_laser)) +extent = [ + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], +] + +# Plot results +plt.figure(figsize=(8, 6)) +plt.subplot(221) +plt.title("PIC field") +plt.imshow(F_laser, extent=extent) +plt.colorbar() +plt.subplot(222) +plt.title("PIC envelope") +plt.imshow(env, extent=extent) +plt.colorbar() +plt.subplot(223) +plt.title("Theory envelope") +plt.imshow(env_theory, extent=extent) +plt.colorbar() +plt.subplot(224) +plt.title("Difference") +plt.imshow(env - env_theory, extent=extent) +plt.colorbar() +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fft2(F_laser) + +freq_rows = np.fft.fftfreq(F_laser.shape[0], dt) +freq_cols = np.fft.fftfreq(F_laser.shape[1], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.sqrt((freq_rows[pos_max[0]]) ** 2 + (freq_cols[pos_max[1]] ** 2)) +exp_freq = c / wavelength + +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_3d.py b/Examples/Tests/laser_injection_from_file/analysis_3d.py index a66f761b9b1..cc19b742134 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_3d.py +++ b/Examples/Tests/laser_injection_from_file/analysis_3d.py @@ -9,160 +9,134 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external lasy file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in 3D, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, epsilon_0 from scipy.signal import hilbert -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + # Function for the envelope def gauss_env(T, X, Y, Z): # Function to compute the theory for the envelope - inv_tau2 = 1./tt/tt - inv_w_2 = 1.0/(w0*w0) - exp_arg = - (X*X)*inv_w_2 - (Y*Y)*inv_w_2- inv_tau2 / c/c * (Z-T*c)*(Z-T*c) + inv_tau2 = 1.0 / tt / tt + inv_w_2 = 1.0 / (w0 * w0) + exp_arg = ( + -(X * X) * inv_w_2 + - (Y * Y) * inv_w_2 + - inv_tau2 / c / c * (Z - T * c) * (Z - T * c) + ) return E_max * np.real(np.exp(exp_arg)) -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - dt = ds.current_time.to_value()/steps - - # Define 3D meshes - x = np.linspace( - ds.domain_left_edge[0], - ds.domain_right_edge[0], - ds.domain_dimensions[0]).v - y = np.linspace( - ds.domain_left_edge[1], - ds.domain_right_edge[1], - ds.domain_dimensions[1]).v - z = np.linspace( - ds.domain_left_edge[ds.dimensionality-1], - ds.domain_right_edge[ds.dimensionality-1], - ds.domain_dimensions[ds.dimensionality-1]).v - X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing='ij') - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), X,Y,Z)+gauss_env(-t_c+ds.current_time.to_value(), X,Y,Z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Ey'].v.squeeze() - env = abs(hilbert(F_laser)) - extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - - F_slice= F_laser[:,F_laser.shape[1]//2, :] - env_slice= env[:,env.shape[1]//2, :] - env_theory_slice= env_theory[:,env_theory.shape[1]//2, :] - - # Plot results - plt.figure(figsize=(8,6)) - plt.subplot(221) - plt.title('PIC field') - plt.imshow(F_slice, extent=extent) - plt.colorbar() - plt.subplot(222) - plt.title('PIC envelope') - plt.imshow(env_slice, extent=extent) - plt.colorbar() - plt.subplot(223) - plt.title('Theory envelope') - plt.imshow(env_theory_slice, extent=extent) - plt.colorbar() - plt.subplot(224) - plt.title('Difference') - plt.imshow(env_slice-env_theory_slice, extent=extent) - plt.colorbar() - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_x = np.fft.fftfreq(F_laser.shape[0],dt) - freq_y = np.fft.fftfreq(F_laser.shape[1],dt) - freq_z = np.fft.fftfreq(F_laser.shape[2],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.sqrt((freq_x[pos_max[0]])**2 + (freq_y[pos_max[1]]**2) + (freq_z[pos_max[2]])**2) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.3d_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000251/", "comp_unf.pdf", 251) - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import GaussianProfile - - # Create a laser using lasy - pol = (1, 0) - profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) - dim = "xyt" - lo = (-25e-6, -25e-6, -20e-15) - hi = (+25e-6, +25e-6, +20e-15) - npoints = (100, 100, 100) - laser = Laser(dim, lo, hi, npoints, profile) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("gaussianlaser3d") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000251/" - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 251 +ds = yt.load(filename) +dt = ds.current_time.to_value() / steps + +# Define 3D meshes +x = np.linspace( + ds.domain_left_edge[0], ds.domain_right_edge[0], ds.domain_dimensions[0] +).v +y = np.linspace( + ds.domain_left_edge[1], ds.domain_right_edge[1], ds.domain_dimensions[1] +).v +z = np.linspace( + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_dimensions[ds.dimensionality - 1], +).v +X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing="ij") + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), X, Y, Z) + gauss_env( + -t_c + ds.current_time.to_value(), X, Y, Z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Ey"].v.squeeze() +env = abs(hilbert(F_laser)) +extent = [ + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], +] + +F_slice = F_laser[:, F_laser.shape[1] // 2, :] +env_slice = env[:, env.shape[1] // 2, :] +env_theory_slice = env_theory[:, env_theory.shape[1] // 2, :] + +# Plot results +plt.figure(figsize=(8, 6)) +plt.subplot(221) +plt.title("PIC field") +plt.imshow(F_slice, extent=extent) +plt.colorbar() +plt.subplot(222) +plt.title("PIC envelope") +plt.imshow(env_slice, extent=extent) +plt.colorbar() +plt.subplot(223) +plt.title("Theory envelope") +plt.imshow(env_theory_slice, extent=extent) +plt.colorbar() +plt.subplot(224) +plt.title("Difference") +plt.imshow(env_slice - env_theory_slice, extent=extent) +plt.colorbar() +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_x = np.fft.fftfreq(F_laser.shape[0], dt) +freq_y = np.fft.fftfreq(F_laser.shape[1], dt) +freq_z = np.fft.fftfreq(F_laser.shape[2], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.sqrt( + (freq_x[pos_max[0]]) ** 2 + (freq_y[pos_max[1]] ** 2) + (freq_z[pos_max[2]]) ** 2 +) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_RZ.py b/Examples/Tests/laser_injection_from_file/analysis_RZ.py deleted file mode 100755 index 3667a1d9419..00000000000 --- a/Examples/Tests/laser_injection_from_file/analysis_RZ.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli -# Remi Lehe, Ilian Kara-Mostefa -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -# This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected -# - Compute the theory for laser envelope at time T -# - Compare theory and simulation in RZ, for both envelope and central frequency - -import glob -import os -import sys - -import matplotlib - -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import numpy as np -from scipy.constants import c, epsilon_0 -from scipy.signal import hilbert - -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -#Maximum acceptable error for this test -relative_error_threshold = 0.065 - -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 - -#Parameters of the gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs - -laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) - -# Function for the envelope -def gauss_env(T, X, Y, Z): - # Function to compute the theory for the envelope - inv_tau2 = 1./tt/tt - inv_w_2 = 1.0/(w0*w0) - exp_arg = - (X*X)*inv_w_2 - (Y*Y)*inv_w_2- inv_tau2 / c/c * (Z-T*c)*(Z-T*c) - return E_max * np.real(np.exp(exp_arg)) - -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - dt = ds.current_time.to_value()/steps - - # Define 3D meshes - x = np.linspace( - ds.domain_left_edge[0], - ds.domain_right_edge[0], - ds.domain_dimensions[0]).v - y = np.linspace( - ds.domain_left_edge[1], - ds.domain_right_edge[1], - ds.domain_dimensions[1]).v - z = np.linspace( - ds.domain_left_edge[ds.dimensionality-1], - ds.domain_right_edge[ds.dimensionality-1], - ds.domain_dimensions[ds.dimensionality-1]).v - X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing='ij') - - # Compute the theory for envelope - env_theory = gauss_env(+t_c-ds.current_time.to_value(), X,Y,Z)+gauss_env(-t_c+ds.current_time.to_value(), X,Y,Z) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Et'].v.squeeze() - env = abs(hilbert(F_laser)) - extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - - env_theory_slice= env_theory[:,env_theory.shape[1]//2, :] - - # Plot results - plt.figure(figsize=(8,6)) - plt.subplot(221) - plt.title('PIC field') - plt.imshow(F_laser, extent=extent) - plt.colorbar() - plt.subplot(222) - plt.title('PIC envelope') - plt.imshow(env, extent=extent) - plt.colorbar() - plt.subplot(223) - plt.title('Theory envelope') - plt.imshow(env_theory_slice, extent=extent) - plt.colorbar() - plt.subplot(224) - plt.title('Difference') - plt.imshow(env-env_theory_slice, extent=extent) - plt.colorbar() - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory_slice)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_x = np.fft.fftfreq(F_laser.shape[0],dt) - freq_z = np.fft.fftfreq(F_laser.shape[1],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.sqrt((freq_x[pos_max[0]])**2 + (freq_z[pos_max[1]])**2) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.RZ_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000252/", "comp_unf.pdf", 252) - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import GaussianProfile - - # Create a laser using lasy - pol = (1, 0) - profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) - dim = "xyt" - lo = (-25e-6, -25e-6, -20e-15) - hi = (+25e-6, +25e-6, +20e-15) - npoints = (100, 100, 100) - laser = Laser(dim, lo, hi, npoints, profile) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("gaussianlaser3d") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000252/" - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/laser_injection_from_file/analysis_default_regression.py b/Examples/Tests/laser_injection_from_file/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py index 44bbd00a7f4..041e2917ece 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py +++ b/Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py @@ -9,174 +9,136 @@ # This file is part of the WarpX automated test suite. It is used to test the -# injection of a laser pulse from an external lasy file. -# -# - Generate an input lasy file with a Laguerre Gaussian laser pulse. -# - Run the WarpX simulation for time T, when the pulse is fully injected +# injection of a laser pulse from an external lasy file: # - Compute the theory for laser envelope at time T # - Compare theory and simulation in RZ, for both envelope and central frequency -import glob -import os import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +import yt from scipy.constants import c, epsilon_0 from scipy.signal import hilbert from scipy.special import genlaguerre -import yt ; yt.funcs.mylog.setLevel(50) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(50) -#Maximum acceptable error for this test +# Maximum acceptable error for this test relative_error_threshold = 0.065 -#Physical parameters -um = 1.e-6 -fs = 1.e-15 -c = 299792458 +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 -#Parameters of the Laguerre Gaussian beam -wavelength = 1.*um -w0 = 12.*um -tt = 10.*fs -t_c = 20.*fs +# Parameters of the Laguerre Gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs laser_energy = 1.0 -E_max = np.sqrt( 2*(2/np.pi)**(3/2)*laser_energy / (epsilon_0*w0**2*c*tt) ) +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + # Function for the envelope def laguerre_env(T, X, Y, Z, p, m): - if m>0: - complex_position= X -1j * Y + if m > 0: + complex_position = X - 1j * Y else: - complex_position= X +1j * Y - inv_w0_2 = 1.0/(w0**2) - inv_tau2 = 1.0/(tt**2) + complex_position = X + 1j * Y + inv_w0_2 = 1.0 / (w0**2) + inv_tau2 = 1.0 / (tt**2) radius = abs(complex_position) - scaled_rad_squared = (radius**2)*inv_w0_2 + scaled_rad_squared = (radius**2) * inv_w0_2 envelope = ( - ( np.sqrt(2) * complex_position / w0 )** m - * genlaguerre(p, m)(2 * scaled_rad_squared) - * np.exp(-scaled_rad_squared) - * np.exp(-( inv_tau2 / (c**2) ) * (Z-T*c)**2) - ) + (np.sqrt(2) * complex_position / w0) ** m + * genlaguerre(p, m)(2 * scaled_rad_squared) + * np.exp(-scaled_rad_squared) + * np.exp(-(inv_tau2 / (c**2)) * (Z - T * c) ** 2) + ) return E_max * np.real(envelope) -def do_analysis(fname, compname, steps): - ds = yt.load(fname) - dt = ds.current_time.to_value()/steps - - # Define 3D meshes - x = np.linspace( - ds.domain_left_edge[0], - ds.domain_right_edge[0], - ds.domain_dimensions[0]).v - y = np.linspace( - ds.domain_left_edge[1], - ds.domain_right_edge[1], - ds.domain_dimensions[1]).v - z = np.linspace( - ds.domain_left_edge[ds.dimensionality-1], - ds.domain_right_edge[ds.dimensionality-1], - ds.domain_dimensions[ds.dimensionality-1]).v - X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing='ij') - - # Compute the theory for envelope - env_theory = laguerre_env(+t_c-ds.current_time.to_value(), X,Y,Z,p=0,m=1)+laguerre_env(-t_c+ds.current_time.to_value(), X,Y,Z,p=0,m=1) - - # Read laser field in PIC simulation, and compute envelope - all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - F_laser = all_data_level_0['boxlib', 'Et'].v.squeeze() - env = abs(hilbert(F_laser)) - extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1], - ds.domain_left_edge[0], ds.domain_right_edge[0] ] - - env_theory_slice= env_theory[:,env_theory.shape[1]//2, :] - - # Plot results - plt.figure(figsize=(8,6)) - plt.subplot(221) - plt.title('PIC field') - plt.imshow(F_laser, extent=extent) - plt.colorbar() - plt.subplot(222) - plt.title('PIC envelope') - plt.imshow(env, extent=extent) - plt.colorbar() - plt.subplot(223) - plt.title('Theory envelope') - plt.imshow(env_theory_slice, extent=extent) - plt.colorbar() - plt.subplot(224) - plt.title('Difference') - plt.imshow(env-env_theory_slice, extent=extent) - plt.colorbar() - plt.tight_layout() - plt.savefig(compname, bbox_inches='tight') - - relative_error_env = np.sum(np.abs(env-env_theory_slice)) / np.sum(np.abs(env)) - print("Relative error envelope: ", relative_error_env) - assert(relative_error_env < relative_error_threshold) - - fft_F_laser = np.fft.fftn(F_laser) - - freq_x = np.fft.fftfreq(F_laser.shape[0],dt) - freq_z = np.fft.fftfreq(F_laser.shape[1],dt) - - pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) - - freq = np.sqrt((freq_x[pos_max[0]])**2 + (freq_z[pos_max[1]])**2) - exp_freq = c/wavelength - relative_error_freq = np.abs(freq-exp_freq)/exp_freq - print("Relative error frequency: ", relative_error_freq) - assert(relative_error_freq < relative_error_threshold) - - - -def launch_analysis(executable): - os.system("./" + executable + " inputs.from_RZ_file_test diag1.file_prefix=diags/plotfiles/plt") - do_analysis("diags/plotfiles/plt000612/", "comp_unf.pdf", 612) - - -def main() : - - from lasy.laser import Laser - from lasy.profiles import CombinedLongitudinalTransverseProfile, GaussianProfile - from lasy.profiles.longitudinal import GaussianLongitudinalProfile - from lasy.profiles.transverse import LaguerreGaussianTransverseProfile - - # Create a Laguerre Gaussian laser in RZ geometry using lasy - pol = (1, 0) - profile = CombinedLongitudinalTransverseProfile( - wavelength,pol,laser_energy, - GaussianLongitudinalProfile(wavelength, tt, t_peak=0), - LaguerreGaussianTransverseProfile(w0, p=0, m=1), - ) - dim = "rt" - lo = (0e-6, -20e-15) - hi = (+25e-6, +20e-15) - npoints = (100,100) - laser = Laser(dim, lo, hi, npoints, profile, n_azimuthal_modes=2) - laser.normalize(laser_energy, kind="energy") - laser.write_to_file("laguerrelaserRZ") - executables = glob.glob("*.ex") - if len(executables) == 1 : - launch_analysis(executables[0]) - else : - assert(False) - - # Do the checksum test - filename_end = "diags/plotfiles/plt000612/" - test_name = "LaserInjectionFromRZLASYFile" - checksumAPI.evaluate_checksum(test_name, filename_end) - print('Passed') - -if __name__ == "__main__": - main() + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 612 +ds = yt.load(filename) +dt = ds.current_time.to_value() / steps + +# Define 3D meshes +x = np.linspace( + ds.domain_left_edge[0], ds.domain_right_edge[0], ds.domain_dimensions[0] +).v +y = np.linspace( + ds.domain_left_edge[1], ds.domain_right_edge[1], ds.domain_dimensions[1] +).v +z = np.linspace( + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_dimensions[ds.dimensionality - 1], +).v +X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing="ij") + +# Compute the theory for envelope +env_theory = laguerre_env( + +t_c - ds.current_time.to_value(), X, Y, Z, p=0, m=1 +) + laguerre_env(-t_c + ds.current_time.to_value(), X, Y, Z, p=0, m=1) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Et"].v.squeeze() +env = abs(hilbert(F_laser)) +extent = [ + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], +] + +env_theory_slice = env_theory[:, env_theory.shape[1] // 2, :] + +# Plot results +plt.figure(figsize=(8, 6)) +plt.subplot(221) +plt.title("PIC field") +plt.imshow(F_laser, extent=extent) +plt.colorbar() +plt.subplot(222) +plt.title("PIC envelope") +plt.imshow(env, extent=extent) +plt.colorbar() +plt.subplot(223) +plt.title("Theory envelope") +plt.imshow(env_theory_slice, extent=extent) +plt.colorbar() +plt.subplot(224) +plt.title("Difference") +plt.imshow(env - env_theory_slice, extent=extent) +plt.colorbar() +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory_slice)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_x = np.fft.fftfreq(F_laser.shape[0], dt) +freq_z = np.fft.fftfreq(F_laser.shape[1], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.sqrt((freq_x[pos_max[0]]) ** 2 + (freq_z[pos_max[1]]) ** 2) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/analysis_rz.py b/Examples/Tests/laser_injection_from_file/analysis_rz.py new file mode 100755 index 00000000000..1ad73b34c4b --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/analysis_rz.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Compute the theory for laser envelope at time T +# - Compare theory and simulation in RZ, for both envelope and central frequency + +import sys + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import yt +from scipy.constants import c, epsilon_0 +from scipy.signal import hilbert + +yt.funcs.mylog.setLevel(50) + +# Maximum acceptable error for this test +relative_error_threshold = 0.065 + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +laser_energy = 1.0 +E_max = np.sqrt( + 2 * (2 / np.pi) ** (3 / 2) * laser_energy / (epsilon_0 * w0**2 * c * tt) +) + + +# Function for the envelope +def gauss_env(T, X, Y, Z): + # Function to compute the theory for the envelope + inv_tau2 = 1.0 / tt / tt + inv_w_2 = 1.0 / (w0 * w0) + exp_arg = ( + -(X * X) * inv_w_2 + - (Y * Y) * inv_w_2 + - inv_tau2 / c / c * (Z - T * c) * (Z - T * c) + ) + return E_max * np.real(np.exp(exp_arg)) + + +filename = sys.argv[1] +compname = "comp_unf.pdf" +steps = 252 +ds = yt.load(filename) +dt = ds.current_time.to_value() / steps + +# Define 3D meshes +x = np.linspace( + ds.domain_left_edge[0], ds.domain_right_edge[0], ds.domain_dimensions[0] +).v +y = np.linspace( + ds.domain_left_edge[1], ds.domain_right_edge[1], ds.domain_dimensions[1] +).v +z = np.linspace( + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_dimensions[ds.dimensionality - 1], +).v +X, Y, Z = np.meshgrid(x, y, z, sparse=False, indexing="ij") + +# Compute the theory for envelope +env_theory = gauss_env(+t_c - ds.current_time.to_value(), X, Y, Z) + gauss_env( + -t_c + ds.current_time.to_value(), X, Y, Z +) + +# Read laser field in PIC simulation, and compute envelope +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_laser = all_data_level_0["boxlib", "Et"].v.squeeze() +env = abs(hilbert(F_laser)) +extent = [ + ds.domain_left_edge[ds.dimensionality - 1], + ds.domain_right_edge[ds.dimensionality - 1], + ds.domain_left_edge[0], + ds.domain_right_edge[0], +] + +env_theory_slice = env_theory[:, env_theory.shape[1] // 2, :] + +# Plot results +plt.figure(figsize=(8, 6)) +plt.subplot(221) +plt.title("PIC field") +plt.imshow(F_laser, extent=extent) +plt.colorbar() +plt.subplot(222) +plt.title("PIC envelope") +plt.imshow(env, extent=extent) +plt.colorbar() +plt.subplot(223) +plt.title("Theory envelope") +plt.imshow(env_theory_slice, extent=extent) +plt.colorbar() +plt.subplot(224) +plt.title("Difference") +plt.imshow(env - env_theory_slice, extent=extent) +plt.colorbar() +plt.tight_layout() +plt.savefig(compname, bbox_inches="tight") + +relative_error_env = np.sum(np.abs(env - env_theory_slice)) / np.sum(np.abs(env)) +print("Relative error envelope: ", relative_error_env) +assert relative_error_env < relative_error_threshold + +fft_F_laser = np.fft.fftn(F_laser) + +freq_x = np.fft.fftfreq(F_laser.shape[0], dt) +freq_z = np.fft.fftfreq(F_laser.shape[1], dt) + +pos_max = np.unravel_index(np.abs(fft_F_laser).argmax(), fft_F_laser.shape) + +freq = np.sqrt((freq_x[pos_max[0]]) ** 2 + (freq_z[pos_max[1]]) ** 2) +exp_freq = c / wavelength +relative_error_freq = np.abs(freq - exp_freq) / exp_freq +print("Relative error frequency: ", relative_error_freq) +assert relative_error_freq < relative_error_threshold diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test deleted file mode 100644 index ffc1865ee0f..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ /dev/null @@ -1,59 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 2.45e-12 -amr.n_cell = 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 1 -geometry.prob_lo = -25.e-6 # physical domain -geometry.prob_hi = 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -# Boost parameters -warpx.gamma_boost = 10 -warpx.boost_direction = z -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = pec -boundary.field_hi = pec - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_laser -lasy_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_laser.direction = 0. 0. 1. # The plane normal direction -lasy_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_laser.profile = from_file -lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" -lasy_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 - -diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz -diag1.diag_type = BackTransformed -diag1.dt_snapshots_lab = 10.e-14 -diag1.num_snapshots_lab = 2 diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test deleted file mode 100644 index 6c392883418..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ /dev/null @@ -1,50 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 40.e-15 -amr.n_cell = 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 1 -geometry.prob_lo = -25.e-6 # physical domain -geometry.prob_hi = 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic -boundary.field_hi = periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_laser -lasy_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_laser.direction = 0. 0. 1. # The plane normal direction -lasy_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_laser.profile = from_file -lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" -lasy_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = -1 -diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test deleted file mode 100644 index e5814471753..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ /dev/null @@ -1,50 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 40.e-15 -amr.n_cell = 32 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -25.e-6 -25.e-6 # physical domain -geometry.prob_hi = 25.e-6 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_laser -lasy_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_laser.direction = 0. 0. 1. # The plane normal direction -lasy_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_laser.profile = from_file -lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" -lasy_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = -1 -diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test_binary b/Examples/Tests/laser_injection_from_file/inputs.2d_test_binary deleted file mode 100644 index fdb19f406ca..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test_binary +++ /dev/null @@ -1,50 +0,0 @@ - -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 250 -amr.n_cell = 672 672 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 2 -geometry.prob_lo = -25.e-6 -25.0e-6 # physical domain -geometry.prob_hi = 25.e-6 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = binary_laser -binary_laser.position = 0. 0. 0. # This point is on the laser plane -binary_laser.direction = 1. 0. 1. # The plane normal direction -binary_laser.polarization = 0. 1. 0. # The main polarization vector -binary_laser.e_max = 1.e12 # Maximum amplitude of the laser field (in V/m) -binary_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -binary_laser.profile = from_file -binary_laser.binary_file_name = "gauss_2d" -binary_laser.time_chunk_size = 50 -binary_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 250 -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test deleted file mode 100644 index ad8159cb650..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ /dev/null @@ -1,50 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 40.e-15 -amr.n_cell = 32 32 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -25.e-6 -25.e-6 -25.e-6 # physical domain -geometry.prob_hi = 25.e-6 25.e-6 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic periodic -boundary.field_hi = periodic periodic periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_laser -lasy_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_laser.direction = 0. 0. 1. # The plane normal direction -lasy_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_laser.profile = from_file -lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" -lasy_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = -1 -diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test deleted file mode 100644 index 2a539883fec..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ /dev/null @@ -1,51 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 40.e-15 -amr.n_cell = 32 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = RZ -geometry.prob_lo = 0.e-6 -25.e-6 # physical domain -geometry.prob_hi = 25.e-6 25.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = none periodic -boundary.field_hi = pec periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -#algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 -warpx.n_rz_azimuthal_modes=2 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_laser -lasy_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_laser.direction = 0. 0. 1. # The plane normal direction -lasy_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_laser.profile = from_file -lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" -lasy_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = -1 -diag1.fields_to_plot = Er Et Ez Br Bz jr jt jz -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test deleted file mode 100644 index f92440188b7..00000000000 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ /dev/null @@ -1,50 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -stop_time = 39.e-15 -amr.n_cell = 32 1024 -amr.max_grid_size = 512 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = RZ -geometry.prob_lo = 0.e-6 -10.e-6 # physical domain -geometry.prob_hi = 25.e-6 10.e-6 -warpx.verbose = 1 -warpx.serialize_initial_conditions = 1 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = none periodic -boundary.field_hi = pec periodic - -################################# -############ NUMERICS ########### -################################# -warpx.cfl = 0.98 -warpx.use_filter = 0 -algo.load_balance_intervals = -1 -warpx.n_rz_azimuthal_modes=3 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############# LASER ############# -################################# -lasers.names = lasy_RZ_laser -lasy_RZ_laser.position = 0. 0. 0. # This point is on the laser plane -lasy_RZ_laser.direction = 0. 0. 1. # The plane normal direction -lasy_RZ_laser.polarization = 0. 1. 0. # The main polarization vector -lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) -lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) -lasy_RZ_laser.profile = from_file -lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" -lasy_RZ_laser.delay = 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = -1 -diag1.fields_to_plot = Er Et Ez Br Bz jr jt jz -diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file new file mode 100644 index 00000000000..1510a8df4c3 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file @@ -0,0 +1,50 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 40.e-15 +amr.n_cell = 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = -25.e-6 # physical domain +geometry.prob_hi = 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic +boundary.field_hi = periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_laser +lasy_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_laser.direction = 0. 0. 1. # The plane normal direction +lasy_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_laser.profile = from_file +lasy_laser.time_chunk_size = 50 +lasy_laser.lasy_file_name = "../test_1d_laser_injection_from_lasy_file_prepare/diags/gaussian_laser_3d_00000.h5" +lasy_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = -1 +diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost new file mode 100644 index 00000000000..d118ce85ae5 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost @@ -0,0 +1,59 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 2.45e-12 +amr.n_cell = 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = -25.e-6 # physical domain +geometry.prob_hi = 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +# Boost parameters +warpx.gamma_boost = 10 +warpx.boost_direction = z +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pec +boundary.field_hi = pec + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_laser +lasy_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_laser.direction = 0. 0. 1. # The plane normal direction +lasy_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_laser.profile = from_file +lasy_laser.time_chunk_size = 50 +lasy_laser.lasy_file_name = "../test_1d_laser_injection_from_lasy_file_boost_prepare/diags/gaussian_laser_3d_00000.h5" +lasy_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 + +diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz +diag1.diag_type = BackTransformed +diag1.dt_snapshots_lab = 10.e-14 +diag1.num_snapshots_lab = 2 diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost_prepare.py new file mode 100755 index 00000000000..f71b87d5fc8 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_boost_prepare.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import GaussianProfile + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +# Create a laser using lasy +profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) +dim = "xyt" +lo = (-25e-6, -25e-6, -20e-15) +hi = (+25e-6, +25e-6, +20e-15) +npoints = (100, 100, 100) +laser = Laser(dim, lo, hi, npoints, profile) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("gaussian_laser_3d") diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_prepare.py new file mode 100755 index 00000000000..902b0c47210 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_1d_laser_injection_from_lasy_file_prepare.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import GaussianProfile + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs + +# Create a laser using lasy +profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) +dim = "xyt" +lo = (-25e-6, -25e-6, -20e-15) +hi = (+25e-6, +25e-6, +20e-15) +npoints = (100, 100, 100) +laser = Laser(dim, lo, hi, npoints, profile) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("gaussian_laser_3d") diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file new file mode 100644 index 00000000000..022da5b0e29 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file @@ -0,0 +1,50 @@ + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 250 +amr.n_cell = 672 672 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = -25.e-6 -25.0e-6 # physical domain +geometry.prob_hi = 25.e-6 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = binary_laser +binary_laser.position = 0. 0. 0. # This point is on the laser plane +binary_laser.direction = 1. 0. 1. # The plane normal direction +binary_laser.polarization = 0. 1. 0. # The main polarization vector +binary_laser.e_max = 1.e12 # Maximum amplitude of the laser field (in V/m) +binary_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +binary_laser.profile = from_file +binary_laser.binary_file_name = "../test_2d_laser_injection_from_binary_file_prepare/gauss_2d" +binary_laser.time_chunk_size = 50 +binary_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 250 +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file_prepare.py new file mode 100755 index 00000000000..d8fe2236ae0 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_binary_file_prepare.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external binary file: +# - Generate an input binary file with a gaussian laser pulse. + +import sys + +import matplotlib + +matplotlib.use("Agg") +import numpy as np +import yt + +yt.funcs.mylog.setLevel(50) + +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") + +# Maximum acceptable error for this test +relative_error_threshold = 0.065 + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 +c = 299792458 + +# Parameters of the gaussian beam +wavelength = 1.0 * um +w0 = 6.0 * um +tt = 10.0 * fs +x_c = 0.0 * um +t_c = 20.0 * fs +foc_dist = 10 * um +E_max = 1e12 +rot_angle = -np.pi / 4.0 + +# Parameters of the tx grid +x_l = -12.0 * um +x_r = 12.0 * um +x_points = 480 +t_l = 0.0 * fs +t_r = 40.0 * fs +t_points = 400 +tcoords = np.linspace(t_l, t_r, t_points) +xcoords = np.linspace(x_l, x_r, x_points) + + +def gauss(T, X, Y, opt): + """Compute the electric field for a Gaussian laser pulse. + This is used to write the binary input file. + """ + + k0 = 2.0 * np.pi / wavelength + inv_tau2 = 1.0 / tt / tt + osc_phase = k0 * c * (T - t_c) + + diff_factor = 1.0 + 1.0j * foc_dist * 2 / (k0 * w0 * w0) + inv_w_2 = 1.0 / (w0 * w0 * diff_factor) + + pre_fact = np.exp(1.0j * osc_phase) + + if opt == "3d": + pre_fact = pre_fact / diff_factor + else: + pre_fact = pre_fact / np.sqrt(diff_factor) + + exp_arg = -(X * X + Y * Y) * inv_w_2 - inv_tau2 * (T - t_c) * (T - t_c) + + return np.real(pre_fact * np.exp(exp_arg)) + + +def write_file(fname, x, y, t, E): + """For a given filename fname, space coordinates x and y, time coordinate t + and field E, write a WarpX-compatible input binary file containing the + profile of the laser pulse. This function should be used in the case + of a uniform spatio-temporal mesh + """ + + with open(fname, "wb") as file: + flag_unif = 1 + file.write(flag_unif.to_bytes(1, byteorder="little")) + file.write((len(t)).to_bytes(4, byteorder="little", signed=False)) + file.write((len(x)).to_bytes(4, byteorder="little", signed=False)) + file.write((len(y)).to_bytes(4, byteorder="little", signed=False)) + file.write(t[0].tobytes()) + file.write(t[-1].tobytes()) + file.write(x[0].tobytes()) + file.write(x[-1].tobytes()) + if len(y) == 1: + file.write(y[0].tobytes()) + else: + file.write(y[0].tobytes()) + file.write(y[-1].tobytes()) + file.write(E.tobytes()) + + +T, X, Y = np.meshgrid(tcoords, xcoords, np.array([0.0]), indexing="ij") +E_t = gauss(T, X, Y, "2d") +write_file("gauss_2d", xcoords, np.array([0.0]), tcoords, E_t) diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file new file mode 100644 index 00000000000..f6a0693a5bd --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file @@ -0,0 +1,50 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 40.e-15 +amr.n_cell = 32 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = -25.e-6 -25.e-6 # physical domain +geometry.prob_hi = 25.e-6 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_laser +lasy_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_laser.direction = 0. 0. 1. # The plane normal direction +lasy_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_laser.profile = from_file +lasy_laser.time_chunk_size = 50 +lasy_laser.lasy_file_name = "../test_2d_laser_injection_from_lasy_file_prepare/diags/gaussian_laser_3d_00000.h5" +lasy_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = -1 +diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file_prepare.py new file mode 100755 index 00000000000..f71b87d5fc8 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_2d_laser_injection_from_lasy_file_prepare.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import GaussianProfile + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +# Create a laser using lasy +profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) +dim = "xyt" +lo = (-25e-6, -25e-6, -20e-15) +hi = (+25e-6, +25e-6, +20e-15) +npoints = (100, 100, 100) +laser = Laser(dim, lo, hi, npoints, profile) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("gaussian_laser_3d") diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file b/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file new file mode 100644 index 00000000000..534ea729886 --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file @@ -0,0 +1,50 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 40.e-15 +amr.n_cell = 32 32 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -25.e-6 -25.e-6 -25.e-6 # physical domain +geometry.prob_hi = 25.e-6 25.e-6 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_laser +lasy_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_laser.direction = 0. 0. 1. # The plane normal direction +lasy_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_laser.profile = from_file +lasy_laser.time_chunk_size = 50 +lasy_laser.lasy_file_name = "../test_3d_laser_injection_from_lasy_file_prepare/diags/gaussian_laser_3d_00000.h5" +lasy_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = -1 +diag1.fields_to_plot = Ex Ey Ez Bx Bz jx jy jz +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file_prepare.py new file mode 100755 index 00000000000..410dcd2c36e --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_3d_laser_injection_from_lasy_file_prepare.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import GaussianProfile + +# Maximum acceptable error for this test +relative_error_threshold = 0.065 + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +# Create a laser using lasy +profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) +dim = "xyt" +lo = (-25e-6, -25e-6, -20e-15) +hi = (+25e-6, +25e-6, +20e-15) +npoints = (100, 100, 100) +laser = Laser(dim, lo, hi, npoints, profile) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("gaussian_laser_3d") diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file new file mode 100644 index 00000000000..a4c87d244fc --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file @@ -0,0 +1,50 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 39.e-15 +amr.n_cell = 32 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = RZ +geometry.prob_lo = 0.e-6 -10.e-6 # physical domain +geometry.prob_hi = 25.e-6 10.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = none periodic +boundary.field_hi = pec periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +algo.load_balance_intervals = -1 +warpx.n_rz_azimuthal_modes=3 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_RZ_laser +lasy_RZ_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_RZ_laser.direction = 0. 0. 1. # The plane normal direction +lasy_RZ_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_RZ_laser.profile = from_file +lasy_RZ_laser.time_chunk_size = 50 +lasy_RZ_laser.lasy_file_name = "../test_rz_laser_injection_from_RZ_lasy_file_prepare/diags/laguerre_laser_RZ_00000.h5" +lasy_RZ_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = -1 +diag1.fields_to_plot = Er Et Ez Br Bz jr jt jz +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file_prepare.py new file mode 100755 index 00000000000..1a1dfabe86e --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_RZ_lasy_file_prepare.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a Laguerre Gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import CombinedLongitudinalTransverseProfile +from lasy.profiles.longitudinal import GaussianLongitudinalProfile +from lasy.profiles.transverse import LaguerreGaussianTransverseProfile + +# Maximum acceptable error for this test +relative_error_threshold = 0.065 + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Laguerre Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +# Create a Laguerre Gaussian laser in RZ geometry using lasy +profile = CombinedLongitudinalTransverseProfile( + wavelength, + pol, + laser_energy, + GaussianLongitudinalProfile(wavelength, tt, t_peak=0), + LaguerreGaussianTransverseProfile(w0, p=0, m=1), +) +dim = "rt" +lo = (0e-6, -20e-15) +hi = (+25e-6, +20e-15) +npoints = (100, 100) +laser = Laser(dim, lo, hi, npoints, profile, n_azimuthal_modes=2) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("laguerre_laser_RZ") diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file new file mode 100644 index 00000000000..cc7100afb9d --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file @@ -0,0 +1,51 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = 40.e-15 +amr.n_cell = 32 1024 +amr.max_grid_size = 512 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = RZ +geometry.prob_lo = 0.e-6 -25.e-6 # physical domain +geometry.prob_hi = 25.e-6 25.e-6 +warpx.verbose = 1 +warpx.serialize_initial_conditions = 1 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = none periodic +boundary.field_hi = pec periodic + +################################# +############ NUMERICS ########### +################################# +warpx.cfl = 0.98 +warpx.use_filter = 0 +#algo.maxwell_solver = ckc +algo.load_balance_intervals = -1 +warpx.n_rz_azimuthal_modes=2 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############# LASER ############# +################################# +lasers.names = lasy_laser +lasy_laser.position = 0. 0. 0. # This point is on the laser plane +lasy_laser.direction = 0. 0. 1. # The plane normal direction +lasy_laser.polarization = 0. 1. 0. # The main polarization vector +lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (in V/m) +lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +lasy_laser.profile = from_file +lasy_laser.time_chunk_size = 50 +lasy_laser.lasy_file_name = "../test_rz_laser_injection_from_lasy_file_prepare/diags/gaussian_laser_3d_00000.h5" +lasy_laser.delay = 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = -1 +diag1.fields_to_plot = Er Et Ez Br Bz jr jt jz +diag1.diag_type = Full diff --git a/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file_prepare.py b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file_prepare.py new file mode 100755 index 00000000000..410dcd2c36e --- /dev/null +++ b/Examples/Tests/laser_injection_from_file/inputs_test_rz_laser_injection_from_lasy_file_prepare.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Andrew Myers, Axel Huebl, Luca Fedeli +# Remi Lehe, Ilian Kara-Mostefa +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This file is part of the WarpX automated test suite. It is used to test the +# injection of a laser pulse from an external lasy file: +# - Generate an input lasy file with a gaussian laser pulse. + +from lasy.laser import Laser +from lasy.profiles import GaussianProfile + +# Maximum acceptable error for this test +relative_error_threshold = 0.065 + +# Physical parameters +um = 1.0e-6 +fs = 1.0e-15 + +# Parameters of the Gaussian beam +wavelength = 1.0 * um +pol = (1, 0) +laser_energy = 1.0 +w0 = 12.0 * um +tt = 10.0 * fs +t_c = 20.0 * fs + +# Create a laser using lasy +profile = GaussianProfile(wavelength, pol, laser_energy, w0, tt, t_peak=0) +dim = "xyt" +lo = (-25e-6, -25e-6, -20e-15) +hi = (+25e-6, +25e-6, +20e-15) +npoints = (100, 100, 100) +laser = Laser(dim, lo, hi, npoints, profile) +laser.normalize(laser_energy, kind="energy") +laser.write_to_file("gaussian_laser_3d") diff --git a/Examples/Tests/laser_on_fine/CMakeLists.txt b/Examples/Tests/laser_on_fine/CMakeLists.txt new file mode 100644 index 00000000000..9d9e48e54be --- /dev/null +++ b/Examples/Tests/laser_on_fine/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_laser_on_fine # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_on_fine # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) diff --git a/Examples/Tests/laser_on_fine/analysis_default_regression.py b/Examples/Tests/laser_on_fine/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/laser_on_fine/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/laser_on_fine/inputs_2d b/Examples/Tests/laser_on_fine/inputs_2d deleted file mode 100644 index 1ffbfcb7e09..00000000000 --- a/Examples/Tests/laser_on_fine/inputs_2d +++ /dev/null @@ -1,69 +0,0 @@ -# Maximum number of time steps -max_step = 500 - -# number of grid points -amr.n_cell = 64 64 - -# The lo and hi ends of grids are multipliers of blocking factor -amr.blocking_factor = 32 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 1 - -warpx.fine_tag_lo = -10.e-6 -0.4e-6 -warpx.fine_tag_hi = 10.e-6 0.4e-6 - -# Geometry -geometry.dims = 2 -geometry.prob_lo = -30.e-6 -1.25e-6 # physical domain -geometry.prob_hi = 30.e-6 1.25e-6 - -# Boundary condition -boundary.field_lo = periodic pml -boundary.field_hi = periodic pml - -# PML -warpx.pml_ncell = 10 - -# Verbosity -warpx.verbose = 1 - -# Algorithms -algo.current_deposition = esirkepov -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -warpx.use_filter = 0 - -# CFL -warpx.cfl = 1.0 - -# Order of particle shape factors -algo.particle_shape = 1 - -# Moving window -warpx.do_moving_window = 0 - -# Laser -lasers.names = laser1 -laser1.prob_lo = -12.e-6 -5.e-6 -laser1.prob_hi = 12.e-6 5.e-6 -laser1.profile = Gaussian -laser1.position = 0. 0. 0.e-6 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 1. 0. 0. # The main polarization vector -laser1.e_max = 16.e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 3.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 15.e-15 # The duration of the laser (in seconds) -laser1.profile_t_peak = 30.e-15 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 10 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divB diff --git a/Examples/Tests/laser_on_fine/inputs_test_2d_laser_on_fine b/Examples/Tests/laser_on_fine/inputs_test_2d_laser_on_fine new file mode 100644 index 00000000000..61c10e81f3e --- /dev/null +++ b/Examples/Tests/laser_on_fine/inputs_test_2d_laser_on_fine @@ -0,0 +1,69 @@ +# Maximum number of time steps +max_step = 50 + +# number of grid points +amr.n_cell = 64 64 + +# The lo and hi ends of grids are multipliers of blocking factor +amr.blocking_factor = 32 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 1 + +warpx.fine_tag_lo = -10.e-6 -0.4e-6 +warpx.fine_tag_hi = 10.e-6 0.4e-6 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -30.e-6 -1.25e-6 # physical domain +geometry.prob_hi = 30.e-6 1.25e-6 + +# Boundary condition +boundary.field_lo = periodic pml +boundary.field_hi = periodic pml + +# PML +warpx.pml_ncell = 10 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Moving window +warpx.do_moving_window = 0 + +# Laser +lasers.names = laser1 +laser1.prob_lo = -12.e-6 -5.e-6 +laser1.prob_hi = 12.e-6 5.e-6 +laser1.profile = Gaussian +laser1.position = 0. 0. 0.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 1. 0. 0. # The main polarization vector +laser1.e_max = 16.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 3.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 15.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 30.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in meters) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 10 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell divB diff --git a/Examples/Tests/load_external_field/CMakeLists.txt b/Examples/Tests/load_external_field/CMakeLists.txt new file mode 100644 index 00000000000..8641f307e16 --- /dev/null +++ b/Examples/Tests/load_external_field/CMakeLists.txt @@ -0,0 +1,62 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_load_external_field_grid_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_load_external_field_grid_picmi.py # inputs + "analysis_3d.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_load_external_field_particle_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_load_external_field_particle_picmi.py # inputs + "analysis_3d.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_load_external_field_grid # name + RZ # dims + 1 # nprocs + inputs_test_rz_load_external_field_grid # inputs + "analysis_rz.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_load_external_field_grid_restart # name + RZ # dims + 1 # nprocs + inputs_test_rz_load_external_field_grid_restart # inputs + "analysis_default_restart.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + test_rz_load_external_field_grid # dependency +) + +add_warpx_test( + test_rz_load_external_field_particles # name + RZ # dims + 1 # nprocs + inputs_test_rz_load_external_field_particles # inputs + "analysis_rz.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_load_external_field_particles_restart # name + RZ # dims + 1 # nprocs + inputs_test_rz_load_external_field_particles_restart # inputs + "analysis_default_restart.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + test_rz_load_external_field_particles # dependency +) diff --git a/Examples/Tests/load_external_field/analysis_3d.py b/Examples/Tests/load_external_field/analysis_3d.py new file mode 100755 index 00000000000..433a4bad5e8 --- /dev/null +++ b/Examples/Tests/load_external_field/analysis_3d.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2022 Yinjian Zhao +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This test tests the external field loading feature. +# A magnetic mirror field is loaded, and a single particle +# in the mirror will be reflected by the magnetic mirror effect. +# At the end of the simulation, the position of the particle +# is compared with known correct results. + +# Possible errors: 3.915833656984999e-9 +# tolerance: 1.0e-8 +# Possible running time: 2.756646401 s + +import sys + +import numpy as np +import yt + +tolerance = 1.0e-8 +x0 = 0.12238072 +y0 = 0.00965394 +z0 = 4.31754477 + +filename = sys.argv[1] + +ds = yt.load(filename) +ad = ds.all_data() +x = ad["proton", "particle_position_x"].to_ndarray() +y = ad["proton", "particle_position_y"].to_ndarray() +z = ad["proton", "particle_position_z"].to_ndarray() + +error = np.min(np.sqrt((x - x0) ** 2 + (y - y0) ** 2 + (z - z0) ** 2)) + +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/load_external_field/analysis_default_regression.py b/Examples/Tests/load_external_field/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/load_external_field/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/load_external_field/analysis_default_restart.py b/Examples/Tests/load_external_field/analysis_default_restart.py new file mode 120000 index 00000000000..0459986eebc --- /dev/null +++ b/Examples/Tests/load_external_field/analysis_default_restart.py @@ -0,0 +1 @@ +../../analysis_default_restart.py \ No newline at end of file diff --git a/Examples/Tests/load_external_field/analysis_rz.py b/Examples/Tests/load_external_field/analysis_rz.py new file mode 100755 index 00000000000..e5601647d4a --- /dev/null +++ b/Examples/Tests/load_external_field/analysis_rz.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2022 Yinjian Zhao +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This test tests the external field loading feature. +# A magnetic mirror field is loaded, and a single particle +# in the mirror will be reflected by the magnetic mirror effect. +# At the end of the simulation, the position of the particle +# is compared with known correct results. + +# Possible errors: 6.235230443866285e-9 +# tolerance: 1.0e-8 +# Possible running time: 0.327827743 s + +import sys + +import numpy as np +import yt + +tolerance = 1.0e-8 +r0 = 0.12402005 +z0 = 4.3632492 + +filename = sys.argv[1] +ds = yt.load(filename) +ad = ds.all_data() +r = ad["proton", "particle_position_x"].to_ndarray() +z = ad["proton", "particle_position_y"].to_ndarray() + +error = np.min(np.sqrt((r - r0) ** 2 + (z - z0) ** 2)) + +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_grid_picmi.py b/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_grid_picmi.py new file mode 100644 index 00000000000..ee24fb667cf --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_grid_picmi.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 40 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +ion_dist = picmi.ParticleListDistribution( + x=0.0, + y=0.2, + z=2.5, + ux=9.5e-05 * constants.c, + uy=0.0 * constants.c, + uz=1.34e-4 * constants.c, + weight=1.0, +) + +ions = picmi.Species( + particle_type="H", + name="proton", + charge="q_e", + mass="m_p", + warpx_do_not_deposit=1, + initial_distribution=ion_dist, +) + +################################# +######## INITIAL FIELD ########## +################################# + +initial_field = picmi.LoadInitialField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False, +) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], +) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=300, + species=[ions], + data_list=["ux", "uy", "uz", "x", "y", "z", "weighting"], +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=300, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], +) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_grid_type="collocated", + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, + particle_shape=particle_shape, +) + +sim.add_applied_field(initial_field) + +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ), +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +# sim.write_input_file('inputs_test_3d_load_external_field_grid') +sim.step(max_steps) diff --git a/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_particle_picmi.py b/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_particle_picmi.py new file mode 100644 index 00000000000..75977a01729 --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_3d_load_external_field_particle_picmi.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 40 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +ion_dist = picmi.ParticleListDistribution( + x=0.0, + y=0.2, + z=2.5, + ux=9.5e-05 * constants.c, + uy=0.0 * constants.c, + uz=1.34e-4 * constants.c, + weight=1.0, +) + +ions = picmi.Species( + particle_type="H", + name="proton", + charge="q_e", + mass="m_p", + warpx_do_not_deposit=1, + initial_distribution=ion_dist, +) + +################################# +######## INITIAL FIELD ########## +################################# + +applied_field = picmi.LoadAppliedField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False, +) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], +) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=300, + species=[ions], + data_list=["ux", "uy", "uz", "x", "y", "z", "weighting"], +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=300, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], +) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_grid_type="collocated", + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, + particle_shape=particle_shape, +) + +sim.add_applied_field(applied_field) + +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ), +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +# sim.write_input_file('inputs_test_3d_load_external_field_particle') +sim.step(max_steps) diff --git a/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid new file mode 100644 index 00000000000..f986add7bf5 --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid @@ -0,0 +1,66 @@ +warpx.abort_on_warning_threshold = medium +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +warpx.B_ext_grid_init_style = "read_from_file" +warpx.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.grid_type = collocated +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 40 40 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = proton +proton.injection_style = "SingleParticle" +proton.single_particle_pos = 0.0 0.2 2.5 +proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 +proton.single_particle_weight = 1.0 +proton.do_not_deposit = 1 +proton.mass = m_p +proton.charge = q_e + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 300 +diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid_restart b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid_restart new file mode 100644 index 00000000000..ed31e697e25 --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_grid_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_rz_load_external_field_grid + +# test input parameters +amr.restart = "../test_rz_load_external_field_grid/diags/chk000150" diff --git a/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles new file mode 100644 index 00000000000..e725ed588b0 --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles @@ -0,0 +1,66 @@ +warpx.abort_on_warning_threshold = medium +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +particles.B_ext_particle_init_style = "read_from_file" +particles.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.grid_type = collocated +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 40 40 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = proton +proton.injection_style = "SingleParticle" +proton.single_particle_pos = 0.0 0.2 2.5 +proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 +proton.single_particle_weight = 1.0 +proton.do_not_deposit = 1 +proton.mass = m_p +proton.charge = q_e + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 300 +diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles_restart b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles_restart new file mode 100644 index 00000000000..7e20f87d6d2 --- /dev/null +++ b/Examples/Tests/load_external_field/inputs_test_rz_load_external_field_particles_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_rz_load_external_field_particles + +# test input parameters +amr.restart = "../test_rz_load_external_field_particles/diags/chk000150" diff --git a/Examples/Tests/magnetostatic_eb/CMakeLists.txt b/Examples/Tests/magnetostatic_eb/CMakeLists.txt new file mode 100644 index 00000000000..5c0a87fd10a --- /dev/null +++ b/Examples/Tests/magnetostatic_eb/CMakeLists.txt @@ -0,0 +1,38 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_magnetostatic_eb # name + 3 # dims + 1 # nprocs + inputs_test_3d_magnetostatic_eb # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_magnetostatic_eb_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_magnetostatic_eb_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_magnetostatic_eb_picmi # name + RZ # dims + 1 # nprocs + inputs_test_rz_magnetostatic_eb_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py deleted file mode 100755 index 8f205724563..00000000000 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2022 S. Eric Clark, LLNL -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -""" -This script tests the magnetostatic solver in a beampipe by initializing a -uniform electron beam with relativistic gamma of 10, and a beam current of 1 kA. -The MLMG soltution from WarpX is computed and compared to the simple analytic -solution. The analytic and simulated values are plotted over one another for the -electric field and magnetic field solutions after taking the gradient of phi and -the curl of the vector potential A. This runs in 3 dimensions, has a PEC boundary -at z=0, and a Neumann boundary at z=1. This exercises the domain boundary conditions -as well as the embedded boundary conditions and will fail if the maximum error is too large. -The script additionally outputs png images with the average fields in a subset of the beampipe -as well as the analytical solution. -""" - -import matplotlib - -matplotlib.use('agg') - -import matplotlib.pyplot as plt -import numpy as np -import scipy.constants as con - -from pywarpx import fields, picmi - -########################## -# physics parameters -########################## - -V_domain_boundary = 0.0 -V_embedded_boundary = 1.0 - -########################## -# numerics parameters -########################## - -dt = 1e-12 - -# --- Nb time steps - -max_steps = 1 - -# --- grid - -nx = 64 -ny = 64 -nz = 64 - -xmin = -0.25 -xmax = 0.25 -ymin = -0.25 -ymax = 0.25 -zmin = 0. -zmax = 1.0 - -########################## -# numerics components -########################## - -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['neumann', 'neumann', 'dirichlet'], - upper_boundary_conditions = ['neumann', 'neumann', 'neumann'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing'], - warpx_potential_lo_z = V_domain_boundary, - warpx_blocking_factor=8, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-7,warpx_magnetostatic=True,warpx_self_fields_verbosity=3 -) - -r_pipe = 0.2 - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="(x**2+y**2-radius**2)", - potential=V_embedded_boundary, - radius = r_pipe -) - - -# Beam Current Density -current = 1000 # A -beam_r = 0.1 # m - -J = current/(np.pi * beam_r**2) -beam_gamma = 10. -vz = con.c*np.sqrt(1. - 1./beam_gamma**2) -n0 = J/(con.e*vz) - - -beam_dist = picmi.AnalyticDistribution( - density_expression='((x**2+y**2)= beam_r and r < r_pipe: - er = -current / (2.*np.pi*r*con.epsilon_0*vz) - else: - er = np.zeros_like(r) - return er - -# compare to region from 0.5*zmax to 0.9*zmax -z_idx = ((z_vec >= 0.5*zmax) & (z_vec < 0.9*zmax)) - -Ex_dat = Ex[...] -Ey_dat = Ey[...] - -Ex_mean = Ex_dat[:,:,z_idx].mean(axis=2).T -Ey_mean = Ey_dat[:,:,z_idx].mean(axis=2).T - -Ex_nodal = Ex_mean -Ey_nodal = Ey_mean - -XM, YM = np.meshgrid(x_vec, y_vec, indexing='xy') -RM = np.sqrt(XM**2 + YM**2) -THM = np.arctan2(YM,XM) - -Er_mean = np.cos(THM) * Ex_nodal + np.sin(THM) * Ey_nodal - -r_vec = np.sqrt(x_vec**2 + y_vec**2) - -r_idx = (RM < 0.95*r_pipe) -r_sub = RM[r_idx] - -plt.figure(1) -plt.plot(r_vec, Er_an(r_vec)) -plt.plot(RM.flatten(), Er_mean.flatten(), '.') -plt.legend(['Analytical', 'Electrostatic']) - -er_err = np.abs(Er_mean[r_idx] - Er_an(r_sub)).max()/np.abs(Er_an(r_sub)).max() - -plt.ylabel('$E_r$ (V/m)') -plt.xlabel('r (m)') -plt.title("Max % Error: {} %".format(er_err*100.)) -plt.tight_layout() -plt.savefig('er_3d.png') - -assert (er_err < 0.05), "Er Max Error increased above 5%" - -######################## -# Check B field -######################## - -Bx = fields.BxWrapper() -By = fields.ByWrapper() - -x_vec = Bx.mesh('x') -y_vec = Bx.mesh('y') -z_vec = Bx.mesh('z') - -dx = x_vec[1] - x_vec[0] -dy = y_vec[1] - y_vec[0] -x_vec = x_vec + dx/2. -y_vec = y_vec + dy/2. - -@np.vectorize -def Bt_an(r): - if r < beam_r: - bt = -current * r * con.mu_0 / (2.*np.pi*beam_r**2) - elif r >= beam_r and r < r_pipe: - bt = -current * con.mu_0 / (2.*np.pi*r) - else: - bt = np.zeros_like(r) - return bt - -# compare to region from 0.25*zmax to 0.75*zmax -z_idx = ((z_vec >= 0.25*zmax) & (z_vec < 0.75*zmax)) -z_sub = z_vec[z_idx] - -Bx_dat = Bx[...] -By_dat = By[...] - -Bx_mean = Bx_dat[:,:,z_idx].mean(axis=2).T -By_mean = By_dat[:,:,z_idx].mean(axis=2).T - -# Interpolate B mesh to nodal points excluding last mesh point -Bx_nodal = (Bx_mean[:-1,1:] + Bx_mean[:-1,:-1])/2. -By_nodal = (By_mean[:-1,:-1] + By_mean[1:,:-1])/2. - -x_vec = x_vec[:-1] -y_vec = y_vec[:-1] - - -XM, YM = np.meshgrid(x_vec, y_vec, indexing='xy') -RM = np.sqrt(XM**2 + YM**2) -THM = np.arctan2(YM,XM) - -Bt_mean = - np.sin(THM) * Bx_nodal + np.cos(THM) * By_nodal - - -r_vec = np.sqrt(x_vec**2 + y_vec**2) - -r_idx = (RM < 0.95*r_pipe) -r_sub = RM[r_idx] - -plt.figure(2) -plt.plot(r_vec, Bt_an(r_vec)) -plt.plot(RM[r_idx].flatten(), Bt_mean[r_idx].flatten(), '.') -plt.legend(['Analytical', 'Magnetostatic']) - -bt_err = np.abs(Bt_mean[r_idx] - Bt_an(r_sub)).max()/np.abs(Bt_an(r_sub)).max() - -plt.ylabel('$B_{\Theta}$ (T)') -plt.xlabel('r (m)') -plt.title("Max % Error: {} %".format(bt_err*100.)) -plt.tight_layout() -plt.savefig('bt_3d.png') - -assert (bt_err < 0.05), "Bt Max Error increased above 5%" diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py deleted file mode 100755 index e46f561f538..00000000000 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2022 S. Eric Clark, LLNL -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -""" -This script tests the magnetostatic solver in a beampipe by initializing a -uniform electron beam with relativistic gamma of 10, and a beam current of 1 kA. -The MLMG soltution from WarpX is computed and compared to the simple analytic -solution. The analytic and simulated values are plotted over one another for the -electric field and magnetic field solutions after taking the gradient of phi and -the curl of the vector potential A. This runs in 2 dimensions in RZ, has a PEC boundary -at z=0, and a Neumann boundary at z=1. This exercises the domain boundary conditions -as well as the embedded boundary conditions and will fail if the maximum error is too large. -The script additionally outputs png images with the average fields in a subset of the beampipe -as well as the analytical solution. -""" - -import matplotlib - -matplotlib.use('agg') - -import matplotlib.pyplot as plt -import numpy as np -import scipy.constants as con - -from pywarpx import fields, picmi - -########################## -# physics parameters -########################## - -V_domain_boundary = 0.0 -V_embedded_boundary = 1.0 - -########################## -# numerics parameters -########################## - -dt = 1e-12 - -# --- Nb time steps - -max_steps = 1 - -# --- grid - -nr = 128 -nz = 128 -rmin = 0. -rmax = 0.25 - -zmin = 0. -zmax = 1 - -r_pipe = 0.2 - -########################## -# numerics components -########################## - -grid = picmi.CylindricalGrid( - number_of_cells = [nr, nz], - lower_bound = [rmin, zmin], - upper_bound = [rmax, zmax], - lower_boundary_conditions = ['none', 'dirichlet'], - upper_boundary_conditions = ['neumann', 'neumann'], - lower_boundary_conditions_particles = ['none', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing'], - warpx_potential_lo_z = V_domain_boundary, - warpx_blocking_factor=8, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-7,warpx_magnetostatic=True,warpx_self_fields_verbosity=3 -) - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="(x**2+y**2-radius**2)", - potential=V_embedded_boundary, - radius = r_pipe -) - - -# Beam Current Density -current = 1000 # A -beam_r = 0.1 # m - -J = current/(np.pi * beam_r**2) -beam_gamma = 10. -vz = con.c*np.sqrt(1. - 1./beam_gamma**2) -n0 = J/(con.e*vz) - - -beam_dist = picmi.AnalyticDistribution( - density_expression='((x**2+y**2)= beam_r and r < r_pipe: - er = -current / (2.*np.pi*r*con.epsilon_0*vz) - else: - er = np.zeros_like(r) - return er - -# compare to region from 0.5*zmax to 0.9*zmax -z_idx = ((z_vec >= 0.5*zmax) & (z_vec < 0.9*zmax)) - -Er_dat = Er[...] - -r_idx = (r_vec < 0.95*r_pipe) -r_sub = r_vec[r_idx] - -# Average Er along z_sub -Er_mean = Er_dat[:,z_idx].mean(axis=1) - -plt.figure(1) -plt.plot(r_vec, Er_an(r_vec)) -plt.plot(r_vec, Er_mean,'--') -plt.legend(['Analytical', 'Electrostatic']) - -er_err = np.abs(Er_mean[r_idx] - Er_an(r_sub)).max()/np.abs(Er_an(r_sub)).max() - -plt.ylabel('$E_r$ (V/m)') -plt.xlabel('r (m)') -plt.title("Max % Error: {} %".format(er_err*100.)) -plt.tight_layout() -plt.savefig('er_rz.png') - -assert (er_err < 0.02), "Er Error increased above 2%" - -######################## -# Check B field -######################## - -Bth = fields.ByWrapper() - -r_vec = Bth.mesh('r') -z_vec = Bth.mesh('z') - -dr = r_vec[1]-r_vec[0] -r_vec = r_vec + dr/2. - -@np.vectorize -def Bth_an(r): - if r < beam_r: - bt = -current * r * con.mu_0 / (2.*np.pi*beam_r**2) - elif r >= beam_r and r < r_pipe: - bt = -current * con.mu_0 / (2.*np.pi*r) - else: - bt = np.zeros_like(r) - return bt - -# compare to region from 0.25*zmax to 0.75*zmax -z_idx = ((z_vec >= 0.25*zmax) & (z_vec < 0.75*zmax)) -z_sub = z_vec[z_idx] - -Bth_dat = Bth[...] - -r_idx = (r_vec < 0.95*r_pipe) -r_sub = r_vec[r_idx] - -# Average Bth along z_idx -Bth_mean = Bth_dat[:,z_idx].mean(axis=1) - -plt.figure(2) -plt.plot(r_vec, Bth_an(r_vec)) -plt.plot(r_vec, Bth_mean,'--') -plt.legend(['Analytical', 'Magnetostatic']) - -bth_err = np.abs(Bth_mean[r_idx] - Bth_an(r_sub)).max()/np.abs(Bth_an(r_sub)).max() - -plt.ylabel('$B_{\Theta}$ (T)') -plt.xlabel('r (m)') -plt.title("Max % Error: {} %".format(bth_err*100.)) -plt.tight_layout() -plt.savefig('bth_rz.png') - -assert (bth_err < 0.02), "Bth Error increased above 2%" diff --git a/Examples/Tests/magnetostatic_eb/analysis_default_regression.py b/Examples/Tests/magnetostatic_eb/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/magnetostatic_eb/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/magnetostatic_eb/analysis_rz.py b/Examples/Tests/magnetostatic_eb/analysis_rz.py deleted file mode 100755 index 00c270e597c..00000000000 --- a/Examples/Tests/magnetostatic_eb/analysis_rz.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import sys - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# Get name of the test -test_name = os.path.split(os.getcwd())[1] - -# Run checksum regression test -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=2.e-6, do_particles=False) -else: - checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) diff --git a/Examples/Tests/magnetostatic_eb/inputs_3d b/Examples/Tests/magnetostatic_eb/inputs_3d deleted file mode 100644 index 60617f7d392..00000000000 --- a/Examples/Tests/magnetostatic_eb/inputs_3d +++ /dev/null @@ -1,51 +0,0 @@ -amr.max_level = 0 -warpx.use_filter = 1 - -max_step = 1 -warpx.const_dt = 1e-12 -amr.n_cell = 64 64 64 -amr.blocking_factor = 8 -amr.max_grid_size = 16 -boundary.field_lo = neumann neumann pec -boundary.field_hi = neumann neumann neumann -boundary.particle_lo = absorbing absorbing absorbing -boundary.particle_hi = absorbing absorbing absorbing -boundary.potential_lo_z = 0.0 -geometry.dims = 3 -geometry.prob_lo = -0.25 -0.25 0.0 -geometry.prob_hi = 0.25 0.25 1.0 - -my_constants.rmax = 0.1 -my_constants.n0 = 666041719496846.5 -my_constants.radius = 0.2 - -warpx.do_electrostatic = labframe-electromagnetostatic -warpx.eb_implicit_function = "(x**2+y**2-radius**2)" -warpx.eb_potential(x,y,z,t) = "1." -warpx.self_fields_required_precision = 1.e-7 - -algo.field_gathering = momentum-conserving -algo.particle_shape = 1 -algo.current_deposition = direct - -particles.species_names = beam -particles.do_tiling = 1 -beam.mass = m_e -beam.charge = -q_e -beam.injection_style = nuniformpercell -beam.initialize_self_fields = 1 -beam.random_theta = 0 -beam.num_particles_per_cell_each_dim = 2 2 2 -beam.momentum_distribution_type = constant -beam.ux = 0.0 -beam.uy = 0.0 -beam.uz = 9.9498743710662 -beam.profile = parse_density_function -beam.density_function(x,y,z) = ((x**2+y**2)= beam_r and r < r_pipe: + er = -current / (2.0 * np.pi * r * con.epsilon_0 * vz) + else: + er = np.zeros_like(r) + return er + + +# compare to region from 0.5*zmax to 0.9*zmax +z_idx = (z_vec >= 0.5 * zmax) & (z_vec < 0.9 * zmax) + +Ex_dat = Ex[...] +Ey_dat = Ey[...] + +Ex_mean = Ex_dat[:, :, z_idx].mean(axis=2).T +Ey_mean = Ey_dat[:, :, z_idx].mean(axis=2).T + +Ex_nodal = Ex_mean +Ey_nodal = Ey_mean + +XM, YM = np.meshgrid(x_vec, y_vec, indexing="xy") +RM = np.sqrt(XM**2 + YM**2) +THM = np.arctan2(YM, XM) + +Er_mean = np.cos(THM) * Ex_nodal + np.sin(THM) * Ey_nodal + +r_vec = np.sqrt(x_vec**2 + y_vec**2) + +r_idx = RM < 0.95 * r_pipe +r_sub = RM[r_idx] + +plt.figure(1) +plt.plot(r_vec, Er_an(r_vec)) +plt.plot(RM.flatten(), Er_mean.flatten(), ".") +plt.legend(["Analytical", "Electrostatic"]) + +er_err = np.abs(Er_mean[r_idx] - Er_an(r_sub)).max() / np.abs(Er_an(r_sub)).max() + +plt.ylabel(r"$E_r$ (V/m)") +plt.xlabel("r (m)") +plt.title("Max % Error: {} %".format(er_err * 100.0)) +plt.tight_layout() +plt.savefig("er_3d.png") + +assert er_err < 0.05, "Er Max Error increased above 5%" + +######################## +# Check B field +######################## + +Bx = fields.BxWrapper() +By = fields.ByWrapper() + +x_vec = Bx.mesh("x") +y_vec = Bx.mesh("y") +z_vec = Bx.mesh("z") + +dx = x_vec[1] - x_vec[0] +dy = y_vec[1] - y_vec[0] +x_vec = x_vec + dx / 2.0 +y_vec = y_vec + dy / 2.0 + + +@np.vectorize +def Bt_an(r): + if r < beam_r: + bt = -current * r * con.mu_0 / (2.0 * np.pi * beam_r**2) + elif r >= beam_r and r < r_pipe: + bt = -current * con.mu_0 / (2.0 * np.pi * r) + else: + bt = np.zeros_like(r) + return bt + + +# compare to region from 0.25*zmax to 0.75*zmax +z_idx = (z_vec >= 0.25 * zmax) & (z_vec < 0.75 * zmax) +z_sub = z_vec[z_idx] + +Bx_dat = Bx[...] +By_dat = By[...] + +Bx_mean = Bx_dat[:, :, z_idx].mean(axis=2).T +By_mean = By_dat[:, :, z_idx].mean(axis=2).T + +# Interpolate B mesh to nodal points excluding last mesh point +Bx_nodal = (Bx_mean[:-1, 1:] + Bx_mean[:-1, :-1]) / 2.0 +By_nodal = (By_mean[:-1, :-1] + By_mean[1:, :-1]) / 2.0 + +x_vec = x_vec[:-1] +y_vec = y_vec[:-1] + + +XM, YM = np.meshgrid(x_vec, y_vec, indexing="xy") +RM = np.sqrt(XM**2 + YM**2) +THM = np.arctan2(YM, XM) + +Bt_mean = -np.sin(THM) * Bx_nodal + np.cos(THM) * By_nodal + + +r_vec = np.sqrt(x_vec**2 + y_vec**2) + +r_idx = RM < 0.95 * r_pipe +r_sub = RM[r_idx] + +plt.figure(2) +plt.plot(r_vec, Bt_an(r_vec)) +plt.plot(RM[r_idx].flatten(), Bt_mean[r_idx].flatten(), ".") +plt.legend(["Analytical", "Magnetostatic"]) + +bt_err = np.abs(Bt_mean[r_idx] - Bt_an(r_sub)).max() / np.abs(Bt_an(r_sub)).max() + +plt.ylabel(r"$B_{\Theta}$ (T)") +plt.xlabel("r (m)") +plt.title("Max % Error: {} %".format(bt_err * 100.0)) +plt.tight_layout() +plt.savefig("bt_3d.png") + +assert bt_err < 0.05, "Bt Max Error increased above 5%" diff --git a/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py b/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py new file mode 100755 index 00000000000..ff7767181f4 --- /dev/null +++ b/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 S. Eric Clark, LLNL +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the magnetostatic solver in a beampipe by initializing a +uniform electron beam with relativistic gamma of 10, and a beam current of 1 kA. +The MLMG soltution from WarpX is computed and compared to the simple analytic +solution. The analytic and simulated values are plotted over one another for the +electric field and magnetic field solutions after taking the gradient of phi and +the curl of the vector potential A. This runs in 2 dimensions in RZ, has a PEC boundary +at z=0, and a Neumann boundary at z=1. This exercises the domain boundary conditions +as well as the embedded boundary conditions and will fail if the maximum error is too large. +The script additionally outputs png images with the average fields in a subset of the beampipe +as well as the analytical solution. +""" + +import matplotlib + +matplotlib.use("agg") + +import matplotlib.pyplot as plt +import numpy as np +import scipy.constants as con + +from pywarpx import fields, picmi + +########################## +# physics parameters +########################## + +V_domain_boundary = 0.0 +V_embedded_boundary = 1.0 + +########################## +# numerics parameters +########################## + +dt = 1e-12 + +# --- Nb time steps + +max_steps = 1 + +# --- grid + +nr = 128 +nz = 128 +rmin = 0.0 +rmax = 0.25 + +zmin = 0.0 +zmax = 1 + +r_pipe = 0.2 + +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=["none", "dirichlet"], + upper_boundary_conditions=["neumann", "neumann"], + lower_boundary_conditions_particles=["none", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing"], + warpx_potential_lo_z=V_domain_boundary, + warpx_blocking_factor=8, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-7, + warpx_magnetostatic=True, + warpx_self_fields_verbosity=3, +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="(x**2+y**2-radius**2)", + potential=V_embedded_boundary, + radius=r_pipe, +) + + +# Beam Current Density +current = 1000 # A +beam_r = 0.1 # m + +J = current / (np.pi * beam_r**2) +beam_gamma = 10.0 +vz = con.c * np.sqrt(1.0 - 1.0 / beam_gamma**2) +n0 = J / (con.e * vz) + + +beam_dist = picmi.AnalyticDistribution( + density_expression="((x**2+y**2)= beam_r and r < r_pipe: + er = -current / (2.0 * np.pi * r * con.epsilon_0 * vz) + else: + er = np.zeros_like(r) + return er + + +# compare to region from 0.5*zmax to 0.9*zmax +z_idx = (z_vec >= 0.5 * zmax) & (z_vec < 0.9 * zmax) + +Er_dat = Er[...] + +r_idx = r_vec < 0.95 * r_pipe +r_sub = r_vec[r_idx] + +# Average Er along z_sub +Er_mean = Er_dat[:, z_idx].mean(axis=1) + +plt.figure(1) +plt.plot(r_vec, Er_an(r_vec)) +plt.plot(r_vec, Er_mean, "--") +plt.legend(["Analytical", "Electrostatic"]) + +er_err = np.abs(Er_mean[r_idx] - Er_an(r_sub)).max() / np.abs(Er_an(r_sub)).max() + +plt.ylabel(r"$E_r$ (V/m)") +plt.xlabel("r (m)") +plt.title("Max % Error: {} %".format(er_err * 100.0)) +plt.tight_layout() +plt.savefig("er_rz.png") + +assert er_err < 0.02, "Er Error increased above 2%" + +######################## +# Check B field +######################## + +Bth = fields.ByWrapper() + +r_vec = Bth.mesh("r") +z_vec = Bth.mesh("z") + +dr = r_vec[1] - r_vec[0] +r_vec = r_vec + dr / 2.0 + + +@np.vectorize +def Bth_an(r): + if r < beam_r: + bt = -current * r * con.mu_0 / (2.0 * np.pi * beam_r**2) + elif r >= beam_r and r < r_pipe: + bt = -current * con.mu_0 / (2.0 * np.pi * r) + else: + bt = np.zeros_like(r) + return bt + + +# compare to region from 0.25*zmax to 0.75*zmax +z_idx = (z_vec >= 0.25 * zmax) & (z_vec < 0.75 * zmax) +z_sub = z_vec[z_idx] + +Bth_dat = Bth[...] + +r_idx = r_vec < 0.95 * r_pipe +r_sub = r_vec[r_idx] + +# Average Bth along z_idx +Bth_mean = Bth_dat[:, z_idx].mean(axis=1) + +plt.figure(2) +plt.plot(r_vec, Bth_an(r_vec)) +plt.plot(r_vec, Bth_mean, "--") +plt.legend(["Analytical", "Magnetostatic"]) + +bth_err = np.abs(Bth_mean[r_idx] - Bth_an(r_sub)).max() / np.abs(Bth_an(r_sub)).max() + +plt.ylabel(r"$B_{\Theta}$ (T)") +plt.xlabel("r (m)") +plt.title("Max % Error: {} %".format(bth_err * 100.0)) +plt.tight_layout() +plt.savefig("bth_rz.png") + +assert bth_err < 0.02, "Bth Error increased above 2%" diff --git a/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt b/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt new file mode 100644 index 00000000000..c02d8abc567 --- /dev/null +++ b/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_FFT) + add_warpx_test( + test_2d_maxwell_hybrid_qed_solver # name + 2 # dims + 2 # nprocs + inputs_test_2d_maxwell_hybrid_qed_solver # inputs + "analysis.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/maxwell_hybrid_qed/analysis.py b/Examples/Tests/maxwell_hybrid_qed/analysis.py new file mode 100755 index 00000000000..88142362372 --- /dev/null +++ b/Examples/Tests/maxwell_hybrid_qed/analysis.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Axel Huebl, Glenn Richardson, Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +import sys + +import numpy as np +import scipy.constants as scc +import yt + +yt.funcs.mylog.setLevel(0) + +# Static electric field and quantum parameters, from the input file. +Es = 1.0e5 +xi = 1.0e-23 + +# Load dataset and get laser field +dsQED = yt.load(sys.argv[1]) +QED_all_data_level_0 = dsQED.covering_grid( + level=0, left_edge=(dsQED.domain_left_edge), dims=dsQED.domain_dimensions +) +EyQED_2d = QED_all_data_level_0["boxlib", "Ey"].v.squeeze() + +# Extract 1D lineout of the laser field +EyQED = EyQED_2d[EyQED_2d.shape[0] // 2, :] + +# Longitudinal resolution +dz = dsQED.domain_width[1].v / dsQED.domain_dimensions[1] + +# Initial position of the laser pulse max (from input file) +z_start = 0.0 +# Final position of the laser pulse max (from plotfile) +z_end = dsQED.domain_left_edge[1].v + np.argmax(EyQED) * dz +# Compute phase velocity and compare with theory +phase_velocity_pic = (z_end - z_start) / dsQED.current_time.v +phase_velocity_theory = scc.c / np.sqrt( + (1.0 + 12.0 * xi * Es**2 / scc.epsilon_0) / (1.0 + 4.0 * xi * Es**2 / scc.epsilon_0) +) +error_percent = ( + 100.0 * np.abs(phase_velocity_pic - phase_velocity_theory) / phase_velocity_theory +) + +# Print and assert correctness +print("Simulation velocity: " + str(phase_velocity_pic)) +print("Theory velocity : " + str(phase_velocity_theory)) +print("error (%) : " + str(error_percent)) +print( + "Theoretical difference between with/without QED (%): " + + str(100 * np.abs(phase_velocity_theory - scc.c) / scc.c) +) +assert error_percent < 1.25 diff --git a/Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py b/Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py deleted file mode 100755 index e74e9c524b9..00000000000 --- a/Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Axel Huebl, Glenn Richardson, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import sys - -import numpy as np -import scipy.constants as scc - -import yt ; yt.funcs.mylog.setLevel(0) - -# Static electric field and quantum parameters, from the input file. -Es = 1.0e5 -xi = 1.0e-23 - -# Load dataset and get laser field -dsQED = yt.load(sys.argv[1]) -QED_all_data_level_0 = dsQED.covering_grid(level=0,left_edge=(dsQED.domain_left_edge), - dims=dsQED.domain_dimensions) -EyQED_2d = QED_all_data_level_0['boxlib', 'Ey'].v.squeeze() - -# Extract 1D lineout of the laser field -EyQED = EyQED_2d[EyQED_2d.shape[0]//2,:] - -# Longitudinal resolution -dz = dsQED.domain_width[1].v/dsQED.domain_dimensions[1] - -# Initial position of the laser pulse max (from input file) -z_start = 0. -# Final position of the laser pulse max (from plotfile) -z_end = dsQED.domain_left_edge[1].v + np.argmax(EyQED) * dz -# Compute phase velocity and compare with theory -phase_velocity_pic = (z_end-z_start)/dsQED.current_time.v -phase_velocity_theory = scc.c/np.sqrt((1.+12.*xi*Es**2/scc.epsilon_0)/(1.+4.*xi*Es**2/scc.epsilon_0)) -error_percent = 100.*np.abs(phase_velocity_pic-phase_velocity_theory)/phase_velocity_theory - -# Print and assert correctness -print('Simulation velocity: ' + str(phase_velocity_pic)) -print('Theory velocity : ' + str(phase_velocity_theory)) -print('error (%) : ' + str(error_percent) ) -print('Theoretical difference between with/without QED (%): ' + str(100*np.abs(phase_velocity_theory-scc.c)/scc.c)) -assert( error_percent < 1.25 ) diff --git a/Examples/Tests/maxwell_hybrid_qed/analysis_default_regression.py b/Examples/Tests/maxwell_hybrid_qed/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/maxwell_hybrid_qed/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/maxwell_hybrid_qed/inputs_2d b/Examples/Tests/maxwell_hybrid_qed/inputs_2d deleted file mode 100644 index 2baa72c0990..00000000000 --- a/Examples/Tests/maxwell_hybrid_qed/inputs_2d +++ /dev/null @@ -1,53 +0,0 @@ -# ############################### -####### GENERAL PARAMETERS ###### -################################# -max_step = 300 -amr.n_cell = 64 1024 -amr.max_grid_size = 4096 -amr.blocking_factor = 8 -geometry.dims = 2 -geometry.prob_lo = -32.e-6 -512.e-6 -geometry.prob_hi = 32.e-6 512.e-6 -amr.max_level = 0 -warpx.grid_type = collocated -warpx.quantum_xi = 1.e-23 - -################################# -####### Boundary Condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = psatd -warpx.verbose = 0 -warpx.use_filter = 1 -warpx.cfl = 1. -warpx.use_hybrid_QED = 1 - -################################# -############ FIELDS ############# -################################# - -my_constants.L = 141.4213562373095e-6 -my_constants.wavelength = 64.e-6 -my_constants.Es = 1.e5 -my_constants.xi = 1.e-23 - - -warpx.E_ext_grid_init_style = parse_E_ext_grid_function -warpx.Ez_external_grid_function(x,y,z) = 0. -warpx.Ex_external_grid_function(x,y,z) = 0. -warpx.Ey_external_grid_function(x,y,z) = "exp(-z**2/L**2)*cos(2*pi*z/wavelength) + Es" - -warpx.B_ext_grid_init_style = parse_B_ext_grid_function -warpx.Bx_external_grid_function(x,y,z)= "-sqrt((1+(12*xi*Es**2)/epsilon0)/(1+(4*xi*Es**2)/epsilon0))*exp(-z**2/L**2)*cos(2*pi*z/wavelength)/clight" -warpx.By_external_grid_function(x,y,z)= 0. -warpx.Bz_external_grid_function(x,y,z) = 0. - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 300 -diag1.diag_type = Full diff --git a/Examples/Tests/maxwell_hybrid_qed/inputs_test_2d_maxwell_hybrid_qed_solver b/Examples/Tests/maxwell_hybrid_qed/inputs_test_2d_maxwell_hybrid_qed_solver new file mode 100644 index 00000000000..8e1091c91e2 --- /dev/null +++ b/Examples/Tests/maxwell_hybrid_qed/inputs_test_2d_maxwell_hybrid_qed_solver @@ -0,0 +1,53 @@ +# ############################### +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 64 1024 +amr.max_grid_size = 4096 +amr.blocking_factor = 8 +geometry.dims = 2 +geometry.prob_lo = -32.e-6 -512.e-6 +geometry.prob_hi = 32.e-6 512.e-6 +amr.max_level = 0 +warpx.grid_type = collocated +warpx.quantum_xi = 1.e-23 + +################################# +####### Boundary Condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = psatd +warpx.verbose = 0 +warpx.use_filter = 1 +warpx.cfl = 0.7071067811865475 +warpx.use_hybrid_QED = 1 + +################################# +############ FIELDS ############# +################################# + +my_constants.L = 141.4213562373095e-6 +my_constants.wavelength = 64.e-6 +my_constants.Es = 1.e5 +my_constants.xi = 1.e-23 + + +warpx.E_ext_grid_init_style = parse_E_ext_grid_function +warpx.Ez_external_grid_function(x,y,z) = 0. +warpx.Ex_external_grid_function(x,y,z) = 0. +warpx.Ey_external_grid_function(x,y,z) = "exp(-z**2/L**2)*cos(2*pi*z/wavelength) + Es" + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function +warpx.Bx_external_grid_function(x,y,z)= "-sqrt((1+(12*xi*Es**2)/epsilon0)/(1+(4*xi*Es**2)/epsilon0))*exp(-z**2/L**2)*cos(2*pi*z/wavelength)/clight" +warpx.By_external_grid_function(x,y,z)= 0. +warpx.Bz_external_grid_function(x,y,z) = 0. + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 300 +diag1.diag_type = Full diff --git a/Examples/Tests/multi_j/inputs_rz b/Examples/Tests/multi_j/inputs_rz deleted file mode 100644 index dd440d60667..00000000000 --- a/Examples/Tests/multi_j/inputs_rz +++ /dev/null @@ -1,119 +0,0 @@ -# Iterations -max_step = 50 - -# Domain -amr.n_cell = 64 128 -amr.max_level = 0 -warpx.numprocs = 1 2 - -# Geometry -geometry.dims = RZ -geometry.prob_lo = 0 -220e-6 -geometry.prob_hi = 200e-6 10e-6 -boundary.field_lo = none damped -boundary.field_hi = none damped - -# Algorithms -algo.current_deposition = direct -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -algo.maxwell_solver = psatd -algo.particle_pusher = vay -algo.particle_shape = 3 - -# Numerics -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1. -warpx.n_rz_azimuthal_modes = 1 -warpx.use_filter = 1 -warpx.verbose = 1 - -warpx.cfl = 1. -#warpx.gamma_boost = 1. -#warpx.boost_direction = z - -warpx.do_dive_cleaning = 1 -warpx.do_divb_cleaning = 1 -warpx.do_multi_J = 1 -warpx.do_multi_J_n_depositions = 2 -psatd.do_time_averaging = 1 - -# PSATD -psatd.update_with_rho = 1 -#psatd.v_galilean = 0. 0. -0.9373391857121336 - -# Particles - -particles.species_names = driver driver_back plasma_e plasma_p -particles.use_fdtd_nci_corr = 0 -particles.rigid_injected_species = driver - -driver.species_type = electron -driver.injection_style = "gaussian_beam" -driver.x_rms = 5e-6 -driver.y_rms = 5e-6 -driver.z_rms = 20.1e-6 -driver.x_m = 0. -driver.y_m = 0. -driver.z_m = -80e-6 -driver.npart = 1000000 -driver.q_tot = -1e-9 -driver.momentum_distribution_type = "gaussian" -driver.ux_m = 0. -driver.uy_m = 0. -driver.uz_m = 2000. -driver.ux_th = 4. -driver.uy_th = 4. -driver.uz_th = 20. -driver.zinject_plane = 2e-3 -driver.rigid_advance = true -driver.initialize_self_fields = 0 -driver.do_symmetrize = 1 -driver.symmetrization_order = 4 - -driver_back.species_type = positron -driver_back.injection_style = "gaussian_beam" -driver_back.x_rms = 5e-6 -driver_back.y_rms = 5e-6 -driver_back.z_rms = 20.1e-6 -driver_back.x_m = 0. -driver_back.y_m = 0. -driver_back.z_m = -80e-6 -driver_back.npart = 1000000 -driver_back.q_tot = 1e-9 -driver_back.momentum_distribution_type = "at_rest" -driver_back.initialize_self_fields = 0 -driver_back.do_symmetrize = 1 -driver_back.symmetrization_order = 4 - -plasma_e.species_type = electron -plasma_e.injection_style = "NUniformPerCell" -plasma_e.zmin = 0. -plasma_e.zmax = 0.5 -plasma_e.xmin = -180e-6 -plasma_e.xmax = 180e-6 -plasma_e.profile = constant -plasma_e.density = 1e23 -plasma_e.num_particles_per_cell_each_dim = 2 2 1 -plasma_e.momentum_distribution_type = "at_rest" -plasma_e.do_continuous_injection = 1 - -plasma_p.species_type = hydrogen -plasma_p.injection_style = "NUniformPerCell" -plasma_p.zmin = 0. -plasma_p.zmax = 0.5 -plasma_p.xmin = -180e-6 -plasma_p.xmax = 180e-6 -plasma_p.profile = constant -plasma_p.density = 1e23 -plasma_p.num_particles_per_cell_each_dim = 2 2 1 -plasma_p.momentum_distribution_type = "at_rest" -plasma_p.do_continuous_injection = 1 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 50 -diag1.diag_type = Full -diag1.fields_to_plot = Er Ez Bt jr jz rho rho_driver rho_plasma_e rho_plasma_p -diag1.species = driver plasma_e plasma_p diff --git a/Examples/Tests/nci_fdtd_stability/CMakeLists.txt b/Examples/Tests/nci_fdtd_stability/CMakeLists.txt new file mode 100644 index 00000000000..9af4034c3ca --- /dev/null +++ b/Examples/Tests/nci_fdtd_stability/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_nci_corrector # name + 2 # dims + 2 # nprocs + inputs_test_2d_nci_corrector # inputs + "analysis_ncicorr.py diags/diag1000600" # analysis + "analysis_default_regression.py --path diags/diag1000600" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_nci_corrector_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_nci_corrector_mr # inputs + "analysis_ncicorr.py diags/diag1000600" # analysis + "analysis_default_regression.py --path diags/diag1000600" # checksum + OFF # dependency +) diff --git a/Examples/Tests/nci_fdtd_stability/analysis_default_regression.py b/Examples/Tests/nci_fdtd_stability/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/nci_fdtd_stability/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py b/Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py index 118b03fed89..290cdea819f 100755 --- a/Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py +++ b/Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py @@ -8,7 +8,6 @@ # License: BSD-3-Clause-LBNL -import os import re import sys @@ -18,35 +17,31 @@ yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - fn = sys.argv[1] -use_MR = re.search( 'nci_correctorMR', fn ) != None +use_MR = re.search("nci_correctorMR", fn) is not None if use_MR: - energy_corrector_off = 5.e32 - energy_threshold = 1.e28 + energy_corrector_off = 5.0e32 + energy_threshold = 1.0e28 else: energy_corrector_off = 1.5e26 - energy_threshold = 1.e24 + energy_threshold = 1.0e24 # Check EB energy after 1000 timesteps filename = sys.argv[1] -ds = yt.load( filename ) -ad0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -ex = ad0['boxlib', 'Ex'].v -ez = ad0['boxlib', 'Ez'].v -by = ad0['boxlib', 'By'].v -energy = np.sum(ex**2 + ez**2 + scc.c**2*by**2) - -print("use_MR: %s" %use_MR) -print("energy if corrector off (from benchmark): %s" %energy_corrector_off) -print("energy threshold (from benchmark): %s" %energy_threshold) -print("energy from this run: %s" %energy) - -assert( energy < energy_threshold ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +ds = yt.load(filename) +ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +ex = ad0["boxlib", "Ex"].v +ez = ad0["boxlib", "Ez"].v +by = ad0["boxlib", "By"].v +energy = np.sum(ex**2 + ez**2 + scc.c**2 * by**2) + +print("use_MR: %s" % use_MR) +print("energy if corrector off (from benchmark): %s" % energy_corrector_off) +print("energy threshold (from benchmark): %s" % energy_threshold) +print("energy from this run: %s" % energy) + +assert energy < energy_threshold diff --git a/Examples/Tests/nci_fdtd_stability/inputs_2d b/Examples/Tests/nci_fdtd_stability/inputs_base_2d similarity index 100% rename from Examples/Tests/nci_fdtd_stability/inputs_2d rename to Examples/Tests/nci_fdtd_stability/inputs_base_2d diff --git a/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector b/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector new file mode 100644 index 00000000000..83d537fd856 --- /dev/null +++ b/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +amr.max_level = 0 +particles.use_fdtd_nci_corr = 1 diff --git a/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector_mr b/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector_mr new file mode 100644 index 00000000000..0f53af0443a --- /dev/null +++ b/Examples/Tests/nci_fdtd_stability/inputs_test_2d_nci_corrector_mr @@ -0,0 +1,9 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +amr.max_level = 1 +amr.n_cell = 64 64 +particles.use_fdtd_nci_corr = 1 +warpx.fine_tag_hi = 20.e-6 20.e-6 +warpx.fine_tag_lo = -20.e-6 -20.e-6 diff --git a/Examples/Tests/nci_psatd_stability/CMakeLists.txt b/Examples/Tests/nci_psatd_stability/CMakeLists.txt new file mode 100644 index 00000000000..210fb13f542 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/CMakeLists.txt @@ -0,0 +1,207 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_FFT) + add_warpx_test( + test_2d_averaged_galilean_psatd # name + 2 # dims + 1 # nprocs + inputs_test_2d_averaged_galilean_psatd # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_averaged_galilean_psatd_hybrid # name + 2 # dims + 2 # nprocs + inputs_test_2d_averaged_galilean_psatd_hybrid # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_comoving_psatd_hybrid # name + 2 # dims + 2 # nprocs + inputs_test_2d_comoving_psatd_hybrid # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000400" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_galilean_psatd # name + 2 # dims + 1 # nprocs + inputs_test_2d_galilean_psatd # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_galilean_psatd_current_correction # name + 2 # dims + 2 # nprocs + inputs_test_2d_galilean_psatd_current_correction # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_galilean_psatd_current_correction_psb # name + 2 # dims + 1 # nprocs + inputs_test_2d_galilean_psatd_current_correction_psb # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_galilean_psatd_hybrid # name + 2 # dims + 2 # nprocs + inputs_test_2d_galilean_psatd_hybrid # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000400" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_averaged_galilean_psatd # name + 3 # dims + 2 # nprocs + inputs_test_3d_averaged_galilean_psatd # inputs + "analysis_galilean.py diags/diag1000160" # analysis + "analysis_default_regression.py --path diags/diag1000160 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_averaged_galilean_psatd_hybrid # name + 3 # dims + 2 # nprocs + inputs_test_3d_averaged_galilean_psatd_hybrid # inputs + "analysis_galilean.py diags/diag1000160" # analysis + "analysis_default_regression.py --path diags/diag1000160 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_galilean_psatd # name + 3 # dims + 2 # nprocs + inputs_test_3d_galilean_psatd # inputs + "analysis_galilean.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_galilean_psatd_current_correction # name + 3 # dims + 2 # nprocs + inputs_test_3d_galilean_psatd_current_correction # inputs + "analysis_galilean.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_galilean_psatd_current_correction_psb # name + 3 # dims + 1 # nprocs + inputs_test_3d_galilean_psatd_current_correction_psb # inputs + "analysis_galilean.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_uniform_plasma_multiJ # name + 3 # dims + 2 # nprocs + inputs_test_3d_uniform_plasma_multiJ # inputs + "analysis_multiJ.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_galilean_psatd # name + RZ # dims + 1 # nprocs + inputs_test_rz_galilean_psatd # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_galilean_psatd_current_correction # name + RZ # dims + 2 # nprocs + inputs_test_rz_galilean_psatd_current_correction # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_galilean_psatd_current_correction_psb # name + RZ # dims + 1 # nprocs + inputs_test_rz_galilean_psatd_current_correction_psb # inputs + "analysis_galilean.py diags/diag1000400" # analysis + "analysis_default_regression.py --path diags/diag1000400 --rtol 1e-8" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_multiJ_psatd # name + RZ # dims + 2 # nprocs + inputs_test_rz_multiJ_psatd # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000025" # checksum + OFF # dependency + ) + label_warpx_test(test_rz_multiJ_psatd slow) +endif() diff --git a/Examples/Tests/nci_psatd_stability/analysis_default_regression.py b/Examples/Tests/nci_psatd_stability/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/nci_psatd_stability/analysis_galilean.py b/Examples/Tests/nci_psatd_stability/analysis_galilean.py index 666d240da8f..43bdaedabbf 100755 --- a/Examples/Tests/nci_psatd_stability/analysis_galilean.py +++ b/Examples/Tests/nci_psatd_stability/analysis_galilean.py @@ -12,16 +12,15 @@ In both cases, the reference energy corresponds to unstable results due to NCI (suppressed by the Galilean PSATD method, without or with averaging, respectively). """ -import os + import re import sys import numpy as np import scipy.constants as scc +import yt -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(0) filename = sys.argv[1] @@ -29,18 +28,19 @@ current_correction = False time_averaging = False periodic_single_box = False -warpx_used_inputs = open('./warpx_used_inputs', 'r').read() -if re.search('geometry.dims\s*=\s*2', warpx_used_inputs): - dims = '2D' -elif re.search('geometry.dims\s*=\s*RZ', warpx_used_inputs): - dims = 'RZ' -elif re.search('geometry.dims\s*=\s*3', warpx_used_inputs): - dims = '3D' -if re.search('psatd.current_correction\s*=\s*1', warpx_used_inputs): +with open("./warpx_used_inputs", "r") as f: + warpx_used_inputs = f.read() +if re.search("geometry.dims\s*=\s*2", warpx_used_inputs): + dims = "2D" +elif re.search("geometry.dims\s*=\s*RZ", warpx_used_inputs): + dims = "RZ" +elif re.search("geometry.dims\s*=\s*3", warpx_used_inputs): + dims = "3D" +if re.search("psatd.current_correction\s*=\s*1", warpx_used_inputs): current_correction = True -if re.search('psatd.do_time_averaging\s*=\s*1', warpx_used_inputs): +if re.search("psatd.do_time_averaging\s*=\s*1", warpx_used_inputs): time_averaging = True -if re.search('psatd.periodic_single_box_fft\s*=\s*1', warpx_used_inputs): +if re.search("psatd.periodic_single_box_fft\s*=\s*1", warpx_used_inputs): periodic_single_box = True ds = yt.load(filename) @@ -48,21 +48,24 @@ # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. -if 'force_periodicity' in dir(ds): ds.force_periodicity() +if "force_periodicity" in dir(ds): + ds.force_periodicity() -all_data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -if dims == 'RZ': - Ex = all_data['boxlib', 'Er'].squeeze().v - Ey = all_data['boxlib', 'Et'].squeeze().v +all_data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +if dims == "RZ": + Ex = all_data["boxlib", "Er"].squeeze().v + Ey = all_data["boxlib", "Et"].squeeze().v else: - Ex = all_data['boxlib', 'Ex'].squeeze().v - Ey = all_data['boxlib', 'Ey'].squeeze().v -Ez = all_data['boxlib', 'Ez'].squeeze().v + Ex = all_data["boxlib", "Ex"].squeeze().v + Ey = all_data["boxlib", "Ey"].squeeze().v +Ez = all_data["boxlib", "Ez"].squeeze().v # Set reference energy values, and tolerances for numerical stability and charge conservation tol_energy = 1e-8 tol_charge = 1e-9 -if dims == '2D': +if dims == "2D": if not current_correction: energy_ref = 35657.41657683263 if current_correction and periodic_single_box: @@ -74,7 +77,7 @@ if time_averaging: energy_ref = 26208.04843478073 tol_energy = 1e-6 -elif dims == 'RZ': +elif dims == "RZ": if not current_correction: energy_ref = 191002.6526271543 if current_correction and periodic_single_box: @@ -82,7 +85,7 @@ if current_correction and not periodic_single_box: energy_ref = 511671.4108624746 tol_charge = 3e-4 -elif dims == '3D': +elif dims == "3D": if not current_correction: energy_ref = 661285.098907683 if current_correction and periodic_single_box: @@ -95,22 +98,19 @@ tol_energy = 1e-4 # Check numerical stability by comparing electric field energy to reference energy -energy = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) +energy = np.sum(scc.epsilon_0 / 2 * (Ex**2 + Ey**2 + Ez**2)) err_energy = energy / energy_ref -print('\nCheck numerical stability:') -print(f'err_energy = {err_energy}') -print(f'tol_energy = {tol_energy}') -assert(err_energy < tol_energy) +print("\nCheck numerical stability:") +print(f"err_energy = {err_energy}") +print(f"tol_energy = {tol_energy}") +assert err_energy < tol_energy # Check charge conservation (relative L-infinity norm of error) with current correction if current_correction: - divE = all_data['boxlib', 'divE'].squeeze().v - rho = all_data['boxlib', 'rho' ].squeeze().v / scc.epsilon_0 + divE = all_data["boxlib", "divE"].squeeze().v + rho = all_data["boxlib", "rho"].squeeze().v / scc.epsilon_0 err_charge = np.amax(np.abs(divE - rho)) / max(np.amax(divE), np.amax(rho)) - print('\nCheck charge conservation:') - print(f'err_charge = {err_charge}') - print(f'tol_charge = {tol_charge}') - assert(err_charge < tol_charge) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, rtol=1.e-8) + print("\nCheck charge conservation:") + print(f"err_charge = {err_charge}") + print(f"tol_charge = {tol_charge}") + assert err_charge < tol_charge diff --git a/Examples/Tests/nci_psatd_stability/analysis_multiJ.py b/Examples/Tests/nci_psatd_stability/analysis_multiJ.py index 1c68b114c1a..19ba722781b 100755 --- a/Examples/Tests/nci_psatd_stability/analysis_multiJ.py +++ b/Examples/Tests/nci_psatd_stability/analysis_multiJ.py @@ -9,15 +9,14 @@ energy corresponds to unstable results due to NCI (suppressed by the use of both J and rho constant in time, and with divergence cleaning). """ -import os + import sys import numpy as np import scipy.constants as scc +import yt -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(0) filename = sys.argv[1] @@ -26,24 +25,24 @@ # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. -if 'force_periodicity' in dir(ds): ds.force_periodicity() +if "force_periodicity" in dir(ds): + ds.force_periodicity() -all_data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Ex = all_data['boxlib', 'Ex'].squeeze().v -Ey = all_data['boxlib', 'Ey'].squeeze().v -Ez = all_data['boxlib', 'Ez'].squeeze().v +all_data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Ex = all_data["boxlib", "Ex"].squeeze().v +Ey = all_data["boxlib", "Ey"].squeeze().v +Ez = all_data["boxlib", "Ez"].squeeze().v # Set reference energy values, and tolerances for numerical stability and charge conservation tol_energy = 1e-8 energy_ref = 66e6 # Check numerical stability by comparing electric field energy to reference energy -energy = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) +energy = np.sum(scc.epsilon_0 / 2 * (Ex**2 + Ey**2 + Ez**2)) err_energy = energy / energy_ref -print('\nCheck numerical stability:') -print(f'err_energy = {err_energy}') -print(f'tol_energy = {tol_energy}') -assert(err_energy < tol_energy) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +print("\nCheck numerical stability:") +print(f"err_energy = {err_energy}") +print(f"tol_energy = {tol_energy}") +assert err_energy < tol_energy diff --git a/Examples/Tests/nci_psatd_stability/inputs_2d_hybrid b/Examples/Tests/nci_psatd_stability/inputs_2d_hybrid deleted file mode 100644 index 90dfd58c4ae..00000000000 --- a/Examples/Tests/nci_psatd_stability/inputs_2d_hybrid +++ /dev/null @@ -1,113 +0,0 @@ -max_step = 400 - -amr.max_level = 0 -amr.n_cell = 48 704 -warpx.numprocs = 1 2 - -geometry.dims = 2 -geometry.prob_lo = -70.e-6 -50.e-6 -geometry.prob_hi = 70.e-6 0.e-6 - -# Boundary condition -boundary.field_lo = periodic damped -boundary.field_hi = periodic damped - -algo.maxwell_solver = psatd -algo.current_deposition = direct -algo.charge_deposition = standard -algo.particle_pusher = vay - -# Order of particle shape factors -algo.particle_shape = 3 - -psatd.use_default_v_galilean = 1 - -warpx.cfl = 1. - -warpx.grid_type = hybrid -warpx.do_current_centering = 0 - -warpx.gamma_boost = 13. -warpx.boost_direction = z - -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 - -warpx.use_filter = 1 - -warpx.serialize_initial_conditions = 1 -warpx.verbose = 1 - -particles.species_names = electrons ions beam -particles.use_fdtd_nci_corr = 0 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 2 2 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.0001 -electrons.uy_th = 0.0001 -electrons.uz_th = 0.0001 -electrons.xmin = -45.e-6 -electrons.xmax = 45.e-6 -electrons.zmin = 0. -electrons.zmax = 15000.e-6 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -electrons.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 2 2 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -45.e-6 -ions.xmax = 45.e-6 -ions.zmin = 0. -ions.zmax = 15000.e-6 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -ions.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 -ions.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 2.e-6 -beam.y_rms = 2.e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -42.e-6 -beam.npart = 1000 -beam.q_tot = -1.e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 200. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 0. -beam.zinject_plane = 0.5e-3 -beam.rigid_advance = true - -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. -0.1e-6 -laser1.direction = 0. 0. 1. -laser1.polarization = 0. 1. 0. -laser1.e_max = 6.e12 -laser1.profile_waist = 30.e-6 -laser1.profile_duration = 26.e-15 -laser1.profile_t_peak = 66.e-15 -laser1.profile_focal_distance = 0.5e-3 -laser1.wavelength = 0.81e-6 - -diagnostics.diags_names = diag1 -diag1.intervals = 400 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Tests/nci_psatd_stability/inputs_2d b/Examples/Tests/nci_psatd_stability/inputs_base_2d similarity index 100% rename from Examples/Tests/nci_psatd_stability/inputs_2d rename to Examples/Tests/nci_psatd_stability/inputs_base_2d diff --git a/Examples/Tests/nci_psatd_stability/inputs_avg_2d b/Examples/Tests/nci_psatd_stability/inputs_base_2d_averaged similarity index 100% rename from Examples/Tests/nci_psatd_stability/inputs_avg_2d rename to Examples/Tests/nci_psatd_stability/inputs_base_2d_averaged diff --git a/Examples/Tests/nci_psatd_stability/inputs_3d b/Examples/Tests/nci_psatd_stability/inputs_base_3d similarity index 100% rename from Examples/Tests/nci_psatd_stability/inputs_3d rename to Examples/Tests/nci_psatd_stability/inputs_base_3d diff --git a/Examples/Tests/nci_psatd_stability/inputs_avg_3d b/Examples/Tests/nci_psatd_stability/inputs_base_3d_averaged similarity index 100% rename from Examples/Tests/nci_psatd_stability/inputs_avg_3d rename to Examples/Tests/nci_psatd_stability/inputs_base_3d_averaged diff --git a/Examples/Tests/nci_psatd_stability/inputs_rz b/Examples/Tests/nci_psatd_stability/inputs_base_rz similarity index 100% rename from Examples/Tests/nci_psatd_stability/inputs_rz rename to Examples/Tests/nci_psatd_stability/inputs_base_rz diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd new file mode 100644 index 00000000000..62f93dbd473 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d_averaged + +# test input parameters +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd_hybrid b/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd_hybrid new file mode 100644 index 00000000000..0ef3668b103 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_averaged_galilean_psatd_hybrid @@ -0,0 +1,9 @@ +# base input parameters +FILE = inputs_base_2d_averaged + +# test input parameters +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.grid_type = hybrid diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_comoving_psatd_hybrid b/Examples/Tests/nci_psatd_stability/inputs_test_2d_comoving_psatd_hybrid new file mode 100644 index 00000000000..32b155cf0b6 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_comoving_psatd_hybrid @@ -0,0 +1,117 @@ +max_step = 400 + +amr.max_level = 0 +amr.n_cell = 48 704 +warpx.numprocs = 1 2 + +geometry.dims = 2 +geometry.prob_lo = -70.e-6 -50.e-6 +geometry.prob_hi = 70.e-6 0.e-6 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = periodic damped +boundary.field_hi = periodic damped + +algo.maxwell_solver = psatd +algo.current_deposition = direct +algo.charge_deposition = standard +algo.particle_pusher = vay + +# Order of particle shape factors +algo.particle_shape = 3 + +psatd.use_default_v_comoving = 1 +psatd.current_correction = 0 + +warpx.cfl = 1. + +warpx.grid_type = hybrid +warpx.do_current_centering = 0 + +warpx.gamma_boost = 13. +warpx.boost_direction = z + +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 + +warpx.use_filter = 1 + +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.abort_on_warning_threshold = medium + +particles.species_names = electrons ions beam +particles.use_fdtd_nci_corr = 0 +particles.rigid_injected_species = beam + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = NUniformPerCell +electrons.num_particles_per_cell_each_dim = 2 2 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = 0.0001 +electrons.uy_th = 0.0001 +electrons.uz_th = 0.0001 +electrons.xmin = -45.e-6 +electrons.xmax = 45.e-6 +electrons.zmin = 0. +electrons.zmax = 15000.e-6 +electrons.profile = "predefined" +electrons.predefined_profile_name = "parabolic_channel" +electrons.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 +electrons.do_continuous_injection = 1 + +ions.charge = q_e +ions.mass = m_p +ions.injection_style = NUniformPerCell +ions.num_particles_per_cell_each_dim = 2 2 +ions.momentum_distribution_type = "at_rest" +ions.xmin = -45.e-6 +ions.xmax = 45.e-6 +ions.zmin = 0. +ions.zmax = 15000.e-6 +ions.profile = "predefined" +ions.predefined_profile_name = "parabolic_channel" +ions.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 +ions.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -42.e-6 +beam.npart = 1000 +beam.q_tot = -1.e-12 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 200. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 0. +beam.zinject_plane = 0.5e-3 +beam.rigid_advance = true + +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -0.1e-6 +laser1.direction = 0. 0. 1. +laser1.polarization = 0. 1. 0. +laser1.e_max = 6.e12 +laser1.profile_waist = 30.e-6 +laser1.profile_duration = 26.e-15 +laser1.profile_t_peak = 66.e-15 +laser1.profile_focal_distance = 0.5e-3 +laser1.wavelength = 0.81e-6 + +diagnostics.diags_names = diag1 +diag1.intervals = 400 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd new file mode 100644 index 00000000000..caebf987434 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_2d + +# test input paramters +algo.current_deposition = direct +psatd.current_correction = 0 +warpx.grid_type = collocated +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction new file mode 100644 index 00000000000..177cf7bcd0c --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_2d + +# test input paramters +amr.blocking_factor = 64 +amr.max_grid_size = 64 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 0 +psatd.update_with_rho = 0 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction_psb b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction_psb new file mode 100644 index 00000000000..437059d6bd8 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_current_correction_psb @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_2d + +# test input paramters +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +psatd.update_with_rho = 0 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_hybrid b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_hybrid new file mode 100644 index 00000000000..501c964353f --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_2d_galilean_psatd_hybrid @@ -0,0 +1,115 @@ +max_step = 400 + +amr.max_level = 0 +amr.n_cell = 48 704 +warpx.numprocs = 1 2 + +geometry.dims = 2 +geometry.prob_lo = -70.e-6 -50.e-6 +geometry.prob_hi = 70.e-6 0.e-6 + +# Boundary condition +boundary.field_lo = periodic damped +boundary.field_hi = periodic damped + +algo.maxwell_solver = psatd +algo.current_deposition = direct +algo.charge_deposition = standard +algo.particle_pusher = vay + +# Order of particle shape factors +algo.particle_shape = 3 + +psatd.use_default_v_galilean = 1 +psatd.current_correction = 0 + +warpx.cfl = 1. + +warpx.grid_type = hybrid +warpx.do_current_centering = 0 + +warpx.gamma_boost = 13. +warpx.boost_direction = z + +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 + +warpx.use_filter = 1 + +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.abort_on_warning_threshold = medium + +particles.species_names = electrons ions beam +particles.use_fdtd_nci_corr = 0 +particles.rigid_injected_species = beam + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = NUniformPerCell +electrons.num_particles_per_cell_each_dim = 2 2 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = 0.0001 +electrons.uy_th = 0.0001 +electrons.uz_th = 0.0001 +electrons.xmin = -45.e-6 +electrons.xmax = 45.e-6 +electrons.zmin = 0. +electrons.zmax = 15000.e-6 +electrons.profile = "predefined" +electrons.predefined_profile_name = "parabolic_channel" +electrons.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 +electrons.do_continuous_injection = 1 + +ions.charge = q_e +ions.mass = m_p +ions.injection_style = NUniformPerCell +ions.num_particles_per_cell_each_dim = 2 2 +ions.momentum_distribution_type = "at_rest" +ions.xmin = -45.e-6 +ions.xmax = 45.e-6 +ions.zmin = 0. +ions.zmax = 15000.e-6 +ions.profile = "predefined" +ions.predefined_profile_name = "parabolic_channel" +ions.predefined_profile_params = 0. 0.5e-3 14.e-3 0.5e-3 30.e-6 1.e24 +ions.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -42.e-6 +beam.npart = 1000 +beam.q_tot = -1.e-12 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 200. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 0. +beam.zinject_plane = 0.5e-3 +beam.rigid_advance = true + +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -0.1e-6 +laser1.direction = 0. 0. 1. +laser1.polarization = 0. 1. 0. +laser1.e_max = 6.e12 +laser1.profile_waist = 30.e-6 +laser1.profile_duration = 26.e-15 +laser1.profile_t_peak = 66.e-15 +laser1.profile_focal_distance = 0.5e-3 +laser1.wavelength = 0.81e-6 + +diagnostics.diags_names = diag1 +diag1.intervals = 400 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd new file mode 100644 index 00000000000..7c978874145 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d_averaged + +# test input parameters +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd_hybrid b/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd_hybrid new file mode 100644 index 00000000000..4996f476854 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_averaged_galilean_psatd_hybrid @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_3d_averaged + +# test input parameters +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium +warpx.grid_type = hybrid diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd new file mode 100644 index 00000000000..3ec82981aea --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +psatd.current_correction = 0 +psatd.v_galilean = 0. 0. 0.99498743710662 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction new file mode 100644 index 00000000000..8b596c9a633 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 0 +psatd.update_with_rho = 0 +psatd.v_galilean = 0. 0. 0.99498743710662 +warpx.numprocs = 1 1 2 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction_psb b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction_psb new file mode 100644 index 00000000000..87ce1b7ed92 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_galilean_psatd_current_correction_psb @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 +psatd.v_galilean = 0. 0. 0.99498743710662 +psatd.update_with_rho = 0 +warpx.numprocs = 1 1 1 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_3d_uniform_plasma_multiJ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_uniform_plasma_multiJ new file mode 100644 index 00000000000..70e9c5e992c --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_3d_uniform_plasma_multiJ @@ -0,0 +1,13 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +diag1.fields_to_plot = Bx By Bz divE Ex Ey Ez F G jx jy jz rho +psatd.J_in_time = constant +psatd.rho_in_time = constant +psatd.solution_type = first-order +warpx.abort_on_warning_threshold = medium +warpx.do_divb_cleaning = 1 +warpx.do_dive_cleaning = 1 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 1 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd new file mode 100644 index 00000000000..30bcfc160cf --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_rz + +# test input paramters +electrons.random_theta = 0 +ions.random_theta = 0 +psatd.current_correction = 0 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction new file mode 100644 index 00000000000..378535e12bc --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction @@ -0,0 +1,10 @@ +# base input parameters +FILE = inputs_base_rz + +# test input paramters +amr.blocking_factor = 32 +amr.max_grid_size = 32 +electrons.random_theta = 0 +ions.random_theta = 0 +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 0 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction_psb b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction_psb new file mode 100644 index 00000000000..6eb35754a90 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_rz_galilean_psatd_current_correction_psb @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_rz + +# test input paramters +electrons.random_theta = 0 +ions.random_theta = 0 +psatd.current_correction = 1 +psatd.periodic_single_box_fft = 1 diff --git a/Examples/Tests/nci_psatd_stability/inputs_test_rz_multiJ_psatd b/Examples/Tests/nci_psatd_stability/inputs_test_rz_multiJ_psatd new file mode 100644 index 00000000000..6350b9aee51 --- /dev/null +++ b/Examples/Tests/nci_psatd_stability/inputs_test_rz_multiJ_psatd @@ -0,0 +1,121 @@ +# Iterations +max_step = 25 + +# Domain +amr.n_cell = 32 64 +amr.max_level = 0 +warpx.numprocs = 1 2 + +# Geometry +geometry.dims = RZ +geometry.prob_lo = 0 -220e-6 +geometry.prob_hi = 200e-6 10e-6 +boundary.field_lo = none damped +boundary.field_hi = none damped + +# Algorithms +algo.current_deposition = direct +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.maxwell_solver = psatd +algo.particle_pusher = vay +algo.particle_shape = 3 + +# Numerics +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1. +warpx.n_rz_azimuthal_modes = 1 +warpx.use_filter = 1 +warpx.verbose = 1 +warpx.abort_on_warning_threshold = medium + +warpx.cfl = 1. +#warpx.gamma_boost = 1. +#warpx.boost_direction = z + +warpx.do_dive_cleaning = 1 +warpx.do_divb_cleaning = 1 +warpx.do_multi_J = 1 +warpx.do_multi_J_n_depositions = 2 +psatd.do_time_averaging = 1 + +# PSATD +psatd.update_with_rho = 1 +#psatd.v_galilean = 0. 0. -0.9373391857121336 +psatd.J_in_time = linear + +# Particles + +particles.species_names = driver driver_back plasma_e plasma_p +particles.use_fdtd_nci_corr = 0 +particles.rigid_injected_species = driver + +driver.species_type = electron +driver.injection_style = "gaussian_beam" +driver.x_rms = 5e-6 +driver.y_rms = 5e-6 +driver.z_rms = 20.1e-6 +driver.x_m = 0. +driver.y_m = 0. +driver.z_m = -80e-6 +driver.npart = 1000000 +driver.q_tot = -1e-9 +driver.momentum_distribution_type = "gaussian" +driver.ux_m = 0. +driver.uy_m = 0. +driver.uz_m = 2000. +driver.ux_th = 4. +driver.uy_th = 4. +driver.uz_th = 20. +driver.zinject_plane = 2e-3 +driver.rigid_advance = true +driver.initialize_self_fields = 0 +driver.do_symmetrize = 1 +driver.symmetrization_order = 4 + +driver_back.species_type = positron +driver_back.injection_style = "gaussian_beam" +driver_back.x_rms = 5e-6 +driver_back.y_rms = 5e-6 +driver_back.z_rms = 20.1e-6 +driver_back.x_m = 0. +driver_back.y_m = 0. +driver_back.z_m = -80e-6 +driver_back.npart = 1000000 +driver_back.q_tot = 1e-9 +driver_back.momentum_distribution_type = "at_rest" +driver_back.initialize_self_fields = 0 +driver_back.do_symmetrize = 1 +driver_back.symmetrization_order = 4 + +plasma_e.species_type = electron +plasma_e.injection_style = "NUniformPerCell" +plasma_e.zmin = 0. +plasma_e.zmax = 0.5 +plasma_e.xmin = -180e-6 +plasma_e.xmax = 180e-6 +plasma_e.profile = constant +plasma_e.density = 1e23 +plasma_e.num_particles_per_cell_each_dim = 2 2 1 +plasma_e.momentum_distribution_type = "at_rest" +plasma_e.do_continuous_injection = 1 + +plasma_p.species_type = hydrogen +plasma_p.injection_style = "NUniformPerCell" +plasma_p.zmin = 0. +plasma_p.zmax = 0.5 +plasma_p.xmin = -180e-6 +plasma_p.xmax = 180e-6 +plasma_p.profile = constant +plasma_p.density = 1e23 +plasma_p.num_particles_per_cell_each_dim = 2 2 1 +plasma_p.momentum_distribution_type = "at_rest" +plasma_p.do_continuous_injection = 1 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 25 +diag1.diag_type = Full +diag1.fields_to_plot = Er Ez Bt jr jz rho rho_driver rho_plasma_e rho_plasma_p +diag1.species = driver plasma_e plasma_p diff --git a/Examples/Tests/nodal_electrostatic/CMakeLists.txt b/Examples/Tests/nodal_electrostatic/CMakeLists.txt new file mode 100644 index 00000000000..026ab1a34bc --- /dev/null +++ b/Examples/Tests/nodal_electrostatic/CMakeLists.txt @@ -0,0 +1,13 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_nodal_electrostatic_solver # name + 3 # dims + 1 # nprocs + inputs_test_3d_nodal_electrostatic_solver # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) +label_warpx_test(test_3d_nodal_electrostatic_solver slow) diff --git a/Examples/Tests/nodal_electrostatic/analysis.py b/Examples/Tests/nodal_electrostatic/analysis.py new file mode 100755 index 00000000000..b6d22d60a79 --- /dev/null +++ b/Examples/Tests/nodal_electrostatic/analysis.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +import numpy as np + +# check that the maximum chi value is small +fname = "diags/reducedfiles/ParticleExtrema_beam_p.txt" +chi_max = np.loadtxt(fname)[:, 19] +assert np.all(chi_max < 2e-8) + +# check that no photons have been produced +fname = "diags/reducedfiles/ParticleNumber.txt" +pho_num = np.loadtxt(fname)[:, 7] +assert pho_num.all() == 0.0 diff --git a/Examples/Tests/nodal_electrostatic/analysis_3d.py b/Examples/Tests/nodal_electrostatic/analysis_3d.py deleted file mode 100755 index 79a2bfdae20..00000000000 --- a/Examples/Tests/nodal_electrostatic/analysis_3d.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -import numpy as np - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# check that the maximum chi value is small -fname = 'diags/reducedfiles/ParticleExtrema_beam_p.txt' -chi_max = np.loadtxt(fname)[:,19] -assert np.all(chi_max < 2e-8) - -# check that no photons have been produced -fname = 'diags/reducedfiles/ParticleNumber.txt' -pho_num = np.loadtxt(fname)[:,7] -assert(pho_num.all()==0.) - -# Checksum regression analysis -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/nodal_electrostatic/analysis_default_regression.py b/Examples/Tests/nodal_electrostatic/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/nodal_electrostatic/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/nodal_electrostatic/inputs_3d b/Examples/Tests/nodal_electrostatic/inputs_3d deleted file mode 100644 index 91732a2c8ff..00000000000 --- a/Examples/Tests/nodal_electrostatic/inputs_3d +++ /dev/null @@ -1,90 +0,0 @@ -# MY CONSTANTS -my_constants.nano = 1e-9 -my_constants.mc2 = m_e*clight*clight -my_constants.GeV = q_e*1.e9 - -my_constants.nx = 128 -my_constants.ny = 128 -my_constants.nz = 128 - -my_constants.sigma = 10*nano -my_constants.gammab = 125.*GeV/(mc2) -my_constants.chargeb = 0.14*nano - -my_constants.Lx = 7*sigma -my_constants.Ly = 7*sigma -my_constants.Lz = 7*sigma - -my_constants.n0 = chargeb / q_e / ((2 * pi)**(3./2.) * sigma*sigma*sigma) -my_constants.dt = sigma/clight/20. - -# GENERAL -max_step = 10 -amr.n_cell = nx ny nz -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz -geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz -warpx.numprocs = 1 1 1 - -# BOUNDARY -boundary.field_lo = PEC PEC PEC -boundary.field_hi = PEC PEC PEC - -# NUMERICS -warpx.do_electrostatic = relativistic -warpx.const_dt = dt -warpx.grid_type = collocated -algo.particle_pusher = vay -algo.particle_shape = 3 - -# SPECIES -particles.species_names = beam_p qsg_p -particles.photon_species = qsg_p -beam_p.species_type = positron -beam_p.injection_style = "NUniformPerCell" -beam_p.num_particles_per_cell_each_dim = 1 1 1 -beam_p.profile = parse_density_function -beam_p.density_function(x,y,z) = "n0 * exp(-x**2/(2*sigma**2)) * exp(-y**2/(2*sigma**2)) * exp(-z**2/(2*sigma**2))" -beam_p.density_min = n0 / 1e2 -beam_p.momentum_distribution_type = "constant" -beam_p.ux = 0.0 -beam_p.uy = 0.0 -beam_p.uz = -gammab -beam_p.initialize_self_fields = 1 -beam_p.self_fields_required_precision = 1e-11 -beam_p.self_fields_max_iters = 20000 -beam_p.do_qed_quantum_sync = 1 -beam_p.qed_quantum_sync_phot_product_species = qsg_p -qsg_p.species_type = "photon" -qsg_p.injection_style = "none" -qsg_p.do_qed_breit_wheeler = 0 - -# QED -qed_qs.chi_min = 0.001 -qed_qs.photon_creation_energy_threshold = 1.0 -qed_qs.lookup_table_mode = builtin -qed_qs.chi_min = 1.e-3 -warpx.do_qed_schwinger = 0. - -# RED DIAGNOSTICS -warpx.reduced_diags_names = ParticleExtrema_beam_p ParticleNumber -ParticleExtrema_beam_p.type = ParticleExtrema -ParticleExtrema_beam_p.intervals = 1 -ParticleExtrema_beam_p.species = beam_p -ParticleNumber.type = ParticleNumber -ParticleNumber.intervals = 1 - -# FULL DIAGNOSTICS -diagnostics.diags_names = diag1 -diag1.intervals = 10 -diag1.diag_type = Full -diag1.write_species = 0 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam_p -diag1.format = plotfile -diag1.particle_fields_to_plot = vx vy vz g -diag1.particle_fields_species = beam_p -diag1.particle_fields.vx(x,y,z,ux,uy,uz) = ux/sqrt(ux*ux+uy*uy+uz*uz) -diag1.particle_fields.vy(x,y,z,ux,uy,uz) = uy/sqrt(ux*ux+uy*uy+uz*uz) -diag1.particle_fields.vz(x,y,z,ux,uy,uz) = uz/sqrt(ux*ux+uy*uy+uz*uz) -diag1.particle_fields.g(x,y,z,ux,uy,uz) = sqrt(ux*ux+uy*uy+uz*uz) diff --git a/Examples/Tests/nodal_electrostatic/inputs_test_3d_nodal_electrostatic_solver b/Examples/Tests/nodal_electrostatic/inputs_test_3d_nodal_electrostatic_solver new file mode 100644 index 00000000000..f1fd206eee3 --- /dev/null +++ b/Examples/Tests/nodal_electrostatic/inputs_test_3d_nodal_electrostatic_solver @@ -0,0 +1,91 @@ +# MY CONSTANTS +my_constants.nano = 1e-9 +my_constants.mc2 = m_e*clight*clight +my_constants.GeV = q_e*1.e9 + +my_constants.nx = 128 +my_constants.ny = 128 +my_constants.nz = 128 + +my_constants.sigma = 10*nano +my_constants.gammab = 125.*GeV/(mc2) +my_constants.chargeb = 0.14*nano + +my_constants.Lx = 7*sigma +my_constants.Ly = 7*sigma +my_constants.Lz = 7*sigma + +my_constants.n0 = chargeb / q_e / ((2 * pi)**(3./2.) * sigma*sigma*sigma) +my_constants.dt = sigma/clight/20. + +# GENERAL +max_step = 10 +amr.n_cell = nx ny nz +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz +warpx.numprocs = 1 1 1 + +# BOUNDARY +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC + +# NUMERICS +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +warpx.abort_on_warning_threshold = high +algo.particle_pusher = vay +algo.particle_shape = 3 + +# SPECIES +particles.species_names = beam_p qsg_p +particles.photon_species = qsg_p +beam_p.species_type = positron +beam_p.injection_style = "NUniformPerCell" +beam_p.num_particles_per_cell_each_dim = 1 1 1 +beam_p.profile = parse_density_function +beam_p.density_function(x,y,z) = "n0 * exp(-x**2/(2*sigma**2)) * exp(-y**2/(2*sigma**2)) * exp(-z**2/(2*sigma**2))" +beam_p.density_min = n0 / 1e2 +beam_p.momentum_distribution_type = "constant" +beam_p.ux = 0.0 +beam_p.uy = 0.0 +beam_p.uz = -gammab +beam_p.initialize_self_fields = 1 +beam_p.self_fields_required_precision = 1e-11 +beam_p.self_fields_max_iters = 20000 +beam_p.do_qed_quantum_sync = 1 +beam_p.qed_quantum_sync_phot_product_species = qsg_p +qsg_p.species_type = "photon" +qsg_p.injection_style = "none" +qsg_p.do_qed_breit_wheeler = 0 + +# QED +qed_qs.chi_min = 0.001 +qed_qs.photon_creation_energy_threshold = 1.0 +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 +warpx.do_qed_schwinger = 0. + +# RED DIAGNOSTICS +warpx.reduced_diags_names = ParticleExtrema_beam_p ParticleNumber +ParticleExtrema_beam_p.type = ParticleExtrema +ParticleExtrema_beam_p.intervals = 1 +ParticleExtrema_beam_p.species = beam_p +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = 1 + +# FULL DIAGNOSTICS +diagnostics.diags_names = diag1 +diag1.intervals = 10 +diag1.diag_type = Full +diag1.write_species = 0 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam_p +diag1.format = plotfile +diag1.particle_fields_to_plot = vx vy vz g +diag1.particle_fields_species = beam_p +diag1.particle_fields.vx(x,y,z,ux,uy,uz) = ux/sqrt(ux*ux+uy*uy+uz*uz) +diag1.particle_fields.vy(x,y,z,ux,uy,uz) = uy/sqrt(ux*ux+uy*uy+uz*uz) +diag1.particle_fields.vz(x,y,z,ux,uy,uz) = uz/sqrt(ux*ux+uy*uy+uz*uz) +diag1.particle_fields.g(x,y,z,ux,uy,uz) = sqrt(ux*ux+uy*uy+uz*uz) diff --git a/Examples/Tests/nuclear_fusion/CMakeLists.txt b/Examples/Tests/nuclear_fusion/CMakeLists.txt new file mode 100644 index 00000000000..74d937601bd --- /dev/null +++ b/Examples/Tests/nuclear_fusion/CMakeLists.txt @@ -0,0 +1,62 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_proton_boron_fusion # name + 2 # dims + 2 # nprocs + inputs_test_2d_proton_boron_fusion # inputs + "analysis_proton_boron_fusion.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_deuterium_deuterium_fusion # name + 3 # dims + 2 # nprocs + inputs_test_3d_deuterium_deuterium_fusion # inputs + "analysis_two_product_fusion.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_deuterium_deuterium_fusion_intraspecies # name + 3 # dims + 1 # nprocs + inputs_test_3d_deuterium_deuterium_fusion_intraspecies # inputs + "analysis_deuterium_deuterium_3d_intraspecies.py" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_deuterium_tritium_fusion # name + 3 # dims + 2 # nprocs + inputs_test_3d_deuterium_tritium_fusion # inputs + "analysis_two_product_fusion.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_proton_boron_fusion # name + 3 # dims + 2 # nprocs + inputs_test_3d_proton_boron_fusion # inputs + "analysis_proton_boron_fusion.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_deuterium_tritium_fusion # name + RZ # dims + 2 # nprocs + inputs_test_rz_deuterium_tritium_fusion # inputs + "analysis_two_product_fusion.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/nuclear_fusion/analysis_default_regression.py b/Examples/Tests/nuclear_fusion/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/nuclear_fusion/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py index 858df0b26a6..2b06a8c3f25 100755 --- a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py +++ b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py @@ -23,35 +23,22 @@ # Nuclear fusion, 32(4), p.611. # DOI: https://doi.org/10.1088/0029-5515/32/4/I07 -import os -import sys - import numpy as np -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Name of the plotfile -fn = sys.argv[1] - # Load data from reduced diagnostics (physical time and neutron weights) -time = np.loadtxt('./reduced_diags/particle_number.txt', usecols=1) -neutron = np.loadtxt('./reduced_diags/particle_number.txt', usecols=9) +time = np.loadtxt("./reduced_diags/particle_number.txt", usecols=1) +neutron = np.loadtxt("./reduced_diags/particle_number.txt", usecols=9) # Compute reactivity in units of cm^3/s as in equation (61) of [1] -dY_dt = np.abs(neutron[-1]-neutron[0])/(time[-1]-time[0]) -delta_ij = 1 # reactants of the same species -nD = 1e26 # density in 1/m^3 -V = (2e-3)**3 # simulation volume in m^3 -sigma = dY_dt*(1+delta_ij)/(nD**2)/V*(1e2)**3 +dY_dt = np.abs(neutron[-1] - neutron[0]) / (time[-1] - time[0]) +delta_ij = 1 # reactants of the same species +nD = 1e26 # density in 1/m^3 +V = (2e-3) ** 3 # simulation volume in m^3 +sigma = dY_dt * (1 + delta_ij) / (nD**2) / V * (1e2) ** 3 sigma_th = 2.603e-18 -error = np.abs(sigma-sigma_th)/sigma_th +error = np.abs(sigma - sigma_th) / sigma_th tolerance = 2e-2 -print('error = ', error) -print('tolerance = ', tolerance) +print("error = ", error) +print("tolerance = ", tolerance) assert error < tolerance - -# Compare checksums with benchmark -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 543eb62484a..917cd86f258 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -5,15 +5,12 @@ # # License: BSD-3-Clause-LBNL -import os +import re import sys -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI import numpy as np import scipy.constants as scc +import yt ## This script performs various checks for the proton boron nuclear fusion module. The simulation ## that we check is made of 5 different tests, each with different proton, boron and alpha species. @@ -59,28 +56,32 @@ ## Please be aware that the relative tolerances are often set empirically in this analysis script, ## so it would not be surprising that some tolerances need to be increased in the future. -default_tol = 1.e-12 # Default relative tolerance +default_tol = 1.0e-12 # Default relative tolerance ## Some physical parameters -keV_to_Joule = scc.e*1e3 -MeV_to_Joule = scc.e*1e6 -barn_to_square_meter = 1.e-28 -m_p = 1.00782503223*scc.m_u # Proton mass -m_b = 11.00930536*scc.m_u # Boron 11 mass -m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.00260325413*scc.m_u # Alpha mass -m_be = 7.94748*scc.m_p # Beryllium 8 mass -Z_boron = 5. -Z_proton = 1. -E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 -E_Gamow_MeV = E_Gamow/MeV_to_Joule -E_Gamow_keV = E_Gamow/keV_to_Joule -E_fusion = 8.59009*MeV_to_Joule # Energy released during p + B -> alpha + Be -E_decay = 0.0918984*MeV_to_Joule # Energy released during Be -> 2*alpha -E_fusion_total = E_fusion + E_decay # Energy released during p + B -> 3*alpha +keV_to_Joule = scc.e * 1e3 +MeV_to_Joule = scc.e * 1e6 +barn_to_square_meter = 1.0e-28 +m_p = 1.00782503223 * scc.m_u # Proton mass +m_b = 11.00930536 * scc.m_u # Boron 11 mass +m_reduced = m_p * m_b / (m_p + m_b) +m_a = 4.00260325413 * scc.m_u # Alpha (He4) mass +m_be = (8.0053095729 + 0.00325283863) * scc.m_u # Be8* mass (3.03 MeV ex. state) +Z_boron = 5.0 +Z_proton = 1.0 +E_Gamow = ( + (Z_boron * Z_proton * np.pi * scc.fine_structure) ** 2 * 2.0 * m_reduced * scc.c**2 +) +E_Gamow_MeV = E_Gamow / MeV_to_Joule +E_Gamow_keV = E_Gamow / keV_to_Joule +E_fusion = 5.55610759 * MeV_to_Joule # Energy released during p + B -> alpha + Be* +E_decay = 3.12600414 * MeV_to_Joule # Energy released during Be* -> 2*alpha +E_fusion_total = E_fusion + E_decay # Energy released during p + B -> 3*alpha ## Checks whether this is the 2D or the 3D test -is_2D = "2D" in sys.argv[1] +with open("./warpx_used_inputs") as warpx_used_inputs: + is_2D = re.search("geometry.dims\s*=\s*2", warpx_used_inputs.read()) +warpx_used_inputs.close() ## Some numerical parameters for this test size_x = 8 @@ -89,44 +90,48 @@ else: size_y = 8 size_z = 16 -dV_total = size_x*size_y*size_z # Total simulation volume +dV_total = size_x * size_y * size_z # Total simulation volume # Volume of a slice corresponding to a single cell in the z direction. In tests 1 and 2, all the # particles of a given species in the same slice have the exact same momentum -dV_slice = size_x*size_y +dV_slice = size_x * size_y if is_2D: - dt = 1./(scc.c*np.sqrt(2.)) + dt = 1.0 / (scc.c * np.sqrt(2.0)) yt_z_string = "particle_position_y" - nppcell_1 = 10000*8 - nppcell_2 = 900*8 + nppcell_1 = 10000 * 8 + nppcell_2 = 900 * 8 else: - dt = 1./(scc.c*np.sqrt(3.)) + dt = 1.0 / (scc.c * np.sqrt(3.0)) yt_z_string = "particle_position_z" nppcell_1 = 10000 nppcell_2 = 900 # In test 1 and 2, the energy in cells number i (in z direction) is typically Energy_step * i**2 -Energy_step = 22.*keV_to_Joule +Energy_step = 22.0 * keV_to_Joule -def is_close(val1, val2, rtol=default_tol, atol=0.): + +def is_close(val1, val2, rtol=default_tol, atol=0.0): ## Wrapper around numpy.isclose, used to override the default tolerances. return np.isclose(val1, val2, rtol=rtol, atol=atol) + def add_existing_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): - data_dict[prefix+"_px_"+suffix] = yt_ad[species_name, "particle_momentum_x"].v - data_dict[prefix+"_py_"+suffix] = yt_ad[species_name, "particle_momentum_y"].v - data_dict[prefix+"_pz_"+suffix] = yt_ad[species_name, "particle_momentum_z"].v - data_dict[prefix+"_w_"+suffix] = yt_ad[species_name, "particle_weight"].v - data_dict[prefix+"_id_"+suffix] = yt_ad[species_name, "particle_id"].v - data_dict[prefix+"_cpu_"+suffix] = yt_ad[species_name, "particle_cpu"].v - data_dict[prefix+"_z_"+suffix] = yt_ad[species_name, yt_z_string].v + data_dict[prefix + "_px_" + suffix] = yt_ad[species_name, "particle_momentum_x"].v + data_dict[prefix + "_py_" + suffix] = yt_ad[species_name, "particle_momentum_y"].v + data_dict[prefix + "_pz_" + suffix] = yt_ad[species_name, "particle_momentum_z"].v + data_dict[prefix + "_w_" + suffix] = yt_ad[species_name, "particle_weight"].v + data_dict[prefix + "_id_" + suffix] = yt_ad[species_name, "particle_id"].v + data_dict[prefix + "_cpu_" + suffix] = yt_ad[species_name, "particle_cpu"].v + data_dict[prefix + "_z_" + suffix] = yt_ad[species_name, yt_z_string].v + def add_empty_species_to_dict(data_dict, species_name, prefix, suffix): - data_dict[prefix+"_px_"+suffix] = np.empty(0) - data_dict[prefix+"_py_"+suffix] = np.empty(0) - data_dict[prefix+"_pz_"+suffix] = np.empty(0) - data_dict[prefix+"_w_"+suffix] = np.empty(0) - data_dict[prefix+"_id_"+suffix] = np.empty(0) - data_dict[prefix+"_cpu_"+suffix] = np.empty(0) - data_dict[prefix+"_z_"+suffix] = np.empty(0) + data_dict[prefix + "_px_" + suffix] = np.empty(0) + data_dict[prefix + "_py_" + suffix] = np.empty(0) + data_dict[prefix + "_pz_" + suffix] = np.empty(0) + data_dict[prefix + "_w_" + suffix] = np.empty(0) + data_dict[prefix + "_id_" + suffix] = np.empty(0) + data_dict[prefix + "_cpu_" + suffix] = np.empty(0) + data_dict[prefix + "_z_" + suffix] = np.empty(0) + def add_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): try: @@ -138,65 +143,79 @@ def add_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): ## entirely fuses into alphas. add_empty_species_to_dict(data_dict, species_name, prefix, suffix) + def check_particle_number_conservation(data): total_w_proton_start = np.sum(data["proton_w_start"]) - total_w_proton_end = np.sum(data["proton_w_end"]) - total_w_boron_start = np.sum(data["boron_w_start"]) - total_w_boron_end = np.sum(data["boron_w_end"]) + total_w_proton_end = np.sum(data["proton_w_end"]) + total_w_boron_start = np.sum(data["boron_w_start"]) + total_w_boron_end = np.sum(data["boron_w_end"]) consumed_proton = total_w_proton_start - total_w_proton_end - consumed_boron = total_w_boron_start - total_w_boron_end - created_alpha = np.sum(data["alpha_w_end"]) - assert(consumed_proton >= 0.) - assert(consumed_boron >= 0.) - assert(created_alpha >= 0.) + consumed_boron = total_w_boron_start - total_w_boron_end + created_alpha = np.sum(data["alpha_w_end"]) + assert consumed_proton >= 0.0 + assert consumed_boron >= 0.0 + assert created_alpha >= 0.0 ## Check that number of consumed proton and consumed boron are equal assert_scale = max(total_w_proton_start, total_w_boron_start) - assert(is_close(consumed_proton, consumed_boron, rtol = 0., atol = default_tol*assert_scale)) + assert is_close( + consumed_proton, consumed_boron, rtol=0.0, atol=default_tol * assert_scale + ) ## Check that number of consumed particles corresponds to number of produced alpha ## Factor 3 is here because each nuclear fusion reaction produces 3 alphas - assert(is_close(total_w_proton_start, total_w_proton_end + created_alpha/3.)) - assert(is_close(total_w_boron_start, total_w_boron_end + created_alpha/3.)) + assert is_close(total_w_proton_start, total_w_proton_end + created_alpha / 3.0) + assert is_close(total_w_boron_start, total_w_boron_end + created_alpha / 3.0) + def compute_energy_array(data, species_name, suffix, m): ## Relativistic computation of kinetic energy for a given species - psq_array = data[species_name+'_px_'+suffix]**2 + data[species_name+'_py_'+suffix]**2 + \ - data[species_name+'_pz_'+suffix]**2 - rest_energy = m*scc.c**2 - return np.sqrt(psq_array*scc.c**2 + rest_energy**2) - rest_energy + psq_array = ( + data[species_name + "_px_" + suffix] ** 2 + + data[species_name + "_py_" + suffix] ** 2 + + data[species_name + "_pz_" + suffix] ** 2 + ) + rest_energy = m * scc.c**2 + return np.sqrt(psq_array * scc.c**2 + rest_energy**2) - rest_energy + def check_energy_conservation(data): proton_energy_start = compute_energy_array(data, "proton", "start", m_p) - proton_energy_end = compute_energy_array(data, "proton", "end", m_p) - boron_energy_start = compute_energy_array(data, "boron", "start", m_b) - boron_energy_end = compute_energy_array(data, "boron", "end", m_b) - alpha_energy_end = compute_energy_array(data, "alpha", "end", m_a) - total_energy_start = np.sum(proton_energy_start*data["proton_w_start"]) + \ - np.sum(boron_energy_start*data["boron_w_start"]) - total_energy_end = np.sum(proton_energy_end*data["proton_w_end"]) + \ - np.sum(boron_energy_end*data["boron_w_end"]) + \ - np.sum(alpha_energy_end*data["alpha_w_end"]) + proton_energy_end = compute_energy_array(data, "proton", "end", m_p) + boron_energy_start = compute_energy_array(data, "boron", "start", m_b) + boron_energy_end = compute_energy_array(data, "boron", "end", m_b) + alpha_energy_end = compute_energy_array(data, "alpha", "end", m_a) + total_energy_start = np.sum(proton_energy_start * data["proton_w_start"]) + np.sum( + boron_energy_start * data["boron_w_start"] + ) + total_energy_end = ( + np.sum(proton_energy_end * data["proton_w_end"]) + + np.sum(boron_energy_end * data["boron_w_end"]) + + np.sum(alpha_energy_end * data["alpha_w_end"]) + ) ## Factor 3 is here because each nuclear fusion reaction produces 3 alphas - n_fusion_reaction = np.sum(data["alpha_w_end"])/3. - assert(is_close(total_energy_end, - total_energy_start + n_fusion_reaction*E_fusion_total, - rtol = 1.e-8)) + n_fusion_reaction = np.sum(data["alpha_w_end"]) / 3.0 + assert is_close( + total_energy_end, + total_energy_start + n_fusion_reaction * E_fusion_total, + rtol=1.0e-8, + ) + def check_momentum_conservation(data): - proton_total_px_start = np.sum(data["proton_px_start"]*data["proton_w_start"]) - proton_total_py_start = np.sum(data["proton_py_start"]*data["proton_w_start"]) - proton_total_pz_start = np.sum(data["proton_pz_start"]*data["proton_w_start"]) - proton_total_px_end = np.sum(data["proton_px_end"]*data["proton_w_end"]) - proton_total_py_end = np.sum(data["proton_py_end"]*data["proton_w_end"]) - proton_total_pz_end = np.sum(data["proton_pz_end"]*data["proton_w_end"]) - boron_total_px_start = np.sum(data["boron_px_start"]*data["boron_w_start"]) - boron_total_py_start = np.sum(data["boron_py_start"]*data["boron_w_start"]) - boron_total_pz_start = np.sum(data["boron_pz_start"]*data["boron_w_start"]) - boron_total_px_end = np.sum(data["boron_px_end"]*data["boron_w_end"]) - boron_total_py_end = np.sum(data["boron_py_end"]*data["boron_w_end"]) - boron_total_pz_end = np.sum(data["boron_pz_end"]*data["boron_w_end"]) - alpha_total_px_end = np.sum(data["alpha_px_end"]*data["alpha_w_end"]) - alpha_total_py_end = np.sum(data["alpha_py_end"]*data["alpha_w_end"]) - alpha_total_pz_end = np.sum(data["alpha_pz_end"]*data["alpha_w_end"]) + proton_total_px_start = np.sum(data["proton_px_start"] * data["proton_w_start"]) + proton_total_py_start = np.sum(data["proton_py_start"] * data["proton_w_start"]) + proton_total_pz_start = np.sum(data["proton_pz_start"] * data["proton_w_start"]) + proton_total_px_end = np.sum(data["proton_px_end"] * data["proton_w_end"]) + proton_total_py_end = np.sum(data["proton_py_end"] * data["proton_w_end"]) + proton_total_pz_end = np.sum(data["proton_pz_end"] * data["proton_w_end"]) + boron_total_px_start = np.sum(data["boron_px_start"] * data["boron_w_start"]) + boron_total_py_start = np.sum(data["boron_py_start"] * data["boron_w_start"]) + boron_total_pz_start = np.sum(data["boron_pz_start"] * data["boron_w_start"]) + boron_total_px_end = np.sum(data["boron_px_end"] * data["boron_w_end"]) + boron_total_py_end = np.sum(data["boron_py_end"] * data["boron_w_end"]) + boron_total_pz_end = np.sum(data["boron_pz_end"] * data["boron_w_end"]) + alpha_total_px_end = np.sum(data["alpha_px_end"] * data["alpha_w_end"]) + alpha_total_py_end = np.sum(data["alpha_py_end"] * data["alpha_w_end"]) + alpha_total_pz_end = np.sum(data["alpha_pz_end"] * data["alpha_w_end"]) total_px_start = proton_total_px_start + boron_total_px_start total_py_start = proton_total_py_start + boron_total_py_start total_pz_start = proton_total_pz_start + boron_total_pz_start @@ -204,42 +223,45 @@ def check_momentum_conservation(data): total_py_end = proton_total_py_end + boron_total_py_end + alpha_total_py_end total_pz_end = proton_total_pz_end + boron_total_pz_end + alpha_total_pz_end ## Absolute tolerance is needed because sometimes the initial momentum is exactly 0 - assert(is_close(total_px_start, total_px_end, atol=1.e-15)) - assert(is_close(total_py_start, total_py_end, atol=1.e-15)) - assert(is_close(total_pz_start, total_pz_end, atol=1.e-15)) + assert is_close(total_px_start, total_px_end, atol=1.0e-15) + assert is_close(total_py_start, total_py_end, atol=1.0e-15) + assert is_close(total_pz_start, total_pz_end, atol=1.0e-15) + def check_id(data): ## Check that all created particles have unique id + cpu identifier (two particles with ## different cpu can have the same id) - complex_id = data["alpha_id_end"] + 1j*data["alpha_cpu_end"] - assert(complex_id.shape == np.unique(complex_id).shape) + complex_id = data["alpha_id_end"] + 1j * data["alpha_cpu_end"] + assert complex_id.shape == np.unique(complex_id).shape + def basic_product_particles_check(data): ## For each nuclear fusion reaction in the code, we create 6 alpha macroparticles. So the ## total number of alpha macroparticles must be a multiple of 6. num_alpha = data["alpha_w_end"].shape[0] - assert(num_alpha%6 == 0) + assert num_alpha % 6 == 0 ## The weight of the 6 macroparticles coming from a single fusion event should be the same. ## We verify this here. - assert(np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][1::6])) - assert(np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][2::6])) - assert(np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][3::6])) - assert(np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][4::6])) - assert(np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][5::6])) + assert np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][1::6]) + assert np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][2::6]) + assert np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][3::6]) + assert np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][4::6]) + assert np.array_equal(data["alpha_w_end"][::6], data["alpha_w_end"][5::6]) ## When we create 6 macroparticles, the first has the exact same momentum as the second, the ## third has the same as the fourth and the fifth has the same as the sixth. We verify this ## here - assert(np.array_equal(data["alpha_px_end"][::6], data["alpha_px_end"][1::6])) - assert(np.array_equal(data["alpha_py_end"][::6], data["alpha_py_end"][1::6])) - assert(np.array_equal(data["alpha_pz_end"][::6], data["alpha_pz_end"][1::6])) - assert(np.array_equal(data["alpha_px_end"][2::6], data["alpha_px_end"][3::6])) - assert(np.array_equal(data["alpha_py_end"][2::6], data["alpha_py_end"][3::6])) - assert(np.array_equal(data["alpha_pz_end"][2::6], data["alpha_pz_end"][3::6])) - assert(np.array_equal(data["alpha_px_end"][4::6], data["alpha_px_end"][5::6])) - assert(np.array_equal(data["alpha_py_end"][4::6], data["alpha_py_end"][5::6])) - assert(np.array_equal(data["alpha_pz_end"][4::6], data["alpha_pz_end"][5::6])) + assert np.array_equal(data["alpha_px_end"][::6], data["alpha_px_end"][1::6]) + assert np.array_equal(data["alpha_py_end"][::6], data["alpha_py_end"][1::6]) + assert np.array_equal(data["alpha_pz_end"][::6], data["alpha_pz_end"][1::6]) + assert np.array_equal(data["alpha_px_end"][2::6], data["alpha_px_end"][3::6]) + assert np.array_equal(data["alpha_py_end"][2::6], data["alpha_py_end"][3::6]) + assert np.array_equal(data["alpha_pz_end"][2::6], data["alpha_pz_end"][3::6]) + assert np.array_equal(data["alpha_px_end"][4::6], data["alpha_px_end"][5::6]) + assert np.array_equal(data["alpha_py_end"][4::6], data["alpha_py_end"][5::6]) + assert np.array_equal(data["alpha_pz_end"][4::6], data["alpha_pz_end"][5::6]) + def generic_check(data): check_particle_number_conservation(data) @@ -248,39 +270,43 @@ def generic_check(data): check_id(data) basic_product_particles_check(data) + def check_isotropy(data, relative_tolerance): ## Checks that the alpha particles are emitted isotropically - average_px_sq = np.average(data["alpha_px_end"]*data["alpha_px_end"]) - average_py_sq = np.average(data["alpha_py_end"]*data["alpha_py_end"]) - average_pz_sq = np.average(data["alpha_pz_end"]*data["alpha_pz_end"]) - assert(is_close(average_px_sq, average_py_sq, rtol = relative_tolerance)) - assert(is_close(average_px_sq, average_pz_sq, rtol = relative_tolerance)) + average_px_sq = np.average(data["alpha_px_end"] * data["alpha_px_end"]) + average_py_sq = np.average(data["alpha_py_end"] * data["alpha_py_end"]) + average_pz_sq = np.average(data["alpha_pz_end"] * data["alpha_pz_end"]) + assert is_close(average_px_sq, average_py_sq, rtol=relative_tolerance) + assert is_close(average_px_sq, average_pz_sq, rtol=relative_tolerance) + def astrophysical_factor_lowE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the low energy fit in the range E < 400 keV ## described in equation (3) of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023) - C0 = 197. + C0 = 197.0 C1 = 0.269 C2 = 2.54e-4 AL = 1.82e4 - EL = 148. + EL = 148.0 dEL = 2.35 - return C0 + C1*E + C2*E**2 + AL/((E-EL)**2 + dEL**2) + return C0 + C1 * E + C2 * E**2 + AL / ((E - EL) ** 2 + dEL**2) + def astrophysical_factor_midE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the mid energy fit in the range ## 400 keV < E < 668 keV described in equation (4) of A. Tentori and F. Belloni, ## Nuclear Fusion, 63, 086001 (2023) - D0 = 346. - D1 = 150. + D0 = 346.0 + D1 = 150.0 D2 = -59.9 D5 = -0.460 - E_400 = 400. - E_100 = 100. - E_norm = (E - E_400)/E_100 - return D0 + D1*E_norm + D2*E_norm**2 + D5*E_norm**5 + E_400 = 400.0 + E_100 = 100.0 + E_norm = (E - E_400) / E_100 + return D0 + D1 * E_norm + D2 * E_norm**2 + D5 * E_norm**5 + def astrophysical_factor_highE(E): ## E is in keV @@ -292,27 +318,36 @@ def astrophysical_factor_highE(E): A2 = 1.36e6 A3 = 3.71e6 E0 = 640.9 - E1 = 1211. - E2 = 2340. - E3 = 3294. + E1 = 1211.0 + E2 = 2340.0 + E3 = 3294.0 dE0 = 85.5 - dE1 = 414. - dE2 = 221. - dE3 = 351. + dE1 = 414.0 + dE2 = 221.0 + dE3 = 351.0 B = 0.381 - return A0/((E-E0)**2 + dE0**2) + A1/((E-E1)**2 + dE1**2) + \ - A2/((E-E2)**2 + dE2**2) + A3/((E-E3)**2 + dE3**2) + B + return ( + A0 / ((E - E0) ** 2 + dE0**2) + + A1 / ((E - E1) ** 2 + dE1**2) + + A2 / ((E - E2) ** 2 + dE2**2) + + A3 / ((E - E3) ** 2 + dE3**2) + + B + ) + def astrophysical_factor(E): ## E is in keV ## Returns astrophysical factor in MeV b using the fits described in A. Tentori ## and F. Belloni, Nuclear Fusion, 63, 086001 (2023) conditions = [E <= 400, E <= 668, E > 668] - choices = [astrophysical_factor_lowE(E), - astrophysical_factor_midE(E), - astrophysical_factor_highE(E)] + choices = [ + astrophysical_factor_lowE(E), + astrophysical_factor_midE(E), + astrophysical_factor_highE(E), + ] return np.select(conditions, choices) + def pb_cross_section_buck_fit(E): ## E is in MeV ## Returns cross section in b using a power law fit of the data presented in Buck et al., @@ -321,52 +356,75 @@ def pb_cross_section_buck_fit(E): ## Cross section at E = E_start_fit = 3.5 MeV cross_section_start_fit = 0.01277998 slope_fit = -2.661840717596765 - return cross_section_start_fit*(E/E_start_fit)**slope_fit + return cross_section_start_fit * (E / E_start_fit) ** slope_fit + def pb_cross_section(E): ## E is in keV ## Returns cross section in b using the fits described in A. Tentori and F. Belloni, ## Nucl. Fusion, 63, 086001 (2023) for E < 9.76 MeV otherwise returns a power law fit ## of the data in Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) - E_MeV = E/1.e3 + E_MeV = E / 1.0e3 conditions = [E <= 9760, E > 9760] - choices = [astrophysical_factor(E)/E_MeV * np.exp(-np.sqrt(E_Gamow_MeV / E_MeV)), - pb_cross_section_buck_fit(E_MeV)] + choices = [ + astrophysical_factor(E) / E_MeV * np.exp(-np.sqrt(E_Gamow_MeV / E_MeV)), + pb_cross_section_buck_fit(E_MeV), + ] return np.select(conditions, choices) + def E_com_to_p_sq_com(m1, m2, E): ## E is the total (kinetic+mass) energy of a two particle (with mass m1 and m2) system in ## its center of mass frame, in J. ## Returns the square norm of the momentum of each particle in that frame. - E_ratio = E/((m1+m2)*scc.c**2) - return m1*m2*scc.c**2 * (E_ratio**2 - 1) + (m1-m2)**2*scc.c**2/4 * (E_ratio - 1./E_ratio)**2 + E_ratio = E / ((m1 + m2) * scc.c**2) + return ( + m1 * m2 * scc.c**2 * (E_ratio**2 - 1) + + (m1 - m2) ** 2 * scc.c**2 / 4 * (E_ratio - 1.0 / E_ratio) ** 2 + ) + def compute_relative_v_com(E): ## E is the kinetic energy of proton+boron in the center of mass frame, in keV ## Returns the relative velocity between proton and boron in this frame, in m/s - E_J = E*keV_to_Joule + (m_p + m_b)*scc.c**2 + E_J = E * keV_to_Joule + (m_p + m_b) * scc.c**2 p_sq = E_com_to_p_sq_com(m_p, m_b, E_J) p = np.sqrt(p_sq) - gamma_p = np.sqrt(1. + p_sq / (m_p*scc.c)**2) - gamma_b = np.sqrt(1. + p_sq / (m_b*scc.c)**2) - v_p = p/(gamma_p*m_p) - v_b = p/(gamma_b*m_b) - return v_p+v_b + gamma_p = np.sqrt(1.0 + p_sq / (m_p * scc.c) ** 2) + gamma_b = np.sqrt(1.0 + p_sq / (m_b * scc.c) ** 2) + v_p = p / (gamma_p * m_p) + v_b = p / (gamma_b * m_b) + return v_p + v_b + def expected_alpha_weight_com(E_com, proton_density, boron_density, dV, dt): ## Computes expected number of produced alpha particles as a function of energy E_com in the ## center of mass frame. E_com is in keV. - assert(np.all(E_com>=0)) + assert np.all(E_com >= 0) ## Case E_com == 0 is handled manually to avoid division by zero conditions = [E_com == 0, E_com > 0] ## Necessary to avoid division by 0 warning when pb_cross_section is evaluated - E_com_never_zero = np.clip(E_com, 1.e-15, None) - choices = [0., pb_cross_section(E_com_never_zero)*compute_relative_v_com(E_com_never_zero)] + E_com_never_zero = np.clip(E_com, 1.0e-15, None) + choices = [ + 0.0, + pb_cross_section(E_com_never_zero) * compute_relative_v_com(E_com_never_zero), + ] sigma_times_vrel = np.select(conditions, choices) ## Factor 3 is here because each fusion reaction produces 3 alphas - return 3.*proton_density*boron_density*sigma_times_vrel*barn_to_square_meter*dV*dt - -def check_macroparticle_number(data, fusion_probability_target_value, num_pair_per_cell): + return ( + 3.0 + * proton_density + * boron_density + * sigma_times_vrel + * barn_to_square_meter + * dV + * dt + ) + + +def check_macroparticle_number( + data, fusion_probability_target_value, num_pair_per_cell +): ## Checks that the number of macroparticles is as expected for the first and second tests ## The first slice 0 < z < 1 does not contribute to alpha creation @@ -374,65 +432,84 @@ def check_macroparticle_number(data, fusion_probability_target_value, num_pair_p ## In these tests, the fusion_multiplier is so high that the fusion probability per pair is ## equal to the parameter fusion_probability_target_value fusion_probability_per_pair = fusion_probability_target_value - expected_fusion_number = numcells*num_pair_per_cell*fusion_probability_per_pair + expected_fusion_number = numcells * num_pair_per_cell * fusion_probability_per_pair ## Each fusion event produces 6 alpha macroparticles - expected_macroparticle_number = 6.*expected_fusion_number - std_macroparticle_number = 6.*np.sqrt(expected_fusion_number) + expected_macroparticle_number = 6.0 * expected_fusion_number + std_macroparticle_number = 6.0 * np.sqrt(expected_fusion_number) actual_macroparticle_number = data["alpha_w_end"].shape[0] # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions - assert(is_close(actual_macroparticle_number, expected_macroparticle_number, rtol = 0., - atol = 5.*std_macroparticle_number)) + assert is_close( + actual_macroparticle_number, + expected_macroparticle_number, + rtol=0.0, + atol=5.0 * std_macroparticle_number, + ) ## used in subsequent function return expected_fusion_number + def p_sq_boron_frame_to_E_COM_frame(p_proton_sq): # Takes the proton square norm of the momentum in the boron rest frame and returns the total # kinetic energy in the center of mass frame. Everything is in SI units. # Total (kinetic + mass) energy in lab frame - E_lab = np.sqrt(p_proton_sq*scc.c**2 + (m_p*scc.c**2)**2) + m_b*scc.c**2 + E_lab = np.sqrt(p_proton_sq * scc.c**2 + (m_p * scc.c**2) ** 2) + m_b * scc.c**2 # Use invariant E**2 - p**2c**2 of 4-momentum norm to compute energy in center of mass frame - E_com = np.sqrt(E_lab**2 - p_proton_sq*scc.c**2) + E_com = np.sqrt(E_lab**2 - p_proton_sq * scc.c**2) # Corresponding kinetic energy - E_com_kin = E_com - (m_b+m_p)*scc.c**2 - return E_com_kin*(p_proton_sq>0.) + E_com_kin = E_com - (m_b + m_p) * scc.c**2 + return E_com_kin * (p_proton_sq > 0.0) + def p_sq_to_kinetic_energy(p_sq, m): ## Returns the kinetic energy of a particle as a function of its squared momentum. ## Everything is in SI units. - return np.sqrt(p_sq*scc.c**2 + (m*scc.c**2)**2) - (m*scc.c**2) + return np.sqrt(p_sq * scc.c**2 + (m * scc.c**2) ** 2) - (m * scc.c**2) + def compute_E_com1(data): ## Computes kinetic energy (in Joule) in the center of frame for the first test ## Square norm of the momentum of proton/boron as a function of cell number in z direction - p_sq = 2.*m_reduced*(Energy_step*np.arange(size_z)**2) + p_sq = 2.0 * m_reduced * (Energy_step * np.arange(size_z) ** 2) return p_sq_to_kinetic_energy(p_sq, m_b) + p_sq_to_kinetic_energy(p_sq, m_p) + def compute_E_com2(data): ## Computes kinetic energy (in Joule) in the center of frame for the second test ## Square norm of the momentum of the proton as a function of cell number in z direction - p_proton_sq = 2.*m_p*(Energy_step*np.arange(size_z)**2) + p_proton_sq = 2.0 * m_p * (Energy_step * np.arange(size_z) ** 2) return p_sq_boron_frame_to_E_COM_frame(p_proton_sq) -def check_alpha_yield(data, expected_fusion_number, E_com, proton_density, boron_density): + +def check_alpha_yield( + data, expected_fusion_number, E_com, proton_density, boron_density +): ## Checks that the fusion yield is as expected for the first and second tests. ## Proton and boron densities are in m^-3. - alpha_weight_theory = expected_alpha_weight_com(E_com/keV_to_Joule, proton_density, - boron_density, dV_slice, dt) - alpha_weight_simulation = np.histogram(data["alpha_z_end"], bins=size_z, range=(0, size_z), - weights = data["alpha_w_end"])[0] + alpha_weight_theory = expected_alpha_weight_com( + E_com / keV_to_Joule, proton_density, boron_density, dV_slice, dt + ) + alpha_weight_simulation = np.histogram( + data["alpha_z_end"], bins=size_z, range=(0, size_z), weights=data["alpha_w_end"] + )[0] ## -1 is here because the first slice 0 < z < 1 does not contribute to alpha creation - expected_fusion_number_per_slice = expected_fusion_number/(size_z-1) - relative_std_alpha_weight = 1./np.sqrt(expected_fusion_number_per_slice) + expected_fusion_number_per_slice = expected_fusion_number / (size_z - 1) + relative_std_alpha_weight = 1.0 / np.sqrt(expected_fusion_number_per_slice) # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions - assert(np.all(is_close(alpha_weight_theory, alpha_weight_simulation, - rtol = 5.*relative_std_alpha_weight))) + assert np.all( + is_close( + alpha_weight_theory, + alpha_weight_simulation, + rtol=5.0 * relative_std_alpha_weight, + ) + ) + def check_initial_energy1(data, E_com): ## In WarpX, the initial momentum of the alphas is computed assuming that the fusion process @@ -461,7 +538,7 @@ def check_initial_energy1(data, E_com): E_kinetic_com_before = E_com[slice_number] ## Total (kinetic + mass) energy in the lab frame after ## proton + boron 11 -> alpha + beryllium 8 - E_total_com_after = E_kinetic_com_before + E_fusion + (m_a + m_be)*scc.c**2 + E_total_com_after = E_kinetic_com_before + E_fusion + (m_a + m_be) * scc.c**2 ## Corresponding momentum norm squared of alpha1/beryllium p_sq_after = E_com_to_p_sq_com(m_a, m_be, E_total_com_after) ## Corresponding kinetic energy for alpha1 @@ -475,16 +552,21 @@ def check_initial_energy1(data, E_com): ## corresponds to an alpha emitted exactly in the (opposite) direction of the beryllium ## in the center of mass frame. This calculation involves solving a polynomial equation of ## order 2 in p_alpha23. - max_p_alpha23 = 0.5*(np.sqrt(p_sq_after) + \ - np.sqrt(4*m_a*energy_alpha2_plus_3_theory - p_sq_after)) - min_p_alpha23 = 0.5*(np.sqrt(p_sq_after) - \ - np.sqrt(4*m_a*energy_alpha2_plus_3_theory - p_sq_after)) - max_energy_alpha23 = max_p_alpha23**2/(2.*m_a) - min_energy_alpha23 = min_p_alpha23**2/(2.*m_a) + max_p_alpha23 = 0.5 * ( + np.sqrt(p_sq_after) + + np.sqrt(4 * m_a * energy_alpha2_plus_3_theory - p_sq_after) + ) + min_p_alpha23 = 0.5 * ( + np.sqrt(p_sq_after) + - np.sqrt(4 * m_a * energy_alpha2_plus_3_theory - p_sq_after) + ) + max_energy_alpha23 = max_p_alpha23**2 / (2.0 * m_a) + min_energy_alpha23 = min_p_alpha23**2 / (2.0 * m_a) ## Get the energy of all alphas in the slice - energy_alpha_slice = energy_alpha_simulation[(z_alpha >= slice_number)* \ - (z_alpha < (slice_number + 1))] + energy_alpha_slice = energy_alpha_simulation[ + (z_alpha >= slice_number) * (z_alpha < (slice_number + 1)) + ] ## Energy of alphas1 (here, first macroparticle of each fusion event) in the slice energy_alpha1_simulation = energy_alpha_slice[::6] ## Energy of alphas2 (here, third macroparticle of each fusion event) in the slice @@ -492,11 +574,25 @@ def check_initial_energy1(data, E_com): ## Energy of alphas3 (here, fifth macroparticle of each fusion event) in the slice energy_alpha3_simulation = energy_alpha_slice[4::6] - assert(np.all(is_close(energy_alpha1_simulation, energy_alpha1_theory, rtol=5.e-8))) - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=1.e-2)) + assert np.all( + is_close(energy_alpha1_simulation, energy_alpha1_theory, rtol=5.0e-8) + ) + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 20 keV of the analytical value + ## The maximum value is checked to be within 1% of the analytical value + assert is_close( + np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=1.0e-2 + ) + assert is_close( + np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-15 + ) + assert is_close( + np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=1.0e-2 + ) + assert is_close( + np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-15 + ) + def check_initial_energy2(data): ## In WarpX, the initial momentum of the alphas is computed assuming that the fusion process @@ -523,9 +619,9 @@ def check_initial_energy2(data): for slice_number in range(1, size_z): ## For simplicity, all the calculations in this function are done nonrelativistically ## Proton kinetic energy in the lab frame before fusion - E_proton_nonrelativistic = Energy_step*slice_number**2 + E_proton_nonrelativistic = Energy_step * slice_number**2 ## Corresponding square norm of proton momentum - p_proton_sq = 2.*m_p*E_proton_nonrelativistic + p_proton_sq = 2.0 * m_p * E_proton_nonrelativistic ## Kinetic energy in the lab frame after ## proton + boron 11 -> alpha + beryllium 8 E_after_fusion = E_proton_nonrelativistic + E_fusion @@ -534,21 +630,29 @@ def check_initial_energy2(data): ## calculation is done by noting that the maximum (minimum) energy corresponds to an alpha ## emitted exactly in the (opposite) direction of the proton in the lab frame. This ## calculation involves solving a polynomial equation of order 2 in p_alpha1. - max_p_alpha1 = (m_a/m_be*np.sqrt(p_proton_sq) + \ - np.sqrt(-m_a/m_be*p_proton_sq + 2.*E_after_fusion*m_a*(m_a/m_be + 1.))) / \ - (m_a/m_be + 1.) - min_p_alpha1 = (m_a/m_be*np.sqrt(p_proton_sq) - \ - np.sqrt(-m_a/m_be*p_proton_sq + 2.*E_after_fusion*m_a*(m_a/m_be + 1.))) / \ - (m_a/m_be + 1.) - max_energy_alpha1 = max_p_alpha1**2/(2*m_a) - min_energy_alpha1 = min_p_alpha1**2/(2*m_a) + max_p_alpha1 = ( + m_a / m_be * np.sqrt(p_proton_sq) + + np.sqrt( + -m_a / m_be * p_proton_sq + + 2.0 * E_after_fusion * m_a * (m_a / m_be + 1.0) + ) + ) / (m_a / m_be + 1.0) + min_p_alpha1 = ( + m_a / m_be * np.sqrt(p_proton_sq) + - np.sqrt( + -m_a / m_be * p_proton_sq + + 2.0 * E_after_fusion * m_a * (m_a / m_be + 1.0) + ) + ) / (m_a / m_be + 1.0) + max_energy_alpha1 = max_p_alpha1**2 / (2 * m_a) + min_energy_alpha1 = min_p_alpha1**2 / (2 * m_a) ## Corresponding max/min kinetic energy of Beryllium in the lab frame max_E_beryllium = E_after_fusion - min_energy_alpha1 min_E_beryllium = E_after_fusion - max_energy_alpha1 ## Corresponding max/min momentum square of Beryllium in the lab frame - max_p_sq_beryllium = 2.*m_be*max_E_beryllium - min_p_sq_beryllium = 2.*m_be*min_E_beryllium + max_p_sq_beryllium = 2.0 * m_be * max_E_beryllium + min_p_sq_beryllium = 2.0 * m_be * min_E_beryllium ## Corresponding max/min kinetic energy in the lab frame for alpha2 + alpha3 after ## Beryllium decay max_energy_alpha2_plus_3 = max_E_beryllium + E_decay @@ -559,16 +663,21 @@ def check_initial_energy2(data): ## to an alpha emitted exactly in the (opposite) direction of a beryllium with energy ## max_E_beryllium (min_E_beryllium). This calculation involves solving a polynomial ## equation of order 2 in p_alpha23. - max_p_alpha23 = 0.5*(np.sqrt(max_p_sq_beryllium) + \ - np.sqrt(4*m_a*max_energy_alpha2_plus_3 - max_p_sq_beryllium)) - min_p_alpha23 = 0.5*(np.sqrt(min_p_sq_beryllium) - \ - np.sqrt(4*m_a*min_energy_alpha2_plus_3 - min_p_sq_beryllium)) - max_energy_alpha23 = max_p_alpha23**2/(2*m_a) - min_energy_alpha23 = min_p_alpha23**2/(2*m_a) + max_p_alpha23 = 0.5 * ( + np.sqrt(max_p_sq_beryllium) + + np.sqrt(4 * m_a * max_energy_alpha2_plus_3 - max_p_sq_beryllium) + ) + min_p_alpha23 = 0.5 * ( + np.sqrt(min_p_sq_beryllium) + - np.sqrt(4 * m_a * min_energy_alpha2_plus_3 - min_p_sq_beryllium) + ) + max_energy_alpha23 = max_p_alpha23**2 / (2 * m_a) + min_energy_alpha23 = min_p_alpha23**2 / (2 * m_a) ## Get the energy of all alphas in the slice - energy_alpha_slice = energy_alpha_simulation[(z_alpha >= slice_number)* \ - (z_alpha < (slice_number + 1))] + energy_alpha_slice = energy_alpha_simulation[ + (z_alpha >= slice_number) * (z_alpha < (slice_number + 1)) + ] ## Energy of alphas1 (here, first macroparticle of each fusion event) in the slice energy_alpha1_simulation = energy_alpha_slice[::6] ## Energy of alphas2 (here, third macroparticle of each fusion event) in the slice @@ -576,48 +685,73 @@ def check_initial_energy2(data): ## Energy of alphas3 (here, fifth macroparticle of each fusion event) in the slice energy_alpha3_simulation = energy_alpha_slice[4::6] - assert(is_close(np.amax(energy_alpha1_simulation), max_energy_alpha1, rtol=1.e-2)) - assert(is_close(np.amin(energy_alpha1_simulation), min_energy_alpha1, rtol=1.e-2)) - ## Tolerance is quite high below because we don't have a lot of alphas to produce good + assert is_close( + np.amax(energy_alpha1_simulation), max_energy_alpha1, rtol=1.0e-2 + ) + assert is_close( + np.amin(energy_alpha1_simulation), min_energy_alpha1, rtol=1.0e-2 + ) + ## Check that the max / min value are comparable to the analytical value + ## The minimum value is checked to be within 200 keV of the analytical value + ## The maximum value is checked to be within 5% of the analytical value + ## Tolerance is quite high because we don't have a lot of alphas to produce good ## statistics and an event like alpha1 emitted exactly in direction of proton & alpha2 ## emitted exactly in direction opposite to Beryllium is somewhat rare. - assert(is_close(np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha2_simulation), min_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=2.5e-1)) - assert(is_close(np.amin(energy_alpha3_simulation), min_energy_alpha23, rtol=2.5e-1)) + assert is_close( + np.amax(energy_alpha2_simulation), max_energy_alpha23, rtol=5e-2 + ) + assert is_close( + np.amin(energy_alpha2_simulation), min_energy_alpha23, atol=3.218e-14 + ) + assert is_close( + np.amax(energy_alpha3_simulation), max_energy_alpha23, rtol=5e-2 + ) + assert is_close( + np.amin(energy_alpha3_simulation), min_energy_alpha23, atol=3.218e-14 + ) + def check_xy_isotropy(data): ## Checks that the alpha particles are emitted isotropically in x and y - average_px_sq = np.average(data["alpha_px_end"]*data["alpha_px_end"]) - average_py_sq = np.average(data["alpha_py_end"]*data["alpha_py_end"]) - average_pz_sq = np.average(data["alpha_pz_end"]*data["alpha_pz_end"]) - assert(is_close(average_px_sq, average_py_sq, rtol = 5.e-2)) - assert(average_pz_sq > average_px_sq) - assert(average_pz_sq > average_py_sq) + average_px_sq = np.average(data["alpha_px_end"] * data["alpha_px_end"]) + average_py_sq = np.average(data["alpha_py_end"] * data["alpha_py_end"]) + average_pz_sq = np.average(data["alpha_pz_end"] * data["alpha_pz_end"]) + assert is_close(average_px_sq, average_py_sq, rtol=5.0e-2) + assert average_pz_sq > average_px_sq + assert average_pz_sq > average_py_sq + def sigmav_thermal_fit_lowE_nonresonant(T): ## Temperature T is in keV ## Returns the nonresonant average of cross section multiplied by relative velocity in m^3/s, ## in the range T <= 70 keV, as described by equations 10-14 of A. Tentori and F. Belloni, ## Nuclear Fusion, 63, 086001 (2023). - E0 = (E_Gamow_keV/4.)**(1./3.) * T**(2./3.) - DE0 = 4.*np.sqrt(T*E0/3.) - C0 = 197.*1.e3 - C1 = 0.269*1.e3 - C2 = 2.54e-4*1.e3 - tau = 3.*E0/T - Seff = C0*(1.+5./(12.*tau)) + C1*(E0+35./36.*T) + C2*(E0**2 + 89./36.*E0*T) + E0 = (E_Gamow_keV / 4.0) ** (1.0 / 3.0) * T ** (2.0 / 3.0) + DE0 = 4.0 * np.sqrt(T * E0 / 3.0) + C0 = 197.0 * 1.0e3 + C1 = 0.269 * 1.0e3 + C2 = 2.54e-4 * 1.0e3 + tau = 3.0 * E0 / T + Seff = ( + C0 * (1.0 + 5.0 / (12.0 * tau)) + + C1 * (E0 + 35.0 / 36.0 * T) + + C2 * (E0**2 + 89.0 / 36.0 * E0 * T) + ) ## nonresonant sigma times vrel, in barn meter per second - sigmav_nr_bmps = np.sqrt(2*T*keV_to_Joule/m_reduced) * DE0*Seff/T**2 * np.exp(-tau) + sigmav_nr_bmps = ( + np.sqrt(2 * T * keV_to_Joule / m_reduced) * DE0 * Seff / T**2 * np.exp(-tau) + ) ## Return result in cubic meter per second - return sigmav_nr_bmps*barn_to_square_meter + return sigmav_nr_bmps * barn_to_square_meter + def sigmav_thermal_fit_lowE_resonant(T): ## Temperature T is in keV ## Returns the resonant average of cross section multiplied by relative velocity in m^3/s, ## in the range T <= 70 keV, as described by equation 15 of A. Tentori and F. Belloni, ## Nuclear Fusion, 63, 086001 (2023). - return 5.41e-21 * np.exp(-148./T) / T**(3./2.) + return 5.41e-21 * np.exp(-148.0 / T) / T ** (3.0 / 2.0) + def sigmav_thermal_fit_lowE(T): ## Temperature T is in keV @@ -626,90 +760,105 @@ def sigmav_thermal_fit_lowE(T): ## The fits are valid for T <= 70 keV. return sigmav_thermal_fit_lowE_nonresonant(T) + sigmav_thermal_fit_lowE_resonant(T) + def expected_alpha_thermal(T, proton_density, boron_density, dV, dt): ## Computes the expected number of produced alpha particles when the protons and borons follow ## a Maxwellian distribution with a temperature T, in keV. This uses the thermal fits described ## in A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fit used here is only valid in the range T <= 70 keV. - assert((T >=0) and (T<=70)) + assert (T >= 0) and (T <= 70) sigma_times_vrel = sigmav_thermal_fit_lowE(T) ## Factor 3 is here because each fusion event produces 3 alphas. - return 3.*proton_density*boron_density*sigma_times_vrel*dV*dt + return 3.0 * proton_density * boron_density * sigma_times_vrel * dV * dt + def check_thermal_alpha_yield(data): ## Checks that the number of alpha particles in test3 is as expected - Temperature = 44. # keV - proton_density = 1.e28 # m^-3 - boron_density = 5.e28 # m^-3 + Temperature = 44.0 # keV + proton_density = 1.0e28 # m^-3 + boron_density = 5.0e28 # m^-3 - alpha_weight_theory = expected_alpha_thermal(Temperature, proton_density, boron_density, - dV_total, dt) + alpha_weight_theory = expected_alpha_thermal( + Temperature, proton_density, boron_density, dV_total, dt + ) alpha_weight_simulation = np.sum(data["alpha_w_end"]) - assert(is_close(alpha_weight_theory, alpha_weight_simulation, rtol = 2.e-1)) + assert is_close(alpha_weight_theory, alpha_weight_simulation, rtol=2.0e-1) + def boron_remains(data): ## Checks whether there remains boron macroparticles at the end of the test n_boron_left = data["boron_w_end"].shape[0] - return (n_boron_left > 0) + return n_boron_left > 0 + def specific_check1(data): - check_isotropy(data, relative_tolerance = 3.e-2) - expected_fusion_number = check_macroparticle_number(data, - fusion_probability_target_value = 0.002, - num_pair_per_cell = nppcell_1) + check_isotropy(data, relative_tolerance=3.0e-2) + expected_fusion_number = check_macroparticle_number( + data, fusion_probability_target_value=0.002, num_pair_per_cell=nppcell_1 + ) E_com = compute_E_com1(data) - check_alpha_yield(data, expected_fusion_number, E_com, proton_density = 1., - boron_density = 1.) + check_alpha_yield( + data, expected_fusion_number, E_com, proton_density=1.0, boron_density=1.0 + ) check_initial_energy1(data, E_com) + def specific_check2(data): check_xy_isotropy(data) ## Only 900 particles pairs per cell here because we ignore the 10% of protons that are at rest - expected_fusion_number = check_macroparticle_number(data, - fusion_probability_target_value = 0.02, - num_pair_per_cell = nppcell_2) + expected_fusion_number = check_macroparticle_number( + data, fusion_probability_target_value=0.02, num_pair_per_cell=nppcell_2 + ) E_com = compute_E_com2(data) - check_alpha_yield(data, expected_fusion_number, E_com, proton_density = 1.e20, - boron_density = 1.e26) + check_alpha_yield( + data, expected_fusion_number, E_com, proton_density=1.0e20, boron_density=1.0e26 + ) check_initial_energy2(data) + def specific_check3(data): - check_isotropy(data, relative_tolerance = 1.e-1) + check_isotropy(data, relative_tolerance=1.0e-1) check_thermal_alpha_yield(data) + def specific_check4(data): ## In test 4, the boron initial density is so small that all borons should have fused within a ## timestep dt. We thus assert that no boron remains at the end of the simulation. - assert(not boron_remains(data)) + assert not boron_remains(data) + def specific_check5(data): ## Test 5 is similar to test 4, expect that the parameter fusion_probability_threshold is ## increased to the point that we should severely underestimate the fusion yield. Consequently, ## there should still be borons at the end of the test, which we verify here. - assert(boron_remains(data)) + assert boron_remains(data) + def check_charge_conservation(rho_start, rho_end): - assert(np.all(is_close(rho_start, rho_end, rtol=2.e-11))) + assert np.all(is_close(rho_start, rho_end, rtol=2.0e-11)) + def main(): filename_end = sys.argv[1] - filename_start = filename_end[:-4] + '0000' + filename_start = filename_end[:-4] + "0000" ds_end = yt.load(filename_end) ds_start = yt.load(filename_start) ad_end = ds_end.all_data() ad_start = ds_start.all_data() - field_data_end = ds_end.covering_grid(level=0, left_edge=ds_end.domain_left_edge, - dims=ds_end.domain_dimensions) - field_data_start = ds_start.covering_grid(level=0, left_edge=ds_start.domain_left_edge, - dims=ds_start.domain_dimensions) + field_data_end = ds_end.covering_grid( + level=0, left_edge=ds_end.domain_left_edge, dims=ds_end.domain_dimensions + ) + field_data_start = ds_start.covering_grid( + level=0, left_edge=ds_start.domain_left_edge, dims=ds_start.domain_dimensions + ) ntests = 5 - for i in range(1, ntests+1): - proton_species = "proton"+str(i) - boron_species = "boron"+str(i) - alpha_species = "alpha"+str(i) + for i in range(1, ntests + 1): + proton_species = "proton" + str(i) + boron_species = "boron" + str(i) + alpha_species = "alpha" + str(i) data = {} add_species_to_dict(ad_start, data, proton_species, "proton", "start") add_species_to_dict(ad_start, data, boron_species, "boron", "start") @@ -721,14 +870,12 @@ def main(): generic_check(data) # Checks that are specific to test number i - eval("specific_check"+str(i)+"(data)") + eval("specific_check" + str(i) + "(data)") rho_start = field_data_start["rho"].to_ndarray() rho_end = field_data_end["rho"].to_ndarray() check_charge_conservation(rho_start, rho_end) - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) if __name__ == "__main__": main() diff --git a/Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py b/Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py index ad8b7f70e10..38ab699a3a4 100755 --- a/Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py @@ -5,16 +5,12 @@ # # License: BSD-3-Clause-LBNL -import os import re import sys -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI import numpy as np import scipy.constants as scc +import yt ## This script performs various checks for the fusion module. The simulation ## that we check is made of 2 different tests, each with different reactant and product species. @@ -46,46 +42,49 @@ ## Please be aware that the relative tolerances are often set empirically in this analysis script, ## so it would not be surprising that some tolerances need to be increased in the future. -default_tol = 1.e-12 # Default relative tolerance +default_tol = 1.0e-12 # Default relative tolerance ## Some physical parameters -keV_to_Joule = scc.e*1e3 -MeV_to_Joule = scc.e*1e6 -barn_to_square_meter = 1.e-28 +keV_to_Joule = scc.e * 1e3 +MeV_to_Joule = scc.e * 1e6 +barn_to_square_meter = 1.0e-28 ## Checks whether this is the 2D or the 3D test -warpx_used_inputs = open('./warpx_used_inputs', 'r').read() -if re.search('geometry.dims = RZ', warpx_used_inputs): +with open("./warpx_used_inputs", "r") as f: + warpx_used_inputs = f.read() +if re.search("geometry.dims = RZ", warpx_used_inputs): is_RZ = True else: is_RZ = False ## Check which kind of test we are doing: D+T or D+D # Define reactants and products -if re.search('tritium', warpx_used_inputs): +if re.search("tritium", warpx_used_inputs): # If tritium appears in the file, than this is the D+T test - reaction_type = 'DT' - reactant_species = ['deuterium', 'tritium'] - product_species = ['helium4', 'neutron'] + reaction_type = "DT" + reactant_species = ["deuterium", "tritium"] + product_species = ["helium4", "neutron"] ntests = 2 - E_fusion = 17.5893*MeV_to_Joule # Energy released during the fusion reaction + E_fusion = 17.5893 * MeV_to_Joule # Energy released during the fusion reaction else: # else, this is the D+D test - reaction_type = 'DD' - reactant_species = ['deuterium', 'hydrogen2'] - product_species = ['helium3', 'neutron'] + reaction_type = "DD" + reactant_species = ["deuterium", "hydrogen2"] + product_species = ["helium3", "neutron"] ntests = 1 - E_fusion = 3.268911e6*MeV_to_Joule + E_fusion = 3.268911e6 * MeV_to_Joule mass = { - 'deuterium': 2.01410177812*scc.m_u, - 'hydrogen2': 2.01410177812*scc.m_u, - 'tritium': 3.0160492779*scc.m_u, - 'helium3': 3.016029*scc.m_u, - 'helium4': 4.00260325413*scc.m_u, - 'neutron': 1.0013784193052508*scc.m_p + "deuterium": 2.01410177812 * scc.m_u, + "hydrogen2": 2.01410177812 * scc.m_u, + "tritium": 3.0160492779 * scc.m_u, + "helium3": 3.016029 * scc.m_u, + "helium4": 4.00260325413 * scc.m_u, + "neutron": 1.0013784193052508 * scc.m_p, } -m_reduced = np.product([mass[s] for s in reactant_species])/np.sum([mass[s] for s in reactant_species]) +m_reduced = np.product([mass[s] for s in reactant_species]) / np.sum( + [mass[s] for s in reactant_species] +) ## Some numerical parameters for this test size_x = 8 @@ -94,10 +93,10 @@ if is_RZ: dV_slice = np.pi * size_x**2 yt_z_string = "particle_position_y" - nppcell_1 = 10000*8 - nppcell_2 = 900*8 + nppcell_1 = 10000 * 8 + nppcell_2 = 900 * 8 else: - dV_slice = size_x*size_y + dV_slice = size_x * size_y yt_z_string = "particle_position_z" nppcell_1 = 10000 nppcell_2 = 900 @@ -105,29 +104,33 @@ # particles of a given species in the same slice have the exact same momentum # In test 1 and 2, the energy in cells number i (in z direction) is typically Energy_step * i**2 -Energy_step = 22.*keV_to_Joule +Energy_step = 22.0 * keV_to_Joule + -def is_close(val1, val2, rtol=default_tol, atol=0.): +def is_close(val1, val2, rtol=default_tol, atol=0.0): ## Wrapper around numpy.isclose, used to override the default tolerances. return np.isclose(val1, val2, rtol=rtol, atol=atol) + def add_existing_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): - data_dict[prefix+"_px_"+suffix] = yt_ad[species_name, "particle_momentum_x"].v - data_dict[prefix+"_py_"+suffix] = yt_ad[species_name, "particle_momentum_y"].v - data_dict[prefix+"_pz_"+suffix] = yt_ad[species_name, "particle_momentum_z"].v - data_dict[prefix+"_w_"+suffix] = yt_ad[species_name, "particle_weight"].v - data_dict[prefix+"_id_"+suffix] = yt_ad[species_name, "particle_id"].v - data_dict[prefix+"_cpu_"+suffix] = yt_ad[species_name, "particle_cpu"].v - data_dict[prefix+"_z_"+suffix] = yt_ad[species_name, yt_z_string].v + data_dict[prefix + "_px_" + suffix] = yt_ad[species_name, "particle_momentum_x"].v + data_dict[prefix + "_py_" + suffix] = yt_ad[species_name, "particle_momentum_y"].v + data_dict[prefix + "_pz_" + suffix] = yt_ad[species_name, "particle_momentum_z"].v + data_dict[prefix + "_w_" + suffix] = yt_ad[species_name, "particle_weight"].v + data_dict[prefix + "_id_" + suffix] = yt_ad[species_name, "particle_id"].v + data_dict[prefix + "_cpu_" + suffix] = yt_ad[species_name, "particle_cpu"].v + data_dict[prefix + "_z_" + suffix] = yt_ad[species_name, yt_z_string].v + def add_empty_species_to_dict(data_dict, species_name, prefix, suffix): - data_dict[prefix+"_px_"+suffix] = np.empty(0) - data_dict[prefix+"_py_"+suffix] = np.empty(0) - data_dict[prefix+"_pz_"+suffix] = np.empty(0) - data_dict[prefix+"_w_"+suffix] = np.empty(0) - data_dict[prefix+"_id_"+suffix] = np.empty(0) - data_dict[prefix+"_cpu_"+suffix] = np.empty(0) - data_dict[prefix+"_z_"+suffix] = np.empty(0) + data_dict[prefix + "_px_" + suffix] = np.empty(0) + data_dict[prefix + "_py_" + suffix] = np.empty(0) + data_dict[prefix + "_pz_" + suffix] = np.empty(0) + data_dict[prefix + "_w_" + suffix] = np.empty(0) + data_dict[prefix + "_id_" + suffix] = np.empty(0) + data_dict[prefix + "_cpu_" + suffix] = np.empty(0) + data_dict[prefix + "_z_" + suffix] = np.empty(0) + def add_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): try: @@ -138,47 +141,67 @@ def add_species_to_dict(yt_ad, data_dict, species_name, prefix, suffix): ## dictionnary. add_empty_species_to_dict(data_dict, species_name, prefix, suffix) + def check_particle_number_conservation(data): # Check consumption of reactants total_w_reactant1_start = np.sum(data[reactant_species[0] + "_w_start"]) - total_w_reactant1_end = np.sum(data[reactant_species[0] + "_w_end"]) + total_w_reactant1_end = np.sum(data[reactant_species[0] + "_w_end"]) total_w_reactant2_start = np.sum(data[reactant_species[1] + "_w_start"]) - total_w_reactant2_end = np.sum(data[reactant_species[1] + "_w_end"]) + total_w_reactant2_end = np.sum(data[reactant_species[1] + "_w_end"]) consumed_reactant1 = total_w_reactant1_start - total_w_reactant1_end consumed_reactant2 = total_w_reactant2_start - total_w_reactant2_end - assert(consumed_reactant1 >= 0.) - assert(consumed_reactant2 >= 0.) + assert consumed_reactant1 >= 0.0 + assert consumed_reactant2 >= 0.0 ## Check that number of consumed reactants are equal assert_scale = max(total_w_reactant1_start, total_w_reactant2_start) - assert(is_close(consumed_reactant1, consumed_reactant2, rtol = 0., atol = default_tol*assert_scale)) + assert is_close( + consumed_reactant1, + consumed_reactant2, + rtol=0.0, + atol=default_tol * assert_scale, + ) # That the number of products corresponds consumed particles for species_name in product_species: created_product = np.sum(data[species_name + "_w_end"]) - assert(created_product >= 0.) - assert(is_close(total_w_reactant1_start, total_w_reactant1_end + created_product)) - assert(is_close(total_w_reactant2_start, total_w_reactant2_end + created_product)) + assert created_product >= 0.0 + assert is_close( + total_w_reactant1_start, total_w_reactant1_end + created_product + ) + assert is_close( + total_w_reactant2_start, total_w_reactant2_end + created_product + ) + def compute_energy_array(data, species_name, suffix, m): ## Relativistic computation of kinetic energy for a given species - psq_array = data[species_name+'_px_'+suffix]**2 + data[species_name+'_py_'+suffix]**2 + \ - data[species_name+'_pz_'+suffix]**2 - rest_energy = m*scc.c**2 - return np.sqrt(psq_array*scc.c**2 + rest_energy**2) - rest_energy + psq_array = ( + data[species_name + "_px_" + suffix] ** 2 + + data[species_name + "_py_" + suffix] ** 2 + + data[species_name + "_pz_" + suffix] ** 2 + ) + rest_energy = m * scc.c**2 + return np.sqrt(psq_array * scc.c**2 + rest_energy**2) - rest_energy + def check_energy_conservation(data): total_energy_start = 0 for species_name in reactant_species: - total_energy_start += np.sum( data[species_name + "_w_start"] * \ - compute_energy_array(data, species_name, "start", mass[species_name]) ) + total_energy_start += np.sum( + data[species_name + "_w_start"] + * compute_energy_array(data, species_name, "start", mass[species_name]) + ) total_energy_end = 0 for species_name in product_species + reactant_species: - total_energy_end += np.sum( data[species_name + "_w_end"] * \ - compute_energy_array(data, species_name, "end", mass[species_name]) ) + total_energy_end += np.sum( + data[species_name + "_w_end"] + * compute_energy_array(data, species_name, "end", mass[species_name]) + ) n_fusion_reaction = np.sum(data[product_species[0] + "_w_end"]) - assert(is_close(total_energy_end, - total_energy_start + n_fusion_reaction*E_fusion, - rtol = 1.e-8)) + assert is_close( + total_energy_end, total_energy_start + n_fusion_reaction * E_fusion, rtol=1.0e-8 + ) + def check_momentum_conservation(data): total_px_start = 0 @@ -186,33 +209,43 @@ def check_momentum_conservation(data): total_pz_start = 0 for species_name in reactant_species: total_px_start += np.sum( - data[species_name+'_px_start'] * data[species_name+'_w_start']) + data[species_name + "_px_start"] * data[species_name + "_w_start"] + ) total_py_start += np.sum( - data[species_name+'_py_start'] * data[species_name+'_w_start']) + data[species_name + "_py_start"] * data[species_name + "_w_start"] + ) total_pz_start += np.sum( - data[species_name+'_pz_start'] * data[species_name+'_w_start']) + data[species_name + "_pz_start"] * data[species_name + "_w_start"] + ) total_px_end = 0 total_py_end = 0 total_pz_end = 0 for species_name in reactant_species + product_species: total_px_end += np.sum( - data[species_name+'_px_end'] * data[species_name+'_w_end']) + data[species_name + "_px_end"] * data[species_name + "_w_end"] + ) total_py_end += np.sum( - data[species_name+'_py_end'] * data[species_name+'_w_end']) + data[species_name + "_py_end"] * data[species_name + "_w_end"] + ) total_pz_end += np.sum( - data[species_name+'_pz_end'] * data[species_name+'_w_end']) + data[species_name + "_pz_end"] * data[species_name + "_w_end"] + ) ## Absolute tolerance is needed because sometimes the initial momentum is exactly 0 - assert(is_close(total_px_start, total_px_end, atol=1.e-15)) - assert(is_close(total_py_start, total_py_end, atol=1.e-15)) - assert(is_close(total_pz_start, total_pz_end, atol=1.e-15)) + assert is_close(total_px_start, total_px_end, atol=1.0e-15) + assert is_close(total_py_start, total_py_end, atol=1.0e-15) + assert is_close(total_pz_start, total_pz_end, atol=1.0e-15) + def check_id(data): ## Check that all created particles have unique id + cpu identifier (two particles with ## different cpu can have the same id) for species_name in product_species: - complex_id = data[species_name + "_id_end"] + 1j*data[species_name + "_cpu_end"] - assert(complex_id.shape == np.unique(complex_id).shape) + complex_id = ( + data[species_name + "_id_end"] + 1j * data[species_name + "_cpu_end"] + ) + assert complex_id.shape == np.unique(complex_id).shape + def generic_check(data): check_particle_number_conservation(data) @@ -220,110 +253,155 @@ def generic_check(data): check_momentum_conservation(data) check_id(data) + def check_isotropy(data, relative_tolerance): ## Checks that the product particles are emitted isotropically for species_name in product_species: - average_px_sq = np.average(data[species_name+"_px_end"]*data[species_name+"_px_end"]) - average_py_sq = np.average(data[species_name+"_py_end"]*data[species_name+"_py_end"]) - average_pz_sq = np.average(data[species_name+"_pz_end"]*data[species_name+"_pz_end"]) - assert(is_close(average_px_sq, average_py_sq, rtol = relative_tolerance)) - assert(is_close(average_px_sq, average_pz_sq, rtol = relative_tolerance)) + average_px_sq = np.average( + data[species_name + "_px_end"] * data[species_name + "_px_end"] + ) + average_py_sq = np.average( + data[species_name + "_py_end"] * data[species_name + "_py_end"] + ) + average_pz_sq = np.average( + data[species_name + "_pz_end"] * data[species_name + "_pz_end"] + ) + assert is_close(average_px_sq, average_py_sq, rtol=relative_tolerance) + assert is_close(average_px_sq, average_pz_sq, rtol=relative_tolerance) + def check_xy_isotropy(data): ## Checks that the product particles are emitted isotropically in x and y for species_name in product_species: - average_px_sq = np.average(data[species_name+"_px_end"]*data[species_name+"_px_end"]) - average_py_sq = np.average(data[species_name+"_py_end"]*data[species_name+"_py_end"]) - average_pz_sq = np.average(data[species_name+"_pz_end"]*data[species_name+"_pz_end"]) - assert(is_close(average_px_sq, average_py_sq, rtol = 5.e-2)) - assert(average_pz_sq > average_px_sq) - assert(average_pz_sq > average_py_sq) - -def cross_section( E_keV ): + average_px_sq = np.average( + data[species_name + "_px_end"] * data[species_name + "_px_end"] + ) + average_py_sq = np.average( + data[species_name + "_py_end"] * data[species_name + "_py_end"] + ) + average_pz_sq = np.average( + data[species_name + "_pz_end"] * data[species_name + "_pz_end"] + ) + assert is_close(average_px_sq, average_py_sq, rtol=5.0e-2) + assert average_pz_sq > average_px_sq + assert average_pz_sq > average_py_sq + + +def cross_section(E_keV): ## Returns cross section in b, using the analytical fits given ## in H.-S. Bosch and G.M. Hale 1992 Nucl. Fusion 32 611 - joule_to_keV = 1.e-3/scc.e - B_G = scc.pi * scc.alpha * np.sqrt( 2.*m_reduced * scc.c**2 * joule_to_keV ); - if reaction_type == 'DT': - A1 = 6.927e4; - A2 = 7.454e8; - A3 = 2.050e6; - A4 = 5.2002e4; - A5 = 0; - B1 = 6.38e1; - B2 = -9.95e-1; - B3 = 6.981e-5; - B4 = 1.728e-4; - elif reaction_type == 'DD': - A1 = 5.3701e4; - A2 = 3.3027e2; - A3 = -1.2706e-1; - A4 = 2.9327e-5; - A5 = -2.5151e-9; - B1 = 0; - B2 = 0; - B3 = 0; - B4 = 0; - - astrophysical_factor = (A1 + E_keV*(A2 + E_keV*(A3 + E_keV*(A4 + E_keV*A5)))) / (1 + E_keV*(B1 + E_keV*(B2 + E_keV*(B3 + E_keV*B4)))); - millibarn_to_barn = 1.e-3; - return millibarn_to_barn * astrophysical_factor/E_keV * np.exp(-B_G/np.sqrt(E_keV)) + joule_to_keV = 1.0e-3 / scc.e + B_G = scc.pi * scc.alpha * np.sqrt(2.0 * m_reduced * scc.c**2 * joule_to_keV) + if reaction_type == "DT": + A1 = 6.927e4 + A2 = 7.454e8 + A3 = 2.050e6 + A4 = 5.2002e4 + A5 = 0 + B1 = 6.38e1 + B2 = -9.95e-1 + B3 = 6.981e-5 + B4 = 1.728e-4 + elif reaction_type == "DD": + A1 = 5.3701e4 + A2 = 3.3027e2 + A3 = -1.2706e-1 + A4 = 2.9327e-5 + A5 = -2.5151e-9 + B1 = 0 + B2 = 0 + B3 = 0 + B4 = 0 + else: + raise RuntimeError(f"Reaction type '{reaction_type}' not implemented.") + + astrophysical_factor = ( + A1 + E_keV * (A2 + E_keV * (A3 + E_keV * (A4 + E_keV * A5))) + ) / (1 + E_keV * (B1 + E_keV * (B2 + E_keV * (B3 + E_keV * B4)))) + millibarn_to_barn = 1.0e-3 + return ( + millibarn_to_barn * astrophysical_factor / E_keV * np.exp(-B_G / np.sqrt(E_keV)) + ) + def E_com_to_p_sq_com(m1, m2, E): ## E is the total (kinetic+mass) energy of a two particle (with mass m1 and m2) system in ## its center of mass frame, in J. ## Returns the square norm of the momentum of each particle in that frame. - E_ratio = E/((m1+m2)*scc.c**2) - return m1*m2*scc.c**2 * (E_ratio**2 - 1) + (m1-m2)**2*scc.c**2/4 * (E_ratio - 1./E_ratio)**2 + E_ratio = E / ((m1 + m2) * scc.c**2) + return ( + m1 * m2 * scc.c**2 * (E_ratio**2 - 1) + + (m1 - m2) ** 2 * scc.c**2 / 4 * (E_ratio - 1.0 / E_ratio) ** 2 + ) + def compute_relative_v_com(E): ## E is the kinetic energy of reactants in the center of mass frame, in keV ## Returns the relative velocity between reactants in this frame, in m/s m0 = mass[reactant_species[0]] m1 = mass[reactant_species[1]] - E_J = E*keV_to_Joule + (m0 + m1)*scc.c**2 + E_J = E * keV_to_Joule + (m0 + m1) * scc.c**2 p_sq = E_com_to_p_sq_com(m0, m1, E_J) p = np.sqrt(p_sq) - gamma0 = np.sqrt(1. + p_sq / (m0*scc.c)**2) - gamma1 = np.sqrt(1. + p_sq / (m1*scc.c)**2) - v0 = p/(gamma0*m0) - v1 = p/(gamma1*m1) - return v0+v1 + gamma0 = np.sqrt(1.0 + p_sq / (m0 * scc.c) ** 2) + gamma1 = np.sqrt(1.0 + p_sq / (m1 * scc.c) ** 2) + v0 = p / (gamma0 * m0) + v1 = p / (gamma1 * m1) + return v0 + v1 + def expected_weight_com(E_com, reactant0_density, reactant1_density, dV, dt): ## Computes expected number of product particles as a function of energy E_com in the ## center of mass frame. E_com is in keV. - assert(np.all(E_com>=0)) + assert np.all(E_com >= 0) ## Case E_com == 0 is handled manually to avoid division by zero conditions = [E_com == 0, E_com > 0] ## Necessary to avoid division by 0 warning when pb_cross_section is evaluated - E_com_never_zero = np.clip(E_com, 1.e-15, None) - choices = [0., cross_section(E_com_never_zero)*compute_relative_v_com(E_com_never_zero)] + E_com_never_zero = np.clip(E_com, 1.0e-15, None) + choices = [ + 0.0, + cross_section(E_com_never_zero) * compute_relative_v_com(E_com_never_zero), + ] sigma_times_vrel = np.select(conditions, choices) - return reactant0_density*reactant1_density*sigma_times_vrel*barn_to_square_meter*dV*dt - -def check_macroparticle_number(data, fusion_probability_target_value, num_pair_per_cell): + return ( + reactant0_density + * reactant1_density + * sigma_times_vrel + * barn_to_square_meter + * dV + * dt + ) + + +def check_macroparticle_number( + data, fusion_probability_target_value, num_pair_per_cell +): ## Checks that the number of macroparticles is as expected for the first and second tests ## The first slice 0 < z < 1 does not contribute to alpha creation if is_RZ: - numcells = size_x*(size_z-1) + numcells = size_x * (size_z - 1) else: - numcells = size_x*size_y*(size_z-1) + numcells = size_x * size_y * (size_z - 1) ## In these tests, the fusion_multiplier is so high that the fusion probability per pair is ## equal to the parameter fusion_probability_target_value fusion_probability_per_pair = fusion_probability_target_value - expected_fusion_number = numcells*num_pair_per_cell*fusion_probability_per_pair - expected_macroparticle_number = 2*expected_fusion_number - std_macroparticle_number = 2*np.sqrt(expected_fusion_number) + expected_fusion_number = numcells * num_pair_per_cell * fusion_probability_per_pair + expected_macroparticle_number = 2 * expected_fusion_number + std_macroparticle_number = 2 * np.sqrt(expected_fusion_number) actual_macroparticle_number = data[product_species[0] + "_w_end"].shape[0] # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions - assert(is_close(actual_macroparticle_number, expected_macroparticle_number, rtol = 0., - atol = 5.*std_macroparticle_number)) + assert is_close( + actual_macroparticle_number, + expected_macroparticle_number, + rtol=0.0, + atol=5.0 * std_macroparticle_number, + ) ## used in subsequent function return expected_fusion_number + def p_sq_reactant1_frame_to_E_COM_frame(p_reactant0_sq): # Takes the reactant0 square norm of the momentum in the reactant1 rest frame and returns the total # kinetic energy in the center of mass frame. Everything is in SI units. @@ -331,108 +409,148 @@ def p_sq_reactant1_frame_to_E_COM_frame(p_reactant0_sq): m1 = mass[reactant_species[1]] # Total (kinetic + mass) energy in lab frame - E_lab = np.sqrt(p_reactant0_sq*scc.c**2 + (m0*scc.c**2)**2) + m1*scc.c**2 + E_lab = np.sqrt(p_reactant0_sq * scc.c**2 + (m0 * scc.c**2) ** 2) + m1 * scc.c**2 # Use invariant E**2 - p**2c**2 of 4-momentum norm to compute energy in center of mass frame - E_com = np.sqrt(E_lab**2 - p_reactant0_sq*scc.c**2) + E_com = np.sqrt(E_lab**2 - p_reactant0_sq * scc.c**2) # Corresponding kinetic energy - E_com_kin = E_com - (m1+m0)*scc.c**2 - return E_com_kin*(p_reactant0_sq>0.) + E_com_kin = E_com - (m1 + m0) * scc.c**2 + return E_com_kin * (p_reactant0_sq > 0.0) + def p_sq_to_kinetic_energy(p_sq, m): ## Returns the kinetic energy of a particle as a function of its squared momentum. ## Everything is in SI units. - return np.sqrt(p_sq*scc.c**2 + (m*scc.c**2)**2) - (m*scc.c**2) + return np.sqrt(p_sq * scc.c**2 + (m * scc.c**2) ** 2) - (m * scc.c**2) + def compute_E_com1(data): ## Computes kinetic energy (in Joule) in the center of frame for the first test ## Square norm of the momentum of reactant as a function of cell number in z direction - p_sq = 2.*m_reduced*(Energy_step*np.arange(size_z)**2) + p_sq = 2.0 * m_reduced * (Energy_step * np.arange(size_z) ** 2) Ekin = 0 for species_name in reactant_species: - Ekin += p_sq_to_kinetic_energy( p_sq, mass[species_name] ) + Ekin += p_sq_to_kinetic_energy(p_sq, mass[species_name]) return Ekin + def compute_E_com2(data): ## Computes kinetic energy (in Joule) in the center of frame for the second test ## Square norm of the momentum of reactant0 as a function of cell number in z direction - p_reactant0_sq = 2.*mass[reactant_species[0]]*(Energy_step*np.arange(size_z)**2) + p_reactant0_sq = ( + 2.0 * mass[reactant_species[0]] * (Energy_step * np.arange(size_z) ** 2) + ) return p_sq_reactant1_frame_to_E_COM_frame(p_reactant0_sq) -def check_fusion_yield(data, expected_fusion_number, E_com, reactant0_density, reactant1_density, dt): + +def check_fusion_yield( + data, expected_fusion_number, E_com, reactant0_density, reactant1_density, dt +): ## Checks that the fusion yield is as expected for the first and second tests. - product_weight_theory = expected_weight_com(E_com/keV_to_Joule, - reactant0_density, reactant1_density, dV_slice, dt) + product_weight_theory = expected_weight_com( + E_com / keV_to_Joule, reactant0_density, reactant1_density, dV_slice, dt + ) for species_name in product_species: - product_weight_simulation = np.histogram(data[species_name+"_z_end"], - bins=size_z, range=(0, size_z), weights = data[species_name+"_w_end"])[0] + product_weight_simulation = np.histogram( + data[species_name + "_z_end"], + bins=size_z, + range=(0, size_z), + weights=data[species_name + "_w_end"], + )[0] ## -1 is here because the first slice 0 < z < 1 does not contribute to fusion - expected_fusion_number_per_slice = expected_fusion_number/(size_z-1) - relative_std_weight = 1./np.sqrt(expected_fusion_number_per_slice) + expected_fusion_number_per_slice = expected_fusion_number / (size_z - 1) + relative_std_weight = 1.0 / np.sqrt(expected_fusion_number_per_slice) # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions - assert(np.all(is_close(product_weight_theory, product_weight_simulation, - rtol = 5.*relative_std_weight))) + assert np.all( + is_close( + product_weight_theory, + product_weight_simulation, + rtol=5.0 * relative_std_weight, + ) + ) + def specific_check1(data, dt): if not is_RZ: - check_isotropy(data, relative_tolerance = 3.e-2) - expected_fusion_number = check_macroparticle_number(data, - fusion_probability_target_value = 0.002, - num_pair_per_cell = nppcell_1) + check_isotropy(data, relative_tolerance=3.0e-2) + expected_fusion_number = check_macroparticle_number( + data, fusion_probability_target_value=0.002, num_pair_per_cell=nppcell_1 + ) E_com = compute_E_com1(data) - check_fusion_yield(data, expected_fusion_number, E_com, reactant0_density = 1., - reactant1_density = 1., dt=dt) + check_fusion_yield( + data, + expected_fusion_number, + E_com, + reactant0_density=1.0, + reactant1_density=1.0, + dt=dt, + ) + def specific_check2(data, dt): check_xy_isotropy(data) ## Only 900 particles pairs per cell here because we ignore the 10% of reactants that are at rest - expected_fusion_number = check_macroparticle_number(data, - fusion_probability_target_value = 0.02, - num_pair_per_cell = nppcell_2) + expected_fusion_number = check_macroparticle_number( + data, fusion_probability_target_value=0.02, num_pair_per_cell=nppcell_2 + ) E_com = compute_E_com2(data) - check_fusion_yield(data, expected_fusion_number, E_com, reactant0_density = 1.e20, - reactant1_density = 1.e26, dt=dt) + check_fusion_yield( + data, + expected_fusion_number, + E_com, + reactant0_density=1.0e20, + reactant1_density=1.0e26, + dt=dt, + ) + def check_charge_conservation(rho_start, rho_end): - assert(np.all(is_close(rho_start, rho_end, rtol=2.e-11))) + assert np.all(is_close(rho_start, rho_end, rtol=2.0e-11)) + def main(): filename_end = sys.argv[1] - filename_start = filename_end[:-4] + '0000' + filename_start = filename_end[:-4] + "0000" ds_end = yt.load(filename_end) ds_start = yt.load(filename_start) ad_end = ds_end.all_data() ad_start = ds_start.all_data() - dt = float(ds_end.current_time - ds_start.current_time) - field_data_end = ds_end.covering_grid(level=0, left_edge=ds_end.domain_left_edge, - dims=ds_end.domain_dimensions) - field_data_start = ds_start.covering_grid(level=0, left_edge=ds_start.domain_left_edge, - dims=ds_start.domain_dimensions) - - for i in range(1, ntests+1): + dt = float(ds_end.current_time - ds_start.current_time) # noqa + field_data_end = ds_end.covering_grid( + level=0, left_edge=ds_end.domain_left_edge, dims=ds_end.domain_dimensions + ) + field_data_start = ds_start.covering_grid( + level=0, left_edge=ds_start.domain_left_edge, dims=ds_start.domain_dimensions + ) + + for i in range(1, ntests + 1): data = {} for species_name in reactant_species: - add_species_to_dict(ad_start, data, species_name+'_'+str(i), species_name, "start") - add_species_to_dict(ad_end, data, species_name+'_'+str(i), species_name, "end") + add_species_to_dict( + ad_start, data, species_name + "_" + str(i), species_name, "start" + ) + add_species_to_dict( + ad_end, data, species_name + "_" + str(i), species_name, "end" + ) for species_name in product_species: - add_species_to_dict(ad_end, data, species_name+'_'+str(i), species_name, "end") + add_species_to_dict( + ad_end, data, species_name + "_" + str(i), species_name, "end" + ) # General checks that are performed for all tests generic_check(data) # Checks that are specific to test number i - eval("specific_check"+str(i)+"(data, dt)") + eval("specific_check" + str(i) + "(data, dt)") rho_start = field_data_start["rho"].to_ndarray() rho_end = field_data_end["rho"].to_ndarray() check_charge_conservation(rho_start, rho_end) - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) if __name__ == "__main__": main() diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d b/Examples/Tests/nuclear_fusion/inputs_test_2d_proton_boron_fusion similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_proton_boron_2d rename to Examples/Tests/nuclear_fusion/inputs_test_2d_proton_boron_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d b/Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_deuterium_fusion similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d rename to Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_deuterium_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies b/Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_deuterium_fusion_intraspecies similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies rename to Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_deuterium_fusion_intraspecies diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_tritium_fusion similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d rename to Examples/Tests/nuclear_fusion/inputs_test_3d_deuterium_tritium_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d b/Examples/Tests/nuclear_fusion/inputs_test_3d_proton_boron_fusion similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_proton_boron_3d rename to Examples/Tests/nuclear_fusion/inputs_test_3d_proton_boron_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_rz b/Examples/Tests/nuclear_fusion/inputs_test_rz_deuterium_tritium_fusion similarity index 100% rename from Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_rz rename to Examples/Tests/nuclear_fusion/inputs_test_rz_deuterium_tritium_fusion diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py deleted file mode 100644 index ec67282fd88..00000000000 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are -# --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script is set up to produce either parallel or -# --- perpendicular (Bernstein) EM modes and can be run in 1d, 2d or 3d -# --- Cartesian geometries. See Section 4.2 and 4.3 of Munoz et al. (2018). -# --- As a CI test only a small number of steps are taken using the 1d version. - -import argparse -import os -import sys - -import dill -import numpy as np -from mpi4py import MPI as mpi - -from pywarpx import callbacks, fields, libwarpx, picmi - -constants = picmi.constants - -comm = mpi.COMM_WORLD - -simulation = picmi.Simulation( - warpx_serialize_initial_conditions=True, - verbose=0 -) - - -class EMModes(object): - '''The following runs a simulation of an uniform plasma at a set - temperature (Te = Ti) with an external magnetic field applied in either the - z-direction (parallel to domain) or x-direction (perpendicular to domain). - The analysis script (in this same directory) analyzes the output field data - for EM modes. This input is based on the EM modes tests as described by - Munoz et al. (2018) and tests done by Scott Nicks at TAE Technologies. - ''' - # Applied field parameters - B0 = 0.25 # Initial magnetic field strength (T) - beta = [0.01, 0.1] # Plasma beta, used to calculate temperature - - # Plasma species parameters - m_ion = [100.0, 400.0] # Ion mass (electron masses) - vA_over_c = [1e-4, 1e-3] # ratio of Alfven speed and the speed of light - - # Spatial domain - Nz = [1024, 1920] # number of cells in z direction - Nx = 8 # number of cells in x (and y) direction for >1 dimensions - - # Temporal domain (if not run as a CI test) - LT = 300.0 # Simulation temporal length (ion cyclotron periods) - - # Numerical parameters - NPPC = [1024, 256, 64] # Seed number of particles per cell - DZ = 1.0 / 10.0 # Cell size (ion skin depths) - DT = [5e-3, 4e-3] # Time step (ion cyclotron periods) - - # Plasma resistivity - used to dampen the mode excitation - eta = [[1e-7, 1e-7], [1e-7, 1e-5], [1e-7, 1e-4]] - # Number of substeps used to update B - substeps = 20 - - def __init__(self, test, dim, B_dir, verbose): - """Get input parameters for the specific case desired.""" - self.test = test - self.dim = int(dim) - self.B_dir = B_dir - self.verbose = verbose or self.test - - # sanity check - assert (dim > 0 and dim < 4), f"{dim}-dimensions not a valid input" - - # get simulation parameters from the defaults given the direction of - # the initial B-field and the dimensionality - self.get_simulation_parameters() - - # calculate various plasma parameters based on the simulation input - self.get_plasma_quantities() - - self.dz = self.DZ * self.l_i - self.Lz = self.Nz * self.dz - self.Lx = self.Nx * self.dz - - self.dt = self.DT * self.t_ci - - if not self.test: - self.total_steps = int(self.LT / self.DT) - else: - # if this is a test case run for only a small number of steps - self.total_steps = 250 - # output diagnostics 20 times per cyclotron period - self.diag_steps = int(1.0/20 / self.DT) - - # dump all the current attributes to a dill pickle file - if comm.rank == 0: - with open(f'sim_parameters.dpkl', 'wb') as f: - dill.dump(self, f) - - # print out plasma parameters - if comm.rank == 0: - print( - f"Initializing simulation with input parameters:\n" - f"\tT = {self.T_plasma:.3f} eV\n" - f"\tn = {self.n_plasma:.1e} m^-3\n" - f"\tB0 = {self.B0:.2f} T\n" - f"\tM/m = {self.m_ion:.0f}\n" - ) - print( - f"Plasma parameters:\n" - f"\tl_i = {self.l_i:.1e} m\n" - f"\tt_ci = {self.t_ci:.1e} s\n" - f"\tv_ti = {self.v_ti:.1e} m/s\n" - f"\tvA = {self.vA:.1e} m/s\n" - ) - print( - f"Numerical parameters:\n" - f"\tdz = {self.dz:.1e} m\n" - f"\tdt = {self.dt:.1e} s\n" - f"\tdiag steps = {self.diag_steps:d}\n" - f"\ttotal steps = {self.total_steps:d}\n" - ) - - self.setup_run() - - def get_simulation_parameters(self): - """Pick appropriate parameters from the defaults given the direction - of the B-field and the simulation dimensionality.""" - if self.B_dir == 'z': - idx = 0 - self.Bx = 0.0 - self.By = 0.0 - self.Bz = self.B0 - elif self.B_dir == 'y': - idx = 1 - self.Bx = 0.0 - self.By = self.B0 - self.Bz = 0.0 - else: - idx = 1 - self.Bx = self.B0 - self.By = 0.0 - self.Bz = 0.0 - - self.beta = self.beta[idx] - self.m_ion = self.m_ion[idx] - self.vA_over_c = self.vA_over_c[idx] - self.Nz = self.Nz[idx] - self.DT = self.DT[idx] - - self.NPPC = self.NPPC[self.dim-1] - self.eta = self.eta[self.dim-1][idx] - - def get_plasma_quantities(self): - """Calculate various plasma parameters based on the simulation input.""" - # Ion mass (kg) - self.M = self.m_ion * constants.m_e - - # Cyclotron angular frequency (rad/s) and period (s) - self.w_ci = constants.q_e * abs(self.B0) / self.M - self.t_ci = 2.0 * np.pi / self.w_ci - - # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi - self.vA = self.vA_over_c * constants.c - self.n_plasma = ( - (self.B0 / self.vA)**2 / (constants.mu0 * (self.M + constants.m_e)) - ) - - # Ion plasma frequency (Hz) - self.w_pi = np.sqrt( - constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) - ) - - # Skin depth (m) - self.l_i = constants.c / self.w_pi - - # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 - self.v_ti = np.sqrt(self.beta / 2.0) * self.vA - - # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) - self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV - - # Larmor radius (m) - self.rho_i = self.v_ti / self.w_ci - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - if self.dim == 1: - grid_object = picmi.Cartesian1DGrid - elif self.dim == 2: - grid_object = picmi.Cartesian2DGrid - else: - grid_object = picmi.Cartesian3DGrid - - self.grid = grid_object( - number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim:], - warpx_max_grid_size=self.Nz, - lower_bound=[-self.Lx/2.0, -self.Lx/2.0, 0][-self.dim:], - upper_bound=[self.Lx/2.0, self.Lx/2.0, self.Lz][-self.dim:], - lower_boundary_conditions=['periodic']*self.dim, - upper_boundary_conditions=['periodic']*self.dim - ) - simulation.time_step_size = self.dt - simulation.max_steps = self.total_steps - simulation.current_deposition_algo = 'direct' - simulation.particle_shape = 1 - simulation.verbose = self.verbose - - ####################################################################### - # Field solver and external field # - ####################################################################### - - self.solver = picmi.HybridPICSolver( - grid=self.grid, - Te=self.T_plasma, n0=self.n_plasma, plasma_resistivity=self.eta, - substeps=self.substeps - ) - simulation.solver = self.solver - - B_ext = picmi.AnalyticInitialField( - Bx_expression=self.Bx, - By_expression=self.By, - Bz_expression=self.Bz - ) - simulation.add_applied_field(B_ext) - - ####################################################################### - # Particle types setup # - ####################################################################### - - self.ions = picmi.Species( - name='ions', charge='q_e', mass=self.M, - initial_distribution=picmi.UniformDistribution( - density=self.n_plasma, - rms_velocity=[self.v_ti]*3, - ) - ) - simulation.add_species( - self.ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, n_macroparticles_per_cell=self.NPPC - ) - ) - - ####################################################################### - # Add diagnostics # - ####################################################################### - - if self.B_dir == 'z': - self.output_file_name = 'par_field_data.txt' - else: - self.output_file_name = 'perp_field_data.txt' - - if self.test: - particle_diag = picmi.ParticleDiagnostic( - name='field_diag', - period=self.total_steps, - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', - # warpx_format = 'openpmd', - # warpx_openpmd_backend = 'h5' - ) - simulation.add_diagnostic(particle_diag) - field_diag = picmi.FieldDiagnostic( - name='field_diag', - grid=self.grid, - period=self.total_steps, - data_list=['B', 'E', 'J_displacement'], - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', - # warpx_format = 'openpmd', - # warpx_openpmd_backend = 'h5' - ) - simulation.add_diagnostic(field_diag) - - if self.B_dir == 'z' or self.dim == 1: - line_diag = picmi.ReducedDiagnostic( - diag_type='FieldProbe', - probe_geometry='Line', - z_probe=0, - z1_probe=self.Lz, - resolution=self.Nz - 1, - name=self.output_file_name[:-4], - period=self.diag_steps, - path='diags/' - ) - simulation.add_diagnostic(line_diag) - else: - # install a custom "reduced diagnostic" to save the average field - callbacks.installafterEsolve(self._record_average_fields) - try: - os.mkdir("diags") - except OSError: - # diags directory already exists - pass - with open(f"diags/{self.output_file_name}", 'w') as f: - f.write( - "[0]step() [1]time(s) [2]z_coord(m) " - "[3]Ez_lev0-(V/m) [4]Bx_lev0-(T) [5]By_lev0-(T)\n" - ) - - ####################################################################### - # Initialize simulation # - ####################################################################### - - simulation.initialize_inputs() - simulation.initialize_warpx() - - def _record_average_fields(self): - """A custom reduced diagnostic to store the average E&M fields in a - similar format as the reduced diagnostic so that the same analysis - script can be used regardless of the simulation dimension. - """ - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if step % self.diag_steps != 0: - return - - Bx_warpx = fields.BxWrapper()[...] - By_warpx = fields.ByWrapper()[...] - Ez_warpx = fields.EzWrapper()[...] - - if libwarpx.amr.ParallelDescriptor.MyProc() != 0: - return - - t = step * self.dt - z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) - - if self.dim == 2: - Ez = np.mean(Ez_warpx[:-1], axis=0) - Bx = np.mean(Bx_warpx[:-1], axis=0) - By = np.mean(By_warpx[:-1], axis=0) - else: - Ez = np.mean(Ez_warpx[:-1, :-1], axis=(0, 1)) - Bx = np.mean(Bx_warpx[:-1], axis=(0, 1)) - By = np.mean(By_warpx[:-1], axis=(0, 1)) - - with open(f"diags/{self.output_file_name}", 'a') as f: - for ii in range(self.Nz): - f.write( - f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {Ez[ii]:+.10e} " - f"{Bx[ii]:+.10e} {By[ii]:+.10e}\n" - ) - - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-d', '--dim', help='Simulation dimension', required=False, type=int, - default=1 -) -parser.add_argument( - '--bdir', help='Direction of the B-field', required=False, - choices=['x', 'y', 'z'], default='z' -) -parser.add_argument( - '-v', '--verbose', help='Verbose output', action='store_true', -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -run = EMModes(test=args.test, dim=args.dim, B_dir=args.bdir, verbose=args.verbose) -simulation.step() diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py deleted file mode 100644 index 5bd1e3518f9..00000000000 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are -# --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script is set up to produce parallel normal EM modes -# --- in a metallic cylinder and is run in RZ geometry. -# --- As a CI test only a small number of steps are taken. - -import argparse -import sys - -import dill -import numpy as np -from mpi4py import MPI as mpi - -from pywarpx import picmi - -constants = picmi.constants - -comm = mpi.COMM_WORLD - -simulation = picmi.Simulation(verbose=0) - - -class CylindricalNormalModes(object): - '''The following runs a simulation of an uniform plasma at a set ion - temperature (and Te = 0) with an external magnetic field applied in the - z-direction (parallel to domain). - The analysis script (in this same directory) analyzes the output field - data for EM modes. - ''' - # Applied field parameters - B0 = 0.5 # Initial magnetic field strength (T) - beta = 0.01 # Plasma beta, used to calculate temperature - - # Plasma species parameters - m_ion = 400.0 # Ion mass (electron masses) - vA_over_c = 5e-3 # ratio of Alfven speed and the speed of light - - # Spatial domain - Nz = 512 # number of cells in z direction - Nr = 128 # number of cells in r direction - - # Temporal domain (if not run as a CI test) - LT = 800.0 # Simulation temporal length (ion cyclotron periods) - - # Numerical parameters - NPPC = 8000 # Seed number of particles per cell - DZ = 0.4 # Cell size (ion skin depths) - DR = 0.4 # Cell size (ion skin depths) - DT = 0.02 # Time step (ion cyclotron periods) - - # Plasma resistivity - used to dampen the mode excitation - eta = 5e-4 - # Number of substeps used to update B - substeps = 20 - - def __init__(self, test, verbose): - """Get input parameters for the specific case desired.""" - self.test = test - self.verbose = verbose or self.test - - # calculate various plasma parameters based on the simulation input - self.get_plasma_quantities() - - if not self.test: - self.total_steps = int(self.LT / self.DT) - else: - # if this is a test case run for only a small number of steps - self.total_steps = 100 - # and make the grid and particle count smaller - self.Nz = 128 - self.Nr = 64 - self.NPPC = 200 - # output diagnostics 5 times per cyclotron period - self.diag_steps = max(10, int(1.0 / 5 / self.DT)) - - self.Lz = self.Nz * self.DZ * self.l_i - self.Lr = self.Nr * self.DR * self.l_i - - self.dt = self.DT * self.t_ci - - # dump all the current attributes to a dill pickle file - if comm.rank == 0: - with open(f'sim_parameters.dpkl', 'wb') as f: - dill.dump(self, f) - - # print out plasma parameters - if comm.rank == 0: - print( - f"Initializing simulation with input parameters:\n" - f"\tT = {self.T_plasma:.3f} eV\n" - f"\tn = {self.n_plasma:.1e} m^-3\n" - f"\tB0 = {self.B0:.2f} T\n" - f"\tM/m = {self.m_ion:.0f}\n" - ) - print( - f"Plasma parameters:\n" - f"\tl_i = {self.l_i:.1e} m\n" - f"\tt_ci = {self.t_ci:.1e} s\n" - f"\tv_ti = {self.v_ti:.1e} m/s\n" - f"\tvA = {self.vA:.1e} m/s\n" - ) - print( - f"Numerical parameters:\n" - f"\tdt = {self.dt:.1e} s\n" - f"\tdiag steps = {self.diag_steps:d}\n" - f"\ttotal steps = {self.total_steps:d}\n", - flush=True - ) - self.setup_run() - - def get_plasma_quantities(self): - """Calculate various plasma parameters based on the simulation input.""" - # Ion mass (kg) - self.M = self.m_ion * constants.m_e - - # Cyclotron angular frequency (rad/s) and period (s) - self.w_ci = constants.q_e * abs(self.B0) / self.M - self.t_ci = 2.0 * np.pi / self.w_ci - - # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi - self.vA = self.vA_over_c * constants.c - self.n_plasma = ( - (self.B0 / self.vA)**2 / (constants.mu0 * (self.M + constants.m_e)) - ) - - # Ion plasma frequency (Hz) - self.w_pi = np.sqrt( - constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) - ) - - # Skin depth (m) - self.l_i = constants.c / self.w_pi - - # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 - self.v_ti = np.sqrt(self.beta / 2.0) * self.vA - - # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) - self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV - - # Larmor radius (m) - self.rho_i = self.v_ti / self.w_ci - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - self.grid = picmi.CylindricalGrid( - number_of_cells=[self.Nr, self.Nz], - warpx_max_grid_size=self.Nz, - lower_bound=[0, -self.Lz/2.0], - upper_bound=[self.Lr, self.Lz/2.0], - lower_boundary_conditions = ['none', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['none', 'periodic'], - upper_boundary_conditions_particles = ['reflecting', 'periodic'] - ) - simulation.time_step_size = self.dt - simulation.max_steps = self.total_steps - simulation.current_deposition_algo = 'direct' - simulation.particle_shape = 1 - simulation.verbose = self.verbose - - ####################################################################### - # Field solver and external field # - ####################################################################### - - self.solver = picmi.HybridPICSolver( - grid=self.grid, - Te=0.0, n0=self.n_plasma, plasma_resistivity=self.eta, - substeps=self.substeps, - n_floor=self.n_plasma*0.05 - ) - simulation.solver = self.solver - - B_ext = picmi.AnalyticInitialField( - Bz_expression=self.B0 - ) - simulation.add_applied_field(B_ext) - - ####################################################################### - # Particle types setup # - ####################################################################### - - self.ions = picmi.Species( - name='ions', charge='q_e', mass=self.M, - initial_distribution=picmi.UniformDistribution( - density=self.n_plasma, - rms_velocity=[self.v_ti]*3, - ) - ) - simulation.add_species( - self.ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, n_macroparticles_per_cell=self.NPPC - ) - ) - - ####################################################################### - # Add diagnostics # - ####################################################################### - - field_diag = picmi.FieldDiagnostic( - name='field_diag', - grid=self.grid, - period=self.diag_steps, - data_list=['B', 'E'], - write_dir='diags', - warpx_file_prefix='field_diags', - warpx_format='openpmd', - warpx_openpmd_backend='h5', - ) - simulation.add_diagnostic(field_diag) - - # add particle diagnostic for checksum - if self.test: - part_diag = picmi.ParticleDiagnostic( - name='diag1', - period=self.total_steps, - species=[self.ions], - data_list=['ux', 'uy', 'uz', 'weighting'], - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_EM_modes_rz_plt' - ) - simulation.add_diagnostic(part_diag) - - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-v', '--verbose', help='Verbose output', action='store_true', -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -run = CylindricalNormalModes(test=args.test, verbose=args.verbose) -simulation.step() diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst deleted file mode 100644 index 034ee5815f0..00000000000 --- a/Examples/Tests/ohm_solver_EM_modes/README.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. _examples-ohm-solver-em-modes: - -Ohm solver: Electromagnetic modes -================================= - -In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the -:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -Fourier analyzed for Alfvén mode excitations. - -Run ---- - -The same input script can be used for 1d, 2d or 3d Cartesian simulations as well -as replicating either the parallel propagating or ion-Bernstein modes as indicated below. - -.. dropdown:: Script ``PICMI_inputs.py`` - - .. literalinclude:: PICMI_inputs.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py``. - -For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. - -.. tab-set:: - - .. tab-item:: Parallel propagating waves - - Execute: - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --bdir z - - .. tab-item:: Perpendicular propagating waves - - Execute: - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y} - -Analyze -------- - -The following script reads the simulation output from the above example, performs -Fourier transforms of the field data and compares the calculated spectrum -to the theoretical dispersions. - -.. dropdown:: Script ``analysis.py`` - - .. literalinclude:: analysis.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. - -Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except -in a region of thermal resonances as indicated on the plot below. - -.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png - :alt: Parallel EM modes in thermal ion plasma - :width: 70% - - Calculated Alvén waves spectrum with the theoretical dispersions overlaid. - -Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. - -.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular modes in thermal ion plasma - :width: 50% - - Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. - -Ohm solver: Cylindrical normal modes -==================================== - -A RZ-geometry example case for normal modes propagating along an applied magnetic -field in a cylinder is also available. The analytical solution for these modes -are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. - -Run ---- - -The following script initializes a thermal plasma in a metallic cylinder with -periodic boundaries at the cylinder ends. - -.. dropdown:: Script ``PICMI_inputs_rz.py`` - - .. literalinclude:: PICMI_inputs_rz.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py``. - -The example can be executed using: - -.. code-block:: bash - - python3 PICMI_inputs_rz.py - -Analyze -------- - -After the simulation completes the following script can be used to analyze the -field evolution and extract the normal mode dispersion relation. It performs a -standard Fourier transform along the cylinder axis and a Hankel transform in the -radial direction. - -.. dropdown:: Script ``analysis_rz.py`` - - .. literalinclude:: analysis_rz.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. - -The following figure was produced with the above analysis script, showing excellent -agreement between the calculated and theoretical dispersion relations. - -.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png - :alt: Normal EM modes in a metallic cylinder - :width: 90% - - Cylindrical normal mode dispersion comparing the calculated spectrum with the - theoretical one. diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis.py b/Examples/Tests/ohm_solver_EM_modes/analysis.py deleted file mode 100755 index ad832dc2f50..00000000000 --- a/Examples/Tests/ohm_solver_EM_modes/analysis.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Analysis script for the hybrid-PIC example producing EM modes. - -import dill -import matplotlib -import matplotlib.pyplot as plt -import numpy as np - -from pywarpx import picmi - -constants = picmi.constants - -matplotlib.rcParams.update({'font.size': 20}) - -# load simulation parameters -with open(f'sim_parameters.dpkl', 'rb') as f: - sim = dill.load(f) - -if sim.B_dir == 'z': - field_idx_dict = {'z': 4, 'Ez': 7, 'Bx': 8, 'By': 9} - data = np.loadtxt("diags/par_field_data.txt", skiprows=1) -else: - if sim.dim == 1: - field_idx_dict = {'z': 4, 'Ez': 7, 'Bx': 8, 'By': 9} - else: - field_idx_dict = {'z': 2, 'Ez': 3, 'Bx': 4, 'By': 5} - data = np.loadtxt("diags/perp_field_data.txt", skiprows=1) - -# step, t, z, Ez, Bx, By = raw_data.T -step = data[:,0] - -num_steps = len(np.unique(step)) - -# get the spatial resolution -resolution = len(np.where(step == 0)[0]) - 1 - -# reshape to separate spatial and time coordinates -sim_data = data.reshape((num_steps, resolution+1, data.shape[1])) - -z_grid = sim_data[1, :, field_idx_dict['z']] -idx = np.argsort(z_grid)[1:] -dz = np.mean(np.diff(z_grid[idx])) -dt = np.mean(np.diff(sim_data[:,0,1])) - -data = np.zeros((num_steps, resolution, 3)) -for i in range(num_steps): - data[i,:,0] = sim_data[i,idx,field_idx_dict['Bx']] - data[i,:,1] = sim_data[i,idx,field_idx_dict['By']] - data[i,:,2] = sim_data[i,idx,field_idx_dict['Ez']] - -print(f"Data file contains {num_steps} time snapshots.") -print(f"Spatial resolution is {resolution}") - -def get_analytic_R_mode(w): - return w / np.sqrt(1.0 + abs(w)) - -def get_analytic_L_mode(w): - return w / np.sqrt(1.0 - abs(w)) - -if sim.B_dir == 'z': - global_norm = ( - 1.0 / (2.0*constants.mu0) - / ((3.0/2)*sim.n_plasma*sim.T_plasma*constants.q_e) - ) -else: - global_norm = ( - constants.ep0 / 2.0 - / ((3.0/2)*sim.n_plasma*sim.T_plasma*constants.q_e) - ) - -if sim.B_dir == 'z': - Bl = (data[:, :, 0] + 1.0j * data[:, :, 1]) / np.sqrt(2.0) - field_kw = np.fft.fftshift(np.fft.fft2(Bl)) -else: - field_kw = np.fft.fftshift(np.fft.fft2(data[:, :, 2])) - -w_norm = sim.w_ci -if sim.B_dir == 'z': - k_norm = 1.0 / sim.l_i -else: - k_norm = 1.0 / sim.rho_i - -k = 2*np.pi * np.fft.fftshift(np.fft.fftfreq(resolution, dz)) / k_norm -w = 2*np.pi * np.fft.fftshift(np.fft.fftfreq(num_steps, dt)) / w_norm -w = -np.flipud(w) - -# aspect = (xmax-xmin)/(ymax-ymin) / aspect_true -extent = [k[0], k[-1], w[0], w[-1]] - -fig, ax1 = plt.subplots(1, 1, figsize=(10, 7.25)) - -if sim.B_dir == 'z' and sim.dim == 1: - vmin = -3 - vmax = 3.5 -else: - vmin = None - vmax = None - -im = ax1.imshow( - np.log10(np.abs(field_kw**2) * global_norm), extent=extent, - aspect="equal", cmap='inferno', vmin=vmin, vmax=vmax -) - -# Colorbars -fig.subplots_adjust(right=0.5) -cbar_ax = fig.add_axes([0.525, 0.15, 0.03, 0.7]) -fig.colorbar(im, cax=cbar_ax, orientation='vertical') - -#cbar_lab = r'$\log_{10}(\frac{|B_{R/L}|^2}{2\mu_0}\frac{2}{3n_0k_BT_e})$' -if sim.B_dir == 'z': - cbar_lab = r'$\log_{10}(\beta_{R/L})$' -else: - cbar_lab = r'$\log_{10}(\varepsilon_0|E_z|^2/(3n_0k_BT_e))$' -cbar_ax.set_ylabel(cbar_lab, rotation=270, labelpad=30) - -if sim.B_dir == 'z': - # plot the L mode - ax1.plot(get_analytic_L_mode(w), np.abs(w), c='limegreen', ls='--', lw=1.25, - label='L mode:\n'+r'$(kl_i)^2=\frac{(\omega/\Omega_i)^2}{1-\omega/\Omega_i}$') - # plot the R mode - ax1.plot(get_analytic_R_mode(w), -np.abs(w), c='limegreen', ls='-.', lw=1.25, - label='R mode:\n'+r'$(kl_i)^2=\frac{(\omega/\Omega_i)^2}{1+\omega/\Omega_i}$') - - ax1.plot(k,1.0+3.0*sim.v_ti/w_norm*k*k_norm, c='limegreen', ls=':', lw=1.25, label = r'$\omega = \Omega_i + 3v_{th,i} k$') - ax1.plot(k,1.0-3.0*sim.v_ti/w_norm*k*k_norm, c='limegreen', ls=':', lw=1.25) - -else: - # digitized values from Munoz et al. (2018) - x = [0.006781609195402272, 0.1321379310344828, 0.2671034482758621, 0.3743678160919539, 0.49689655172413794, 0.6143908045977011, 0.766022988505747, 0.885448275862069, 1.0321149425287355, 1.193862068965517, 1.4417701149425288, 1.7736781609195402] - y = [-0.033194664836814436, 0.5306857657503109, 1.100227301968521, 1.5713856842646996, 2.135780760818287, 2.675601492473303, 3.3477291246729854, 3.8469357121413563, 4.4317021915340735, 5.1079898786293265, 6.10275764463696, 7.310074194793499] - ax1.plot(x, y, c='limegreen', ls='-.', lw=1.5, label="X mode") - - x = [3.9732873563218387, 3.6515862068965514, 3.306275862068966, 2.895655172413793, 2.4318850574712645, 2.0747586206896553, 1.8520229885057473, 1.6589195402298849, 1.4594942528735633, 1.2911724137931033, 1.1551264367816092, 1.0335402298850576, 0.8961149425287356, 0.7419770114942528, 0.6141379310344828, 0.4913103448275862] - y = [1.1145945018655916, 1.1193978642192393, 1.1391259596002916, 1.162971222713042, 1.1986533430544237, 1.230389844319595, 1.2649997855641806, 1.3265857528841618, 1.3706737573444268, 1.4368486511986962, 1.4933310460179268, 1.5485268259210019, 1.6386327572157655, 1.7062658146416778, 1.7828194021529358, 1.8533687867221342] - ax1.plot(x, y, c='limegreen', ls=':', lw=2, label="Bernstein modes") - - x = [3.9669885057471266, 3.6533333333333333, 3.3213563218390805, 2.9646896551724136, 2.6106436781609195, 2.2797011494252875, 1.910919540229885, 1.6811724137931034, 1.4499540229885057, 1.2577011494252872, 1.081057471264368, 0.8791494252873564, 0.7153103448275862] - y = [2.2274306300124374, 2.2428271218424327, 2.272505039241755, 2.3084873697302397, 2.3586224642964364, 2.402667581592829, 2.513873997512545, 2.5859673199811297, 2.6586610627439207, 2.7352146502551786, 2.8161427284813656, 2.887850066475104, 2.9455761890466183] - ax1.plot(x, y, c='limegreen', ls=':', lw=2) - - x = [3.9764137931034487, 3.702022988505747, 3.459793103448276, 3.166712643678161, 2.8715862068965516, 2.5285057471264367, 2.2068505747126435, 1.9037011494252871, 1.6009885057471265, 1.3447816091954023, 1.1538850574712645, 0.9490114942528736] - y = [3.3231976669382854, 3.34875841660591, 3.378865205643951, 3.424454260839731, 3.474160483767209, 3.522194107303684, 3.6205343740618434, 3.7040356821203417, 3.785435519149119, 3.868851052879873, 3.9169704507440923, 3.952481022429987] - ax1.plot(x, y, c='limegreen', ls=':', lw=2) - - x = [3.953609195402299, 3.7670114942528734, 3.5917471264367817, 3.39735632183908, 3.1724137931034484, 2.9408045977011494, 2.685977011494253, 2.4593563218390804, 2.2203218390804595, 2.0158850574712646, 1.834183908045977, 1.6522758620689655, 1.4937471264367814, 1.3427586206896551, 1.2075402298850575] - y = [4.427971008277223, 4.458335120298495, 4.481579963117039, 4.495861388686366, 4.544581206844791, 4.587425483552773, 4.638160998413175, 4.698631899472488, 4.757987734271133, 4.813955483123902, 4.862332203971352, 4.892481880173264, 4.9247759145687695, 4.947934983059571, 4.953124329888064] - ax1.plot(x, y, c='limegreen', ls=':', lw=2) - -# ax1.legend(loc='upper left') -fig.legend(loc=7, fontsize=18) - -if sim.B_dir == 'z': - ax1.set_xlabel(r'$k l_i$') - ax1.set_title('$B_{R/L} = B_x \pm iB_y$') - fig.suptitle("Parallel EM modes") - ax1.set_xlim(-3, 3) - ax1.set_ylim(-6, 3) - dir_str = 'par' -else: - ax1.set_xlabel(r'$k \rho_i$') - ax1.set_title('$E_z(k, \omega)$') - fig.suptitle(f"Perpendicular EM modes (ion Bernstein) - {sim.dim}D") - ax1.set_xlim(-3, 3) - ax1.set_ylim(0, 8) - dir_str = 'perp' - -ax1.set_ylabel(r'$\omega / \Omega_i$') - -plt.savefig( - f"spectrum_{dir_str}_{sim.dim}d_{sim.substeps}_substeps_{sim.eta}_eta.png", - bbox_inches='tight' -) -if not sim.test: - plt.show() - -if sim.test: - import os - import sys - sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - import checksumAPI - - # this will be the name of the plot file - fn = sys.argv[1] - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py deleted file mode 100755 index 58e615c5332..00000000000 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Analysis script for the hybrid-PIC example producing EM modes. - -import dill -import matplotlib.pyplot as plt -import numpy as np -import scipy.fft as fft -from matplotlib import colors -from openpmd_viewer import OpenPMDTimeSeries -from scipy.interpolate import RegularGridInterpolator -from scipy.special import j1, jn, jn_zeros - -from pywarpx import picmi - -constants = picmi.constants - -# load simulation parameters -with open(f'sim_parameters.dpkl', 'rb') as f: - sim = dill.load(f) - -diag_dir = "diags/field_diags" - -ts = OpenPMDTimeSeries(diag_dir, check_all_files=True) - -def transform_spatially(data_for_transform): - # interpolate from regular r-grid to special r-grid - interp = RegularGridInterpolator( - (info.z, info.r), data_for_transform, - method='linear' - ) - data_interp = interp((zg, rg)) - - # Applying manual hankel in r - # Fmz = np.sum(proj*data_for_transform, axis=(2,3)) - Fmz = np.einsum('ijkl,kl->ij', proj, data_interp) - # Standard fourier in z - Fmn = fft.fftshift(fft.fft(Fmz, axis=1), axes=1) - return Fmn - -def process(it): - print(f"Processing iteration {it}", flush=True) - field, info = ts.get_field('E', 'y', iteration=it) - F_k = transform_spatially(field) - return F_k - -# grab the first iteration to get the grids -Bz, info = ts.get_field('B', 'z', iteration=0) - -nr = len(info.r) -nz = len(info.z) - -nkr = 12 # number of radial modes to solve for - -r_max = np.max(info.r) - -# create r-grid with points spaced out according to zeros of the Bessel function -r_grid = jn_zeros(1, nr) / jn_zeros(1, nr)[-1] * r_max - -zg, rg = np.meshgrid(info.z, r_grid) - -# Setup Hankel Transform -j_1M = jn_zeros(1, nr)[-1] -r_modes = np.arange(nkr) - -A = ( - 4.0 * np.pi * r_max**2 / j_1M**2 - * j1(np.outer(jn_zeros(1, max(r_modes)+1)[r_modes], jn_zeros(1, nr)) / j_1M) - / jn(2 ,jn_zeros(1, nr))**2 -) - -# No transformation for z -B = np.identity(nz) - -# combine projection arrays -proj = np.einsum('ab,cd->acbd', A, B) - -results = np.zeros((len(ts.t), nkr, nz), dtype=complex) -for ii, it in enumerate(ts.iterations): - results[ii] = process(it) - -# now Fourier transform in time -F_kw = fft.fftshift(fft.fft(results, axis=0), axes=0) - -dz = info.z[1] - info.z[0] -kz = 2*np.pi*fft.fftshift(fft.fftfreq(F_kw[0].shape[1], dz)) -dt = ts.iterations[1] - ts.iterations[0] -omega = 2*np.pi*fft.fftshift(fft.fftfreq(F_kw.shape[0], sim.dt*dt)) - -# Save data for future plotting purposes -np.savez( - "diags/spectrograms.npz", - F_kw=F_kw, dz=dz, kz=kz, dt=dt, omega=omega -) - -# plot the resulting dispersions -k = np.linspace(0, 250, 500) -kappa = k * sim.l_i - -fig, axes = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(6.75, 5)) - -vmin = [2e-3, 1.5e-3, 7.5e-4, 5e-4] -vmax = 1.0 - -# plot m = 1 -for ii, m in enumerate([1, 3, 6, 8]): - ax = axes.flatten()[ii] - ax.set_title(f"m = {m}", fontsize=11) - m -= 1 - pm1 = ax.pcolormesh( - kz*sim.l_i, omega/sim.w_ci, - abs(F_kw[:, m, :])/np.max(abs(F_kw[:, m, :])), - norm=colors.LogNorm(vmin=vmin[ii], vmax=vmax), - cmap='inferno' - ) - cb = fig.colorbar(pm1, ax=ax) - cb.set_label(r'Normalized $E_\theta(k_z, m, \omega)$') - - # Get dispersion relation - see for example - # T. Stix, Waves in Plasmas (American Inst. of Physics, 1992), Chap 6, Sec 2 - nu_m = jn_zeros(1, m+1)[-1] / sim.Lr - R2 = 0.5 * (nu_m**2 * (1.0 + kappa**2) + k**2 * (kappa**2 + 2.0)) - P4 = k**2 * (nu_m**2 + k**2) - omega_fast = sim.vA * np.sqrt(R2 + np.sqrt(R2**2 - P4)) - omega_slow = sim.vA * np.sqrt(R2 - np.sqrt(R2**2 - P4)) - # Upper right corner - ax.plot(k*sim.l_i, omega_fast/sim.w_ci, 'w--', label = f"$\omega_{{fast}}$") - ax.plot(k*sim.l_i, omega_slow/sim.w_ci, color='white', linestyle='--', label = f"$\omega_{{slow}}$") - # Thermal resonance - thermal_res = sim.w_ci + 3*sim.v_ti*k - ax.plot(k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "$\omega = \Omega_i + 3v_{th,i}k$") - ax.plot(-k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "") - thermal_res = sim.w_ci - 3*sim.v_ti*k - ax.plot(k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "$\omega = \Omega_i + 3v_{th,i}k$") - ax.plot(-k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "") - - -for ax in axes.flatten(): - ax.set_xlim(-1.75, 1.75) - ax.set_ylim(0, 1.6) - -axes[0, 0].set_ylabel('$\omega/\Omega_{ci}$') -axes[1, 0].set_ylabel('$\omega/\Omega_{ci}$') -axes[1, 0].set_xlabel('$k_zl_i$') -axes[1, 1].set_xlabel('$k_zl_i$') - -plt.savefig('normal_modes_disp.png', dpi=600) -if not sim.test: - plt.show() -else: - plt.close() - - # check if power spectrum sampling match earlier results - amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) - print("Amplitude sample: ", amps) - assert np.allclose( - amps, np.array([ 61.02377286, 19.80026021, 100.47687017, 10.83331295]) - ) - -if sim.test: - import os - import sys - sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - import checksumAPI - - # this will be the name of the plot file - fn = sys.argv[1] - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-6) diff --git a/Examples/Tests/ohm_solver_cylinder_compression/CMakeLists.txt b/Examples/Tests/ohm_solver_cylinder_compression/CMakeLists.txt new file mode 100644 index 00000000000..c813d669fa6 --- /dev/null +++ b/Examples/Tests/ohm_solver_cylinder_compression/CMakeLists.txt @@ -0,0 +1,24 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_ohm_solver_cylinder_compression_picmi # name + 3 # dims + 2 # nprocs + "inputs_test_3d_ohm_solver_cylinder_compression_picmi.py --test" # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000020 --rtol 1e-6" # checksum + OFF # dependency +) +label_warpx_test(test_3d_ohm_solver_cylinder_compression_picmi slow) + +add_warpx_test( + test_rz_ohm_solver_cylinder_compression_picmi # name + RZ # dims + 2 # nprocs + "inputs_test_rz_ohm_solver_cylinder_compression_picmi.py --test" # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000020 --rtol 1e-6" # output + OFF # dependency +) +label_warpx_test(test_rz_ohm_solver_cylinder_compression_picmi slow) diff --git a/Examples/Tests/ohm_solver_cylinder_compression/analysis_default_regression.py b/Examples/Tests/ohm_solver_cylinder_compression/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ohm_solver_cylinder_compression/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_3d_ohm_solver_cylinder_compression_picmi.py b/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_3d_ohm_solver_cylinder_compression_picmi.py new file mode 100644 index 00000000000..4f05fd15d83 --- /dev/null +++ b/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_3d_ohm_solver_cylinder_compression_picmi.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script demonstrates the use of this model to +# --- simulate adiabatic compression of a plasma cylinder initialized from an +# --- analytical Grad-Shafranov solution. + +import argparse +import shutil +import sys +from pathlib import Path + +import numpy as np +import openpmd_api as io +from mpi4py import MPI as mpi + +from pywarpx import fields, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=False) + + +class PlasmaCylinderCompression(object): + # B0 is chosen with all other quantities scaled by it + n0 = 1e20 + T_i = 10 # eV + T_e = 0 + p0 = n0 * constants.q_e * T_i + + B0 = np.sqrt(2 * constants.mu0 * p0) # Initial magnetic field strength (T) + + # Do a 2x uniform B-field compression + dB = B0 + + # Flux Conserver radius + R_c = 0.5 + + # Plasma Radius (These values control the analytical GS solution) + R_p = 0.25 + delta_p = 0.025 + + # Domain parameters + LX = 2.0 * R_c * 1.05 # m + LY = 2.0 * R_c * 1.05 + LZ = 0.5 # m + + LT = 10 # ion cyclotron periods + DT = 1e-3 # ion cyclotron periods + + # Resolution parameters + NX = 256 + NY = 256 + NZ = 128 + + # Starting number of particles per cell + NPPC = 100 + + # Number of substeps used to update B + substeps = 20 + + def Bz(self, r): + return np.sqrt( + self.B0**2 + - 2.0 + * constants.mu0 + * self.n0 + * constants.q_e + * self.T_i + / (1.0 + np.exp((r - self.R_p) / self.delta_p)) + ) + + def __init__(self, test, verbose): + self.test = test + self.verbose = verbose or self.test + + self.Lx = self.LX + self.Ly = self.LY + self.Lz = self.LZ + + self.DX = self.LX / self.NX + self.DY = self.LY / self.NY + self.DZ = self.LZ / self.NZ + + if comm.rank == 0: + # Write uniform compression dataset to OpenPMD to exercise reading openPMD data + # for the time varying external fields + xvec = np.linspace(-self.LX, self.LX, num=2 * self.NX) + yvec = np.linspace(-self.LY, self.LY, num=2 * self.NY) + zvec = np.linspace(-self.LZ, self.LZ, num=2 * self.NZ) + XM, YM, ZM = np.meshgrid(xvec, yvec, zvec, indexing="ij") + + RM = np.sqrt(XM**2 + YM**2) + + Ax_data = -0.5 * YM * self.dB + Ay_data = 0.5 * XM * self.dB + Az_data = np.zeros_like(RM) + + # Write vector potential to file to exercise field loading via OpenPMD + series = io.Series("Afield.h5", io.Access.create) + + it = series.iterations[0] + + A = it.meshes["A"] + A.grid_spacing = [self.DX, self.DY, self.DZ] + A.grid_global_offset = [-self.LX, -self.LY, -self.LZ] + A.grid_unit_SI = 1.0 + A.axis_labels = ["x", "y", "z"] + A.data_order = "C" + A.unit_dimension = { + io.Unit_Dimension.M: 1.0, + io.Unit_Dimension.T: -2.0, + io.Unit_Dimension.I: -1.0, + io.Unit_Dimension.L: -1.0, + } + + Ax = A["x"] + Ay = A["y"] + Az = A["z"] + + Ax.position = [0.0, 0.0] + Ay.position = [0.0, 0.0] + Az.position = [0.0, 0.0] + + Ax_dataset = io.Dataset(Ax_data.dtype, Ax_data.shape) + + Ay_dataset = io.Dataset(Ay_data.dtype, Ay_data.shape) + + Az_dataset = io.Dataset(Az_data.dtype, Az_data.shape) + + Ax.reset_dataset(Ax_dataset) + Ay.reset_dataset(Ay_dataset) + Az.reset_dataset(Az_dataset) + + Ax.store_chunk(Ax_data) + Ay.store_chunk(Ay_data) + Az.store_chunk(Az_data) + + series.flush() + series.close() + + comm.Barrier() + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.dt = self.DT * self.t_ci + + # run very low resolution as a CI test + if self.test: + self.total_steps = 20 + self.diag_steps = self.total_steps // 5 + self.NX = 64 + self.NY = 64 + self.NZ = 32 + else: + self.total_steps = int(self.LT / self.DT) + self.diag_steps = 100 + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tTi = {self.T_i:.1f} eV\n" + f"\tn0 = {self.n0:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n", + f"\tDX/DY = {self.DX / self.l_i:.3f} c/w_pi\n" + f"\tDZ = {self.DZ / self.l_i:.3f} c/w_pi\n", + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.vi_th:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.Lz / self.NZ:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + + # Ion mass (kg) + self.M = constants.m_p + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n0 / (self.M * constants.ep0)) + + # Ion skin depth (m) + self.l_i = constants.c / self.w_pi + + # # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = abs(self.B0) / np.sqrt( + constants.mu0 * self.n0 * (constants.m_e + self.M) + ) + + # calculate thermal speeds + self.vi_th = np.sqrt(self.T_i * constants.q_e / self.M) + + # Ion Larmor radius (m) + self.rho_i = self.vi_th / self.w_ci + + def load_fields(self): + Bx = fields.BxFPExternalWrapper(include_ghosts=False) + By = fields.ByFPExternalWrapper(include_ghosts=False) + Bz = fields.BzFPExternalWrapper(include_ghosts=False) + + Bx[:, :] = 0.0 + By[:, :] = 0.0 + + XM, YM, ZM = np.meshgrid( + Bz.mesh("x"), Bz.mesh("y"), Bz.mesh("z"), indexing="ij" + ) + + RM = np.sqrt(XM**2 + YM**2) + + Bz[:, :] = self.Bz(RM) + comm.Barrier() + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + # Create grid + self.grid = picmi.Cartesian3DGrid( + number_of_cells=[self.NX, self.NY, self.NZ], + lower_bound=[-0.5 * self.Lx, -0.5 * self.Ly, -0.5 * self.Lz], + upper_bound=[0.5 * self.Lx, 0.5 * self.Ly, 0.5 * self.Lz], + lower_boundary_conditions=["dirichlet", "dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "periodic"], + warpx_max_grid_size=self.NZ, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.use_filter = True + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + # External Field definition. Sigmoid starting around 2.5 us + A_ext = { + "uniform": { + "read_from_file": True, + "path": "Afield.h5", + "A_time_external_function": "1/(1+exp(5*(1-(t-t0_ramp)*sqrt(2)/tau_ramp)))", + } + } + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + gamma=1.0, + Te=self.T_e, + n0=self.n0, + n_floor=0.05 * self.n0, + plasma_resistivity="if(rho<=rho_floor,eta_v,eta_p)", + plasma_hyper_resistivity=1e-8, + substeps=self.substeps, + A_external=A_ext, + tau_ramp=20e-6, + t0_ramp=5e-6, + rho_floor=0.05 * self.n0 * constants.q_e, + eta_p=1e-8, + eta_v=1e-3, + ) + simulation.solver = self.solver + + simulation.embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="(x**2+y**2-R_w**2)", R_w=self.R_c + ) + + # Add field loader callback + B_ext = picmi.LoadInitialFieldFromPython( + load_from_python=self.load_fields, + warpx_do_divb_cleaning_external=True, + load_B=True, + load_E=False, + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + r_omega = "(sqrt(x*x+y*y)*q_e*B0/m_p)" + dlnndr = "((-1/delta_p)/(1+exp(-(sqrt(x*x+y*y)-R_p)/delta_p)))" + vth = f"0.5*(-{r_omega}+sqrt({r_omega}*{r_omega}+4*q_e*T_i*{dlnndr}/m_p))" + + momentum_expr = [f"y*{vth}", f"-x*{vth}", "0"] + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.AnalyticDistribution( + density_expression="n0_p/(1+exp((sqrt(x*x+y*y)-R_p)/delta_p))", + momentum_expressions=momentum_expr, + warpx_momentum_spread_expressions=[f"{str(self.vi_th)}"] * 3, + warpx_density_min=0.01 * self.n0, + R_p=self.R_p, + delta_p=self.delta_p, + n0_p=self.n0, + B0=self.B0, + T_i=self.T_i, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + if self.test: + particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=self.diag_steps, + species=[self.ions], + data_list=["ux", "uy", "uz", "x", "z", "weighting"], + write_dir="diags", + warpx_format="plotfile", + ) + simulation.add_diagnostic(particle_diag) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=self.diag_steps, + data_list=["B", "E", "rho"], + write_dir="diags", + warpx_format="plotfile", + ) + simulation.add_diagnostic(field_diag) + + ####################################################################### + # Initialize # + ####################################################################### + + if comm.rank == 0: + if Path.exists(Path("diags")): + shutil.rmtree("diags") + Path("diags").mkdir(parents=True, exist_ok=True) + + # Initialize inputs and WarpX instance + simulation.initialize_inputs() + simulation.initialize_warpx() + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = PlasmaCylinderCompression(test=args.test, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_rz_ohm_solver_cylinder_compression_picmi.py b/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_rz_ohm_solver_cylinder_compression_picmi.py new file mode 100644 index 00000000000..8c65f88ae79 --- /dev/null +++ b/Examples/Tests/ohm_solver_cylinder_compression/inputs_test_rz_ohm_solver_cylinder_compression_picmi.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script demonstrates the use of this model to +# --- simulate adiabatic compression of a plasma cylinder initialized from an +# --- analytical Grad-Shafranov solution. + +import argparse +import shutil +import sys +from pathlib import Path + +import numpy as np +import openpmd_api as io +from mpi4py import MPI as mpi + +from pywarpx import fields, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=False) + + +class PlasmaCylinderCompression(object): + # B0 is chosen with all other quantities scaled by it + n0 = 1e20 + T_i = 10 # eV + T_e = 0 + p0 = n0 * constants.q_e * T_i + + B0 = np.sqrt(2 * constants.mu0 * p0) # External magnetic field strength (T) + + # Do a 2x uniform B-field compression + dB = B0 + + # Flux Conserver radius + R_c = 0.5 + + # Plasma Radius (These values control the analytical GS solution) + R_p = 0.25 + delta_p = 0.025 + + # Domain parameters + LR = R_c # m + LZ = 0.25 * R_c # m + + LT = 10 # ion cyclotron periods + DT = 1e-3 # ion cyclotron periods + + # Resolution parameters + NR = 128 + NZ = 32 + + # Starting number of particles per cell + NPPC = 100 + + # Number of substeps used to update B + substeps = 20 + + def Bz(self, r): + return np.sqrt( + self.B0**2 + - 2.0 + * constants.mu0 + * self.n0 + * constants.q_e + * self.T_i + / (1.0 + np.exp((r - self.R_p) / self.delta_p)) + ) + + def __init__(self, test, verbose): + self.test = test + self.verbose = verbose or self.test + + self.Lr = self.LR + self.Lz = self.LZ + + self.DR = self.LR / self.NR + self.DZ = self.LZ / self.NZ + + # Write A to OpenPMD for a uniform B field to exercise file based loader + if comm.rank == 0: + mvec = np.array([0]) + rvec = np.linspace(0, 2 * self.LR, num=2 * self.NR) + zvec = np.linspace(-self.LZ, self.LZ, num=2 * self.NZ) + MM, RM, ZM = np.meshgrid(mvec, rvec, zvec, indexing="ij") + + # Write uniform compression dataset to OpenPMD to exercise reading openPMD data + # for the time varying external fields + Ar_data = np.zeros_like(RM) + Az_data = np.zeros_like(RM) + + # Zero padded outside of domain + At_data = 0.5 * RM * self.dB + + # Write vector potential to file to exercise field loading via + series = io.Series("Afield.h5", io.Access.create) + + it = series.iterations[0] + + A = it.meshes["A"] + A.geometry = io.Geometry.thetaMode + A.geometry_parameters = "m=0" + A.grid_spacing = [self.DR, self.DZ] + A.grid_global_offset = [0.0, -self.LZ] + A.grid_unit_SI = 1.0 + A.axis_labels = ["r", "z"] + A.data_order = "C" + A.unit_dimension = { + io.Unit_Dimension.M: 1.0, + io.Unit_Dimension.T: -2.0, + io.Unit_Dimension.I: -1.0, + io.Unit_Dimension.L: -1.0, + } + + Ar = A["r"] + At = A["t"] + Az = A["z"] + + Ar.position = [0.0, 0.0] + At.position = [0.0, 0.0] + Az.position = [0.0, 0.0] + + Ar_dataset = io.Dataset(Ar_data.dtype, Ar_data.shape) + + At_dataset = io.Dataset(At_data.dtype, At_data.shape) + + Az_dataset = io.Dataset(Az_data.dtype, Az_data.shape) + + Ar.reset_dataset(Ar_dataset) + At.reset_dataset(At_dataset) + Az.reset_dataset(Az_dataset) + + Ar.store_chunk(Ar_data) + At.store_chunk(At_data) + Az.store_chunk(Az_data) + + series.flush() + series.close() + + comm.Barrier() + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.dt = self.DT * self.t_ci + + # run very low resolution as a CI test + if self.test: + self.total_steps = 20 + self.diag_steps = self.total_steps // 5 + self.NR = 64 + self.NZ = 16 + else: + self.total_steps = int(self.LT / self.DT) + self.diag_steps = 100 + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tTi = {self.T_i:.1f} eV\n" + f"\tn0 = {self.n0:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n", + f"\tDR = {self.DR / self.l_i:.3f} c/w_pi\n" + f"\tDZ = {self.DZ / self.l_i:.3f} c/w_pi\n", + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.vi_th:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.Lz / self.NZ:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + + # Ion mass (kg) + self.M = constants.m_p + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n0 / (self.M * constants.ep0)) + + # Ion skin depth (m) + self.l_i = constants.c / self.w_pi + + # # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = abs(self.B0) / np.sqrt( + constants.mu0 * self.n0 * (constants.m_e + self.M) + ) + + # calculate thermal speeds + self.vi_th = np.sqrt(self.T_i * constants.q_e / self.M) + + # Ion Larmor radius (m) + self.rho_i = self.vi_th / self.w_ci + + def load_fields(self): + Br = fields.BxFPExternalWrapper(include_ghosts=False) + Bt = fields.ByFPExternalWrapper(include_ghosts=False) + Bz = fields.BzFPExternalWrapper(include_ghosts=False) + + Br[:, :] = 0.0 + Bt[:, :] = 0.0 + + RM, ZM = np.meshgrid(Bz.mesh("r"), Bz.mesh("z"), indexing="ij") + + Bz[:, :] = self.Bz(RM) * (RM <= self.R_c) + comm.Barrier() + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + # Create grid + self.grid = picmi.CylindricalGrid( + number_of_cells=[self.NR, self.NZ], + lower_bound=[0.0, -self.Lz / 2.0], + upper_bound=[self.Lr, self.Lz / 2.0], + lower_boundary_conditions=["none", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["none", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + warpx_max_grid_size=self.NZ, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.use_filter = True + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + # External Field definition. Sigmoid starting around 2.5 us + A_ext = { + "uniform": { + "read_from_file": True, + "path": "Afield.h5", + "A_time_external_function": "1/(1+exp(5*(1-(t-t0_ramp)*sqrt(2)/tau_ramp)))", + } + } + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + gamma=1.0, + Te=self.T_e, + n0=self.n0, + n_floor=0.05 * self.n0, + plasma_resistivity="if(rho<=rho_floor,eta_v,eta_p)", + plasma_hyper_resistivity=1e-8, + substeps=self.substeps, + A_external=A_ext, + tau_ramp=20e-6, + t0_ramp=5e-6, + rho_floor=0.05 * self.n0 * constants.q_e, + eta_p=1e-8, + eta_v=1e-3, + ) + simulation.solver = self.solver + + # Add field loader callback + B_ext = picmi.LoadInitialFieldFromPython( + load_from_python=self.load_fields, + warpx_do_divb_cleaning_external=True, + load_B=True, + load_E=False, + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + r_omega = "(sqrt(x*x+y*y)*q_e*B0/m_p)" + dlnndr = "((-1/delta_p)/(1+exp(-(sqrt(x*x+y*y)-R_p)/delta_p)))" + vth = f"0.5*(-{r_omega}+sqrt({r_omega}*{r_omega}+4*q_e*T_i*{dlnndr}/m_p))" + + momentum_expr = [f"y*{vth}", f"-x*{vth}", "0"] + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.AnalyticDistribution( + density_expression="n0_p/(1+exp((sqrt(x*x+y*y)-R_p)/delta_p))", + momentum_expressions=momentum_expr, + warpx_momentum_spread_expressions=[f"{str(self.vi_th)}"] * 3, + warpx_density_min=0.01 * self.n0, + R_p=self.R_p, + delta_p=self.delta_p, + n0_p=self.n0, + B0=self.B0, + T_i=self.T_i, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + if self.test: + particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=self.diag_steps, + species=[self.ions], + data_list=["ux", "uy", "uz", "x", "z", "weighting"], + write_dir="diags", + warpx_format="plotfile", + ) + simulation.add_diagnostic(particle_diag) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=self.diag_steps, + data_list=["B", "E", "rho"], + write_dir="diags", + warpx_format="plotfile", + ) + simulation.add_diagnostic(field_diag) + + ####################################################################### + # Initialize # + ####################################################################### + + if comm.rank == 0: + if Path.exists(Path("diags")): + shutil.rmtree("diags") + Path("diags").mkdir(parents=True, exist_ok=True) + + # Initialize inputs and WarpX instance + simulation.initialize_inputs() + simulation.initialize_warpx() + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = PlasmaCylinderCompression(test=args.test, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt b/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt new file mode 100644 index 00000000000..03843fe29f6 --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt @@ -0,0 +1,23 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_ohm_solver_em_modes_picmi # name + 1 # dims + 2 # nprocs + "inputs_test_1d_ohm_solver_em_modes_picmi.py --test --dim 1 --bdir z" # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/field_diag000250" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_ohm_solver_em_modes_picmi # name + RZ # dims + 2 # nprocs + "inputs_test_rz_ohm_solver_em_modes_picmi.py --test" # inputs + "analysis_rz.py" # analysis + "analysis_default_regression.py --path diags/diag1000100 --rtol 1e-6" # checksum + OFF # dependency +) +label_warpx_test(test_rz_ohm_solver_em_modes_picmi slow) diff --git a/Examples/Tests/ohm_solver_em_modes/README.rst b/Examples/Tests/ohm_solver_em_modes/README.rst new file mode 100644 index 00000000000..24d95d2bcb8 --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/README.rst @@ -0,0 +1,119 @@ +.. _examples-ohm-solver-em-modes: + +Ohm solver: Electromagnetic modes +================================= + +In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the +:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are +Fourier analyzed for Alfvén mode excitations. + +Run +--- + +The same input script can be used for 1d, 2d or 3d Cartesian simulations as well +as replicating either the parallel propagating or ion-Bernstein modes as indicated below. + +.. dropdown:: Script ``inputs_test_1d_ohm_solver_em_modes_picmi.py`` + + .. literalinclude:: inputs_test_1d_ohm_solver_em_modes_picmi.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/inputs_test_1d_ohm_solver_em_modes_picmi.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Parallel propagating waves + + Execute: + + .. code-block:: bash + + python3 inputs_test_1d_ohm_solver_em_modes_picmi.py -dim {1/2/3} --bdir z + + .. tab-item:: Perpendicular propagating waves + + Execute: + + .. code-block:: bash + + python3 inputs_test_1d_ohm_solver_em_modes_picmi.py -dim {1/2/3} --bdir {x/y} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and compares the calculated spectrum +to the theoretical dispersions. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. + +Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except +in a region of thermal resonances as indicated on the plot below. + +.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png + :alt: Parallel EM modes in thermal ion plasma + :width: 70% + + Calculated Alvén waves spectrum with the theoretical dispersions overlaid. + +Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. + +.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png + :alt: Perpendicular modes in thermal ion plasma + :width: 50% + + Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. + +Ohm solver: Cylindrical normal modes +==================================== + +A RZ-geometry example case for normal modes propagating along an applied magnetic +field in a cylinder is also available. The analytical solution for these modes +are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +Run +--- + +The following script initializes a thermal plasma in a metallic cylinder with +periodic boundaries at the cylinder ends. + +.. dropdown:: Script ``inputs_test_rz_ohm_solver_em_modes_picmi.py`` + + .. literalinclude:: inputs_test_rz_ohm_solver_em_modes_picmi.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/inputs_test_rz_ohm_solver_em_modes_picmi.py``. + +The example can be executed using: + +.. code-block:: bash + + python3 inputs_test_rz_ohm_solver_em_modes_picmi.py + +Analyze +------- + +After the simulation completes the following script can be used to analyze the +field evolution and extract the normal mode dispersion relation. It performs a +standard Fourier transform along the cylinder axis and a Hankel transform in the +radial direction. + +.. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. + +The following figure was produced with the above analysis script, showing excellent +agreement between the calculated and theoretical dispersion relations. + +.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png + :alt: Normal EM modes in a metallic cylinder + :width: 90% + + Cylindrical normal mode dispersion comparing the calculated spectrum with the + theoretical one. diff --git a/Examples/Tests/ohm_solver_em_modes/analysis.py b/Examples/Tests/ohm_solver_em_modes/analysis.py new file mode 100755 index 00000000000..e2075944932 --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/analysis.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +# +# --- Analysis script for the hybrid-PIC example producing EM modes. + +import dill +import matplotlib +import matplotlib.pyplot as plt +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +matplotlib.rcParams.update({"font.size": 20}) + +# load simulation parameters +with open("sim_parameters.dpkl", "rb") as f: + sim = dill.load(f) + +if sim.B_dir == "z": + field_idx_dict = {"z": 4, "Ez": 7, "Bx": 8, "By": 9} + data = np.loadtxt("diags/par_field_data.txt", skiprows=1) +else: + if sim.dim == 1: + field_idx_dict = {"z": 4, "Ez": 7, "Bx": 8, "By": 9} + else: + field_idx_dict = {"z": 2, "Ez": 3, "Bx": 4, "By": 5} + data = np.loadtxt("diags/perp_field_data.txt", skiprows=1) + +# step, t, z, Ez, Bx, By = raw_data.T +step = data[:, 0] + +num_steps = len(np.unique(step)) + +# get the spatial resolution +resolution = len(np.where(step == 0)[0]) - 1 + +# reshape to separate spatial and time coordinates +sim_data = data.reshape((num_steps, resolution + 1, data.shape[1])) + +z_grid = sim_data[1, :, field_idx_dict["z"]] +idx = np.argsort(z_grid)[1:] +dz = np.mean(np.diff(z_grid[idx])) +dt = np.mean(np.diff(sim_data[:, 0, 1])) + +data = np.zeros((num_steps, resolution, 3)) +for i in range(num_steps): + data[i, :, 0] = sim_data[i, idx, field_idx_dict["Bx"]] + data[i, :, 1] = sim_data[i, idx, field_idx_dict["By"]] + data[i, :, 2] = sim_data[i, idx, field_idx_dict["Ez"]] + +print(f"Data file contains {num_steps} time snapshots.") +print(f"Spatial resolution is {resolution}") + + +def get_analytic_R_mode(w): + return w / np.sqrt(1.0 + abs(w)) + + +def get_analytic_L_mode(w): + return w / np.sqrt(1.0 - abs(w)) + + +if sim.B_dir == "z": + global_norm = ( + 1.0 + / (2.0 * constants.mu0) + / ((3.0 / 2) * sim.n_plasma * sim.T_plasma * constants.q_e) + ) +else: + global_norm = ( + constants.ep0 / 2.0 / ((3.0 / 2) * sim.n_plasma * sim.T_plasma * constants.q_e) + ) + +if sim.B_dir == "z": + Bl = (data[:, :, 0] + 1.0j * data[:, :, 1]) / np.sqrt(2.0) + field_kw = np.fft.fftshift(np.fft.fft2(Bl)) +else: + field_kw = np.fft.fftshift(np.fft.fft2(data[:, :, 2])) + +w_norm = sim.w_ci +if sim.B_dir == "z": + k_norm = 1.0 / sim.l_i +else: + k_norm = 1.0 / sim.rho_i + +k = 2 * np.pi * np.fft.fftshift(np.fft.fftfreq(resolution, dz)) / k_norm +w = 2 * np.pi * np.fft.fftshift(np.fft.fftfreq(num_steps, dt)) / w_norm +w = -np.flipud(w) + +# aspect = (xmax-xmin)/(ymax-ymin) / aspect_true +extent = [k[0], k[-1], w[0], w[-1]] + +fig, ax1 = plt.subplots(1, 1, figsize=(10, 7.25)) + +if sim.B_dir == "z" and sim.dim == 1: + vmin = -3 + vmax = 3.5 +else: + vmin = None + vmax = None + +im = ax1.imshow( + np.log10(np.abs(field_kw**2) * global_norm), + extent=extent, + aspect="equal", + cmap="inferno", + vmin=vmin, + vmax=vmax, +) + +# Colorbars +fig.subplots_adjust(right=0.5) +cbar_ax = fig.add_axes([0.525, 0.15, 0.03, 0.7]) +fig.colorbar(im, cax=cbar_ax, orientation="vertical") + +# cbar_lab = r'$\log_{10}(\frac{|B_{R/L}|^2}{2\mu_0}\frac{2}{3n_0k_BT_e})$' +if sim.B_dir == "z": + cbar_lab = r"$\log_{10}(\beta_{R/L})$" +else: + cbar_lab = r"$\log_{10}(\varepsilon_0|E_z|^2/(3n_0k_BT_e))$" +cbar_ax.set_ylabel(cbar_lab, rotation=270, labelpad=30) + +if sim.B_dir == "z": + # plot the L mode + ax1.plot( + get_analytic_L_mode(w), + np.abs(w), + c="limegreen", + ls="--", + lw=1.25, + label="L mode:\n" + r"$(kl_i)^2=\frac{(\omega/\Omega_i)^2}{1-\omega/\Omega_i}$", + ) + # plot the R mode + ax1.plot( + get_analytic_R_mode(w), + -np.abs(w), + c="limegreen", + ls="-.", + lw=1.25, + label="R mode:\n" + r"$(kl_i)^2=\frac{(\omega/\Omega_i)^2}{1+\omega/\Omega_i}$", + ) + + ax1.plot( + k, + 1.0 + 3.0 * sim.v_ti / w_norm * k * k_norm, + c="limegreen", + ls=":", + lw=1.25, + label=r"$\omega = \Omega_i + 3v_{th,i} k$", + ) + ax1.plot( + k, 1.0 - 3.0 * sim.v_ti / w_norm * k * k_norm, c="limegreen", ls=":", lw=1.25 + ) + +else: + # digitized values from Munoz et al. (2018) + x = [ + 0.006781609195402272, + 0.1321379310344828, + 0.2671034482758621, + 0.3743678160919539, + 0.49689655172413794, + 0.6143908045977011, + 0.766022988505747, + 0.885448275862069, + 1.0321149425287355, + 1.193862068965517, + 1.4417701149425288, + 1.7736781609195402, + ] + y = [ + -0.033194664836814436, + 0.5306857657503109, + 1.100227301968521, + 1.5713856842646996, + 2.135780760818287, + 2.675601492473303, + 3.3477291246729854, + 3.8469357121413563, + 4.4317021915340735, + 5.1079898786293265, + 6.10275764463696, + 7.310074194793499, + ] + ax1.plot(x, y, c="limegreen", ls="-.", lw=1.5, label="X mode") + + x = [ + 3.9732873563218387, + 3.6515862068965514, + 3.306275862068966, + 2.895655172413793, + 2.4318850574712645, + 2.0747586206896553, + 1.8520229885057473, + 1.6589195402298849, + 1.4594942528735633, + 1.2911724137931033, + 1.1551264367816092, + 1.0335402298850576, + 0.8961149425287356, + 0.7419770114942528, + 0.6141379310344828, + 0.4913103448275862, + ] + y = [ + 1.1145945018655916, + 1.1193978642192393, + 1.1391259596002916, + 1.162971222713042, + 1.1986533430544237, + 1.230389844319595, + 1.2649997855641806, + 1.3265857528841618, + 1.3706737573444268, + 1.4368486511986962, + 1.4933310460179268, + 1.5485268259210019, + 1.6386327572157655, + 1.7062658146416778, + 1.7828194021529358, + 1.8533687867221342, + ] + ax1.plot(x, y, c="limegreen", ls=":", lw=2, label="Bernstein modes") + + x = [ + 3.9669885057471266, + 3.6533333333333333, + 3.3213563218390805, + 2.9646896551724136, + 2.6106436781609195, + 2.2797011494252875, + 1.910919540229885, + 1.6811724137931034, + 1.4499540229885057, + 1.2577011494252872, + 1.081057471264368, + 0.8791494252873564, + 0.7153103448275862, + ] + y = [ + 2.2274306300124374, + 2.2428271218424327, + 2.272505039241755, + 2.3084873697302397, + 2.3586224642964364, + 2.402667581592829, + 2.513873997512545, + 2.5859673199811297, + 2.6586610627439207, + 2.7352146502551786, + 2.8161427284813656, + 2.887850066475104, + 2.9455761890466183, + ] + ax1.plot(x, y, c="limegreen", ls=":", lw=2) + + x = [ + 3.9764137931034487, + 3.702022988505747, + 3.459793103448276, + 3.166712643678161, + 2.8715862068965516, + 2.5285057471264367, + 2.2068505747126435, + 1.9037011494252871, + 1.6009885057471265, + 1.3447816091954023, + 1.1538850574712645, + 0.9490114942528736, + ] + y = [ + 3.3231976669382854, + 3.34875841660591, + 3.378865205643951, + 3.424454260839731, + 3.474160483767209, + 3.522194107303684, + 3.6205343740618434, + 3.7040356821203417, + 3.785435519149119, + 3.868851052879873, + 3.9169704507440923, + 3.952481022429987, + ] + ax1.plot(x, y, c="limegreen", ls=":", lw=2) + + x = [ + 3.953609195402299, + 3.7670114942528734, + 3.5917471264367817, + 3.39735632183908, + 3.1724137931034484, + 2.9408045977011494, + 2.685977011494253, + 2.4593563218390804, + 2.2203218390804595, + 2.0158850574712646, + 1.834183908045977, + 1.6522758620689655, + 1.4937471264367814, + 1.3427586206896551, + 1.2075402298850575, + ] + y = [ + 4.427971008277223, + 4.458335120298495, + 4.481579963117039, + 4.495861388686366, + 4.544581206844791, + 4.587425483552773, + 4.638160998413175, + 4.698631899472488, + 4.757987734271133, + 4.813955483123902, + 4.862332203971352, + 4.892481880173264, + 4.9247759145687695, + 4.947934983059571, + 4.953124329888064, + ] + ax1.plot(x, y, c="limegreen", ls=":", lw=2) + +# ax1.legend(loc='upper left') +fig.legend(loc=7, fontsize=18) + +if sim.B_dir == "z": + ax1.set_xlabel(r"$k l_i$") + ax1.set_title("$B_{R/L} = B_x \pm iB_y$") + fig.suptitle("Parallel EM modes") + ax1.set_xlim(-3, 3) + ax1.set_ylim(-6, 3) + dir_str = "par" +else: + ax1.set_xlabel(r"$k \rho_i$") + ax1.set_title("$E_z(k, \omega)$") + fig.suptitle(f"Perpendicular EM modes (ion Bernstein) - {sim.dim}D") + ax1.set_xlim(-3, 3) + ax1.set_ylim(0, 8) + dir_str = "perp" + +ax1.set_ylabel(r"$\omega / \Omega_i$") + +plt.savefig( + f"spectrum_{dir_str}_{sim.dim}d_{sim.substeps}_substeps_{sim.eta}_eta.png", + bbox_inches="tight", +) +if not sim.test: + plt.show() diff --git a/Examples/Tests/ohm_solver_em_modes/analysis_default_regression.py b/Examples/Tests/ohm_solver_em_modes/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ohm_solver_em_modes/analysis_rz.py b/Examples/Tests/ohm_solver_em_modes/analysis_rz.py new file mode 100755 index 00000000000..7cd5086c408 --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/analysis_rz.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# +# --- Analysis script for the hybrid-PIC example producing EM modes. + +import dill +import matplotlib.pyplot as plt +import numpy as np +import scipy.fft as fft +from matplotlib import colors +from openpmd_viewer import OpenPMDTimeSeries +from scipy.interpolate import RegularGridInterpolator +from scipy.special import j1, jn, jn_zeros + +from pywarpx import picmi + +constants = picmi.constants + +# load simulation parameters +with open("sim_parameters.dpkl", "rb") as f: + sim = dill.load(f) + +diag_dir = "diags/field_diags" + +ts = OpenPMDTimeSeries(diag_dir, check_all_files=True) + + +def transform_spatially(data_for_transform): + # interpolate from regular r-grid to special r-grid + interp = RegularGridInterpolator( + (info.z, info.r), data_for_transform, method="linear" + ) + data_interp = interp((zg, rg)) + + # Applying manual hankel in r + # Fmz = np.sum(proj*data_for_transform, axis=(2,3)) + Fmz = np.einsum("ijkl,kl->ij", proj, data_interp) + # Standard fourier in z + Fmn = fft.fftshift(fft.fft(Fmz, axis=1), axes=1) + return Fmn + + +def process(it): + print(f"Processing iteration {it}", flush=True) + field, info = ts.get_field("E", "y", iteration=it) + F_k = transform_spatially(field) + return F_k + + +# grab the first iteration to get the grids +Bz, info = ts.get_field("B", "z", iteration=0) + +nr = len(info.r) +nz = len(info.z) + +nkr = 12 # number of radial modes to solve for + +r_max = np.max(info.r) + +# create r-grid with points spaced out according to zeros of the Bessel function +r_grid = jn_zeros(1, nr) / jn_zeros(1, nr)[-1] * r_max + +zg, rg = np.meshgrid(info.z, r_grid) + +# Setup Hankel Transform +j_1M = jn_zeros(1, nr)[-1] +r_modes = np.arange(nkr) + +A = ( + 4.0 + * np.pi + * r_max**2 + / j_1M**2 + * j1(np.outer(jn_zeros(1, max(r_modes) + 1)[r_modes], jn_zeros(1, nr)) / j_1M) + / jn(2, jn_zeros(1, nr)) ** 2 +) + +# No transformation for z +B = np.identity(nz) + +# combine projection arrays +proj = np.einsum("ab,cd->acbd", A, B) + +results = np.zeros((len(ts.t), nkr, nz), dtype=complex) +for ii, it in enumerate(ts.iterations): + results[ii] = process(it) + +# now Fourier transform in time +F_kw = fft.fftshift(fft.fft(results, axis=0), axes=0) + +dz = info.z[1] - info.z[0] +kz = 2 * np.pi * fft.fftshift(fft.fftfreq(F_kw[0].shape[1], dz)) +dt = ts.iterations[1] - ts.iterations[0] +omega = 2 * np.pi * fft.fftshift(fft.fftfreq(F_kw.shape[0], sim.dt * dt)) + +# Save data for future plotting purposes +np.savez("diags/spectrograms.npz", F_kw=F_kw, dz=dz, kz=kz, dt=dt, omega=omega) + +# plot the resulting dispersions +k = np.linspace(0, 250, 500) +kappa = k * sim.l_i + +fig, axes = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(6.75, 5)) + +vmin = [2e-3, 1.5e-3, 7.5e-4, 5e-4] +vmax = 1.0 + +# plot m = 1 +for ii, m in enumerate([1, 3, 6, 8]): + ax = axes.flatten()[ii] + ax.set_title(f"m = {m}", fontsize=11) + m -= 1 + pm1 = ax.pcolormesh( + kz * sim.l_i, + omega / sim.w_ci, + abs(F_kw[:, m, :]) / np.max(abs(F_kw[:, m, :])), + norm=colors.LogNorm(vmin=vmin[ii], vmax=vmax), + cmap="inferno", + ) + cb = fig.colorbar(pm1, ax=ax) + cb.set_label(r"Normalized $E_\theta(k_z, m, \omega)$") + + # Get dispersion relation - see for example + # T. Stix, Waves in Plasmas (American Inst. of Physics, 1992), Chap 6, Sec 2 + nu_m = jn_zeros(1, m + 1)[-1] / sim.Lr + R2 = 0.5 * (nu_m**2 * (1.0 + kappa**2) + k**2 * (kappa**2 + 2.0)) + P4 = k**2 * (nu_m**2 + k**2) + omega_fast = sim.vA * np.sqrt(R2 + np.sqrt(R2**2 - P4)) + omega_slow = sim.vA * np.sqrt(R2 - np.sqrt(R2**2 - P4)) + # Upper right corner + ax.plot(k * sim.l_i, omega_fast / sim.w_ci, "w--", label="$\omega_{fast}$") + ax.plot( + k * sim.l_i, + omega_slow / sim.w_ci, + color="white", + linestyle="--", + label="$\omega_{slow}$", + ) + # Thermal resonance + thermal_res = sim.w_ci + 3 * sim.v_ti * k + ax.plot( + k * sim.l_i, + thermal_res / sim.w_ci, + color="magenta", + linestyle="--", + label="$\omega = \Omega_i + 3v_{th,i}k$", + ) + ax.plot( + -k * sim.l_i, thermal_res / sim.w_ci, color="magenta", linestyle="--", label="" + ) + thermal_res = sim.w_ci - 3 * sim.v_ti * k + ax.plot( + k * sim.l_i, + thermal_res / sim.w_ci, + color="magenta", + linestyle="--", + label="$\omega = \Omega_i + 3v_{th,i}k$", + ) + ax.plot( + -k * sim.l_i, thermal_res / sim.w_ci, color="magenta", linestyle="--", label="" + ) + + +for ax in axes.flatten(): + ax.set_xlim(-1.75, 1.75) + ax.set_ylim(0, 1.6) + +axes[0, 0].set_ylabel("$\omega/\Omega_{ci}$") +axes[1, 0].set_ylabel("$\omega/\Omega_{ci}$") +axes[1, 0].set_xlabel("$k_zl_i$") +axes[1, 1].set_xlabel("$k_zl_i$") + +plt.savefig("normal_modes_disp.png", dpi=600) +if not sim.test: + plt.show() +else: + plt.close() + + # check if power spectrum sampling match earlier results + amps = np.abs(F_kw[2, 1, len(kz) // 2 - 2 : len(kz) // 2 + 2]) + print("Amplitude sample: ", amps) + assert np.allclose( + amps, np.array([59.23850009, 19.26746169, 92.65794174, 10.83627164]) + ) diff --git a/Examples/Tests/ohm_solver_em_modes/inputs_test_1d_ohm_solver_em_modes_picmi.py b/Examples/Tests/ohm_solver_em_modes/inputs_test_1d_ohm_solver_em_modes_picmi.py new file mode 100644 index 00000000000..ac0c2369c0e --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/inputs_test_1d_ohm_solver_em_modes_picmi.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script is set up to produce either parallel or +# --- perpendicular (Bernstein) EM modes and can be run in 1d, 2d or 3d +# --- Cartesian geometries. See Section 4.2 and 4.3 of Munoz et al. (2018). +# --- As a CI test only a small number of steps are taken using the 1d version. + +import argparse +import os +import sys + +import dill +import numpy as np +from mpi4py import MPI as mpi + +from pywarpx import callbacks, fields, libwarpx, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=0) + + +class EMModes(object): + """The following runs a simulation of an uniform plasma at a set + temperature (Te = Ti) with an external magnetic field applied in either the + z-direction (parallel to domain) or x-direction (perpendicular to domain). + The analysis script (in this same directory) analyzes the output field data + for EM modes. This input is based on the EM modes tests as described by + Munoz et al. (2018) and tests done by Scott Nicks at TAE Technologies. + """ + + # Applied field parameters + B0 = 0.25 # Initial magnetic field strength (T) + beta = [0.01, 0.1] # Plasma beta, used to calculate temperature + + # Plasma species parameters + m_ion = [100.0, 400.0] # Ion mass (electron masses) + vA_over_c = [1e-4, 1e-3] # ratio of Alfven speed and the speed of light + + # Spatial domain + Nz = [1024, 1920] # number of cells in z direction + Nx = 8 # number of cells in x (and y) direction for >1 dimensions + + # Temporal domain (if not run as a CI test) + LT = 300.0 # Simulation temporal length (ion cyclotron periods) + + # Numerical parameters + NPPC = [1024, 256, 64] # Seed number of particles per cell + DZ = 1.0 / 10.0 # Cell size (ion skin depths) + DT = [5e-3, 4e-3] # Time step (ion cyclotron periods) + + # Plasma resistivity - used to dampen the mode excitation + eta = [[1e-7, 1e-7], [1e-7, 1e-5], [1e-7, 1e-4]] + # Number of substeps used to update B + substeps = 20 + + def __init__(self, test, dim, B_dir, verbose): + """Get input parameters for the specific case desired.""" + self.test = test + self.dim = int(dim) + self.B_dir = B_dir + self.verbose = verbose or self.test + + # sanity check + assert dim > 0 and dim < 4, f"{dim}-dimensions not a valid input" + + # get simulation parameters from the defaults given the direction of + # the initial B-field and the dimensionality + self.get_simulation_parameters() + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.dz = self.DZ * self.l_i + self.Lz = self.Nz * self.dz + self.Lx = self.Nx * self.dz + + self.dt = self.DT * self.t_ci + + if not self.test: + self.total_steps = int(self.LT / self.DT) + else: + # if this is a test case run for only a small number of steps + self.total_steps = 250 + # output diagnostics 20 times per cyclotron period + self.diag_steps = int(1.0 / 20 / self.DT) + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT = {self.T_plasma:.3f} eV\n" + f"\tn = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.v_ti:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.dz:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_simulation_parameters(self): + """Pick appropriate parameters from the defaults given the direction + of the B-field and the simulation dimensionality.""" + if self.B_dir == "z": + idx = 0 + self.Bx = 0.0 + self.By = 0.0 + self.Bz = self.B0 + elif self.B_dir == "y": + idx = 1 + self.Bx = 0.0 + self.By = self.B0 + self.Bz = 0.0 + else: + idx = 1 + self.Bx = self.B0 + self.By = 0.0 + self.Bz = 0.0 + + self.beta = self.beta[idx] + self.m_ion = self.m_ion[idx] + self.vA_over_c = self.vA_over_c[idx] + self.Nz = self.Nz[idx] + self.DT = self.DT[idx] + + self.NPPC = self.NPPC[self.dim - 1] + self.eta = self.eta[self.dim - 1][idx] + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = self.vA_over_c * constants.c + self.n_plasma = (self.B0 / self.vA) ** 2 / ( + constants.mu0 * (self.M + constants.m_e) + ) + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n_plasma / (self.M * constants.ep0)) + + # Skin depth (m) + self.l_i = constants.c / self.w_pi + + # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 + self.v_ti = np.sqrt(self.beta / 2.0) * self.vA + + # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) + self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV + + # Larmor radius (m) + self.rho_i = self.v_ti / self.w_ci + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + if self.dim == 1: + grid_object = picmi.Cartesian1DGrid + elif self.dim == 2: + grid_object = picmi.Cartesian2DGrid + else: + grid_object = picmi.Cartesian3DGrid + + self.grid = grid_object( + number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim :], + warpx_max_grid_size=self.Nz, + lower_bound=[-self.Lx / 2.0, -self.Lx / 2.0, 0][-self.dim :], + upper_bound=[self.Lx / 2.0, self.Lx / 2.0, self.Lz][-self.dim :], + lower_boundary_conditions=["periodic"] * self.dim, + upper_boundary_conditions=["periodic"] * self.dim, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + Te=self.T_plasma, + n0=self.n_plasma, + plasma_resistivity=self.eta, + substeps=self.substeps, + ) + simulation.solver = self.solver + + B_ext = picmi.AnalyticInitialField( + Bx_expression=self.Bx, By_expression=self.By, Bz_expression=self.Bz + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_plasma, + rms_velocity=[self.v_ti] * 3, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + if self.B_dir == "z": + self.output_file_name = "par_field_data.txt" + else: + self.output_file_name = "perp_field_data.txt" + + if self.test: + particle_diag = picmi.ParticleDiagnostic( + name="field_diag", + period=self.total_steps, + # warpx_format = 'openpmd', + # warpx_openpmd_backend = 'h5' + ) + simulation.add_diagnostic(particle_diag) + field_diag = picmi.FieldDiagnostic( + name="field_diag", + grid=self.grid, + period=self.total_steps, + data_list=["B", "E", "J_displacement"], + # warpx_format = 'openpmd', + # warpx_openpmd_backend = 'h5' + ) + simulation.add_diagnostic(field_diag) + + if self.B_dir == "z" or self.dim == 1: + line_diag = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + probe_geometry="Line", + z_probe=0, + z1_probe=self.Lz, + resolution=self.Nz - 1, + name=self.output_file_name[:-4], + period=self.diag_steps, + path="diags/", + ) + simulation.add_diagnostic(line_diag) + else: + # install a custom "reduced diagnostic" to save the average field + callbacks.installafterEsolve(self._record_average_fields) + try: + os.mkdir("diags") + except OSError: + # diags directory already exists + pass + with open(f"diags/{self.output_file_name}", "w") as f: + f.write( + "[0]step() [1]time(s) [2]z_coord(m) " + "[3]Ez_lev0-(V/m) [4]Bx_lev0-(T) [5]By_lev0-(T)\n" + ) + + ####################################################################### + # Initialize simulation # + ####################################################################### + + simulation.initialize_inputs() + simulation.initialize_warpx() + + def _record_average_fields(self): + """A custom reduced diagnostic to store the average E&M fields in a + similar format as the reduced diagnostic so that the same analysis + script can be used regardless of the simulation dimension. + """ + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if step % self.diag_steps != 0: + return + + Bx_warpx = fields.BxWrapper()[...] + By_warpx = fields.ByWrapper()[...] + Ez_warpx = fields.EzWrapper()[...] + + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: + return + + t = step * self.dt + z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) + + if self.dim == 2: + Ez = np.mean(Ez_warpx[:-1], axis=0) + Bx = np.mean(Bx_warpx[:-1], axis=0) + By = np.mean(By_warpx[:-1], axis=0) + else: + Ez = np.mean(Ez_warpx[:-1, :-1], axis=(0, 1)) + Bx = np.mean(Bx_warpx[:-1], axis=(0, 1)) + By = np.mean(By_warpx[:-1], axis=(0, 1)) + + with open(f"diags/{self.output_file_name}", "a") as f: + for ii in range(self.Nz): + f.write( + f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {Ez[ii]:+.10e} " + f"{Bx[ii]:+.10e} {By[ii]:+.10e}\n" + ) + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-d", "--dim", help="Simulation dimension", required=False, type=int, default=1 +) +parser.add_argument( + "--bdir", + help="Direction of the B-field", + required=False, + choices=["x", "y", "z"], + default="z", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = EMModes(test=args.test, dim=args.dim, B_dir=args.bdir, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/ohm_solver_em_modes/inputs_test_rz_ohm_solver_em_modes_picmi.py b/Examples/Tests/ohm_solver_em_modes/inputs_test_rz_ohm_solver_em_modes_picmi.py new file mode 100644 index 00000000000..ba922dbdc9f --- /dev/null +++ b/Examples/Tests/ohm_solver_em_modes/inputs_test_rz_ohm_solver_em_modes_picmi.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script is set up to produce parallel normal EM modes +# --- in a metallic cylinder and is run in RZ geometry. +# --- As a CI test only a small number of steps are taken. + +import argparse +import sys + +import dill +import numpy as np +from mpi4py import MPI as mpi + +from pywarpx import picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(verbose=0) + + +class CylindricalNormalModes(object): + """The following runs a simulation of an uniform plasma at a set ion + temperature (and Te = 0) with an external magnetic field applied in the + z-direction (parallel to domain). + The analysis script (in this same directory) analyzes the output field + data for EM modes. + """ + + # Applied field parameters + B0 = 0.5 # Initial magnetic field strength (T) + beta = 0.01 # Plasma beta, used to calculate temperature + + # Plasma species parameters + m_ion = 400.0 # Ion mass (electron masses) + vA_over_c = 5e-3 # ratio of Alfven speed and the speed of light + + # Spatial domain + Nz = 512 # number of cells in z direction + Nr = 128 # number of cells in r direction + + # Temporal domain (if not run as a CI test) + LT = 800.0 # Simulation temporal length (ion cyclotron periods) + + # Numerical parameters + NPPC = 8000 # Seed number of particles per cell + DZ = 0.4 # Cell size (ion skin depths) + DR = 0.4 # Cell size (ion skin depths) + DT = 0.02 # Time step (ion cyclotron periods) + + # Plasma resistivity - used to dampen the mode excitation + eta = 5e-4 + # Number of substeps used to update B + substeps = 20 + + def __init__(self, test, verbose): + """Get input parameters for the specific case desired.""" + self.test = test + self.verbose = verbose or self.test + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + if not self.test: + self.total_steps = int(self.LT / self.DT) + else: + # if this is a test case run for only a small number of steps + self.total_steps = 100 + # and make the grid and particle count smaller + self.Nz = 128 + self.Nr = 64 + self.NPPC = 200 + # output diagnostics 5 times per cyclotron period + self.diag_steps = max(10, int(1.0 / 5 / self.DT)) + + self.Lz = self.Nz * self.DZ * self.l_i + self.Lr = self.Nr * self.DR * self.l_i + + self.dt = self.DT * self.t_ci + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT = {self.T_plasma:.3f} eV\n" + f"\tn = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.v_ti:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n", + flush=True, + ) + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = self.vA_over_c * constants.c + self.n_plasma = (self.B0 / self.vA) ** 2 / ( + constants.mu0 * (self.M + constants.m_e) + ) + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n_plasma / (self.M * constants.ep0)) + + # Skin depth (m) + self.l_i = constants.c / self.w_pi + + # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 + self.v_ti = np.sqrt(self.beta / 2.0) * self.vA + + # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) + self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV + + # Larmor radius (m) + self.rho_i = self.v_ti / self.w_ci + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + self.grid = picmi.CylindricalGrid( + number_of_cells=[self.Nr, self.Nz], + warpx_max_grid_size=self.Nz, + lower_bound=[0, -self.Lz / 2.0], + upper_bound=[self.Lr, self.Lz / 2.0], + lower_boundary_conditions=["none", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["none", "periodic"], + upper_boundary_conditions_particles=["reflecting", "periodic"], + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + Te=0.0, + n0=self.n_plasma, + plasma_resistivity=self.eta, + substeps=self.substeps, + n_floor=self.n_plasma * 0.05, + ) + simulation.solver = self.solver + + B_ext = picmi.AnalyticInitialField(Bz_expression=self.B0) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_plasma, + rms_velocity=[self.v_ti] * 3, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + field_diag = picmi.FieldDiagnostic( + name="field_diag", + grid=self.grid, + period=self.diag_steps, + data_list=["B", "E"], + write_dir="diags", + warpx_file_prefix="field_diags", + warpx_format="openpmd", + warpx_openpmd_backend="h5", + ) + simulation.add_diagnostic(field_diag) + + # add particle diagnostic for checksum + if self.test: + part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=self.total_steps, + species=[self.ions], + data_list=["ux", "uy", "uz", "weighting"], + ) + simulation.add_diagnostic(part_diag) + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = CylindricalNormalModes(test=args.test, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt b/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt new file mode 100644 index 00000000000..a57ef7bb922 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt @@ -0,0 +1,13 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_ohm_solver_landau_damping_picmi # name + 2 # dims + 2 # nprocs + "inputs_test_2d_ohm_solver_landau_damping_picmi.py --test --dim 2 --temp_ratio 0.1" # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency +) +label_warpx_test(test_2d_ohm_solver_landau_damping_picmi slow) diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py deleted file mode 100644 index d4430502f42..00000000000 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are -# --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script simulates ion Landau damping as described -# --- in section 4.5 of Munoz et al. (2018). - -import argparse -import os -import sys -import time - -import dill -import numpy as np -from mpi4py import MPI as mpi - -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi - -constants = picmi.constants - -comm = mpi.COMM_WORLD - -simulation = picmi.Simulation( - warpx_serialize_initial_conditions=True, - verbose=0 -) - - -class IonLandauDamping(object): - '''This input is based on the ion Landau damping test as described by - Munoz et al. (2018). - ''' - # Applied field parameters - B0 = 0.1 # Initial magnetic field strength (T) - beta = 2.0 # Plasma beta, used to calculate temperature - - # Plasma species parameters - m_ion = 100.0 # Ion mass (electron masses) - vA_over_c = 1e-3 # ratio of Alfven speed and the speed of light - - # Spatial domain - Nz = 256 # number of cells in z direction - Nx = 4 # number of cells in x (and y) direction for >1 dimensions - - # Temporal domain (if not run as a CI test) - LT = 40.0 # Simulation temporal length (ion cyclotron periods) - - # Numerical parameters - NPPC = [8192, 4096, 1024] # Seed number of particles per cell - DZ = 1.0 / 6.0 # Cell size (ion skin depths) - DT = 1e-3 # Time step (ion cyclotron periods) - - # density perturbation strength - epsilon = 0.03 - - # Plasma resistivity - used to dampen the mode excitation - eta = 1e-7 - # Number of substeps used to update B - substeps = 10 - - - def __init__(self, test, dim, m, T_ratio, verbose): - """Get input parameters for the specific case desired.""" - self.test = test - self.dim = int(dim) - self.m = m - self.T_ratio = T_ratio - self.verbose = verbose or self.test - - # sanity check - assert (dim > 0 and dim < 4), f"{dim}-dimensions not a valid input" - - # calculate various plasma parameters based on the simulation input - self.get_plasma_quantities() - - self.dz = self.DZ * self.l_i - self.Lz = self.Nz * self.dz - self.Lx = self.Nx * self.dz - - diag_period = 1 / 16.0 # Output interval (ion cyclotron periods) - self.diag_steps = int(diag_period / self.DT) - - self.total_steps = int(np.ceil(self.LT / self.DT)) - # if this is a test case run for only 100 steps - if self.test: - self.total_steps = 100 - - self.dt = self.DT / self.w_ci # self.DT * self.t_ci - - # dump all the current attributes to a dill pickle file - if comm.rank == 0: - with open('sim_parameters.dpkl', 'wb') as f: - dill.dump(self, f) - - # print out plasma parameters - if comm.rank == 0: - print( - f"Initializing simulation with input parameters:\n" - f"\tT = {self.T_plasma*1e-3:.1f} keV\n" - f"\tn = {self.n_plasma:.1e} m^-3\n" - f"\tB0 = {self.B0:.2f} T\n" - f"\tM/m = {self.m_ion:.0f}\n" - ) - print( - f"Plasma parameters:\n" - f"\tl_i = {self.l_i:.1e} m\n" - f"\tt_ci = {self.t_ci:.1e} s\n" - f"\tv_ti = {self.v_ti:.1e} m/s\n" - f"\tvA = {self.vA:.1e} m/s\n" - ) - print( - f"Numerical parameters:\n" - f"\tdz = {self.dz:.1e} m\n" - f"\tdt = {self.dt:.1e} s\n" - f"\tdiag steps = {self.diag_steps:d}\n" - f"\ttotal steps = {self.total_steps:d}\n" - ) - - self.setup_run() - - def get_plasma_quantities(self): - """Calculate various plasma parameters based on the simulation input.""" - # Ion mass (kg) - self.M = self.m_ion * constants.m_e - - # Cyclotron angular frequency (rad/s) and period (s) - self.w_ci = constants.q_e * abs(self.B0) / self.M - self.t_ci = 2.0 * np.pi / self.w_ci - - # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi - self.vA = self.vA_over_c * constants.c - self.n_plasma = ( - (self.B0 / self.vA)**2 / (constants.mu0 * (self.M + constants.m_e)) - ) - - # Ion plasma frequency (Hz) - self.w_pi = np.sqrt( - constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) - ) - - # Skin depth (m) - self.l_i = constants.c / self.w_pi - - # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 - self.v_ti = np.sqrt(self.beta / 2.0) * self.vA - - # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) - self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV - - # Larmor radius (m) - self.rho_i = self.v_ti / self.w_ci - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - if self.dim == 1: - grid_object = picmi.Cartesian1DGrid - elif self.dim == 2: - grid_object = picmi.Cartesian2DGrid - else: - grid_object = picmi.Cartesian3DGrid - - self.grid = grid_object( - number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim:], - warpx_max_grid_size=self.Nz, - lower_bound=[-self.Lx/2.0, -self.Lx/2.0, 0][-self.dim:], - upper_bound=[self.Lx/2.0, self.Lx/2.0, self.Lz][-self.dim:], - lower_boundary_conditions=['periodic']*self.dim, - upper_boundary_conditions=['periodic']*self.dim, - warpx_blocking_factor=4 - ) - simulation.time_step_size = self.dt - simulation.max_steps = self.total_steps - simulation.current_deposition_algo = 'direct' - simulation.particle_shape = 1 - simulation.verbose = self.verbose - - ####################################################################### - # Field solver and external field # - ####################################################################### - - self.solver = picmi.HybridPICSolver( - grid=self.grid, gamma=1.0, - Te=self.T_plasma/self.T_ratio, - n0=self.n_plasma, - plasma_resistivity=self.eta, substeps=self.substeps - ) - simulation.solver = self.solver - - ####################################################################### - # Particle types setup # - ####################################################################### - - k_m = 2.0*np.pi*self.m / self.Lz - self.ions = picmi.Species( - name='ions', charge='q_e', mass=self.M, - initial_distribution=picmi.AnalyticDistribution( - density_expression=f"{self.n_plasma}*(1+{self.epsilon}*cos({k_m}*z))", - rms_velocity=[self.v_ti]*3 - ) - ) - simulation.add_species( - self.ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, n_macroparticles_per_cell=self.NPPC[self.dim-1] - ) - ) - - ####################################################################### - # Add diagnostics # - ####################################################################### - - callbacks.installafterstep(self.text_diag) - - if self.test: - particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=100, - write_dir='.', - species=[self.ions], - data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], - warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', - ) - simulation.add_diagnostic(particle_diag) - field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=self.grid, - period=100, - write_dir='.', - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', - ) - simulation.add_diagnostic(field_diag) - - self.output_file_name = 'field_data.txt' - # install a custom "reduced diagnostic" to save the average field - callbacks.installafterEsolve(self._record_average_fields) - try: - os.mkdir("diags") - except OSError: - # diags directory already exists - pass - with open(f"diags/{self.output_file_name}", 'w') as f: - f.write("[0]step() [1]time(s) [2]z_coord(m) [3]Ez_lev0-(V/m)\n") - - self.prev_time = time.time() - self.start_time = self.prev_time - self.prev_step = 0 - - ####################################################################### - # Initialize simulation # - ####################################################################### - - simulation.initialize_inputs() - simulation.initialize_warpx() - - # get ion particle container wrapper - self.ion_part_container = particle_containers.ParticleContainerWrapper( - 'ions' - ) - - def text_diag(self): - """Diagnostic function to print out timing data and particle numbers.""" - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if step % (self.total_steps // 10) != 0: - return - - wall_time = time.time() - self.prev_time - steps = step - self.prev_step - step_rate = steps / wall_time - - status_dict = { - 'step': step, - 'nplive ions': self.ion_part_container.nps, - 'wall_time': wall_time, - 'step_rate': step_rate, - "diag_steps": self.diag_steps, - 'iproc': None - } - - diag_string = ( - "Step #{step:6d}; " - "{nplive ions} core ions; " - "{wall_time:6.1f} s wall time; " - "{step_rate:4.2f} steps/s" - ) - - if libwarpx.amr.ParallelDescriptor.MyProc() == 0: - print(diag_string.format(**status_dict)) - - self.prev_time = time.time() - self.prev_step = step - - def _record_average_fields(self): - """A custom reduced diagnostic to store the average E&M fields in a - similar format as the reduced diagnostic so that the same analysis - script can be used regardless of the simulation dimension. - """ - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if step % self.diag_steps != 0: - return - - Ez_warpx = fields.EzWrapper()[...] - - if libwarpx.amr.ParallelDescriptor.MyProc() != 0: - return - - t = step * self.dt - z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) - - if self.dim == 1: - Ez = Ez_warpx - elif self.dim == 2: - Ez = np.mean(Ez_warpx, axis=0) - else: - Ez = np.mean(Ez_warpx, axis=(0, 1)) - - with open(f"diags/{self.output_file_name}", 'a') as f: - for ii in range(self.Nz): - f.write( - f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {Ez[ii]:+.10e}\n" - ) - - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-d', '--dim', help='Simulation dimension', required=False, type=int, - default=1 -) -parser.add_argument( - '-m', help='Mode number to excite', required=False, type=int, - default=4 -) -parser.add_argument( - '--temp_ratio', help='Ratio of ion to electron temperature', required=False, - type=float, default=1.0/3 -) -parser.add_argument( - '-v', '--verbose', help='Verbose output', action='store_true', -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -run = IonLandauDamping( - test=args.test, dim=args.dim, m=args.m, T_ratio=args.temp_ratio, - verbose=args.verbose -) -simulation.step() diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst index dd4f94b4edf..d54f4e16aa6 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -14,17 +14,17 @@ Run The same input script can be used for 1d, 2d or 3d simulations and to sweep different temperature ratios. -.. dropdown:: Script ``PICMI_inputs.py`` +.. dropdown:: Script ``inputs_test_2d_ohm_solver_landau_damping_picmi.py`` - .. literalinclude:: PICMI_inputs.py + .. literalinclude:: inputs_test_2d_ohm_solver_landau_damping_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py``. + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/inputs_test_2d_ohm_solver_landau_damping_picmi.py``. For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. .. code-block:: bash - python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + python3 inputs_test_2d_ohm_solver_landau_damping_picmi.py -dim {1/2/3} --temp_ratio {value} Analyze ------- diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py index ade7187ebe3..bd193260f2f 100755 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py @@ -11,39 +11,56 @@ constants = picmi.constants -matplotlib.rcParams.update({'font.size': 20}) +matplotlib.rcParams.update({"font.size": 20}) # load simulation parameters -with open(f'sim_parameters.dpkl', 'rb') as f: +with open("sim_parameters.dpkl", "rb") as f: sim = dill.load(f) # theoretical damping rates were taken from Fig. 14b of Munoz et al. -theoretical_damping_rate = np.array([ - [0.09456706, 0.05113443], [0.09864177, 0.05847507], - [0.10339559, 0.0659153 ], [0.10747029, 0.07359366], - [0.11290323, 0.08256106], [0.11833616, 0.09262114], - [0.12580645, 0.10541121], [0.13327674, 0.11825558], - [0.14006791, 0.13203098], [0.14889643, 0.14600538], - [0.15772496, 0.16379615], [0.16791171, 0.18026693], - [0.17606112, 0.19650209], [0.18828523, 0.21522808], - [0.19983022, 0.23349062], [0.21273345, 0.25209216], - [0.22835314, 0.27877403], [0.24465195, 0.30098317], - [0.25959253, 0.32186286], [0.27657046, 0.34254601], - [0.29626486, 0.36983567], [0.3139219 , 0.38984826], - [0.33157895, 0.40897973], [0.35195246, 0.43526107], - [0.37368421, 0.45662113], [0.39745331, 0.47902942], - [0.44974533, 0.52973074], [0.50747029, 0.57743925], - [0.57334465, 0.63246726], [0.64193548, 0.67634255] -]) +theoretical_damping_rate = np.array( + [ + [0.09456706, 0.05113443], + [0.09864177, 0.05847507], + [0.10339559, 0.0659153], + [0.10747029, 0.07359366], + [0.11290323, 0.08256106], + [0.11833616, 0.09262114], + [0.12580645, 0.10541121], + [0.13327674, 0.11825558], + [0.14006791, 0.13203098], + [0.14889643, 0.14600538], + [0.15772496, 0.16379615], + [0.16791171, 0.18026693], + [0.17606112, 0.19650209], + [0.18828523, 0.21522808], + [0.19983022, 0.23349062], + [0.21273345, 0.25209216], + [0.22835314, 0.27877403], + [0.24465195, 0.30098317], + [0.25959253, 0.32186286], + [0.27657046, 0.34254601], + [0.29626486, 0.36983567], + [0.3139219, 0.38984826], + [0.33157895, 0.40897973], + [0.35195246, 0.43526107], + [0.37368421, 0.45662113], + [0.39745331, 0.47902942], + [0.44974533, 0.52973074], + [0.50747029, 0.57743925], + [0.57334465, 0.63246726], + [0.64193548, 0.67634255], + ] +) expected_gamma = np.interp( sim.T_ratio, theoretical_damping_rate[:, 0], theoretical_damping_rate[:, 1] ) data = np.loadtxt("diags/field_data.txt", skiprows=1) -field_idx_dict = {'z': 2, 'Ez': 3} +field_idx_dict = {"z": 2, "Ez": 3} -step = data[:,0] +step = data[:, 0] num_steps = len(np.unique(step)) @@ -51,16 +68,16 @@ resolution = len(np.where(step == 0)[0]) - 1 # reshape to separate spatial and time coordinates -sim_data = data.reshape((num_steps, resolution+1, data.shape[1])) +sim_data = data.reshape((num_steps, resolution + 1, data.shape[1])) -z_grid = sim_data[1, :, field_idx_dict['z']] +z_grid = sim_data[1, :, field_idx_dict["z"]] idx = np.argsort(z_grid)[1:] dz = np.mean(np.diff(z_grid[idx])) -dt = np.mean(np.diff(sim_data[:,0,1])) +dt = np.mean(np.diff(sim_data[:, 0, 1])) data = np.zeros((num_steps, resolution)) for i in range(num_steps): - data[i,:] = sim_data[i,idx,field_idx_dict['Ez']] + data[i, :] = sim_data[i, idx, field_idx_dict["Ez"]] print(f"Data file contains {num_steps} time snapshots.") print(f"Spatial resolution is {resolution}") @@ -72,36 +89,25 @@ # Plot the 4th Fourier mode fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) -t_points = np.arange(num_steps)*dt*t_norm +t_points = np.arange(num_steps) * dt * t_norm ax1.plot( - t_points, np.abs(field_kt[:, sim.m] / field_kt[0, sim.m]), 'r', - label=f'$T_i/T_e$ = {sim.T_ratio:.2f}' + t_points, + np.abs(field_kt[:, sim.m] / field_kt[0, sim.m]), + "r", + label=f"$T_i/T_e$ = {sim.T_ratio:.2f}", ) # Plot a line showing the expected damping rate t_points = t_points[np.where(t_points < 8)] -ax1.plot( - t_points, np.exp(-t_points*expected_gamma), 'k--', lw=2 -) +ax1.plot(t_points, np.exp(-t_points * expected_gamma), "k--", lw=2) ax1.grid() ax1.legend() -ax1.set_yscale('log') -ax1.set_ylabel('$|E_z|/E_0$') -ax1.set_xlabel('t $(k_mv_{th,i})$') +ax1.set_yscale("log") +ax1.set_ylabel("$|E_z|/E_0$") +ax1.set_xlabel("t $(k_mv_{th,i})$") ax1.set_xlim(0, 18) ax1.set_title(f"Ion Landau damping - {sim.dim}d") plt.tight_layout() plt.savefig(f"diags/ion_Landau_damping_T_ratio_{sim.T_ratio}.png") - -if sim.test: - import os - import sys - sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - import checksumAPI - - # this will be the name of the plot file - fn = sys.argv[1] - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis_default_regression.py b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/inputs_test_2d_ohm_solver_landau_damping_picmi.py b/Examples/Tests/ohm_solver_ion_Landau_damping/inputs_test_2d_ohm_solver_landau_damping_picmi.py new file mode 100644 index 00000000000..320d36785db --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/inputs_test_2d_ohm_solver_landau_damping_picmi.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script simulates ion Landau damping as described +# --- in section 4.5 of Munoz et al. (2018). + +import argparse +import os +import sys +import time + +import dill +import numpy as np +from mpi4py import MPI as mpi + +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=0) + + +class IonLandauDamping(object): + """This input is based on the ion Landau damping test as described by + Munoz et al. (2018). + """ + + # Applied field parameters + B0 = 0.1 # Initial magnetic field strength (T) + beta = 2.0 # Plasma beta, used to calculate temperature + + # Plasma species parameters + m_ion = 100.0 # Ion mass (electron masses) + vA_over_c = 1e-3 # ratio of Alfven speed and the speed of light + + # Spatial domain + Nz = 256 # number of cells in z direction + Nx = 4 # number of cells in x (and y) direction for >1 dimensions + + # Temporal domain (if not run as a CI test) + LT = 40.0 # Simulation temporal length (ion cyclotron periods) + + # Numerical parameters + NPPC = [8192, 4096, 1024] # Seed number of particles per cell + DZ = 1.0 / 6.0 # Cell size (ion skin depths) + DT = 1e-3 # Time step (ion cyclotron periods) + + # density perturbation strength + epsilon = 0.03 + + # Plasma resistivity - used to dampen the mode excitation + eta = 1e-7 + # Number of substeps used to update B + substeps = 10 + + def __init__(self, test, dim, m, T_ratio, verbose): + """Get input parameters for the specific case desired.""" + self.test = test + self.dim = int(dim) + self.m = m + self.T_ratio = T_ratio + self.verbose = verbose or self.test + + # sanity check + assert dim > 0 and dim < 4, f"{dim}-dimensions not a valid input" + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.dz = self.DZ * self.l_i + self.Lz = self.Nz * self.dz + self.Lx = self.Nx * self.dz + + diag_period = 1 / 16.0 # Output interval (ion cyclotron periods) + self.diag_steps = int(diag_period / self.DT) + + self.total_steps = int(np.ceil(self.LT / self.DT)) + # if this is a test case run for only 100 steps + if self.test: + self.total_steps = 100 + + self.dt = self.DT / self.w_ci # self.DT * self.t_ci + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT = {self.T_plasma * 1e-3:.1f} keV\n" + f"\tn = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.v_ti:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.dz:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = self.vA_over_c * constants.c + self.n_plasma = (self.B0 / self.vA) ** 2 / ( + constants.mu0 * (self.M + constants.m_e) + ) + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n_plasma / (self.M * constants.ep0)) + + # Skin depth (m) + self.l_i = constants.c / self.w_pi + + # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 + self.v_ti = np.sqrt(self.beta / 2.0) * self.vA + + # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) + self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV + + # Larmor radius (m) + self.rho_i = self.v_ti / self.w_ci + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + if self.dim == 1: + grid_object = picmi.Cartesian1DGrid + elif self.dim == 2: + grid_object = picmi.Cartesian2DGrid + else: + grid_object = picmi.Cartesian3DGrid + + self.grid = grid_object( + number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim :], + warpx_max_grid_size=self.Nz, + lower_bound=[-self.Lx / 2.0, -self.Lx / 2.0, 0][-self.dim :], + upper_bound=[self.Lx / 2.0, self.Lx / 2.0, self.Lz][-self.dim :], + lower_boundary_conditions=["periodic"] * self.dim, + upper_boundary_conditions=["periodic"] * self.dim, + warpx_blocking_factor=4, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + gamma=1.0, + Te=self.T_plasma / self.T_ratio, + n0=self.n_plasma, + plasma_resistivity=self.eta, + substeps=self.substeps, + ) + simulation.solver = self.solver + + ####################################################################### + # Particle types setup # + ####################################################################### + + k_m = 2.0 * np.pi * self.m / self.Lz + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.AnalyticDistribution( + density_expression=f"{self.n_plasma}*(1+{self.epsilon}*cos({k_m}*z))", + rms_velocity=[self.v_ti] * 3, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC[self.dim - 1] + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + callbacks.installafterstep(self.text_diag) + + if self.test: + particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=100, + species=[self.ions], + data_list=["ux", "uy", "uz", "x", "z", "weighting"], + ) + simulation.add_diagnostic(particle_diag) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=100, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], + ) + simulation.add_diagnostic(field_diag) + + self.output_file_name = "field_data.txt" + # install a custom "reduced diagnostic" to save the average field + callbacks.installafterEsolve(self._record_average_fields) + try: + os.mkdir("diags") + except OSError: + # diags directory already exists + pass + with open(f"diags/{self.output_file_name}", "w") as f: + f.write("[0]step() [1]time(s) [2]z_coord(m) [3]Ez_lev0-(V/m)\n") + + self.prev_time = time.time() + self.start_time = self.prev_time + self.prev_step = 0 + + ####################################################################### + # Initialize simulation # + ####################################################################### + + simulation.initialize_inputs() + simulation.initialize_warpx() + + # get ion particle container wrapper + self.ion_part_container = particle_containers.ParticleContainerWrapper("ions") + + def text_diag(self): + """Diagnostic function to print out timing data and particle numbers.""" + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if step % (self.total_steps // 10) != 0: + return + + wall_time = time.time() - self.prev_time + steps = step - self.prev_step + step_rate = steps / wall_time + + status_dict = { + "step": step, + "nplive ions": self.ion_part_container.nps, + "wall_time": wall_time, + "step_rate": step_rate, + "diag_steps": self.diag_steps, + "iproc": None, + } + + diag_string = ( + "Step #{step:6d}; " + "{nplive ions} core ions; " + "{wall_time:6.1f} s wall time; " + "{step_rate:4.2f} steps/s" + ) + + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: + print(diag_string.format(**status_dict)) + + self.prev_time = time.time() + self.prev_step = step + + def _record_average_fields(self): + """A custom reduced diagnostic to store the average E&M fields in a + similar format as the reduced diagnostic so that the same analysis + script can be used regardless of the simulation dimension. + """ + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if step % self.diag_steps != 0: + return + + Ez_warpx = fields.EzWrapper()[...] + + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: + return + + t = step * self.dt + z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) + + if self.dim == 1: + Ez = Ez_warpx + elif self.dim == 2: + Ez = np.mean(Ez_warpx, axis=0) + else: + Ez = np.mean(Ez_warpx, axis=(0, 1)) + + with open(f"diags/{self.output_file_name}", "a") as f: + for ii in range(self.Nz): + f.write(f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {Ez[ii]:+.10e}\n") + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-d", "--dim", help="Simulation dimension", required=False, type=int, default=1 +) +parser.add_argument( + "-m", help="Mode number to excite", required=False, type=int, default=4 +) +parser.add_argument( + "--temp_ratio", + help="Ratio of ion to electron temperature", + required=False, + type=float, + default=1.0 / 3, +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = IonLandauDamping( + test=args.test, + dim=args.dim, + m=args.m, + T_ratio=args.temp_ratio, + verbose=args.verbose, +) +simulation.step() diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt b/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt new file mode 100644 index 00000000000..288f8e32e53 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt @@ -0,0 +1,13 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_ohm_solver_ion_beam_picmi # name + 1 # dims + 2 # nprocs + "inputs_test_1d_ohm_solver_ion_beam_picmi.py --test --dim 1 --resonant" # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1002500" # checksum + OFF # dependency +) +label_warpx_test(test_1d_ohm_solver_ion_beam_picmi slow) diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py deleted file mode 100644 index 4f8b5edcc3e..00000000000 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are -# --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script simulates an ion beam instability wherein a -# --- low density ion beam interacts with background plasma. See Section 6.5 of -# --- Matthews (1994) and Section 4.4 of Munoz et al. (2018). - -import argparse -import os -import sys -import time - -import dill -import numpy as np -from mpi4py import MPI as mpi - -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi - -constants = picmi.constants - -comm = mpi.COMM_WORLD - -simulation = picmi.Simulation( - warpx_serialize_initial_conditions=True, - verbose=0 -) - - -class HybridPICBeamInstability(object): - '''This input is based on the ion beam R instability test as described by - Munoz et al. (2018). - ''' - # Applied field parameters - B0 = 0.25 # Initial magnetic field strength (T) - beta = 1.0 # Plasma beta, used to calculate temperature - - # Plasma species parameters - m_ion = 100.0 # Ion mass (electron masses) - vA_over_c = 1e-4 # ratio of Alfven speed and the speed of light - - # Spatial domain - Nz = 1024 # number of cells in z direction - Nx = 8 # number of cells in x (and y) direction for >1 dimensions - - # Temporal domain (if not run as a CI test) - LT = 120.0 # Simulation temporal length (ion cyclotron periods) - - # Numerical parameters - NPPC = [1024, 256, 64] # Seed number of particles per cell - DZ = 1.0 / 4.0 # Cell size (ion skin depths) - DT = 0.01 # Time step (ion cyclotron periods) - - # Plasma resistivity - used to dampen the mode excitation - eta = 1e-7 - # Number of substeps used to update B - substeps = 10 - - # Beam parameters - n_beam = [0.02, 0.1] - U_bc = 10.0 # relative drifts between beam and core in Alfven speeds - - def __init__(self, test, dim, resonant, verbose): - """Get input parameters for the specific case desired.""" - self.test = test - self.dim = int(dim) - self.resonant = resonant - self.verbose = verbose or self.test - - # sanity check - assert (dim > 0 and dim < 4), f"{dim}-dimensions not a valid input" - - # calculate various plasma parameters based on the simulation input - self.get_plasma_quantities() - - self.n_beam = self.n_beam[1 - int(resonant)] - self.u_beam = 1.0 / (1.0 + self.n_beam) * self.U_bc * self.vA - self.u_c = -1.0 * self.n_beam / (1.0 + self.n_beam) * self.U_bc * self.vA - self.n_beam = self.n_beam * self.n_plasma - - self.dz = self.DZ * self.l_i - self.Lz = self.Nz * self.dz - self.Lx = self.Nx * self.dz - - if self.dim == 3: - self.volume = self.Lx * self.Lx * self.Lz - self.N_cells = self.Nx * self.Nx * self.Nz - elif self.dim == 2: - self.volume = self.Lx * self.Lz - self.N_cells = self.Nx * self.Nz - else: - self.volume = self.Lz - self.N_cells = self.Nz - - diag_period = 1 / 4.0 # Output interval (ion cyclotron periods) - self.diag_steps = int(diag_period / self.DT) - - # if this is a test case run for only 25 cyclotron periods - if self.test: - self.LT = 25.0 - - self.total_steps = int(np.ceil(self.LT / self.DT)) - - self.dt = self.DT / self.w_ci - - # dump all the current attributes to a dill pickle file - if comm.rank == 0: - with open('sim_parameters.dpkl', 'wb') as f: - dill.dump(self, f) - - # print out plasma parameters - if comm.rank == 0: - print( - f"Initializing simulation with input parameters:\n" - f"\tT = {self.T_plasma*1e-3:.1f} keV\n" - f"\tn = {self.n_plasma:.1e} m^-3\n" - f"\tB0 = {self.B0:.2f} T\n" - f"\tM/m = {self.m_ion:.0f}\n" - ) - print( - f"Plasma parameters:\n" - f"\tl_i = {self.l_i:.1e} m\n" - f"\tt_ci = {self.t_ci:.1e} s\n" - f"\tv_ti = {self.v_ti:.1e} m/s\n" - f"\tvA = {self.vA:.1e} m/s\n" - ) - print( - f"Numerical parameters:\n" - f"\tdz = {self.dz:.1e} m\n" - f"\tdt = {self.dt:.1e} s\n" - f"\tdiag steps = {self.diag_steps:d}\n" - f"\ttotal steps = {self.total_steps:d}\n" - ) - - self.setup_run() - - def get_plasma_quantities(self): - """Calculate various plasma parameters based on the simulation input.""" - # Ion mass (kg) - self.M = self.m_ion * constants.m_e - - # Cyclotron angular frequency (rad/s) and period (s) - self.w_ci = constants.q_e * abs(self.B0) / self.M - self.t_ci = 2.0 * np.pi / self.w_ci - - # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi - self.vA = self.vA_over_c * constants.c - self.n_plasma = ( - (self.B0 / self.vA)**2 / (constants.mu0 * (self.M + constants.m_e)) - ) - - # Ion plasma frequency (Hz) - self.w_pi = np.sqrt( - constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) - ) - - # Skin depth (m) - self.l_i = constants.c / self.w_pi - - # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 - self.v_ti = np.sqrt(self.beta / 2.0) * self.vA - - # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) - self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV - - # Larmor radius (m) - self.rho_i = self.v_ti / self.w_ci - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - if self.dim == 1: - grid_object = picmi.Cartesian1DGrid - elif self.dim == 2: - grid_object = picmi.Cartesian2DGrid - else: - grid_object = picmi.Cartesian3DGrid - - self.grid = grid_object( - number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim:], - warpx_max_grid_size=self.Nz, - lower_bound=[-self.Lx/2.0, -self.Lx/2.0, 0][-self.dim:], - upper_bound=[self.Lx/2.0, self.Lx/2.0, self.Lz][-self.dim:], - lower_boundary_conditions=['periodic']*self.dim, - upper_boundary_conditions=['periodic']*self.dim - ) - simulation.time_step_size = self.dt - simulation.max_steps = self.total_steps - simulation.current_deposition_algo = 'direct' - simulation.particle_shape = 1 - simulation.verbose = self.verbose - - ####################################################################### - # Field solver and external field # - ####################################################################### - - self.solver = picmi.HybridPICSolver( - grid=self.grid, gamma=1.0, - Te=self.T_plasma/10.0, - n0=self.n_plasma+self.n_beam, - plasma_resistivity=self.eta, substeps=self.substeps - ) - simulation.solver = self.solver - - B_ext = picmi.AnalyticInitialField( - Bx_expression=0.0, - By_expression=0.0, - Bz_expression=self.B0 - ) - simulation.add_applied_field(B_ext) - - ####################################################################### - # Particle types setup # - ####################################################################### - - self.ions = picmi.Species( - name='ions', charge='q_e', mass=self.M, - initial_distribution=picmi.UniformDistribution( - density=self.n_plasma, - rms_velocity=[self.v_ti]*3, - directed_velocity=[0, 0, self.u_c] - ) - ) - simulation.add_species( - self.ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, n_macroparticles_per_cell=self.NPPC[self.dim-1] - ) - ) - self.beam_ions = picmi.Species( - name='beam_ions', charge='q_e', mass=self.M, - initial_distribution=picmi.UniformDistribution( - density=self.n_beam, - rms_velocity=[self.v_ti]*3, - directed_velocity=[0, 0, self.u_beam] - ) - ) - simulation.add_species( - self.beam_ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, - n_macroparticles_per_cell=self.NPPC[self.dim-1]/2 - ) - ) - - ####################################################################### - # Add diagnostics # - ####################################################################### - - callbacks.installafterstep(self.energy_diagnostic) - callbacks.installafterstep(self.text_diag) - - if self.test: - part_diag = picmi.ParticleDiagnostic( - name='diag1', - period=1250, - species=[self.ions, self.beam_ions], - data_list = ['ux', 'uy', 'uz', 'z', 'weighting'], - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', - ) - simulation.add_diagnostic(part_diag) - field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=self.grid, - period=1250, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', - ) - simulation.add_diagnostic(field_diag) - - # output the full particle data at t*w_ci = 40 - step = int(40.0 / self.DT) - parts_diag = picmi.ParticleDiagnostic( - name='parts_diag', - period=f"{step}:{step}", - species=[self.ions, self.beam_ions], - write_dir='diags', - warpx_file_prefix='Python_hybrid_PIC_plt', - warpx_format = 'openpmd', - warpx_openpmd_backend = 'h5' - ) - simulation.add_diagnostic(parts_diag) - - self.output_file_name = 'field_data.txt' - if self.dim == 1: - line_diag = picmi.ReducedDiagnostic( - diag_type='FieldProbe', - probe_geometry='Line', - z_probe=0, - z1_probe=self.Lz, - resolution=self.Nz - 1, - name=self.output_file_name[:-4], - period=self.diag_steps, - path='diags/' - ) - simulation.add_diagnostic(line_diag) - else: - # install a custom "reduced diagnostic" to save the average field - callbacks.installafterEsolve(self._record_average_fields) - try: - os.mkdir("diags") - except OSError: - # diags directory already exists - pass - with open(f"diags/{self.output_file_name}", 'w') as f: - f.write("[0]step() [1]time(s) [2]z_coord(m) [3]By_lev0-(T)\n") - - - ####################################################################### - # Initialize simulation # - ####################################################################### - - simulation.initialize_inputs() - simulation.initialize_warpx() - - # create particle container wrapper for the ion species to access - # particle data - self.ion_container_wrapper = particle_containers.ParticleContainerWrapper( - self.ions.name - ) - self.beam_ion_container_wrapper = particle_containers.ParticleContainerWrapper( - self.beam_ions.name - ) - - def _create_data_arrays(self): - self.prev_time = time.time() - self.start_time = self.prev_time - self.prev_step = 0 - - if libwarpx.amr.ParallelDescriptor.MyProc() == 0: - # allocate arrays for storing energy values - self.energy_vals = np.zeros((self.total_steps//self.diag_steps, 4)) - - def text_diag(self): - """Diagnostic function to print out timing data and particle numbers.""" - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if not hasattr(self, "prev_time"): - self._create_data_arrays() - - if step % (self.total_steps // 10) != 0: - return - - wall_time = time.time() - self.prev_time - steps = step - self.prev_step - step_rate = steps / wall_time - - status_dict = { - 'step': step, - 'nplive beam ions': self.ion_container_wrapper.nps, - 'nplive ions': self.beam_ion_container_wrapper.nps, - 'wall_time': wall_time, - 'step_rate': step_rate, - "diag_steps": self.diag_steps, - 'iproc': None - } - - diag_string = ( - "Step #{step:6d}; " - "{nplive beam ions} beam ions; " - "{nplive ions} core ions; " - "{wall_time:6.1f} s wall time; " - "{step_rate:4.2f} steps/s" - ) - - if libwarpx.amr.ParallelDescriptor.MyProc() == 0: - print(diag_string.format(**status_dict)) - - self.prev_time = time.time() - self.prev_step = step - - def energy_diagnostic(self): - """Diagnostic to get the total, magnetic and kinetic energies in the - simulation.""" - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if step % self.diag_steps != 1: - return - - idx = (step - 1) // self.diag_steps - - if not hasattr(self, "prev_time"): - self._create_data_arrays() - - # get the simulation energies - Ec_par, Ec_perp = self._get_kinetic_energy(self.ion_container_wrapper) - Eb_par, Eb_perp = self._get_kinetic_energy(self.beam_ion_container_wrapper) - - if libwarpx.amr.ParallelDescriptor.MyProc() != 0: - return - - self.energy_vals[idx, 0] = Ec_par - self.energy_vals[idx, 1] = Ec_perp - self.energy_vals[idx, 2] = Eb_par - self.energy_vals[idx, 3] = Eb_perp - - if step == self.total_steps: - np.save('diags/energies.npy', run.energy_vals) - - def _get_kinetic_energy(self, container_wrapper): - """Utility function to retrieve the total kinetic energy in the - simulation.""" - try: - ux = np.concatenate(container_wrapper.get_particle_ux()) - uy = np.concatenate(container_wrapper.get_particle_uy()) - uz = np.concatenate(container_wrapper.get_particle_uz()) - w = np.concatenate(container_wrapper.get_particle_weight()) - except ValueError: - return 0.0, 0.0 - - my_E_perp = 0.5 * self.M * np.sum(w * (ux**2 + uy**2)) - E_perp = comm.allreduce(my_E_perp, op=mpi.SUM) - - my_E_par = 0.5 * self.M * np.sum(w * uz**2) - E_par = comm.allreduce(my_E_par, op=mpi.SUM) - - return E_par, E_perp - - def _record_average_fields(self): - """A custom reduced diagnostic to store the average E&M fields in a - similar format as the reduced diagnostic so that the same analysis - script can be used regardless of the simulation dimension. - """ - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if step % self.diag_steps != 0: - return - - By_warpx = fields.BxWrapper()[...] - - if libwarpx.amr.ParallelDescriptor.MyProc() != 0: - return - - t = step * self.dt - z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) - - if self.dim == 2: - By = np.mean(By_warpx[:-1], axis=0) - else: - By = np.mean(By_warpx[:-1], axis=(0, 1)) - - with open(f"diags/{self.output_file_name}", 'a') as f: - for ii in range(self.Nz): - f.write( - f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {By[ii]:+.10e}\n" - ) - - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-d', '--dim', help='Simulation dimension', required=False, type=int, - default=1 -) -parser.add_argument( - '-r', '--resonant', help='Run the resonant case', required=False, - action='store_true', -) -parser.add_argument( - '-v', '--verbose', help='Verbose output', action='store_true', -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -run = HybridPICBeamInstability( - test=args.test, dim=args.dim, resonant=args.resonant, verbose=args.verbose -) -simulation.step() diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst index 59469cf4aa9..49b5f1ac0a0 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -13,11 +13,11 @@ Run The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant condition as indicated below. -.. dropdown:: Script ``PICMI_inputs.py`` +.. dropdown:: Script ``inputs_test_1d_ohm_solver_ion_beam_picmi.py`` - .. literalinclude:: PICMI_inputs.py + .. literalinclude:: inputs_test_1d_ohm_solver_ion_beam_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py``. + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/inputs_test_1d_ohm_solver_ion_beam_picmi.py``. For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. @@ -29,7 +29,7 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. code-block:: bash - python3 PICMI_inputs.py -dim {1/2/3} --resonant + python3 inputs_test_1d_ohm_solver_ion_beam_picmi.py -dim {1/2/3} --resonant .. tab-item:: Non-resonant case @@ -37,7 +37,7 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. code-block:: bash - python3 PICMI_inputs.py -dim {1/2/3} + python3 inputs_test_1d_ohm_solver_ion_beam_picmi.py -dim {1/2/3} Analyze ------- diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index 6a57b1c1046..978b8b9a731 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -12,24 +12,24 @@ constants = picmi.constants -matplotlib.rcParams.update({'font.size': 20}) +matplotlib.rcParams.update({"font.size": 20}) # load simulation parameters -with open(f'sim_parameters.dpkl', 'rb') as f: +with open("sim_parameters.dpkl", "rb") as f: sim = dill.load(f) if sim.resonant: - resonant_str = 'resonant' + resonant_str = "resonant" else: - resonant_str = 'non resonant' + resonant_str = "non resonant" data = np.loadtxt("diags/field_data.txt", skiprows=1) if sim.dim == 1: - field_idx_dict = {'z': 4, 'By': 8} + field_idx_dict = {"z": 4, "By": 8} else: - field_idx_dict = {'z': 2, 'By': 3} + field_idx_dict = {"z": 2, "By": 3} -step = data[:,0] +step = data[:, 0] num_steps = len(np.unique(step)) @@ -37,16 +37,16 @@ resolution = len(np.where(step == 0)[0]) - 1 # reshape to separate spatial and time coordinates -sim_data = data.reshape((num_steps, resolution+1, data.shape[1])) +sim_data = data.reshape((num_steps, resolution + 1, data.shape[1])) -z_grid = sim_data[1, :, field_idx_dict['z']] +z_grid = sim_data[1, :, field_idx_dict["z"]] idx = np.argsort(z_grid)[1:] dz = np.mean(np.diff(z_grid[idx])) -dt = np.mean(np.diff(sim_data[:,0,1])) +dt = np.mean(np.diff(sim_data[:, 0, 1])) data = np.zeros((num_steps, resolution)) for i in range(num_steps): - data[i,:] = sim_data[i,idx,field_idx_dict['By']] + data[i, :] = sim_data[i, idx, field_idx_dict["By"]] print(f"Data file contains {num_steps} time snapshots.") print(f"Spatial resolution is {resolution}") @@ -54,36 +54,48 @@ # Create the stack time plot fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) -max_val = np.max(np.abs(data[:,:]/sim.B0)) +max_val = np.max(np.abs(data[:, :] / sim.B0)) -extent = [0, sim.Lz/sim.l_i, 0, num_steps*dt*sim.w_ci] # num_steps*dt/sim.t_ci] +extent = [0, sim.Lz / sim.l_i, 0, num_steps * dt * sim.w_ci] # num_steps*dt/sim.t_ci] im = ax1.imshow( - data[:,:]/sim.B0, extent=extent, origin='lower', - cmap='seismic', vmin=-max_val, vmax=max_val, aspect="equal", + data[:, :] / sim.B0, + extent=extent, + origin="lower", + cmap="seismic", + vmin=-max_val, + vmax=max_val, + aspect="equal", ) # Colorbar fig.subplots_adjust(right=0.825) cbar_ax = fig.add_axes([0.85, 0.2, 0.03, 0.6]) -fig.colorbar(im, cax=cbar_ax, orientation='vertical', label='$B_y/B_0$') +fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="$B_y/B_0$") ax1.set_xlabel("$x/l_i$") ax1.set_ylabel("$t \Omega_i$ (rad)") ax1.set_title(f"Ion beam R instability - {resonant_str} case") -plt.savefig(f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}.png") +plt.savefig( + f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}.png" +) plt.close() if sim.resonant: - # Plot the 4th, 5th and 6th Fourier modes field_kt = np.fft.fft(data[:, :], axis=1) - k = 2*np.pi * np.fft.fftfreq(resolution, dz) * sim.l_i + k = 2 * np.pi * np.fft.fftfreq(resolution, dz) * sim.l_i - t_grid = np.arange(num_steps)*dt*sim.w_ci - plt.plot(t_grid, np.abs(field_kt[:, 4] / sim.B0), 'r', label=f'm = 4, $kl_i={k[4]:.2f}$') - plt.plot(t_grid, np.abs(field_kt[:, 5] / sim.B0), 'b', label=f'm = 5, $kl_i={k[5]:.2f}$') - plt.plot(t_grid, np.abs(field_kt[:, 6] / sim.B0), 'k', label=f'm = 6, $kl_i={k[6]:.2f}$') + t_grid = np.arange(num_steps) * dt * sim.w_ci + plt.plot( + t_grid, np.abs(field_kt[:, 4] / sim.B0), "r", label=f"m = 4, $kl_i={k[4]:.2f}$" + ) + plt.plot( + t_grid, np.abs(field_kt[:, 5] / sim.B0), "b", label=f"m = 5, $kl_i={k[5]:.2f}$" + ) + plt.plot( + t_grid, np.abs(field_kt[:, 6] / sim.B0), "k", label=f"m = 6, $kl_i={k[6]:.2f}$" + ) # The theoretical growth rates for the 4th, 5th and 6th Fourier modes of # the By-field was obtained from Fig. 12a of Munoz et al. @@ -97,94 +109,117 @@ idx = np.where((t_grid > 10) & (t_grid < 40)) t_points = t_grid[idx] - A4 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 4] / sim.B0)) - t_points*gamma4)) - plt.plot(t_points, A4*np.exp(t_points*gamma4), 'r--', lw=3) - A5 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 5] / sim.B0)) - t_points*gamma5)) - plt.plot(t_points, A5*np.exp(t_points*gamma5), 'b--', lw=3) - A6 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 6] / sim.B0)) - t_points*gamma6)) - plt.plot(t_points, A6*np.exp(t_points*gamma6), 'k--', lw=3) + A4 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 4] / sim.B0)) - t_points * gamma4)) + plt.plot(t_points, A4 * np.exp(t_points * gamma4), "r--", lw=3) + A5 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 5] / sim.B0)) - t_points * gamma5)) + plt.plot(t_points, A5 * np.exp(t_points * gamma5), "b--", lw=3) + A6 = np.exp(np.mean(np.log(np.abs(field_kt[idx, 6] / sim.B0)) - t_points * gamma6)) + plt.plot(t_points, A6 * np.exp(t_points * gamma6), "k--", lw=3) plt.grid() plt.legend() - plt.yscale('log') - plt.ylabel('$|B_y/B_0|$') - plt.xlabel('$t\Omega_i$ (rad)') + plt.yscale("log") + plt.ylabel("$|B_y/B_0|$") + plt.xlabel("$t\Omega_i$ (rad)") plt.tight_layout() - plt.savefig(f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_low_modes.png") + plt.savefig( + f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_low_modes.png" + ) plt.close() # check if the growth rate matches expectation - m4_rms_error = np.sqrt(np.mean( - (np.abs(field_kt[idx, 4] / sim.B0) - A4*np.exp(t_points*gamma4))**2 - )) - m5_rms_error = np.sqrt(np.mean( - (np.abs(field_kt[idx, 5] / sim.B0) - A5*np.exp(t_points*gamma5))**2 - )) - m6_rms_error = np.sqrt(np.mean( - (np.abs(field_kt[idx, 6] / sim.B0) - A6*np.exp(t_points*gamma6))**2 - )) + m4_rms_error = np.sqrt( + np.mean( + (np.abs(field_kt[idx, 4] / sim.B0) - A4 * np.exp(t_points * gamma4)) ** 2 + ) + ) + m5_rms_error = np.sqrt( + np.mean( + (np.abs(field_kt[idx, 5] / sim.B0) - A5 * np.exp(t_points * gamma5)) ** 2 + ) + ) + m6_rms_error = np.sqrt( + np.mean( + (np.abs(field_kt[idx, 6] / sim.B0) - A6 * np.exp(t_points * gamma6)) ** 2 + ) + ) print("Growth rate RMS errors:") print(f" m = 4: {m4_rms_error:.3e}") print(f" m = 5: {m5_rms_error:.3e}") print(f" m = 6: {m6_rms_error:.3e}") if not sim.test: - with h5py.File('diags/Python_hybrid_PIC_plt/openpmd_004000.h5', 'r') as data: + with h5py.File("diags/Python_hybrid_PIC_plt/openpmd_004000.h5", "r") as data: + timestep = str(np.squeeze([key for key in data["data"].keys()])) - timestep = str(np.squeeze([key for key in data['data'].keys()])) - - z = np.array(data['data'][timestep]['particles']['ions']['position']['z']) - vy = np.array(data['data'][timestep]['particles']['ions']['momentum']['y']) - w = np.array(data['data'][timestep]['particles']['ions']['weighting']) + z = np.array(data["data"][timestep]["particles"]["ions"]["position"]["z"]) + vy = np.array(data["data"][timestep]["particles"]["ions"]["momentum"]["y"]) + w = np.array(data["data"][timestep]["particles"]["ions"]["weighting"]) fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) im = ax1.hist2d( - z/sim.l_i, vy/sim.M/sim.vA, weights=w, density=True, - range=[[0, 250], [-10, 10]], bins=250, cmin=1e-5 + z / sim.l_i, + vy / sim.M / sim.vA, + weights=w, + density=True, + range=[[0, 250], [-10, 10]], + bins=250, + cmin=1e-5, ) # Colorbar fig.subplots_adjust(bottom=0.15, right=0.815) cbar_ax = fig.add_axes([0.83, 0.2, 0.03, 0.6]) - fig.colorbar(im[3], cax=cbar_ax, orientation='vertical', format='%.0e', label='$f(z, v_y)$') + fig.colorbar( + im[3], cax=cbar_ax, orientation="vertical", format="%.0e", label="$f(z, v_y)$" + ) ax1.set_xlabel("$x/l_i$") ax1.set_ylabel("$v_{y}/v_A$") ax1.set_title(f"Ion beam R instability - {resonant_str} case") - plt.savefig(f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_core_phase_space.png") + plt.savefig( + f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_core_phase_space.png" + ) plt.close() - with h5py.File('diags/Python_hybrid_PIC_plt/openpmd_004000.h5', 'r') as data: + with h5py.File("diags/Python_hybrid_PIC_plt/openpmd_004000.h5", "r") as data: + timestep = str(np.squeeze([key for key in data["data"].keys()])) - timestep = str(np.squeeze([key for key in data['data'].keys()])) - - z = np.array(data['data'][timestep]['particles']['beam_ions']['position']['z']) - vy = np.array(data['data'][timestep]['particles']['beam_ions']['momentum']['y']) - w = np.array(data['data'][timestep]['particles']['beam_ions']['weighting']) + z = np.array(data["data"][timestep]["particles"]["beam_ions"]["position"]["z"]) + vy = np.array(data["data"][timestep]["particles"]["beam_ions"]["momentum"]["y"]) + w = np.array(data["data"][timestep]["particles"]["beam_ions"]["weighting"]) fig, ax1 = plt.subplots(1, 1, figsize=(10, 5)) im = ax1.hist2d( - z/sim.l_i, vy/sim.M/sim.vA, weights=w, density=True, - range=[[0, 250], [-10, 10]], bins=250, cmin=1e-5 + z / sim.l_i, + vy / sim.M / sim.vA, + weights=w, + density=True, + range=[[0, 250], [-10, 10]], + bins=250, + cmin=1e-5, ) # Colorbar fig.subplots_adjust(bottom=0.15, right=0.815) cbar_ax = fig.add_axes([0.83, 0.2, 0.03, 0.6]) - fig.colorbar(im[3], cax=cbar_ax, orientation='vertical', format='%.0e', label='$f(z, v_y)$') + fig.colorbar( + im[3], cax=cbar_ax, orientation="vertical", format="%.0e", label="$f(z, v_y)$" + ) ax1.set_xlabel("$x/l_i$") ax1.set_ylabel("$v_{y}/v_A$") ax1.set_title(f"Ion beam R instability - {resonant_str} case") - plt.savefig(f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_beam_phase_space.png") + plt.savefig( + f"diags/ion_beam_R_instability_{resonant_str}_eta_{sim.eta}_substeps_{sim.substeps}_beam_phase_space.png" + ) plt.show() if sim.test: - # physics based check - these error tolerances are not set from theory # but from the errors that were present when the test was created. If these # assert's fail, the full benchmark should be rerun (same as the test but @@ -195,14 +230,3 @@ assert np.isclose(m4_rms_error, 1.515, atol=0.01) assert np.isclose(m5_rms_error, 0.718, atol=0.01) assert np.isclose(m6_rms_error, 0.357, atol=0.01) - - # checksum check - import os - import sys - sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - import checksumAPI - - # this will be the name of the plot file - fn = sys.argv[1] - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis_default_regression.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/inputs_test_1d_ohm_solver_ion_beam_picmi.py b/Examples/Tests/ohm_solver_ion_beam_instability/inputs_test_1d_ohm_solver_ion_beam_picmi.py new file mode 100644 index 00000000000..52160038831 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/inputs_test_1d_ohm_solver_ion_beam_picmi.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script simulates an ion beam instability wherein a +# --- low density ion beam interacts with background plasma. See Section 6.5 of +# --- Matthews (1994) and Section 4.4 of Munoz et al. (2018). + +import argparse +import os +import sys +import time + +import dill +import numpy as np +from mpi4py import MPI as mpi + +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=0) + + +class HybridPICBeamInstability(object): + """This input is based on the ion beam R instability test as described by + Munoz et al. (2018). + """ + + # Applied field parameters + B0 = 0.25 # Initial magnetic field strength (T) + beta = 1.0 # Plasma beta, used to calculate temperature + + # Plasma species parameters + m_ion = 100.0 # Ion mass (electron masses) + vA_over_c = 1e-4 # ratio of Alfven speed and the speed of light + + # Spatial domain + Nz = 1024 # number of cells in z direction + Nx = 8 # number of cells in x (and y) direction for >1 dimensions + + # Temporal domain (if not run as a CI test) + LT = 120.0 # Simulation temporal length (ion cyclotron periods) + + # Numerical parameters + NPPC = [1024, 256, 64] # Seed number of particles per cell + DZ = 1.0 / 4.0 # Cell size (ion skin depths) + DT = 0.01 # Time step (ion cyclotron periods) + + # Plasma resistivity - used to dampen the mode excitation + eta = 1e-7 + # Number of substeps used to update B + substeps = 10 + + # Beam parameters + n_beam = [0.02, 0.1] + U_bc = 10.0 # relative drifts between beam and core in Alfven speeds + + def __init__(self, test, dim, resonant, verbose): + """Get input parameters for the specific case desired.""" + self.test = test + self.dim = int(dim) + self.resonant = resonant + self.verbose = verbose or self.test + + # sanity check + assert dim > 0 and dim < 4, f"{dim}-dimensions not a valid input" + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.n_beam = self.n_beam[1 - int(resonant)] + self.u_beam = 1.0 / (1.0 + self.n_beam) * self.U_bc * self.vA + self.u_c = -1.0 * self.n_beam / (1.0 + self.n_beam) * self.U_bc * self.vA + self.n_beam = self.n_beam * self.n_plasma + + self.dz = self.DZ * self.l_i + self.Lz = self.Nz * self.dz + self.Lx = self.Nx * self.dz + + if self.dim == 3: + self.volume = self.Lx * self.Lx * self.Lz + self.N_cells = self.Nx * self.Nx * self.Nz + elif self.dim == 2: + self.volume = self.Lx * self.Lz + self.N_cells = self.Nx * self.Nz + else: + self.volume = self.Lz + self.N_cells = self.Nz + + diag_period = 1 / 4.0 # Output interval (ion cyclotron periods) + self.diag_steps = int(diag_period / self.DT) + + # if this is a test case run for only 25 cyclotron periods + if self.test: + self.LT = 25.0 + + self.total_steps = int(np.ceil(self.LT / self.DT)) + + self.dt = self.DT / self.w_ci + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT = {self.T_plasma * 1e-3:.1f} keV\n" + f"\tn = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.v_ti:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.dz:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = self.vA_over_c * constants.c + self.n_plasma = (self.B0 / self.vA) ** 2 / ( + constants.mu0 * (self.M + constants.m_e) + ) + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n_plasma / (self.M * constants.ep0)) + + # Skin depth (m) + self.l_i = constants.c / self.w_pi + + # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 + self.v_ti = np.sqrt(self.beta / 2.0) * self.vA + + # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) + self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV + + # Larmor radius (m) + self.rho_i = self.v_ti / self.w_ci + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + if self.dim == 1: + grid_object = picmi.Cartesian1DGrid + elif self.dim == 2: + grid_object = picmi.Cartesian2DGrid + else: + grid_object = picmi.Cartesian3DGrid + + self.grid = grid_object( + number_of_cells=[self.Nx, self.Nx, self.Nz][-self.dim :], + warpx_max_grid_size=self.Nz, + lower_bound=[-self.Lx / 2.0, -self.Lx / 2.0, 0][-self.dim :], + upper_bound=[self.Lx / 2.0, self.Lx / 2.0, self.Lz][-self.dim :], + lower_boundary_conditions=["periodic"] * self.dim, + upper_boundary_conditions=["periodic"] * self.dim, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + gamma=1.0, + Te=self.T_plasma / 10.0, + n0=self.n_plasma + self.n_beam, + plasma_resistivity=self.eta, + substeps=self.substeps, + ) + simulation.solver = self.solver + + B_ext = picmi.AnalyticInitialField( + Bx_expression=0.0, By_expression=0.0, Bz_expression=self.B0 + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_plasma, + rms_velocity=[self.v_ti] * 3, + directed_velocity=[0, 0, self.u_c], + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC[self.dim - 1] + ), + ) + self.beam_ions = picmi.Species( + name="beam_ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_beam, + rms_velocity=[self.v_ti] * 3, + directed_velocity=[0, 0, self.u_beam], + ), + ) + simulation.add_species( + self.beam_ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC[self.dim - 1] / 2 + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + callbacks.installafterstep(self.energy_diagnostic) + callbacks.installafterstep(self.text_diag) + + if self.test: + part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=1250, + species=[self.ions, self.beam_ions], + data_list=["ux", "uy", "uz", "z", "weighting"], + ) + simulation.add_diagnostic(part_diag) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=1250, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], + ) + simulation.add_diagnostic(field_diag) + + # output the full particle data at t*w_ci = 40 + step = int(40.0 / self.DT) + parts_diag = picmi.ParticleDiagnostic( + name="parts_diag", + period=f"{step}:{step}", + species=[self.ions, self.beam_ions], + write_dir="diags", + warpx_file_prefix="Python_hybrid_PIC_plt", + warpx_format="openpmd", + warpx_openpmd_backend="h5", + ) + simulation.add_diagnostic(parts_diag) + + self.output_file_name = "field_data.txt" + if self.dim == 1: + line_diag = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + probe_geometry="Line", + z_probe=0, + z1_probe=self.Lz, + resolution=self.Nz - 1, + name=self.output_file_name[:-4], + period=self.diag_steps, + path="diags/", + ) + simulation.add_diagnostic(line_diag) + else: + # install a custom "reduced diagnostic" to save the average field + callbacks.installafterEsolve(self._record_average_fields) + try: + os.mkdir("diags") + except OSError: + # diags directory already exists + pass + with open(f"diags/{self.output_file_name}", "w") as f: + f.write("[0]step() [1]time(s) [2]z_coord(m) [3]By_lev0-(T)\n") + + ####################################################################### + # Initialize simulation # + ####################################################################### + + simulation.initialize_inputs() + simulation.initialize_warpx() + + # create particle container wrapper for the ion species to access + # particle data + self.ion_container_wrapper = particle_containers.ParticleContainerWrapper( + self.ions.name + ) + self.beam_ion_container_wrapper = particle_containers.ParticleContainerWrapper( + self.beam_ions.name + ) + + def _create_data_arrays(self): + self.prev_time = time.time() + self.start_time = self.prev_time + self.prev_step = 0 + + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: + # allocate arrays for storing energy values + self.energy_vals = np.zeros((self.total_steps // self.diag_steps, 4)) + + def text_diag(self): + """Diagnostic function to print out timing data and particle numbers.""" + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if not hasattr(self, "prev_time"): + self._create_data_arrays() + + if step % (self.total_steps // 10) != 0: + return + + wall_time = time.time() - self.prev_time + steps = step - self.prev_step + step_rate = steps / wall_time + + status_dict = { + "step": step, + "nplive beam ions": self.ion_container_wrapper.nps, + "nplive ions": self.beam_ion_container_wrapper.nps, + "wall_time": wall_time, + "step_rate": step_rate, + "diag_steps": self.diag_steps, + "iproc": None, + } + + diag_string = ( + "Step #{step:6d}; " + "{nplive beam ions} beam ions; " + "{nplive ions} core ions; " + "{wall_time:6.1f} s wall time; " + "{step_rate:4.2f} steps/s" + ) + + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: + print(diag_string.format(**status_dict)) + + self.prev_time = time.time() + self.prev_step = step + + def energy_diagnostic(self): + """Diagnostic to get the total, magnetic and kinetic energies in the + simulation.""" + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if step % self.diag_steps != 1: + return + + idx = (step - 1) // self.diag_steps + + if not hasattr(self, "prev_time"): + self._create_data_arrays() + + # get the simulation energies + Ec_par, Ec_perp = self._get_kinetic_energy(self.ion_container_wrapper) + Eb_par, Eb_perp = self._get_kinetic_energy(self.beam_ion_container_wrapper) + + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: + return + + self.energy_vals[idx, 0] = Ec_par + self.energy_vals[idx, 1] = Ec_perp + self.energy_vals[idx, 2] = Eb_par + self.energy_vals[idx, 3] = Eb_perp + + if step == self.total_steps: + np.save("diags/energies.npy", run.energy_vals) + + def _get_kinetic_energy(self, container_wrapper): + """Utility function to retrieve the total kinetic energy in the + simulation.""" + try: + ux = np.concatenate(container_wrapper.get_particle_ux()) + uy = np.concatenate(container_wrapper.get_particle_uy()) + uz = np.concatenate(container_wrapper.get_particle_uz()) + w = np.concatenate(container_wrapper.get_particle_weight()) + except ValueError: + return 0.0, 0.0 + + my_E_perp = 0.5 * self.M * np.sum(w * (ux**2 + uy**2)) + E_perp = comm.allreduce(my_E_perp, op=mpi.SUM) + + my_E_par = 0.5 * self.M * np.sum(w * uz**2) + E_par = comm.allreduce(my_E_par, op=mpi.SUM) + + return E_par, E_perp + + def _record_average_fields(self): + """A custom reduced diagnostic to store the average E&M fields in a + similar format as the reduced diagnostic so that the same analysis + script can be used regardless of the simulation dimension. + """ + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if step % self.diag_steps != 0: + return + + By_warpx = fields.BxWrapper()[...] + + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: + return + + t = step * self.dt + z_vals = np.linspace(0, self.Lz, self.Nz, endpoint=False) + + if self.dim == 2: + By = np.mean(By_warpx[:-1], axis=0) + else: + By = np.mean(By_warpx[:-1], axis=(0, 1)) + + with open(f"diags/{self.output_file_name}", "a") as f: + for ii in range(self.Nz): + f.write(f"{step:05d} {t:.10e} {z_vals[ii]:.10e} {By[ii]:+.10e}\n") + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-d", "--dim", help="Simulation dimension", required=False, type=int, default=1 +) +parser.add_argument( + "-r", + "--resonant", + help="Run the resonant case", + required=False, + action="store_true", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = HybridPICBeamInstability( + test=args.test, dim=args.dim, resonant=args.resonant, verbose=args.verbose +) +simulation.step() diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt b/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt new file mode 100644 index 00000000000..02b4f5e3cb9 --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_ohm_solver_magnetic_reconnection_picmi # name + 2 # dims + 2 # nprocs + "inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py --test" # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py deleted file mode 100644 index 56173893b7f..00000000000 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are -# --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script demonstrates the use of this model to -# --- simulate magnetic reconnection in a force-free sheet. The setup is based -# --- on the problem described in Le et al. (2016) -# --- https://aip.scitation.org/doi/10.1063/1.4943893. - -import argparse -import shutil -import sys -from pathlib import Path - -import dill -import numpy as np -from mpi4py import MPI as mpi - -from pywarpx import callbacks, fields, libwarpx, picmi - -constants = picmi.constants - -comm = mpi.COMM_WORLD - -simulation = picmi.Simulation( - warpx_serialize_initial_conditions=True, - verbose=0 -) - - -class ForceFreeSheetReconnection(object): - - # B0 is chosen with all other quantities scaled by it - B0 = 0.1 # Initial magnetic field strength (T) - - # Physical parameters - m_ion = 400.0 # Ion mass (electron masses) - - beta_e = 0.1 - Bg = 0.3 # times B0 - guiding field - dB = 0.01 # times B0 - initial perturbation to seed reconnection - - T_ratio = 5.0 # T_i / T_e - - # Domain parameters - LX = 40 # ion skin depths - LZ = 20 # ion skin depths - - LT = 50 # ion cyclotron periods - DT = 1e-3 # ion cyclotron periods - - # Resolution parameters - NX = 512 - NZ = 512 - - # Starting number of particles per cell - NPPC = 400 - - # Plasma resistivity - used to dampen the mode excitation - eta = 6e-3 # normalized resistivity - # Number of substeps used to update B - substeps = 20 - - def __init__(self, test, verbose): - - self.test = test - self.verbose = verbose or self.test - - # calculate various plasma parameters based on the simulation input - self.get_plasma_quantities() - - self.Lx = self.LX * self.l_i - self.Lz = self.LZ * self.l_i - - self.dt = self.DT * self.t_ci - - # run very low resolution as a CI test - if self.test: - self.total_steps = 20 - self.diag_steps = self.total_steps // 5 - self.NX = 128 - self.NZ = 128 - else: - self.total_steps = int(self.LT / self.DT) - self.diag_steps = self.total_steps // 200 - - # Initial magnetic field - self.Bg *= self.B0 - self.dB *= self.B0 - self.Bx = ( - f"{self.B0}*tanh(z*{1.0/self.l_i})" - f"+{-self.dB*self.Lx/(2.0*self.Lz)}*cos({2.0*np.pi/self.Lx}*x)" - f"*sin({np.pi/self.Lz}*z)" - ) - self.By = ( - f"sqrt({self.Bg**2 + self.B0**2}-" - f"({self.B0}*tanh(z*{1.0/self.l_i}))**2)" - ) - self.Bz = f"{self.dB}*sin({2.0*np.pi/self.Lx}*x)*cos({np.pi/self.Lz}*z)" - - self.J0 = self.B0 / constants.mu0 / self.l_i - - # dump all the current attributes to a dill pickle file - if comm.rank == 0: - with open(f'sim_parameters.dpkl', 'wb') as f: - dill.dump(self, f) - - # print out plasma parameters - if comm.rank == 0: - print( - f"Initializing simulation with input parameters:\n" - f"\tTi = {self.Ti*1e-3:.1f} keV\n" - f"\tn0 = {self.n_plasma:.1e} m^-3\n" - f"\tB0 = {self.B0:.2f} T\n" - f"\tM/m = {self.m_ion:.0f}\n" - ) - print( - f"Plasma parameters:\n" - f"\tl_i = {self.l_i:.1e} m\n" - f"\tt_ci = {self.t_ci:.1e} s\n" - f"\tv_ti = {self.vi_th:.1e} m/s\n" - f"\tvA = {self.vA:.1e} m/s\n" - ) - print( - f"Numerical parameters:\n" - f"\tdz = {self.Lz/self.NZ:.1e} m\n" - f"\tdt = {self.dt:.1e} s\n" - f"\tdiag steps = {self.diag_steps:d}\n" - f"\ttotal steps = {self.total_steps:d}\n" - ) - - self.setup_run() - - def get_plasma_quantities(self): - """Calculate various plasma parameters based on the simulation input.""" - - # Ion mass (kg) - self.M = self.m_ion * constants.m_e - - # Cyclotron angular frequency (rad/s) and period (s) - self.w_ce = constants.q_e * abs(self.B0) / constants.m_e - self.w_ci = constants.q_e * abs(self.B0) / self.M - self.t_ci = 2.0 * np.pi / self.w_ci - - # Electron plasma frequency: w_pe / omega_ce = 2 is given - self.w_pe = 2.0 * self.w_ce - - # calculate plasma density based on electron plasma frequency - self.n_plasma = ( - self.w_pe**2 * constants.m_e * constants.ep0 / constants.q_e**2 - ) - - # Ion plasma frequency (Hz) - self.w_pi = np.sqrt( - constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) - ) - - # Ion skin depth (m) - self.l_i = constants.c / self.w_pi - - # # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi - self.vA = abs(self.B0) / np.sqrt( - constants.mu0 * self.n_plasma * (constants.m_e + self.M) - ) - - # calculate Te based on beta - self.Te = ( - self.beta_e * self.B0**2 / (2.0 * constants.mu0 * self.n_plasma) - / constants.q_e - ) - self.Ti = self.Te * self.T_ratio - - # calculate thermal speeds - self.ve_th = np.sqrt(self.Te * constants.q_e / constants.m_e) - self.vi_th = np.sqrt(self.Ti * constants.q_e / self.M) - - # Ion Larmor radius (m) - self.rho_i = self.vi_th / self.w_ci - - # Reference resistivity (Malakit et al.) - self.eta0 = self.l_i * self.vA / (constants.ep0 * constants.c**2) - - def setup_run(self): - """Setup simulation components.""" - - ####################################################################### - # Set geometry and boundary conditions # - ####################################################################### - - # Create grid - self.grid = picmi.Cartesian2DGrid( - number_of_cells=[self.NX, self.NZ], - lower_bound=[-self.Lx/2.0, -self.Lz/2.0], - upper_bound=[self.Lx/2.0, self.Lz/2.0], - lower_boundary_conditions=['periodic', 'dirichlet'], - upper_boundary_conditions=['periodic', 'dirichlet'], - lower_boundary_conditions_particles=['periodic', 'reflecting'], - upper_boundary_conditions_particles=['periodic', 'reflecting'], - warpx_max_grid_size=self.NZ - ) - simulation.time_step_size = self.dt - simulation.max_steps = self.total_steps - simulation.current_deposition_algo = 'direct' - simulation.particle_shape = 1 - simulation.use_filter = False - simulation.verbose = self.verbose - - ####################################################################### - # Field solver and external field # - ####################################################################### - - self.solver = picmi.HybridPICSolver( - grid=self.grid, gamma=1.0, - Te=self.Te, n0=self.n_plasma, n_floor=0.1*self.n_plasma, - plasma_resistivity=self.eta*self.eta0, - substeps=self.substeps - ) - simulation.solver = self.solver - - B_ext = picmi.AnalyticInitialField( - Bx_expression=self.Bx, - By_expression=self.By, - Bz_expression=self.Bz - ) - simulation.add_applied_field(B_ext) - - ####################################################################### - # Particle types setup # - ####################################################################### - - self.ions = picmi.Species( - name='ions', charge='q_e', mass=self.M, - initial_distribution=picmi.UniformDistribution( - density=self.n_plasma, - rms_velocity=[self.vi_th]*3, - ) - ) - simulation.add_species( - self.ions, - layout=picmi.PseudoRandomLayout( - grid=self.grid, - n_macroparticles_per_cell=self.NPPC - ) - ) - - ####################################################################### - # Add diagnostics # - ####################################################################### - - callbacks.installafterEsolve(self.check_fields) - - if self.test: - particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=self.total_steps, - write_dir='.', - species=[self.ions], - data_list=['ux', 'uy', 'uz', 'x', 'z', 'weighting'], - warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', - # warpx_format='openpmd', - # warpx_openpmd_backend='h5', - ) - simulation.add_diagnostic(particle_diag) - field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=self.grid, - period=self.total_steps, - data_list=['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez'], - write_dir='.', - warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', - # warpx_format='openpmd', - # warpx_openpmd_backend='h5', - ) - simulation.add_diagnostic(field_diag) - - - # reduced diagnostics for reconnection rate calculation - # create a 2 l_i box around the X-point on which to measure - # magnetic flux changes - plane = picmi.ReducedDiagnostic( - diag_type="FieldProbe", - name='plane', - period=self.diag_steps, - path='diags/', - extension='dat', - probe_geometry='Plane', - resolution=60, - x_probe=0.0, z_probe=0.0, detector_radius=self.l_i, - target_up_x=0, target_up_z=1.0 - ) - simulation.add_diagnostic(plane) - - ####################################################################### - # Initialize # - ####################################################################### - - if comm.rank == 0: - if Path.exists(Path("diags")): - shutil.rmtree("diags") - Path("diags/fields").mkdir(parents=True, exist_ok=True) - - # Initialize inputs and WarpX instance - simulation.initialize_inputs() - simulation.initialize_warpx() - - def check_fields(self): - - step = simulation.extension.warpx.getistep(lev=0) - 1 - - if not (step == 1 or step%self.diag_steps == 0): - return - - rho = fields.RhoFPWrapper(include_ghosts=False)[:,:] - Jiy = fields.JyFPWrapper(include_ghosts=False)[...] / self.J0 - Jy = fields.JyFPAmpereWrapper(include_ghosts=False)[...] / self.J0 - Bx = fields.BxFPWrapper(include_ghosts=False)[...] / self.B0 - By = fields.ByFPWrapper(include_ghosts=False)[...] / self.B0 - Bz = fields.BzFPWrapper(include_ghosts=False)[...] / self.B0 - - if libwarpx.amr.ParallelDescriptor.MyProc() != 0: - return - - # save the fields to file - with open(f"diags/fields/fields_{step:06d}.npz", 'wb') as f: - np.savez(f, rho=rho, Jiy=Jiy, Jy=Jy, Bx=Bx, By=By, Bz=Bz) - -########################## -# parse input parameters -########################## - -parser = argparse.ArgumentParser() -parser.add_argument( - '-t', '--test', help='toggle whether this script is run as a short CI test', - action='store_true', -) -parser.add_argument( - '-v', '--verbose', help='Verbose output', action='store_true', -) -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1]+left - -run = ForceFreeSheetReconnection(test=args.test, verbose=args.verbose) -simulation.step() diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst index 943b5bd0248..5181a6381d8 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -12,11 +12,11 @@ Run The following **Python** script configures and launches the simulation. -.. dropdown:: Script ``PICMI_inputs.py`` +.. dropdown:: Script ``inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py`` - .. literalinclude:: PICMI_inputs.py + .. literalinclude:: inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py``. Running the full simulation should take about 4 hours if executed on 1 V100 GPU. For `MPI-parallel `__ runs, prefix these lines with @@ -24,7 +24,7 @@ For `MPI-parallel `__ runs, prefix these lines with .. code-block:: bash - python3 PICMI_inputs.py + python3 inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py Analyze ------- diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py index 0fb1c05ae1a..84ab7140fd4 100755 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py @@ -9,10 +9,10 @@ import numpy as np from matplotlib import colors -plt.rcParams.update({'font.size': 20}) +plt.rcParams.update({"font.size": 20}) # load simulation parameters -with open(f'sim_parameters.dpkl', 'rb') as f: +with open("sim_parameters.dpkl", "rb") as f: sim = dill.load(f) x_idx = 2 @@ -20,11 +20,11 @@ Ey_idx = 6 Bx_idx = 8 -plane_data = np.loadtxt(f'diags/plane.dat', skiprows=1) +plane_data = np.loadtxt("diags/plane.dat", skiprows=1) -steps = np.unique(plane_data[:,0]) +steps = np.unique(plane_data[:, 0]) num_steps = len(steps) -num_cells = plane_data.shape[0]//num_steps +num_cells = plane_data.shape[0] // num_steps plane_data = plane_data.reshape((num_steps, num_cells, plane_data.shape[1])) @@ -33,13 +33,13 @@ plt.plot( times / sim.t_ci, - np.mean(plane_data[:,:,Ey_idx], axis=1) / (sim.vA * sim.B0), - 'o-' + np.mean(plane_data[:, :, Ey_idx], axis=1) / (sim.vA * sim.B0), + "o-", ) plt.grid() -plt.xlabel(r'$t/\tau_{c,i}$') -plt.ylabel('$/v_AB_0$') +plt.xlabel(r"$t/\tau_{c,i}$") +plt.ylabel("$/v_AB_0$") plt.title("Reconnection rate") plt.tight_layout() plt.savefig("diags/reconnection_rate.png") @@ -52,10 +52,10 @@ fig, axes = plt.subplots(3, 1, sharex=True, figsize=(7, 9)) for ax in axes.flatten(): - ax.set_aspect('equal') - ax.set_ylabel('$z/l_i$') + ax.set_aspect("equal") + ax.set_ylabel("$z/l_i$") - axes[2].set_xlabel('$x/l_i$') + axes[2].set_xlabel("$x/l_i$") datafiles = sorted(glob.glob("diags/fields/*.npz")) num_steps = len(datafiles) @@ -63,47 +63,52 @@ data0 = np.load(datafiles[0]) sX = axes[0].imshow( - data0['Jy'].T, origin='lower', - norm=colors.TwoSlopeNorm(vmin=-0.6, vcenter=0., vmax=1.6), - extent=[0, sim.LX, -sim.LZ/2, sim.LZ/2], - cmap=plt.cm.RdYlBu_r + data0["Jy"].T, + origin="lower", + norm=colors.TwoSlopeNorm(vmin=-0.6, vcenter=0.0, vmax=1.6), + extent=[0, sim.LX, -sim.LZ / 2, sim.LZ / 2], + cmap=plt.cm.RdYlBu_r, ) # axes[0].set_ylim(-5, 5) - cb = plt.colorbar(sX, ax=axes[0], label='$J_y/J_0$') - cb.ax.set_yscale('linear') + cb = plt.colorbar(sX, ax=axes[0], label="$J_y/J_0$") + cb.ax.set_yscale("linear") cb.ax.set_yticks([-0.5, 0.0, 0.75, 1.5]) sY = axes[1].imshow( - data0['By'].T, origin='lower', extent=[0, sim.LX, -sim.LZ/2, sim.LZ/2], - cmap=plt.cm.plasma + data0["By"].T, + origin="lower", + extent=[0, sim.LX, -sim.LZ / 2, sim.LZ / 2], + cmap=plt.cm.plasma, ) # axes[1].set_ylim(-5, 5) - cb = plt.colorbar(sY, ax=axes[1], label='$B_y/B_0$') - cb.ax.set_yscale('linear') + cb = plt.colorbar(sY, ax=axes[1], label="$B_y/B_0$") + cb.ax.set_yscale("linear") sZ = axes[2].imshow( - data0['Bz'].T, origin='lower', extent=[0, sim.LX, -sim.LZ/2, sim.LZ/2], + data0["Bz"].T, + origin="lower", + extent=[0, sim.LX, -sim.LZ / 2, sim.LZ / 2], # norm=colors.TwoSlopeNorm(vmin=-0.02, vcenter=0., vmax=0.02), - cmap=plt.cm.RdBu + cmap=plt.cm.RdBu, ) - cb = plt.colorbar(sZ, ax=axes[2], label='$B_z/B_0$') - cb.ax.set_yscale('linear') + cb = plt.colorbar(sZ, ax=axes[2], label="$B_z/B_0$") + cb.ax.set_yscale("linear") # plot field lines - x_grid = np.linspace(0, sim.LX, data0['Bx'][:-1].shape[0]) - z_grid = np.linspace(-sim.LZ/2.0, sim.LZ/2.0, data0['Bx'].shape[1]) + x_grid = np.linspace(0, sim.LX, data0["Bx"][:-1].shape[0]) + z_grid = np.linspace(-sim.LZ / 2.0, sim.LZ / 2.0, data0["Bx"].shape[1]) n_lines = 10 start_x = np.zeros(n_lines) - start_x[:n_lines//2] = sim.LX - start_z = np.linspace(-sim.LZ/2.0*0.9, sim.LZ/2.0*0.9, n_lines) + start_x[: n_lines // 2] = sim.LX + start_z = np.linspace(-sim.LZ / 2.0 * 0.9, sim.LZ / 2.0 * 0.9, n_lines) step_size = 1.0 / 100.0 def get_field_lines(Bx, Bz): field_line_coords = [] Bx_interp = interpolate.interp2d(x_grid, z_grid, Bx[:-1].T) - Bz_interp = interpolate.interp2d(x_grid, z_grid, Bz[:,:-1].T) + Bz_interp = interpolate.interp2d(x_grid, z_grid, Bz[:, :-1].T) for kk, z in enumerate(start_z): path_x = [start_x[kk]] @@ -111,7 +116,7 @@ def get_field_lines(Bx, Bz): ii = 0 while ii < 10000: - ii+=1 + ii += 1 Bx = Bx_interp(path_x[-1], path_z[-1])[0] Bz = Bz_interp(path_x[-1], path_z[-1])[0] @@ -128,7 +133,12 @@ def get_field_lines(Bx, Bz): x_new = path_x[-1] + dx z_new = path_z[-1] + dz - if np.isnan(x_new) or x_new <= 0 or x_new > sim.LX or abs(z_new) > sim.LZ/2: + if ( + np.isnan(x_new) + or x_new <= 0 + or x_new > sim.LX + or abs(z_new) > sim.LZ / 2 + ): break path_x.append(x_new) @@ -138,47 +148,39 @@ def get_field_lines(Bx, Bz): return field_line_coords field_lines = [] - for path in get_field_lines(data0['Bx'], data0['Bz']): + for path in get_field_lines(data0["Bx"], data0["Bz"]): path_x = path[0] path_z = path[1] - l, = axes[2].plot(path_x, path_z, '--', color='k') + (ln,) = axes[2].plot(path_x, path_z, "--", color="k") # draws arrows on the field lines # if path_x[10] > path_x[0]: axes[2].arrow( - path_x[50], path_z[50], - path_x[250]-path_x[50], path_z[250]-path_z[50], - shape='full', length_includes_head=True, lw=0, head_width=1.0, - color='g' + path_x[50], + path_z[50], + path_x[250] - path_x[50], + path_z[250] - path_z[50], + shape="full", + length_includes_head=True, + lw=0, + head_width=1.0, + color="g", ) - field_lines.append(l) + field_lines.append(ln) def animate(i): data = np.load(datafiles[i]) - sX.set_array(data['Jy'].T) - sY.set_array(data['By'].T) - sZ.set_array(data['Bz'].T) - sZ.set_clim(-np.max(abs(data['Bz'])), np.max(abs(data['Bz']))) + sX.set_array(data["Jy"].T) + sY.set_array(data["By"].T) + sZ.set_array(data["Bz"].T) + sZ.set_clim(-np.max(abs(data["Bz"])), np.max(abs(data["Bz"]))) - for ii, path in enumerate(get_field_lines(data['Bx'], data['Bz'])): + for ii, path in enumerate(get_field_lines(data["Bx"], data["Bz"])): path_x = path[0] path_z = path[1] field_lines[ii].set_data(path_x, path_z) - anim = FuncAnimation( - fig, animate, frames=num_steps-1, repeat=True - ) + anim = FuncAnimation(fig, animate, frames=num_steps - 1, repeat=True) writervideo = FFMpegWriter(fps=14) - anim.save('diags/mag_reconnection.mp4', writer=writervideo) - -if sim.test: - import os - import sys - sys.path.insert(1, '../../../../warpx/Regression/Checksum/') - import checksumAPI - - # this will be the name of the plot file - fn = sys.argv[1] - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn) + anim.save("diags/mag_reconnection.mp4", writer=writervideo) diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/analysis_default_regression.py b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py b/Examples/Tests/ohm_solver_magnetic_reconnection/inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py new file mode 100644 index 00000000000..2ddbd0df93d --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script demonstrates the use of this model to +# --- simulate magnetic reconnection in a force-free sheet. The setup is based +# --- on the problem described in Le et al. (2016) +# --- https://aip.scitation.org/doi/10.1063/1.4943893. + +import argparse +import shutil +import sys +from pathlib import Path + +import dill +import numpy as np +from mpi4py import MPI as mpi + +from pywarpx import callbacks, fields, libwarpx, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(warpx_serialize_initial_conditions=True, verbose=0) + + +class ForceFreeSheetReconnection(object): + # B0 is chosen with all other quantities scaled by it + B0 = 0.1 # Initial magnetic field strength (T) + + # Physical parameters + m_ion = 400.0 # Ion mass (electron masses) + + beta_e = 0.1 + Bg = 0.3 # times B0 - guiding field + dB = 0.01 # times B0 - initial perturbation to seed reconnection + + T_ratio = 5.0 # T_i / T_e + + # Domain parameters + LX = 40 # ion skin depths + LZ = 20 # ion skin depths + + LT = 50 # ion cyclotron periods + DT = 1e-3 # ion cyclotron periods + + # Resolution parameters + NX = 512 + NZ = 512 + + # Starting number of particles per cell + NPPC = 400 + + # Plasma resistivity - used to dampen the mode excitation + eta = 6e-3 # normalized resistivity + # Number of substeps used to update B + substeps = 20 + + def __init__(self, test, verbose): + self.test = test + self.verbose = verbose or self.test + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + self.Lx = self.LX * self.l_i + self.Lz = self.LZ * self.l_i + + self.dt = self.DT * self.t_ci + + # run very low resolution as a CI test + if self.test: + self.total_steps = 20 + self.diag_steps = self.total_steps // 5 + self.NX = 128 + self.NZ = 128 + else: + self.total_steps = int(self.LT / self.DT) + self.diag_steps = self.total_steps // 200 + + # Initial magnetic field + self.Bg *= self.B0 + self.dB *= self.B0 + self.Bx = ( + f"{self.B0}*tanh(z*{1.0 / self.l_i})" + f"+{-self.dB * self.Lx / (2.0 * self.Lz)}*cos({2.0 * np.pi / self.Lx}*x)" + f"*sin({np.pi / self.Lz}*z)" + ) + self.By = ( + f"sqrt({self.Bg**2 + self.B0**2}-({self.B0}*tanh(z*{1.0 / self.l_i}))**2)" + ) + self.Bz = f"{self.dB}*sin({2.0 * np.pi / self.Lx}*x)*cos({np.pi / self.Lz}*z)" + + self.J0 = self.B0 / constants.mu0 / self.l_i + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open("sim_parameters.dpkl", "wb") as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tTi = {self.Ti * 1e-3:.1f} keV\n" + f"\tn0 = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.vi_th:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdz = {self.Lz / self.NZ:.1e} m\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n" + ) + + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ce = constants.q_e * abs(self.B0) / constants.m_e + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Electron plasma frequency: w_pe / omega_ce = 2 is given + self.w_pe = 2.0 * self.w_ce + + # calculate plasma density based on electron plasma frequency + self.n_plasma = self.w_pe**2 * constants.m_e * constants.ep0 / constants.q_e**2 + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt(constants.q_e**2 * self.n_plasma / (self.M * constants.ep0)) + + # Ion skin depth (m) + self.l_i = constants.c / self.w_pi + + # # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = abs(self.B0) / np.sqrt( + constants.mu0 * self.n_plasma * (constants.m_e + self.M) + ) + + # calculate Te based on beta + self.Te = ( + self.beta_e + * self.B0**2 + / (2.0 * constants.mu0 * self.n_plasma) + / constants.q_e + ) + self.Ti = self.Te * self.T_ratio + + # calculate thermal speeds + self.ve_th = np.sqrt(self.Te * constants.q_e / constants.m_e) + self.vi_th = np.sqrt(self.Ti * constants.q_e / self.M) + + # Ion Larmor radius (m) + self.rho_i = self.vi_th / self.w_ci + + # Reference resistivity (Malakit et al.) + self.eta0 = self.l_i * self.vA / (constants.ep0 * constants.c**2) + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + # Create grid + self.grid = picmi.Cartesian2DGrid( + number_of_cells=[self.NX, self.NZ], + lower_bound=[-self.Lx / 2.0, -self.Lz / 2.0], + upper_bound=[self.Lx / 2.0, self.Lz / 2.0], + lower_boundary_conditions=["periodic", "dirichlet"], + upper_boundary_conditions=["periodic", "dirichlet"], + lower_boundary_conditions_particles=["periodic", "reflecting"], + upper_boundary_conditions_particles=["periodic", "reflecting"], + warpx_max_grid_size=self.NZ, + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = "direct" + simulation.particle_shape = 1 + simulation.use_filter = False + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + gamma=1.0, + Te=self.Te, + n0=self.n_plasma, + n_floor=0.1 * self.n_plasma, + plasma_resistivity=self.eta * self.eta0, + substeps=self.substeps, + ) + simulation.solver = self.solver + + B_ext = picmi.AnalyticInitialField( + Bx_expression=self.Bx, By_expression=self.By, Bz_expression=self.Bz + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.ions = picmi.Species( + name="ions", + charge="q_e", + mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_plasma, + rms_velocity=[self.vi_th] * 3, + ), + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ), + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + callbacks.installafterEsolve(self.check_fields) + + if self.test: + particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=self.total_steps, + species=[self.ions], + data_list=["ux", "uy", "uz", "x", "z", "weighting"], + # warpx_format='openpmd', + # warpx_openpmd_backend='h5', + ) + simulation.add_diagnostic(particle_diag) + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=self.total_steps, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez"], + # warpx_format='openpmd', + # warpx_openpmd_backend='h5', + ) + simulation.add_diagnostic(field_diag) + + # reduced diagnostics for reconnection rate calculation + # create a 2 l_i box around the X-point on which to measure + # magnetic flux changes + plane = picmi.ReducedDiagnostic( + diag_type="FieldProbe", + name="plane", + period=self.diag_steps, + path="diags/", + extension="dat", + probe_geometry="Plane", + resolution=60, + x_probe=0.0, + z_probe=0.0, + detector_radius=self.l_i, + target_up_x=0, + target_up_z=1.0, + ) + simulation.add_diagnostic(plane) + + ####################################################################### + # Initialize # + ####################################################################### + + if comm.rank == 0: + if Path.exists(Path("diags")): + shutil.rmtree("diags") + Path("diags/fields").mkdir(parents=True, exist_ok=True) + + # Initialize inputs and WarpX instance + simulation.initialize_inputs() + simulation.initialize_warpx() + + def check_fields(self): + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if not (step == 1 or step % self.diag_steps == 0): + return + + rho = fields.RhoFPWrapper(include_ghosts=False)[:, :] + Jiy = fields.JyFPWrapper(include_ghosts=False)[...] / self.J0 + Jy = fields.JyFPPlasmaWrapper(include_ghosts=False)[...] / self.J0 + Bx = fields.BxFPWrapper(include_ghosts=False)[...] / self.B0 + By = fields.ByFPWrapper(include_ghosts=False)[...] / self.B0 + Bz = fields.BzFPWrapper(include_ghosts=False)[...] / self.B0 + + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: + return + + # save the fields to file + with open(f"diags/fields/fields_{step:06d}.npz", "wb") as f: + np.savez(f, rho=rho, Jiy=Jiy, Jy=Jy, Bx=Bx, By=By, Bz=Bz) + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + "-t", + "--test", + help="toggle whether this script is run as a short CI test", + action="store_true", +) +parser.add_argument( + "-v", + "--verbose", + help="Verbose output", + action="store_true", +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +run = ForceFreeSheetReconnection(test=args.test, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt new file mode 100644 index 00000000000..94fe240263c --- /dev/null +++ b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_FFT) + add_warpx_test( + test_3d_open_bc_poisson_solver # name + 3 # dims + 2 # nprocs + inputs_test_3d_open_bc_poisson_solver # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000001 --rtol 1e-2" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_open_bc_poisson_solver_sliced # name + 3 # dims + 2 # nprocs + inputs_test_3d_open_bc_poisson_solver_sliced # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1000001 --rtol 1e-2" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/open_bc_poisson_solver/analysis.py b/Examples/Tests/open_bc_poisson_solver/analysis.py new file mode 100755 index 00000000000..fda9da96cf6 --- /dev/null +++ b/Examples/Tests/open_bc_poisson_solver/analysis.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import os + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import epsilon_0, pi +from scipy.special import erf + +sigmaz = 300e-6 +sigmax = 516e-9 +sigmay = 7.7e-9 +Q = -3.2e-9 + + +def w(z): + return np.exp(-(z**2)) * (1 + erf(1.0j * z)) + + +def evaluate_E(x, y, z): + """Basseti-Erskine formula https://cds.cern.ch/record/122227/files/198005132.pdf""" + den = np.sqrt(2 * (sigmax**2 - sigmay**2)) + arg1 = (x + 1j * y) / den + term1 = w(arg1) + arg2 = (x * sigmay / sigmax + 1j * y * sigmax / sigmay) / den + term2 = -np.exp(-(x**2) / (2 * sigmax**2) - y**2 / (2 * sigmay**2)) * w(arg2) + factor = ( + Q + / (2.0 * np.sqrt(2.0) * pi * epsilon_0 * sigmaz * den) + * np.exp(-(z**2) / (2 * sigmaz**2)) + ) + E_complex = factor * (term1 + term2) + return E_complex.imag, E_complex.real + + +path = os.path.join("diags", "diag2") +ts = OpenPMDTimeSeries(path) + +Ex, info = ts.get_field(field="E", coord="x", iteration=0, plot=False) +Ey, info = ts.get_field(field="E", coord="y", iteration=0, plot=False) + +grid_x = info.x[1:-1] +grid_y = info.y[1:-1] +grid_z = info.z[1:-1] + +hnx = int(0.5 * len(grid_x)) +hny = int(0.5 * len(grid_y)) + +# Compare theoretical and WarpX Ex, Ey fields for every z +for k, z in enumerate(grid_z, start=1): + Ex_warpx = Ex[k, hny, 1:-1] + Ey_warpx = Ey[k, 1:-1, hnx] + + Ex_theory = evaluate_E(grid_x, 0.0, z)[0] + Ey_theory = evaluate_E(0.0, grid_y, z)[1] + + assert np.allclose(Ex_warpx, Ex_theory, rtol=0.032, atol=0) + assert np.allclose(Ey_warpx, Ey_theory, rtol=0.029, atol=0) diff --git a/Examples/Tests/open_bc_poisson_solver/analysis_default_regression.py b/Examples/Tests/open_bc_poisson_solver/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/open_bc_poisson_solver/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver similarity index 100% rename from Examples/Tests/openbc_poisson_solver/inputs_3d rename to Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver diff --git a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_sliced b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_sliced new file mode 100644 index 00000000000..e2639c59e74 --- /dev/null +++ b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_sliced @@ -0,0 +1,3 @@ +FILE = inputs_test_3d_open_bc_poisson_solver + +warpx.use_2d_slices_fft_solver = 1 diff --git a/Examples/Tests/openbc_poisson_solver/analysis.py b/Examples/Tests/openbc_poisson_solver/analysis.py deleted file mode 100755 index dc7b5dca8bd..00000000000 --- a/Examples/Tests/openbc_poisson_solver/analysis.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -import numpy as np -from openpmd_viewer import OpenPMDTimeSeries -from scipy.constants import epsilon_0, pi -from scipy.special import erf - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -sigmaz = 300e-6 -sigmax = 516e-9 -sigmay = 7.7e-9 -Q = -3.2e-9 - -def w(z): - return np.exp(-z**2) * ( 1 + erf(1.j*z) ) - -def evaluate_E(x, y, z): - ''' Basseti-Erskine formula https://cds.cern.ch/record/122227/files/198005132.pdf ''' - den = np.sqrt(2*(sigmax**2-sigmay**2)) - arg1 = (x+1j*y)/den - term1 = w(arg1) - arg2 = (x*sigmay/sigmax + 1j*y*sigmax/sigmay)/den - term2 = -np.exp(-x**2/(2*sigmax**2)-y**2/(2*sigmay**2))*w(arg2) - factor = Q/(2.*np.sqrt(2.)*pi*epsilon_0*sigmaz*den)*np.exp(-z**2/(2*sigmaz**2)) - E_complex = factor*(term1 + term2) - return E_complex.imag, E_complex.real - - -fn = sys.argv[1] - -path=os.path.join('diags', 'diag2') -ts = OpenPMDTimeSeries(path) - -Ex, info = ts.get_field(field='E', coord='x', iteration=0, plot=False) -Ey, info = ts.get_field(field='E', coord='y', iteration=0, plot=False) - -grid_x = info.x[1:-1] -grid_y = info.y[1:-1] -grid_z = info.z[1:-1] - -hnx = int(0.5*len(grid_x)) -hny = int(0.5*len(grid_y)) - -# Compare theoretical and WarpX Ex, Ey fields for every z -for k, z in enumerate(grid_z,start=1): - Ex_warpx = Ex[k,hny,1:-1] - Ey_warpx = Ey[k,1:-1,hnx] - - Ex_theory = evaluate_E(grid_x, 0., z)[0] - Ey_theory = evaluate_E(0., grid_y, z)[1] - - assert(np.allclose(Ex_warpx, Ex_theory, rtol=0.032, atol=0)) - assert(np.allclose(Ey_warpx, Ey_theory, rtol=0.029, atol=0)) - - -# Get name of the test -test_name = os.path.split(os.getcwd())[1] - -# Run checksum regression test -checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-2) diff --git a/Examples/Tests/openpmd_rz/analysis_openpmd_rz.py b/Examples/Tests/openpmd_rz/analysis_openpmd_rz.py deleted file mode 100755 index 247c4ac61a0..00000000000 --- a/Examples/Tests/openpmd_rz/analysis_openpmd_rz.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import openpmd_api as io - -series = io.Series("LaserAccelerationRZ_opmd_plt/openpmd_%T.h5", io.Access.read_only) - -assert len(series.iterations) == 3, 'improper number of iterations stored' - -ii = series.iterations[20] - -assert len(ii.meshes) == 8, 'improper number of meshes' - -# select j_t -jt = ii.meshes['j']['t'] - -# this is in C (Python) order; r is the fastest varying index -(Nm, Nz, Nr) = jt.shape - -assert Nm == 3, 'Wrong number of angular modes stored or possible incorrect ordering when flushed' -assert Nr == 64, 'Wrong number of radial points stored or possible incorrect ordering when flushed' -assert Nz == 512, 'Wrong number of z points stored or possible incorrect ordering when flushed' - -assert ii.meshes['part_per_grid'][io.Mesh_Record_Component.SCALAR].shape == [512,64], 'problem with part_per_grid' -assert ii.meshes['rho_electrons'][io.Mesh_Record_Component.SCALAR].shape == [3, 512, 64], 'problem with rho_electrons' - - -### test that openpmd+RZ -### 1. creates rho per species correctly -### 2. orders these appropriately -rhoe_mesh = ii.meshes['rho_electrons'] -rhob_mesh = ii.meshes['rho_beam'] -dz, dr = rhoe_mesh.grid_spacing -zmin, rmin = rhoe_mesh.grid_global_offset - -rhoe = rhoe_mesh[io.Mesh_Record_Component.SCALAR][:] -rhob = rhob_mesh[io.Mesh_Record_Component.SCALAR][:] -series.flush() -nm, nz, nr = rhoe.shape -zlist = zmin + dz * np.arange(nz) -rhoe0 = rhoe[0] # 0 mode -rhob0 = rhob[0] # 0 mode - -electron_meanz = np.sum(np.dot(zlist, rhoe0))/ np.sum(rhoe0) -beam_meanz = np.sum(np.dot(zlist, rhob0))/ np.sum(rhob0) - -assert ((electron_meanz > 0) and (beam_meanz < 0)), 'problem with openPMD+RZ. Maybe openPMD+RZ mixed up the order of rho_ diagnostics?' diff --git a/Examples/Tests/particle_boundary_interaction/CMakeLists.txt b/Examples/Tests/particle_boundary_interaction/CMakeLists.txt new file mode 100644 index 00000000000..56739cf636b --- /dev/null +++ b/Examples/Tests/particle_boundary_interaction/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_rz_particle_boundary_interaction_picmi # name + RZ # dims + 2 # nprocs + inputs_test_rz_particle_boundary_interaction_picmi.py # inputs + "analysis.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py deleted file mode 100644 index df4a4579e2f..00000000000 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 -# @Eya Dammak supervised by @Remi Lehe, 2024 -# --- Input file for particle-boundary interaction testing in RZ. -# --- This input is a simple case of reflection -# --- of one electron on the surface of a sphere. -import numpy as np - -from pywarpx import callbacks, particle_containers, picmi - -########################## -# numerics parameters -########################## - -dt = 1.0e-11 - -# --- Nb time steps - -max_steps = 23 -diagnostic_interval = 1 - -# --- grid - -nr = 64 -nz= 64 - -rmin = 0.0 -rmax = 2 -zmin = -2 -zmax = 2 - -########################## -# numerics components -########################## - -grid = picmi.CylindricalGrid( - number_of_cells = [nr, nz], - n_azimuthal_modes = 1, - lower_bound = [rmin, zmin], - upper_bound = [rmax, zmax], - lower_boundary_conditions = ['none', 'dirichlet'], - upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['none', 'reflecting'], - upper_boundary_conditions_particles = ['absorbing', 'reflecting'] -) - - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', - warpx_absolute_tolerance=1e-7 -) - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="-(x**2+y**2+z**2-radius**2)", - radius = 0.2 -) - -########################## -# physics components -########################## - -#one particle -e_dist=picmi.ParticleListDistribution(x=0.0, y=0.0, z=-0.25, ux=0.5e10, uy=0.0, uz=1.0e10, weight=1) - -electrons = picmi.Species( - particle_type='electron', name='electrons', initial_distribution=e_dist, warpx_save_particles_at_eb=1 -) - -########################## -# diagnostics -########################## - -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = diagnostic_interval, - data_list = ['Er', 'Ez', 'phi', 'rho','rho_electrons'], - warpx_format = 'openpmd', - write_dir = '.', - warpx_file_prefix = 'particle_boundary_interaction_plt' -) - -part_diag = picmi.ParticleDiagnostic(name = 'diag1', - period = diagnostic_interval, - species = [electrons], - warpx_format = 'openpmd', - write_dir = '.', - warpx_file_prefix = 'particle_boundary_interaction_plt' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver=solver, - time_step_size = dt, - max_steps = max_steps, - warpx_embedded_boundary=embedded_boundary, - warpx_amrex_the_arena_is_managed=1, -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[10, 1, 1], grid=grid - ) -) -sim.add_diagnostic(part_diag) -sim.add_diagnostic(field_diag) - -sim.initialize_inputs() -sim.initialize_warpx() - -########################## -# python particle data access -########################## - -def concat( list_of_arrays ): - if len(list_of_arrays) == 0: - # Return a 1d array of size 0 - return np.empty(0) - else: - return np.concatenate( list_of_arrays ) - - -def mirror_reflection(): - buffer = particle_containers.ParticleBoundaryBufferWrapper() #boundary buffer - - #STEP 1: extract the different parameters of the boundary buffer (normal, time, position) - lev = 0 # level 0 (no mesh refinement here) - delta_t = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'deltaTimeScraped', lev)) - r = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'x', lev)) - theta = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'theta', lev)) - z = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'z', lev)) - x= r*np.cos(theta) #from RZ coordinates to 3D coordinates - y= r*np.sin(theta) - ux = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'ux', lev)) - uy = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'uy', lev)) - uz = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'uz', lev)) - w = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'w', lev)) - nx = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'nx', lev)) - ny = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'ny', lev)) - nz = concat(buffer.get_particle_boundary_buffer("electrons", 'eb', 'nz', lev)) - - #STEP 2: use these parameters to inject particle from the same position in the plasma - elect_pc = particle_containers.ParticleContainerWrapper('electrons') #general particle container - - ####this part is specific to the case of simple reflection. - un=ux*nx+uy*ny+uz*nz - ux_reflect=-2*un*nx+ux #for a "mirror reflection" u(sym)=-2(u.n)n+u - uy_reflect=-2*un*ny+uy - uz_reflect=-2*un*nz+uz - elect_pc.add_particles( - x=x + (dt-delta_t)*ux_reflect, y=y + (dt-delta_t)*uy_reflect, z=z + (dt-delta_t)*uz_reflect, - ux=ux_reflect, uy=uy_reflect, uz=uz_reflect, - w=w - ) #adds the particle in the general particle container at the next step - #### Can be modified depending on the model of interaction. - - buffer.clear_buffer() #reinitialise the boundary buffer - -callbacks.installafterstep(mirror_reflection) #mirror_reflection is called at the next step - # using the new particle container modified at the last step - -########################## -# simulation run -########################## - -sim.step(max_steps) #the whole process is done "max_steps" times diff --git a/Examples/Tests/particle_boundary_interaction/analysis.py b/Examples/Tests/particle_boundary_interaction/analysis.py index ff83cc1fed7..2e8b57cb1e6 100755 --- a/Examples/Tests/particle_boundary_interaction/analysis.py +++ b/Examples/Tests/particle_boundary_interaction/analysis.py @@ -5,9 +5,9 @@ The sphere is centered on O and has a radius of 0.2 (EB) The electron is initially at: (0,0,-0.25) and moves with a velocity: (0.5e10,0,1.0e10) with a time step of 1e-11. -An input file PICMI_inputs_rz.py is used. +An input file inputs_test_rz_particle_boundary_interaction_picmi.py is used. """ -import os + import sys import numpy as np @@ -15,37 +15,34 @@ from openpmd_viewer import OpenPMDTimeSeries yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # Open plotfile specified in command line filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') - -ts = OpenPMDTimeSeries('./particle_boundary_interaction_plt') +ts = OpenPMDTimeSeries(filename) -it=ts.iterations -x,y,z=ts.get_particle(['x','y','z'], species='electrons', iteration=it[-1]) +it = ts.iterations +x, y, z = ts.get_particle(["x", "y", "z"], species="electrons", iteration=it[-1]) # Analytical results calculated -x_analytic=0.03532 -y_analytic=0.00000 -z_analytic=-0.20531 +x_analytic = 0.03532 +y_analytic = 0.00000 +z_analytic = -0.20531 -print('NUMERICAL coordinates of the point of contact:') -print('x=%5.5f, y=%5.5f, z=%5.5f' % (x[0], y[0], z[0])) -print('\n') -print('ANALYTICAL coordinates of the point of contact:') -print('x=%5.5f, y=%5.5f, z=%5.5f' % (x_analytic, y_analytic, z_analytic)) +print("NUMERICAL coordinates of the point of contact:") +print(f"x={x[0]:5.5f}, y={y[0]:5.5f}, z={z[0]:5.5f}") +print("\n") +print("ANALYTICAL coordinates of the point of contact:") +print(f"x={x_analytic:5.5f}, y={y_analytic:5.5f}, z={z_analytic:5.5f}") -tolerance=1e-5 +tolerance = 1e-5 -diff_x=np.abs((x[0]-x_analytic)/x_analytic) -diff_z=np.abs((z[0]-z_analytic)/z_analytic) +rel_err_x = np.abs((x[0] - x_analytic) / x_analytic) +rel_err_z = np.abs((z[0] - z_analytic) / z_analytic) -print('\n') -print("percentage error for x = %5.4f %%" %(diff_x *100)) -print("percentage error for z = %5.4f %%" %(diff_z *100)) +print("\n") +print(f"Relative percentage error for x = {rel_err_x * 100:5.4f} %") +print(f"Relative percentage error for z = {rel_err_z * 100:5.4f} %") -assert (diff_x < tolerance) and (y[0] < 1e-8) and (diff_z < tolerance), 'Test particle_boundary_interaction did not pass' +assert (rel_err_x < tolerance) and (y[0] < 1e-8) and (rel_err_z < tolerance), ( + "Test particle_boundary_interaction did not pass" +) diff --git a/Examples/Tests/particle_boundary_interaction/analysis_default_regression.py b/Examples/Tests/particle_boundary_interaction/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_boundary_interaction/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_boundary_interaction/inputs_test_rz_particle_boundary_interaction_picmi.py b/Examples/Tests/particle_boundary_interaction/inputs_test_rz_particle_boundary_interaction_picmi.py new file mode 100644 index 00000000000..4b491ac6873 --- /dev/null +++ b/Examples/Tests/particle_boundary_interaction/inputs_test_rz_particle_boundary_interaction_picmi.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# @Eya Dammak supervised by @Remi Lehe, 2024 +# --- Input file for particle-boundary interaction testing in RZ. +# --- This input is a simple case of reflection +# --- of one electron on the surface of a sphere. +import numpy as np + +from pywarpx import callbacks, particle_containers, picmi + +########################## +# numerics parameters +########################## + +dt = 1.0e-11 + +# --- Nb time steps + +max_steps = 23 +diagnostic_interval = 1 + +# --- grid + +nr = 64 +nz = 64 + +rmin = 0.0 +rmax = 2 +zmin = -2 +zmax = 2 + +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + n_azimuthal_modes=1, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=["none", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["none", "reflecting"], + upper_boundary_conditions_particles=["absorbing", "reflecting"], +) + + +solver = picmi.ElectrostaticSolver( + grid=grid, method="Multigrid", warpx_absolute_tolerance=1e-7 +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-(x**2+y**2+z**2-radius**2)", radius=0.2 +) + +########################## +# physics components +########################## + +# one particle +e_dist = picmi.ParticleListDistribution( + x=0.0, y=0.0, z=-0.25, ux=0.5e10, uy=0.0, uz=1.0e10, weight=1 +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=e_dist, + warpx_save_particles_at_eb=1, +) + +########################## +# diagnostics +########################## + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_interval, + data_list=["Er", "Ez", "phi", "rho", "rho_electrons"], + warpx_format="openpmd", +) + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_interval, + species=[electrons], + warpx_format="openpmd", +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=dt, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + warpx_amrex_the_arena_is_managed=1, +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout(n_macroparticle_per_cell=[10, 1, 1], grid=grid), +) +sim.add_diagnostic(part_diag) +sim.add_diagnostic(field_diag) + +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# python particle data access +########################## + + +def concat(list_of_arrays): + if len(list_of_arrays) == 0: + # Return a 1d array of size 0 + return np.empty(0) + else: + return np.concatenate(list_of_arrays) + + +def mirror_reflection(): + buffer = particle_containers.ParticleBoundaryBufferWrapper() # boundary buffer + + # STEP 1: extract the different parameters of the boundary buffer (normal, time, position) + lev = 0 # level 0 (no mesh refinement here) + delta_t = concat( + buffer.get_particle_boundary_buffer("electrons", "eb", "deltaTimeScraped", lev) + ) + r = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "x", lev)) + theta = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "theta", lev)) + z = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "z", lev)) + x = r * np.cos(theta) # from RZ coordinates to 3D coordinates + y = r * np.sin(theta) + ux = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "ux", lev)) + uy = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "uy", lev)) + uz = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "uz", lev)) + w = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "w", lev)) + nx = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "nx", lev)) + ny = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "ny", lev)) + nz = concat(buffer.get_particle_boundary_buffer("electrons", "eb", "nz", lev)) + + # STEP 2: use these parameters to inject particle from the same position in the plasma + elect_pc = particle_containers.ParticleContainerWrapper( + "electrons" + ) # general particle container + + ####this part is specific to the case of simple reflection. + un = ux * nx + uy * ny + uz * nz + ux_reflect = -2 * un * nx + ux # for a "mirror reflection" u(sym)=-2(u.n)n+u + uy_reflect = -2 * un * ny + uy + uz_reflect = -2 * un * nz + uz + elect_pc.add_particles( + x=x + (dt - delta_t) * ux_reflect, + y=y + (dt - delta_t) * uy_reflect, + z=z + (dt - delta_t) * uz_reflect, + ux=ux_reflect, + uy=uy_reflect, + uz=uz_reflect, + w=w, + ) # adds the particle in the general particle container at the next step + #### Can be modified depending on the model of interaction. + + buffer.clear_buffer() # reinitialise the boundary buffer + + +callbacks.installafterstep( + mirror_reflection +) # mirror_reflection is called at the next step +# using the new particle container modified at the last step + +########################## +# simulation run +########################## + +sim.step(max_steps) # the whole process is done "max_steps" times diff --git a/Examples/Tests/particle_boundary_process/CMakeLists.txt b/Examples/Tests/particle_boundary_process/CMakeLists.txt new file mode 100644 index 00000000000..d99121afea0 --- /dev/null +++ b/Examples/Tests/particle_boundary_process/CMakeLists.txt @@ -0,0 +1,24 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_particle_reflection_picmi # name + 2 # dims + 1 # nprocs + inputs_test_2d_particle_reflection_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +if(WarpX_EB) + add_warpx_test( + test_3d_particle_absorption # name + 3 # dims + 2 # nprocs + inputs_test_3d_particle_absorption # inputs + "analysis_absorption.py diags/diag1000060" # analysis + "analysis_default_regression.py --path diags/diag1000060" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py deleted file mode 100755 index bb1ebd73082..00000000000 --- a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file to test particle reflection off an absorbing boundary - -from pywarpx import particle_containers, picmi - -constants = picmi.constants - -########################## -# numerics parameters -########################## - -dt = 7.5e-12 - -# --- Nb time steps - -max_steps = 10 - -# --- grid - -nx = 64 -nz = 64 - -xmin = -125e-6 -zmin = -149e-6 -xmax = 125e-6 -zmax = 1e-6 - - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, nz], - lower_bound = [xmin, zmin], - upper_bound = [xmax, zmax], - lower_boundary_conditions = ['dirichlet', 'dirichlet'], - upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['open', 'absorbing'], - upper_boundary_conditions_particles = ['open', 'absorbing'], - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6, - warpx_self_fields_verbosity=0 -) - -#embedded_boundary = picmi.EmbeddedBoundary( -# implicit_function="-max(max(x-12.5e-6,-12.5e-6-x),max(z+6.15e-5,-8.65e-5-z))" -#) - -########################## -# physics components -########################## - -uniform_plasma_elec = picmi.UniformDistribution( - density = 1e15, # number of electrons per m^3 - lower_bound = [-1e-5, -1e-5, -125e-6], - upper_bound = [1e-5, 1e-5, -120e-6], - directed_velocity = [0., 0., 5e6] # uth the std of the (unitless) momentum -) - -electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=uniform_plasma_elec, - warpx_save_particles_at_zhi=1, - warpx_save_particles_at_zlo=1, - warpx_reflection_model_zhi="0.5" -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - write_dir = '.', - warpx_file_prefix = 'Python_particle_reflection_plt' -) -field_diag = picmi.FieldDiagnostic( - grid=grid, - name = 'diag1', - data_list=['E'], - period = 10, - write_dir = '.', - warpx_file_prefix = 'Python_particle_reflection_plt' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - # warpx_embedded_boundary=embedded_boundary, - verbose = 1 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[5, 2], grid=grid - ) -) -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -########################## -# simulation run -########################## - -sim.step(max_steps) - -################################################ -# check that the wrappers to access the particle -# buffer functions as intended -################################################ - -buffer = particle_containers.ParticleBoundaryBufferWrapper() - -n = buffer.get_particle_boundary_buffer_size("electrons", 'z_hi') -print("Number of electrons in upper buffer:", n) -assert n == 63 - -n = buffer.get_particle_boundary_buffer_size("electrons", 'z_lo') -print("Number of electrons in lower buffer:", n) -assert n == 67 - -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_hi', 'stepScraped', 0) -for arr in scraped_steps: - # print(arr) - assert all(arr == 4) - -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_lo', 'stepScraped', 0) -for arr in scraped_steps: - # print(arr) - assert all(arr == 8) diff --git a/Examples/Tests/particle_boundary_process/analysis_absorption.py b/Examples/Tests/particle_boundary_process/analysis_absorption.py index 9029cc60214..498a456d871 100755 --- a/Examples/Tests/particle_boundary_process/analysis_absorption.py +++ b/Examples/Tests/particle_boundary_process/analysis_absorption.py @@ -1,19 +1,22 @@ #!/usr/bin/env python3 -import yt - # This test shoots a beam of electrons at cubic embedded boundary geometry # At time step 40, none of the particles have hit the boundary yet. At time # step 60, all of them should have been absorbed by the boundary. In the # absence of the cube, none of the particles would have had time to exit # the problem domain yet. +import sys + +import yt + # all particles are still there -ds40 = yt.load("particle_absorption_plt000040") -np40 = ds40.index.particle_headers['electrons'].num_particles -assert(np40 == 612) +ds40 = yt.load("diags/diag1000040") +np40 = ds40.index.particle_headers["electrons"].num_particles +assert np40 == 612 # all particles have been removed -ds60 = yt.load("particle_absorption_plt000060") -np60 = ds60.index.particle_headers['electrons'].num_particles -assert(np60 == 0) +filename = sys.argv[1] +ds60 = yt.load(filename) +np60 = ds60.index.particle_headers["electrons"].num_particles +assert np60 == 0 diff --git a/Examples/Tests/particle_boundary_process/analysis_default_regression.py b/Examples/Tests/particle_boundary_process/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_boundary_process/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_boundary_process/analysis_reflection.py b/Examples/Tests/particle_boundary_process/analysis_reflection.py deleted file mode 100755 index a6402976549..00000000000 --- a/Examples/Tests/particle_boundary_process/analysis_reflection.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 Modern Electron -# -# License: BSD-3-Clause-LBNL - -# This script just checks that the PICMI file executed successfully. -# If it did there will be a plotfile for the final step. - -import yt - -plotfile = 'Python_particle_reflection_plt000010' -ds = yt.load( plotfile ) # noqa - -assert True diff --git a/Examples/Tests/particle_boundary_process/inputs_test_2d_particle_reflection_picmi.py b/Examples/Tests/particle_boundary_process/inputs_test_2d_particle_reflection_picmi.py new file mode 100755 index 00000000000..ef1b7d45e1a --- /dev/null +++ b/Examples/Tests/particle_boundary_process/inputs_test_2d_particle_reflection_picmi.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# +# --- Input file to test particle reflection off an absorbing boundary + +from pywarpx import particle_containers, picmi + +constants = picmi.constants + +########################## +# numerics parameters +########################## + +dt = 7.5e-12 + +# --- Nb time steps + +max_steps = 10 + +# --- grid + +nx = 64 +nz = 64 + +xmin = -125e-6 +zmin = -149e-6 +xmax = 125e-6 +zmax = 1e-6 + + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["open", "absorbing"], + upper_boundary_conditions_particles=["open", "absorbing"], + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-6, + warpx_self_fields_verbosity=0, +) + +# embedded_boundary = picmi.EmbeddedBoundary( +# implicit_function="-max(max(x-12.5e-6,-12.5e-6-x),max(z+6.15e-5,-8.65e-5-z))" +# ) + +########################## +# physics components +########################## + +uniform_plasma_elec = picmi.UniformDistribution( + density=1e15, # number of electrons per m^3 + lower_bound=[-1e-5, -1e-5, -125e-6], + upper_bound=[1e-5, 1e-5, -120e-6], + directed_velocity=[0.0, 0.0, 5e6], # uth the std of the (unitless) momentum +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_plasma_elec, + warpx_save_particles_at_zhi=1, + warpx_save_particles_at_zlo=1, + warpx_reflection_model_zhi="0.5", +) + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, +) +field_diag = picmi.FieldDiagnostic( + grid=grid, + name="diag1", + data_list=["E"], + period=10, +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=dt, + max_steps=max_steps, + # warpx_embedded_boundary=embedded_boundary, + verbose=1, +) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[5, 2], grid=grid) +) +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +########################## +# simulation run +########################## + +sim.step(max_steps) + +################################################ +# check that the wrappers to access the particle +# buffer functions as intended +################################################ + +buffer = particle_containers.ParticleBoundaryBufferWrapper() + +n = buffer.get_particle_boundary_buffer_size("electrons", "z_hi") +print("Number of electrons in upper buffer:", n) +assert n == 63 + +n = buffer.get_particle_boundary_buffer_size("electrons", "z_lo") +print("Number of electrons in lower buffer:", n) +assert n == 67 + +scraped_steps = buffer.get_particle_boundary_buffer( + "electrons", "z_hi", "stepScraped", 0 +) +for arr in scraped_steps: + # print(arr) + assert all(arr == 4) + +scraped_steps = buffer.get_particle_boundary_buffer( + "electrons", "z_lo", "stepScraped", 0 +) +for arr in scraped_steps: + # print(arr) + assert all(arr == 8) diff --git a/Examples/Tests/particle_boundary_process/inputs_absorption b/Examples/Tests/particle_boundary_process/inputs_test_3d_particle_absorption similarity index 100% rename from Examples/Tests/particle_boundary_process/inputs_absorption rename to Examples/Tests/particle_boundary_process/inputs_test_3d_particle_absorption diff --git a/Examples/Tests/particle_boundary_scrape/CMakeLists.txt b/Examples/Tests/particle_boundary_scrape/CMakeLists.txt new file mode 100644 index 00000000000..5d2c52f30e0 --- /dev/null +++ b/Examples/Tests/particle_boundary_scrape/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_particle_scrape # name + 3 # dims + 2 # nprocs + inputs_test_3d_particle_scrape # inputs + "analysis_scrape.py diags/diag1000060" # analysis + "analysis_default_regression.py --path diags/diag1000060" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_particle_scrape_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_particle_scrape_picmi.py # inputs + "analysis_scrape.py diags/diag1000060" # analysis + "analysis_default_regression.py --path diags/diag1000060" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py deleted file mode 100755 index e5a9a58f597..00000000000 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file to test the particle scraper and the Python wrappers -# --- to access the buffer of scraped particles. - -import numpy as np - -from pywarpx import libwarpx, particle_containers, picmi - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 60 -diagnostic_intervals = 20 - -# --- Grid -nx = 64 -ny = 64 -nz = 128 - -cfl = 0.99 - -xmin = -125e-6 -ymin = -125e-6 -zmin = -149e-6 -xmax = 125e-6 -ymax = 125e-6 -zmax = 1e-6 - -########################## -# physics components -########################## - -uniform_plasma_elec = picmi.UniformDistribution( - density = 1e23, # number of electrons per m^3 - lower_bound = [-1e-5, -1e-5, -149e-6], - upper_bound = [1e-5, 1e-5, -129e-6], - directed_velocity = [0., 0., 2000.*picmi.constants.c] # uth the std of the (unitless) momentum -) - -electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=uniform_plasma_elec, - warpx_save_particles_at_xhi=1, warpx_save_particles_at_eb=1 -) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions=['none', 'none', 'none'], - upper_boundary_conditions=['none', 'none', 'none'], - lower_boundary_conditions_particles=['open', 'open', 'open'], - upper_boundary_conditions_particles=['open', 'open', 'open'], - warpx_max_grid_size = 32 -) - -solver = picmi.ElectromagneticSolver( - grid=grid, cfl=cfl -) - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="-max(max(max(x-12.5e-6,-12.5e-6-x),max(y-12.5e-6,-12.5e-6-y)),max(z-(-6.15e-5),-8.65e-5-z))" -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = diagnostic_intervals, - write_dir = '.', - warpx_file_prefix = 'Python_particle_scrape_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz'], - write_dir = '.', - warpx_file_prefix = 'Python_particle_scrape_plt' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - warpx_embedded_boundary=embedded_boundary, - verbose=True, - warpx_load_balance_intervals=40, - warpx_load_balance_efficiency_ratio_threshold=0.9 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[1, 1, 1], grid=grid - ) -) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -########################## -# simulation run -########################## - -# sim.write_input_file(file_name = 'inputs_from_PICMI') -sim.step(max_steps) - -################################################ -# check that the wrappers to access the particle -# buffer functions as intended -################################################ - -from mpi4py import MPI as mpi - -my_id = libwarpx.amr.ParallelDescriptor.MyProc() - -particle_buffer = particle_containers.ParticleBoundaryBufferWrapper() - -n = particle_buffer.get_particle_boundary_buffer_size("electrons", 'eb') -print(f"Number of electrons in buffer (proc #{my_id}): {n}") -assert n == 612 - -scraped_steps = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'stepScraped', 0) -for arr in scraped_steps: - assert all(np.array(arr, copy=False) > 40) - -weights = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'w', 0) -n = sum(len(arr) for arr in weights) -print(f"Number of electrons in this proc's buffer (proc #{my_id}): {n}") -n_sum = mpi.COMM_WORLD.allreduce(n, op=mpi.SUM) -assert n_sum == 612 - -particle_buffer.clear_buffer() -# confirm that the buffer was cleared -n = particle_buffer.get_particle_boundary_buffer_size("electrons", 'eb') -print(f"Number of electrons in buffer (proc #{my_id}): {n}") -assert n == 0 diff --git a/Examples/Tests/particle_boundary_scrape/analysis_default_regression.py b/Examples/Tests/particle_boundary_scrape/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_boundary_scrape/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_boundary_scrape/analysis_scrape.py b/Examples/Tests/particle_boundary_scrape/analysis_scrape.py index bf1de62bf0f..1b3a97da228 100755 --- a/Examples/Tests/particle_boundary_scrape/analysis_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/analysis_scrape.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from pathlib import Path + +import sys import yt @@ -11,19 +12,13 @@ # the problem domain yet. # all particles are still there -if Path("particle_scrape_plt000040").is_dir(): - filename = "particle_scrape_plt000040" -else: - filename = "Python_particle_scrape_plt000040" +filename = "diags/diag1000040" ds40 = yt.load(filename) -np40 = ds40.index.particle_headers['electrons'].num_particles -assert(np40 == 612) +np40 = ds40.index.particle_headers["electrons"].num_particles +assert np40 == 612 # all particles have been removed -if Path("particle_scrape_plt000060").is_dir(): - filename = "particle_scrape_plt000060" -else: - filename = "Python_particle_scrape_plt000060" +filename = sys.argv[1] ds60 = yt.load(filename) -np60 = ds60.index.particle_headers['electrons'].num_particles -assert(np60 == 0) +np60 = ds60.index.particle_headers["electrons"].num_particles +assert np60 == 0 diff --git a/Examples/Tests/particle_boundary_scrape/inputs_scrape b/Examples/Tests/particle_boundary_scrape/inputs_test_3d_particle_scrape similarity index 100% rename from Examples/Tests/particle_boundary_scrape/inputs_scrape rename to Examples/Tests/particle_boundary_scrape/inputs_test_3d_particle_scrape diff --git a/Examples/Tests/particle_boundary_scrape/inputs_test_3d_particle_scrape_picmi.py b/Examples/Tests/particle_boundary_scrape/inputs_test_3d_particle_scrape_picmi.py new file mode 100755 index 00000000000..1be71d4c397 --- /dev/null +++ b/Examples/Tests/particle_boundary_scrape/inputs_test_3d_particle_scrape_picmi.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# --- Input file to test the particle scraper and the Python wrappers +# --- to access the buffer of scraped particles. + +import numpy as np + +from pywarpx import libwarpx, particle_containers, picmi + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 60 +diagnostic_intervals = 20 + +# --- Grid +nx = 64 +ny = 64 +nz = 128 + +cfl = 0.99 + +xmin = -125e-6 +ymin = -125e-6 +zmin = -149e-6 +xmax = 125e-6 +ymax = 125e-6 +zmax = 1e-6 + +########################## +# physics components +########################## + +uniform_plasma_elec = picmi.UniformDistribution( + density=1e23, # number of electrons per m^3 + lower_bound=[-1e-5, -1e-5, -149e-6], + upper_bound=[1e-5, 1e-5, -129e-6], + directed_velocity=[ + 0.0, + 0.0, + 2000.0 * picmi.constants.c, + ], # uth the std of the (unitless) momentum +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_plasma_elec, + warpx_save_particles_at_xhi=1, + warpx_save_particles_at_eb=1, +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["none", "none", "none"], + upper_boundary_conditions=["none", "none", "none"], + lower_boundary_conditions_particles=["open", "open", "open"], + upper_boundary_conditions_particles=["open", "open", "open"], + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=cfl) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-max(max(max(x-12.5e-6,-12.5e-6-x),max(y-12.5e-6,-12.5e-6-y)),max(z-(-6.15e-5),-8.65e-5-z))" +) + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["Ex", "Ey", "Ez", "Bx", "By", "Bz"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + verbose=True, + warpx_load_balance_intervals=40, + warpx_load_balance_efficiency_ratio_threshold=0.9, +) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[1, 1, 1], grid=grid) +) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +########################## +# simulation run +########################## + +# sim.write_input_file(file_name = 'inputs_from_PICMI') +sim.step(max_steps) + +################################################ +# check that the wrappers to access the particle +# buffer functions as intended +################################################ + +from mpi4py import MPI as mpi + +my_id = libwarpx.amr.ParallelDescriptor.MyProc() + +particle_buffer = particle_containers.ParticleBoundaryBufferWrapper() + +n = particle_buffer.get_particle_boundary_buffer_size("electrons", "eb") +print(f"Number of electrons in buffer (proc #{my_id}): {n}") +assert n == 612 + +scraped_steps = particle_buffer.get_particle_boundary_buffer( + "electrons", "eb", "stepScraped", 0 +) +for arr in scraped_steps: + assert all(np.array(arr, copy=False) > 40) + +weights = particle_buffer.get_particle_boundary_buffer("electrons", "eb", "w", 0) +n = sum(len(arr) for arr in weights) +print(f"Number of electrons in this proc's buffer (proc #{my_id}): {n}") +n_sum = mpi.COMM_WORLD.allreduce(n, op=mpi.SUM) +assert n_sum == 612 + +particle_buffer.clear_buffer() +# confirm that the buffer was cleared +n = particle_buffer.get_particle_boundary_buffer_size("electrons", "eb") +print(f"Number of electrons in buffer (proc #{my_id}): {n}") +assert n == 0 diff --git a/Examples/Tests/particle_data_python/CMakeLists.txt b/Examples/Tests/particle_data_python/CMakeLists.txt new file mode 100644 index 00000000000..6bae89fd41b --- /dev/null +++ b/Examples/Tests/particle_data_python/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_particle_attr_access_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_particle_attr_access_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_particle_attr_access_unique_picmi # name + 2 # dims + 2 # nprocs + "inputs_test_2d_particle_attr_access_picmi.py --unique" # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_prev_positions_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_prev_positions_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py deleted file mode 100755 index 572871b8ed5..00000000000 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import sys - -import numpy as np - -from pywarpx import callbacks, libwarpx, particle_containers, picmi - -# Create the parser and add the argument -parser = argparse.ArgumentParser() -parser.add_argument( - '-u', '--unique', action='store_true', - help="Whether injected particles should be treated as unique" -) - -# Parse the input -args, left = parser.parse_known_args() -sys.argv = sys.argv[:1] + left - -########################## -# numerics parameters -########################## - -dt = 7.5e-10 - -# --- Nb time steps - -max_steps = 10 - -# --- grid - -nx = 64 -ny = 64 - -xmin = 0 -xmax = 0.03 -ymin = 0 -ymax = 0.03 - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - moving_window_velocity = None, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6, - warpx_self_fields_verbosity=0 -) - -########################## -# physics components -########################## - -electrons = picmi.Species( - particle_type='electron', name='electrons' -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - write_dir = '.', - warpx_file_prefix = f"Python_particle_attr_access_{'unique_' if args.unique else ''}plt" -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 10, - data_list = ['phi'], - write_dir = '.', - warpx_file_prefix = f"Python_particle_attr_access_{'unique_' if args.unique else ''}plt" -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - verbose = 1 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[0, 0], grid=grid - ) -) -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -sim.initialize_inputs() -sim.initialize_warpx() - -########################## -# python particle data access -########################## - -# set numpy random seed so that the particle properties generated -# below will be reproducible from run to run -np.random.seed(30025025) - -elec_wrapper = particle_containers.ParticleContainerWrapper('electrons') -elec_wrapper.add_real_comp('newPid') - -my_id = libwarpx.amr.ParallelDescriptor.MyProc() - -def add_particles(): - - nps = 10 * (my_id + 1) - x = np.linspace(0.005, 0.025, nps) - y = np.zeros(nps) - z = np.linspace(0.005, 0.025, nps) - ux = np.random.normal(loc=0, scale=1e3, size=nps) - uy = np.random.normal(loc=0, scale=1e3, size=nps) - uz = np.random.normal(loc=0, scale=1e3, size=nps) - w = np.ones(nps) * 2.0 - newPid = 5.0 - - elec_wrapper.add_particles( - x=x, y=y, z=z, ux=ux, uy=uy, uz=uz, - w=w, newPid=newPid, - unique_particles=args.unique - ) - -callbacks.installbeforestep(add_particles) - -########################## -# simulation run -########################## - -sim.step(max_steps - 1) - -########################## -# check that the new PIDs -# are properly set -########################## - -assert (elec_wrapper.nps == 270 / (2 - args.unique)) -assert (elec_wrapper.particle_container.get_comp_index('w') == 2) -assert (elec_wrapper.particle_container.get_comp_index('newPid') == 6) - -new_pid_vals = elec_wrapper.get_particle_real_arrays('newPid', 0) -for vals in new_pid_vals: - assert np.allclose(vals, 5) - -########################## -# take the final sim step -########################## - -sim.step(1) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py deleted file mode 100755 index 5de9879f0f8..00000000000 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Input file to test the saving of old particle positions - -import numpy as np - -from pywarpx import particle_containers, picmi - -constants = picmi.constants - -########################## -# numerics parameters -########################## - -dt = 7.5e-10 - -# --- Nb time steps - -max_steps = 10 - -# --- grid - -nx = 64 -nz = 64 - -xmin = 0 -xmax = 0.03 -zmin = 0 -zmax = 0.03 - - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, nz], - lower_bound = [xmin, zmin], - upper_bound = [xmax, zmax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - moving_window_velocity = None, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6, - warpx_self_fields_verbosity=0 -) - -########################## -# physics components -########################## - -uniform_plasma_elec = picmi.UniformDistribution( - density = 1e15, - upper_bound = [None] * 3, - rms_velocity = [np.sqrt(constants.kb * 1e3 / constants.m_e)] * 3, - directed_velocity = [0.] * 3 -) - -electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=uniform_plasma_elec, - warpx_save_previous_position=True -) - -########################## -# diagnostics -########################## - -part_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - species=[electrons], - write_dir = '.', - warpx_file_prefix = 'Python_prev_positions_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - data_list=['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - period = 10, - grid=grid, - write_dir = '.', - warpx_file_prefix = 'Python_prev_positions_plt' -) -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - verbose = 1 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[1, 1], grid=grid - ) -) -sim.add_diagnostic(part_diag) -sim.add_diagnostic(field_diag) -########################## -# simulation run -########################## - -sim.step(max_steps - 1) - -########################## -# check that the new PIDs -# exist -########################## - -elec_wrapper = particle_containers.ParticleContainerWrapper('electrons') -elec_count = elec_wrapper.nps - -# check that the runtime attributes have the right indices -assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 6) -assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 7) - -# sanity check that the prev_z values are reasonable and -# that the correct number of values are returned -prev_z_vals = elec_wrapper.get_particle_real_arrays('prev_z', 0) -running_count = 0 - -for z_vals in prev_z_vals: - running_count += len(z_vals) - assert np.all(z_vals < zmax) - -assert running_count == elec_wrapper.get_particle_count(True) - -########################## -# take the final sim step -########################## - -sim.step(1) diff --git a/Examples/Tests/particle_data_python/analysis.py b/Examples/Tests/particle_data_python/analysis.py deleted file mode 100755 index 0b0cce3295b..00000000000 --- a/Examples/Tests/particle_data_python/analysis.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 Modern Electron -# -# License: BSD-3-Clause-LBNL - -# This script just checks that the PICMI file executed successfully. -# If it did there will be a plotfile for the final step. - -import sys - -step = int(sys.argv[1][-5:]) - -assert step == 10 diff --git a/Examples/Tests/particle_data_python/analysis_default_regression.py b/Examples/Tests/particle_data_python/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_data_python/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_data_python/inputs_test_2d_particle_attr_access_picmi.py b/Examples/Tests/particle_data_python/inputs_test_2d_particle_attr_access_picmi.py new file mode 100755 index 00000000000..0d8c2ac209b --- /dev/null +++ b/Examples/Tests/particle_data_python/inputs_test_2d_particle_attr_access_picmi.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +import argparse +import sys + +import numpy as np + +from pywarpx import callbacks, libwarpx, particle_containers, picmi + +# Create the parser and add the argument +parser = argparse.ArgumentParser() +parser.add_argument( + "-u", + "--unique", + action="store_true", + help="Whether injected particles should be treated as unique", +) + +# Parse the input +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1] + left + +########################## +# numerics parameters +########################## + +dt = 7.5e-10 + +# --- Nb time steps + +max_steps = 10 + +# --- grid + +nx = 64 +ny = 64 + +xmin = 0 +xmax = 0.03 +ymin = 0 +ymax = 0.03 + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + moving_window_velocity=None, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-6, + warpx_self_fields_verbosity=0, +) + +########################## +# physics components +########################## + +electrons = picmi.Species(particle_type="electron", name="electrons") + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=["phi"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver=solver, time_step_size=dt, max_steps=max_steps, verbose=1) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[0, 0], grid=grid) +) +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# python particle data access +########################## + +# set numpy random seed so that the particle properties generated +# below will be reproducible from run to run +np.random.seed(30025025) + +elec_wrapper = particle_containers.ParticleContainerWrapper("electrons") +elec_wrapper.add_real_comp("newPid") + +my_id = libwarpx.amr.ParallelDescriptor.MyProc() + + +def add_particles(): + nps = 10 * (my_id + 1) + x = np.linspace(0.005, 0.025, nps) + y = np.zeros(nps) + z = np.linspace(0.005, 0.025, nps) + ux = np.random.normal(loc=0, scale=1e3, size=nps) + uy = np.random.normal(loc=0, scale=1e3, size=nps) + uz = np.random.normal(loc=0, scale=1e3, size=nps) + w = np.ones(nps) * 2.0 + newPid = 5.0 + + elec_wrapper.add_particles( + x=x, + y=y, + z=z, + ux=ux, + uy=uy, + uz=uz, + w=w, + newPid=newPid, + unique_particles=args.unique, + ) + + +callbacks.installbeforestep(add_particles) + +########################## +# simulation run +########################## + +sim.step(max_steps - 1) + +########################## +# check that the new PIDs +# are properly set +########################## + +assert elec_wrapper.nps == 270 / (2 - args.unique) +assert elec_wrapper.particle_container.get_real_comp_index("w") == 2 +assert elec_wrapper.particle_container.get_real_comp_index("newPid") == 6 + +new_pid_vals = elec_wrapper.get_particle_real_arrays("newPid", 0) +for vals in new_pid_vals: + assert np.allclose(vals, 5) + +########################## +# take the final sim step +########################## + +sim.step(1) diff --git a/Examples/Tests/particle_data_python/inputs_test_2d_prev_positions_picmi.py b/Examples/Tests/particle_data_python/inputs_test_2d_prev_positions_picmi.py new file mode 100755 index 00000000000..c15409edb0c --- /dev/null +++ b/Examples/Tests/particle_data_python/inputs_test_2d_prev_positions_picmi.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# --- Input file to test the saving of old particle positions + +import numpy as np + +from pywarpx import particle_containers, picmi + +constants = picmi.constants + +########################## +# numerics parameters +########################## + +dt = 7.5e-10 + +# --- Nb time steps + +max_steps = 10 + +# --- grid + +nx = 64 +nz = 64 + +xmin = 0 +xmax = 0.03 +zmin = 0 +zmax = 0.03 + + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=["dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + moving_window_velocity=None, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-6, + warpx_self_fields_verbosity=0, +) + +########################## +# physics components +########################## + +uniform_plasma_elec = picmi.UniformDistribution( + density=1e15, + upper_bound=[None] * 3, + rms_velocity=[np.sqrt(constants.kb * 1e3 / constants.m_e)] * 3, + directed_velocity=[0.0] * 3, +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_plasma_elec, + warpx_save_previous_position=True, +) + +########################## +# diagnostics +########################## + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, + species=[electrons], +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], + period=10, + grid=grid, +) +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver=solver, time_step_size=dt, max_steps=max_steps, verbose=1) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[1, 1], grid=grid) +) +sim.add_diagnostic(part_diag) +sim.add_diagnostic(field_diag) +########################## +# simulation run +########################## + +sim.step(max_steps - 1) + +########################## +# check that the new PIDs +# exist +########################## + +elec_wrapper = particle_containers.ParticleContainerWrapper("electrons") +elec_count = elec_wrapper.nps + +# check that the runtime attributes have the right indices +assert elec_wrapper.particle_container.get_real_comp_index("prev_x") == 6 +assert elec_wrapper.particle_container.get_real_comp_index("prev_z") == 7 + +# sanity check that the prev_z values are reasonable and +# that the correct number of values are returned +prev_z_vals = elec_wrapper.get_particle_real_arrays("prev_z", 0) +running_count = 0 + +for z_vals in prev_z_vals: + running_count += len(z_vals) + assert np.all(z_vals < zmax) + +assert running_count == elec_wrapper.get_particle_count(True) + +########################## +# take the final sim step +########################## + +sim.step(1) diff --git a/Examples/Tests/particle_fields_diags/CMakeLists.txt b/Examples/Tests/particle_fields_diags/CMakeLists.txt new file mode 100644 index 00000000000..414d303629a --- /dev/null +++ b/Examples/Tests/particle_fields_diags/CMakeLists.txt @@ -0,0 +1,23 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_particle_fields_diags # name + 3 # dims + 2 # nprocs + inputs_test_3d_particle_fields_diags # inputs + "analysis_particle_diags.py diags/diag1000200" # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) + +# FIXME +#add_warpx_test( +# test_3d_particle_fields_diags_single_precision # name +# 3 # dims +# 2 # nprocs +# inputs_test_3d_particle_fields_diags # inputs +# "analysis_particle_diags_single.py diags/diag1000200" # analysis +# "analysis_default_regression.py --path diags/diag1000200 --rtol 1e-3" # checksum +# OFF # dependency +#) diff --git a/Examples/Tests/particle_fields_diags/analysis_default_regression.py b/Examples/Tests/particle_fields_diags/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_fields_diags/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_fields_diags/analysis_particle_diags.py b/Examples/Tests/particle_fields_diags/analysis_particle_diags.py index 3a77cbfc571..78ed9137e79 100755 --- a/Examples/Tests/particle_fields_diags/analysis_particle_diags.py +++ b/Examples/Tests/particle_fields_diags/analysis_particle_diags.py @@ -13,4 +13,4 @@ import analysis_particle_diags_impl as an -an.do_analysis(single_precision = False) +an.do_analysis(single_precision=False) diff --git a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py index 5e54fc42d87..f59e0aed8bf 100755 --- a/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py +++ b/Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py @@ -11,7 +11,6 @@ # Various particle and field quantities are written to file using the reduced diagnostics # and compared with the corresponding quantities computed from the data in the plotfiles. -import os import sys import numpy as np @@ -19,23 +18,22 @@ import yt from scipy.constants import c, e, m_e, m_p -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -def do_analysis(single_precision = False): +def do_analysis(single_precision=False): fn = sys.argv[1] ds = yt.load(fn) ad = ds.all_data() - ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) + ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions + ) - opmd = io.Series('diags/openpmd/openpmd_%T.h5', io.Access.read_only) + opmd = io.Series("diags/openpmd/openpmd_%T.h5", io.Access.read_only) opmd_i = opmd.iterations[200] - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Part 1: get results from plotfiles (label '_yt') - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Quantities computed from plotfiles values_yt = dict() @@ -44,11 +42,11 @@ def do_analysis(single_precision = False): dx = domain_size / ds.domain_dimensions # Electrons - x = ad['electrons', 'particle_position_x'].to_ndarray() - y = ad['electrons', 'particle_position_y'].to_ndarray() - z = ad['electrons', 'particle_position_z'].to_ndarray() - uz = ad['electrons', 'particle_momentum_z'].to_ndarray() / m_e / c - w = ad['electrons', 'particle_weight'].to_ndarray() + x = ad["electrons", "particle_position_x"].to_ndarray() + y = ad["electrons", "particle_position_y"].to_ndarray() + z = ad["electrons", "particle_position_z"].to_ndarray() + uz = ad["electrons", "particle_momentum_z"].to_ndarray() / m_e / c + w = ad["electrons", "particle_weight"].to_ndarray() filt = uz < 0 x_ind = ((x - ds.domain_left_edge[0].value) / dx[0]).astype(int) @@ -63,27 +61,27 @@ def do_analysis(single_precision = False): wavg_filt = np.zeros(ds.domain_dimensions) for i_p in range(len(x)): - zavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * w[i_p] - uzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] - zuzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] - wavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] - uzavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] - wavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] * filt[i_p] + zavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * w[i_p] + uzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] + zuzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] + wavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] + uzavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] + wavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] * filt[i_p] wavg_adj = np.where(wavg == 0, 1, wavg) wavg_filt_adj = np.where(wavg_filt == 0, 1, wavg_filt) - values_yt['electrons: zavg'] = zavg / wavg_adj - values_yt['electrons: uzavg'] = uzavg / wavg_adj - values_yt['electrons: zuzavg'] = zuzavg / wavg_adj - values_yt['electrons: uzavg_filt'] = uzavg_filt / wavg_filt_adj - values_yt['electrons: jz'] = e*uzavg + values_yt["electrons: zavg"] = zavg / wavg_adj + values_yt["electrons: uzavg"] = uzavg / wavg_adj + values_yt["electrons: zuzavg"] = zuzavg / wavg_adj + values_yt["electrons: uzavg_filt"] = uzavg_filt / wavg_filt_adj + values_yt["electrons: jz"] = e * uzavg # protons - x = ad['protons', 'particle_position_x'].to_ndarray() - y = ad['protons', 'particle_position_y'].to_ndarray() - z = ad['protons', 'particle_position_z'].to_ndarray() - uz = ad['protons', 'particle_momentum_z'].to_ndarray() / m_p / c - w = ad['protons', 'particle_weight'].to_ndarray() + x = ad["protons", "particle_position_x"].to_ndarray() + y = ad["protons", "particle_position_y"].to_ndarray() + z = ad["protons", "particle_position_z"].to_ndarray() + uz = ad["protons", "particle_momentum_z"].to_ndarray() / m_p / c + w = ad["protons", "particle_weight"].to_ndarray() filt = uz < 0 x_ind = ((x - ds.domain_left_edge[0].value) / dx[0]).astype(int) @@ -98,27 +96,27 @@ def do_analysis(single_precision = False): wavg_filt = np.zeros(ds.domain_dimensions) for i_p in range(len(x)): - zavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * w[i_p] - uzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] - zuzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] - wavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] - uzavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] - wavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] * filt[i_p] + zavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * w[i_p] + uzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] + zuzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] + wavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] + uzavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] + wavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] * filt[i_p] wavg_adj = np.where(wavg == 0, 1, wavg) wavg_filt_adj = np.where(wavg_filt == 0, 1, wavg_filt) - values_yt['protons: zavg'] = zavg / wavg_adj - values_yt['protons: uzavg'] = uzavg / wavg_adj - values_yt['protons: zuzavg'] = zuzavg / wavg_adj - values_yt['protons: uzavg_filt'] = uzavg_filt / wavg_filt_adj - values_yt['protons: jz'] = e*uzavg + values_yt["protons: zavg"] = zavg / wavg_adj + values_yt["protons: uzavg"] = uzavg / wavg_adj + values_yt["protons: zuzavg"] = zuzavg / wavg_adj + values_yt["protons: uzavg_filt"] = uzavg_filt / wavg_filt_adj + values_yt["protons: jz"] = e * uzavg # Photons (momentum in units of m_e c) - x = ad['photons', 'particle_position_x'].to_ndarray() - y = ad['photons', 'particle_position_y'].to_ndarray() - z = ad['photons', 'particle_position_z'].to_ndarray() - uz = ad['photons', 'particle_momentum_z'].to_ndarray() / m_e / c - w = ad['photons', 'particle_weight'].to_ndarray() + x = ad["photons", "particle_position_x"].to_ndarray() + y = ad["photons", "particle_position_y"].to_ndarray() + z = ad["photons", "particle_position_z"].to_ndarray() + uz = ad["photons", "particle_momentum_z"].to_ndarray() / m_e / c + w = ad["photons", "particle_weight"].to_ndarray() filt = uz < 0 x_ind = ((x - ds.domain_left_edge[0].value) / dx[0]).astype(int) @@ -133,91 +131,113 @@ def do_analysis(single_precision = False): wavg_filt = np.zeros(ds.domain_dimensions) for i_p in range(len(x)): - zavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * w[i_p] - uzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] - zuzavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] - wavg[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] - uzavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] - wavg_filt[x_ind[i_p],y_ind[i_p],z_ind[i_p]] += w[i_p] * filt[i_p] + zavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * w[i_p] + uzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] + zuzavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += z[i_p] * uz[i_p] * w[i_p] + wavg[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] + uzavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += uz[i_p] * w[i_p] * filt[i_p] + wavg_filt[x_ind[i_p], y_ind[i_p], z_ind[i_p]] += w[i_p] * filt[i_p] wavg_adj = np.where(wavg == 0, 1, wavg) wavg_filt_adj = np.where(wavg_filt == 0, 1, wavg_filt) - values_yt['photons: zavg'] = zavg / wavg_adj - values_yt['photons: uzavg'] = uzavg / wavg_adj - values_yt['photons: zuzavg'] = zuzavg / wavg_adj - values_yt['photons: uzavg_filt'] = uzavg_filt / wavg_filt_adj - values_yt['photons: jz'] = e*uzavg - + values_yt["photons: zavg"] = zavg / wavg_adj + values_yt["photons: uzavg"] = uzavg / wavg_adj + values_yt["photons: zuzavg"] = zuzavg / wavg_adj + values_yt["photons: uzavg_filt"] = uzavg_filt / wavg_filt_adj + values_yt["photons: jz"] = e * uzavg values_rd = dict() # Load reduced particle diagnostic data from plotfiles - values_rd['electrons: zavg'] = ad0[('boxlib','z_electrons')] - values_rd['protons: zavg'] = ad0[('boxlib','z_protons')] - values_rd['photons: zavg'] = ad0[('boxlib','z_photons')] + values_rd["electrons: zavg"] = ad0[("boxlib", "z_electrons")] + values_rd["protons: zavg"] = ad0[("boxlib", "z_protons")] + values_rd["photons: zavg"] = ad0[("boxlib", "z_photons")] - values_rd['electrons: uzavg'] = ad0[('boxlib','uz_electrons')] - values_rd['protons: uzavg'] = ad0[('boxlib','uz_protons')] - values_rd['photons: uzavg'] = ad0[('boxlib','uz_photons')] + values_rd["electrons: uzavg"] = ad0[("boxlib", "uz_electrons")] + values_rd["protons: uzavg"] = ad0[("boxlib", "uz_protons")] + values_rd["photons: uzavg"] = ad0[("boxlib", "uz_photons")] - values_rd['electrons: zuzavg'] = ad0[('boxlib','zuz_electrons')] - values_rd['protons: zuzavg'] = ad0[('boxlib','zuz_protons')] - values_rd['photons: zuzavg'] = ad0[('boxlib','zuz_photons')] + values_rd["electrons: zuzavg"] = ad0[("boxlib", "zuz_electrons")] + values_rd["protons: zuzavg"] = ad0[("boxlib", "zuz_protons")] + values_rd["photons: zuzavg"] = ad0[("boxlib", "zuz_photons")] - values_rd['electrons: uzavg_filt'] = ad0[('boxlib','uz_filt_electrons')] - values_rd['protons: uzavg_filt'] = ad0[('boxlib','uz_filt_protons')] - values_rd['photons: uzavg_filt'] = ad0[('boxlib','uz_filt_photons')] + values_rd["electrons: uzavg_filt"] = ad0[("boxlib", "uz_filt_electrons")] + values_rd["protons: uzavg_filt"] = ad0[("boxlib", "uz_filt_protons")] + values_rd["photons: uzavg_filt"] = ad0[("boxlib", "uz_filt_photons")] - values_rd['electrons: jz'] = ad0[('boxlib','jz_electrons')] - values_rd['protons: jz'] = ad0[('boxlib','jz_protons')] - values_rd['photons: jz'] = ad0[('boxlib','jz_photons')] + values_rd["electrons: jz"] = ad0[("boxlib", "jz_electrons")] + values_rd["protons: jz"] = ad0[("boxlib", "jz_protons")] + values_rd["photons: jz"] = ad0[("boxlib", "jz_photons")] values_opmd = dict() # Load reduced particle diagnostic data from OPMD output - values_opmd['electrons: zavg'] = opmd_i.meshes['z_electrons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['protons: zavg'] = opmd_i.meshes['z_protons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['photons: zavg'] = opmd_i.meshes['z_photons'][io.Mesh_Record_Component.SCALAR].load_chunk() - - values_opmd['electrons: uzavg'] = opmd_i.meshes['uz_electrons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['protons: uzavg'] = opmd_i.meshes['uz_protons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['photons: uzavg'] = opmd_i.meshes['uz_photons'][io.Mesh_Record_Component.SCALAR].load_chunk() - - values_opmd['electrons: zuzavg'] = opmd_i.meshes['zuz_electrons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['protons: zuzavg'] = opmd_i.meshes['zuz_protons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['photons: zuzavg'] = opmd_i.meshes['zuz_photons'][io.Mesh_Record_Component.SCALAR].load_chunk() - - values_opmd['electrons: uzavg_filt'] = opmd_i.meshes['uz_filt_electrons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['protons: uzavg_filt'] = opmd_i.meshes['uz_filt_protons'][io.Mesh_Record_Component.SCALAR].load_chunk() - values_opmd['photons: uzavg_filt'] = opmd_i.meshes['uz_filt_photons'][io.Mesh_Record_Component.SCALAR].load_chunk() - - values_opmd['electrons: jz'] = opmd_i.meshes['j_electrons']['z'].load_chunk() - values_opmd['protons: jz'] = opmd_i.meshes['j_protons']['z'].load_chunk() - values_opmd['photons: jz'] = opmd_i.meshes['j_photons']['z'].load_chunk() + values_opmd["electrons: zavg"] = opmd_i.meshes["z_electrons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["protons: zavg"] = opmd_i.meshes["z_protons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["photons: zavg"] = opmd_i.meshes["z_photons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + + values_opmd["electrons: uzavg"] = opmd_i.meshes["uz_electrons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["protons: uzavg"] = opmd_i.meshes["uz_protons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["photons: uzavg"] = opmd_i.meshes["uz_photons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + + values_opmd["electrons: zuzavg"] = opmd_i.meshes["zuz_electrons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["protons: zuzavg"] = opmd_i.meshes["zuz_protons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["photons: zuzavg"] = opmd_i.meshes["zuz_photons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + + values_opmd["electrons: uzavg_filt"] = opmd_i.meshes["uz_filt_electrons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["protons: uzavg_filt"] = opmd_i.meshes["uz_filt_protons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + values_opmd["photons: uzavg_filt"] = opmd_i.meshes["uz_filt_photons"][ + io.Mesh_Record_Component.SCALAR + ].load_chunk() + + values_opmd["electrons: jz"] = opmd_i.meshes["j_electrons"]["z"].load_chunk() + values_opmd["protons: jz"] = opmd_i.meshes["j_protons"]["z"].load_chunk() + values_opmd["photons: jz"] = opmd_i.meshes["j_photons"]["z"].load_chunk() opmd.flush() del opmd - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Part 3: compare values from plotfiles and diagnostics and print output - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- error_plt = dict() error_opmd = dict() tolerance = 5e-3 if single_precision else 1e-12 - # if single precision, increase tolerance from default value - check_tolerance = 5e-3 if single_precision else 1e-9 for k in values_yt.keys(): # check that the zeros line up, since we'll be ignoring them in the error calculation - assert(np.all((values_yt[k] == 0) == (values_rd[k] == 0))) - error_plt[k] = np.max(abs(values_yt[k] - values_rd[k])[values_yt[k] != 0] / abs(values_yt[k])[values_yt[k] != 0]) - print(k, 'relative error plotfile = ', error_plt[k]) - assert(error_plt[k] < tolerance) - assert(np.all((values_yt[k] == 0) == (values_opmd[k].T == 0))) - error_opmd[k] = np.max(abs(values_yt[k] - values_opmd[k].T)[values_yt[k] != 0] / abs(values_yt[k])[values_yt[k] != 0]) - assert(error_opmd[k] < tolerance) - print(k, 'relative error openPMD = ', error_opmd[k]) - - - - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, fn, rtol=check_tolerance) + assert np.all((values_yt[k] == 0) == (values_rd[k] == 0)) + error_plt[k] = np.max( + abs(values_yt[k] - values_rd[k])[values_yt[k] != 0] + / abs(values_yt[k])[values_yt[k] != 0] + ) + print(k, "relative error plotfile = ", error_plt[k]) + assert error_plt[k] < tolerance + assert np.all((values_yt[k] == 0) == (values_opmd[k].T == 0)) + error_opmd[k] = np.max( + abs(values_yt[k] - values_opmd[k].T)[values_yt[k] != 0] + / abs(values_yt[k])[values_yt[k] != 0] + ) + assert error_opmd[k] < tolerance + print(k, "relative error openPMD = ", error_opmd[k]) diff --git a/Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py b/Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py index 56d98831e66..7efbe4e39d4 100755 --- a/Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py +++ b/Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py @@ -13,4 +13,4 @@ import analysis_particle_diags_impl as an -an.do_analysis(single_precision = True) +an.do_analysis(single_precision=True) diff --git a/Examples/Tests/particle_fields_diags/inputs b/Examples/Tests/particle_fields_diags/inputs_test_3d_particle_fields_diags similarity index 100% rename from Examples/Tests/particle_fields_diags/inputs rename to Examples/Tests/particle_fields_diags/inputs_test_3d_particle_fields_diags diff --git a/Examples/Tests/particle_pusher/CMakeLists.txt b/Examples/Tests/particle_pusher/CMakeLists.txt new file mode 100644 index 00000000000..cd414316b67 --- /dev/null +++ b/Examples/Tests/particle_pusher/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_particle_pusher # name + 3 # dims + 1 # nprocs + inputs_test_3d_particle_pusher # inputs + "analysis.py diags/diag1010000" # analysis + "analysis_default_regression.py --path diags/diag1010000" # checksum + OFF # dependency +) diff --git a/Examples/Tests/particle_pusher/analysis.py b/Examples/Tests/particle_pusher/analysis.py new file mode 100755 index 00000000000..ae7b2054c28 --- /dev/null +++ b/Examples/Tests/particle_pusher/analysis.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Yinjian Zhao +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This script tests the particle pusher (HC) +# using a force-free field, +# in which position x should remain 0. +# An initial velocity Vy corresponding to +# Lorentz factor = 20 is used. +# Bz is fixed at 1 T. +# Ex = -Vy*Bz. + +# Possible errors: +# Boris: 2321.3958529 +# Vay: 0.00010467 +# HC: 0.00011403 +# tolerance: 0.001 +# Possible running time: ~ 4.0 s + +import sys + +import yt + +tolerance = 0.001 + +filename = sys.argv[1] +ds = yt.load(filename) +ad = ds.all_data() +x = ad["particle_position_x"].to_ndarray() + +print("error = ", abs(x)) +print("tolerance = ", tolerance) +assert abs(x) < tolerance diff --git a/Examples/Tests/particle_pusher/analysis_default_regression.py b/Examples/Tests/particle_pusher/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_pusher/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_pusher/analysis_pusher.py b/Examples/Tests/particle_pusher/analysis_pusher.py deleted file mode 100755 index 0d9e3a743f0..00000000000 --- a/Examples/Tests/particle_pusher/analysis_pusher.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Yinjian Zhao -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -# This script tests the particle pusher (HC) -# using a force-free field, -# in which position x should remain 0. -# An initial velocity Vy corresponding to -# Lorentz factor = 20 is used. -# Bz is fixed at 1 T. -# Ex = -Vy*Bz. - -# Possible errors: -# Boris: 2321.3958529 -# Vay: 0.00010467 -# HC: 0.00011403 -# tolerance: 0.001 -# Possible running time: ~ 4.0 s - -import os -import sys - -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -tolerance = 0.001 - -filename = sys.argv[1] -ds = yt.load( filename ) -ad = ds.all_data() -x = ad['particle_position_x'].to_ndarray() - -print('error = ', abs(x)) -print('tolerance = ', tolerance) -assert(abs(x) < tolerance) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particle_pusher/inputs_3d b/Examples/Tests/particle_pusher/inputs_test_3d_particle_pusher similarity index 100% rename from Examples/Tests/particle_pusher/inputs_3d rename to Examples/Tests/particle_pusher/inputs_test_3d_particle_pusher diff --git a/Examples/Tests/particle_thermal_boundary/CMakeLists.txt b/Examples/Tests/particle_thermal_boundary/CMakeLists.txt new file mode 100644 index 00000000000..2f0cbba4ee6 --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_particle_thermal_boundary # name + 2 # dims + 2 # nprocs + inputs_test_2d_particle_thermal_boundary # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1002000" # checksum + OFF # dependency +) diff --git a/Examples/Tests/particle_thermal_boundary/analysis.py b/Examples/Tests/particle_thermal_boundary/analysis.py new file mode 100755 index 00000000000..81f8a73b474 --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/analysis.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2023-2024 Revathi Jambunathan +# +# This file is part of WarpX +# +# License: BSD-3-Clause-LBNL + +""" +The script checks to see the growth in total field energy and total particle energy + +The input file in 2D initializes uniform plasma (electrons,ions) with +thermal boundary condition. We do not expect the particle energy to increase +beyond 2% in the time that it takes all particles to cross the domain boundary +""" + +import numpy as np + +FE_rdiag = "./diags/reducedfiles/EF.txt" +init_Fenergy = np.loadtxt(FE_rdiag)[1, 2] +final_Fenergy = np.loadtxt(FE_rdiag)[-1, 2] +assert final_Fenergy / init_Fenergy < 40 +assert final_Fenergy < 5.0e-5 + +PE_rdiag = "./diags/reducedfiles/EN.txt" +init_Penergy = np.loadtxt(PE_rdiag)[0, 2] +final_Penergy = np.loadtxt(PE_rdiag)[-1, 2] +assert abs(final_Penergy - init_Penergy) / init_Penergy < 0.02 diff --git a/Examples/Tests/particle_thermal_boundary/analysis_2d.py b/Examples/Tests/particle_thermal_boundary/analysis_2d.py deleted file mode 100755 index db14479af2c..00000000000 --- a/Examples/Tests/particle_thermal_boundary/analysis_2d.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2023-2024 Revathi Jambunathan -# -# This file is part of WarpX -# -# License: BSD-3-Clause-LBNL - -""" -The script checks to see the growth in total field energy and total particle energy - -The input file in 2D initializes uniform plasma (electrons,ions) with -thermal boundary condition. We do not expect the particle energy to increase -beyond 2% in the time that it takes all particles to cross the domain boundary -""" - -import os -import sys - -import numpy as np - -sys.path.insert(1,'../../../../warpx/Regression/Checksum/') -import checksumAPI - -FE_rdiag = './diags/reducedfiles/EF.txt' -init_Fenergy = np.loadtxt(FE_rdiag)[1,2] -final_Fenergy = np.loadtxt(FE_rdiag)[-1,2] -assert(final_Fenergy/init_Fenergy < 40) -assert(final_Fenergy < 5.e-5) - -PE_rdiag = './diags/reducedfiles/EN.txt' -init_Penergy = np.loadtxt(PE_rdiag)[0,2] -final_Penergy = np.loadtxt(PE_rdiag)[-1,2] -assert( abs(final_Penergy - init_Penergy)/init_Penergy < 0.02) -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particle_thermal_boundary/analysis_default_regression.py b/Examples/Tests/particle_thermal_boundary/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particle_thermal_boundary/inputs_2d b/Examples/Tests/particle_thermal_boundary/inputs_test_2d_particle_thermal_boundary similarity index 100% rename from Examples/Tests/particle_thermal_boundary/inputs_2d rename to Examples/Tests/particle_thermal_boundary/inputs_test_2d_particle_thermal_boundary diff --git a/Examples/Tests/particles_in_pml/CMakeLists.txt b/Examples/Tests/particles_in_pml/CMakeLists.txt new file mode 100644 index 00000000000..4f150c6d4e4 --- /dev/null +++ b/Examples/Tests/particles_in_pml/CMakeLists.txt @@ -0,0 +1,43 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_particles_in_pml # name + 2 # dims + 2 # nprocs + inputs_test_2d_particles_in_pml # inputs + "analysis_particles_in_pml.py diags/diag1000180" # analysis + "analysis_default_regression.py --path diags/diag1000180" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_particles_in_pml_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_particles_in_pml_mr # inputs + "analysis_particles_in_pml.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_particles_in_pml # name + 3 # dims + 2 # nprocs + inputs_test_3d_particles_in_pml # inputs + "analysis_particles_in_pml.py diags/diag1000120" # analysis + "analysis_default_regression.py --path diags/diag1000120" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_particles_in_pml_mr # name + 3 # dims + 2 # nprocs + inputs_test_3d_particles_in_pml_mr # inputs + "analysis_particles_in_pml.py diags/diag1000200" # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) +label_warpx_test(test_3d_particles_in_pml_mr slow) diff --git a/Examples/Tests/particles_in_pml/analysis_default_regression.py b/Examples/Tests/particles_in_pml/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/particles_in_pml/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/particles_in_pml/analysis_particles_in_pml.py b/Examples/Tests/particles_in_pml/analysis_particles_in_pml.py index 1d1a8959edd..63ef4c7d2ff 100755 --- a/Examples/Tests/particles_in_pml/analysis_particles_in_pml.py +++ b/Examples/Tests/particles_in_pml/analysis_particles_in_pml.py @@ -17,21 +17,19 @@ PML, this test fails, since the particles leave a spurious charge, with associated fields, behind them. """ -import os + import sys import yt yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # Open plotfile specified in command line filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) # When extracting the fields, choose the right dimensions -dimensions = [ n_pts for n_pts in ds.domain_dimensions ] +dimensions = [n_pts for n_pts in ds.domain_dimensions] if ds.max_level == 1: dimensions[0] *= 2 dimensions[1] *= 2 @@ -39,12 +37,14 @@ dimensions[2] *= 2 # Check that the field is low enough -ad0 = ds.covering_grid(level=ds.max_level, left_edge=ds.domain_left_edge, dims=dimensions) -Ex_array = ad0[('mesh','Ex')].to_ndarray() -Ey_array = ad0[('mesh','Ey')].to_ndarray() -Ez_array = ad0[('mesh','Ez')].to_ndarray() +ad0 = ds.covering_grid( + level=ds.max_level, left_edge=ds.domain_left_edge, dims=dimensions +) +Ex_array = ad0[("mesh", "Ex")].to_ndarray() +Ey_array = ad0[("mesh", "Ey")].to_ndarray() +Ez_array = ad0[("mesh", "Ez")].to_ndarray() max_Efield = max(Ex_array.max(), Ey_array.max(), Ez_array.max()) -print( "max_Efield = %s" %max_Efield ) +print("max_Efield = %s" % max_Efield) # The field associated with the particle does not have # the same amplitude in 2d and 3d @@ -63,6 +63,3 @@ print("tolerance_abs: " + str(tolerance_abs)) assert max_Efield < tolerance_abs - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particles_in_pml/inputs_2d b/Examples/Tests/particles_in_pml/inputs_test_2d_particles_in_pml similarity index 100% rename from Examples/Tests/particles_in_pml/inputs_2d rename to Examples/Tests/particles_in_pml/inputs_test_2d_particles_in_pml diff --git a/Examples/Tests/particles_in_pml/inputs_mr_2d b/Examples/Tests/particles_in_pml/inputs_test_2d_particles_in_pml_mr similarity index 100% rename from Examples/Tests/particles_in_pml/inputs_mr_2d rename to Examples/Tests/particles_in_pml/inputs_test_2d_particles_in_pml_mr diff --git a/Examples/Tests/particles_in_pml/inputs_3d b/Examples/Tests/particles_in_pml/inputs_test_3d_particles_in_pml similarity index 100% rename from Examples/Tests/particles_in_pml/inputs_3d rename to Examples/Tests/particles_in_pml/inputs_test_3d_particles_in_pml diff --git a/Examples/Tests/particles_in_pml/inputs_mr_3d b/Examples/Tests/particles_in_pml/inputs_test_3d_particles_in_pml_mr similarity index 100% rename from Examples/Tests/particles_in_pml/inputs_mr_3d rename to Examples/Tests/particles_in_pml/inputs_test_3d_particles_in_pml_mr diff --git a/Examples/Tests/pass_mpi_communicator/CMakeLists.txt b/Examples/Tests/pass_mpi_communicator/CMakeLists.txt new file mode 100644 index 00000000000..4f5a8b4965c --- /dev/null +++ b/Examples/Tests/pass_mpi_communicator/CMakeLists.txt @@ -0,0 +1,16 @@ +# Add tests (alphabetical order) ############################################## +# + +# TODO +# - Enable in pyAMReX (https://github.com/AMReX-Codes/pyamrex/issues/163) +# - Enable related lines in input script +# - Enable analysis script +add_warpx_test( + test_2d_pass_mpi_comm_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_pass_mpi_comm_picmi.py # inputs + OFF # analysis + OFF # checksum + OFF # dependency +) diff --git a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py deleted file mode 100755 index 66f259da2ef..00000000000 --- a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -# -# --- Test if MPI_communicator is correctly passed from Python to CPP -# --- This test should be run with MPI enabled -# --- Inputs taken from langmuir_2d. Runs 10 steps, and will check -# --- if the correct amount of processors are initialized in AMReX. - -from mpi4py import MPI - -from pywarpx import picmi - -constants = picmi.constants - -########################## -# MPI communicator setup -########################## - -# split processor 0 into separate communicator from others -comm_world = MPI.COMM_WORLD -rank = comm_world.Get_rank() -if rank == 0: - color = 0 -else: - color = 1 -new_comm = comm_world.Split(color) - -########################## -# physics parameters -########################## - -# different communicators will be passed different plasma density -plasma_density = [1.e18, 1.e19] -plasma_xmin = 0. -plasma_x_velocity = 0.1*constants.c - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 10 -diagnostic_intervals = "::10" - -# --- Grid -nx = 64 -ny = 64 - -xmin = -20.e-6 -ymin = -20.e-6 -xmax = +20.e-6 -ymax = +20.e-6 - -number_per_cell_each_dim = [2,2] - -########################## -# physics components -########################## - -uniform_plasma = picmi.UniformDistribution(density = plasma_density[color], - upper_bound = [0., None, None], - directed_velocity = [0.1*constants.c, 0., 0.]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=uniform_plasma) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['periodic', 'periodic'], - upper_boundary_conditions = ['periodic', 'periodic'], - moving_window_velocity = [0., 0., 0.], - warpx_max_grid_size = 32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.) - -########################## -# diagnostics -########################## - -field_diag = picmi.FieldDiagnostic(name = f'diag{color + 1}', - grid = grid, - period = diagnostic_intervals, - data_list = ['Ex', 'Jx'], - write_dir = '.', - warpx_file_prefix = f'Python_pass_mpi_comm_plt{color + 1}_') - -part_diag = picmi.ParticleDiagnostic(name = f'diag{color + 1}', - period = diagnostic_intervals, - species = [electrons], - data_list = ['weighting', 'ux']) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - warpx_current_deposition_algo = 'direct') - -sim.add_species(electrons, - layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) - -sim.add_diagnostic(field_diag) -sim.add_diagnostic(part_diag) - -########################## -# simulation run -########################## - -# TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again -# https://github.com/AMReX-Codes/pyamrex/issues/163 - -#sim.step(max_steps, mpi_comm=new_comm) - -########################## -# test -########################## - -# NOTE: Some tests are done in this input PICMI file. These tests -# check that amrex initialized the correct amount of procs and -# that the procs ranks in amrex are correct. -# If any of these tests fail, the terminal will print that the -# program crashed. - -# TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again -# https://github.com/AMReX-Codes/pyamrex/issues/163 -#comm_world_size = comm_world.size -#new_comm_size = new_comm.size - -#if color == 0: -# # verify that communicator contains correct number of procs (1) -# assert sim.extension.getNProcs() == comm_world_size - 1 -# assert sim.extension.getNProcs() == new_comm_size - -#else: -# # verify that amrex initialized with 1 fewer proc than comm world -# assert sim.extension.getNProcs() == comm_world_size - 1 -# assert sim.extension.getNProcs() == new_comm_size - - # verify that amrex proc ranks are offset by -1 from - # world comm proc ranks -# assert libwarpx.amr.ParallelDescriptor.MyProc() == rank - 1 diff --git a/Examples/Tests/pass_mpi_communicator/analysis.py b/Examples/Tests/pass_mpi_communicator/analysis.py index 9a622943127..cfac572c1b9 100755 --- a/Examples/Tests/pass_mpi_communicator/analysis.py +++ b/Examples/Tests/pass_mpi_communicator/analysis.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 # This is a script that analyses the simulation results from -# the script `PICMI_inputs_2d`. +# the script `inputs_test_2d_pass_mpi_comm_picmi.py`. import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import yt yt.funcs.mylog.setLevel(50) import numpy as np -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") import checksum # this will be the name of the first plot file @@ -25,56 +25,54 @@ test_name2 = fn2[:-10] -checksum1 = checksum.Checksum(test_name1, fn1, do_fields=True, - do_particles=True) +checksum1 = checksum.Checksum(test_name1, fn1, do_fields=True, do_particles=True) -checksum2 = checksum.Checksum(test_name2, fn2, do_fields=True, - do_particles=True) +checksum2 = checksum.Checksum(test_name2, fn2, do_fields=True, do_particles=True) -rtol=1.e-9 -atol=1.e-40 +rtol = 1.0e-9 +atol = 1.0e-40 # Evaluate checksums against each other, adapted from # Checksum.evaluate() method # Dictionaries have same outer keys (levels, species)? -if (checksum1.data.keys() != checksum2.data.keys()): - print("ERROR: plotfile 1 and plotfile 2 checksums " - "have different outer keys:") +if checksum1.data.keys() != checksum2.data.keys(): + print("ERROR: plotfile 1 and plotfile 2 checksums have different outer keys:") print("Plot1: %s" % checksum1.data.keys()) print("Plot2: %s" % checksum2.data.keys()) sys.exit(1) # Dictionaries have same inner keys (field and particle quantities)? for key1 in checksum1.data.keys(): - if (checksum1.data[key1].keys() != checksum2.data[key1].keys()): - print("ERROR: plotfile 1 and plotfile 2 checksums have " - "different inner keys:") + if checksum1.data[key1].keys() != checksum2.data[key1].keys(): + print("ERROR: plotfile 1 and plotfile 2 checksums have different inner keys:") print("Common outer keys: %s" % checksum2.data.keys()) - print("Plotfile 1 inner keys in %s: %s" - % (key1, checksum1.data[key1].keys())) - print("Plotfile 2 inner keys in %s: %s" - % (key1, checksum2.data[key1].keys())) + print("Plotfile 1 inner keys in %s: %s" % (key1, checksum1.data[key1].keys())) + print("Plotfile 2 inner keys in %s: %s" % (key1, checksum2.data[key1].keys())) sys.exit(1) # Dictionaries have same values? checksums_same = False for key1 in checksum1.data.keys(): for key2 in checksum1.data[key1].keys(): - passed = np.isclose(checksum2.data[key1][key2], - checksum1.data[key1][key2], - rtol=rtol, atol=atol) + passed = np.isclose( + checksum2.data[key1][key2], checksum1.data[key1][key2], rtol=rtol, atol=atol + ) # skip over these, since they will be the same if communicators # have same number of procs if key2 in ["particle_cpu", "particle_id", "particle_position_y"]: continue if passed: - print("ERROR: plotfile 1 and plotfile 2 checksums have " - "same values for key [%s,%s]" % (key1, key2)) - print("Plotfile 1: [%s,%s] %.15e" - % (key1, key2, checksum1.data[key1][key2])) - print("Plotfile 2: [%s,%s] %.15e" - % (key1, key2, checksum2.data[key1][key2])) + print( + "ERROR: plotfile 1 and plotfile 2 checksums have " + "same values for key [%s,%s]" % (key1, key2) + ) + print( + "Plotfile 1: [%s,%s] %.15e" % (key1, key2, checksum1.data[key1][key2]) + ) + print( + "Plotfile 2: [%s,%s] %.15e" % (key1, key2, checksum2.data[key1][key2]) + ) checksums_same = True if checksums_same: sys.exit(1) diff --git a/Examples/Tests/pass_mpi_communicator/analysis_default_regression.py b/Examples/Tests/pass_mpi_communicator/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/pass_mpi_communicator/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/pass_mpi_communicator/inputs_test_2d_pass_mpi_comm_picmi.py b/Examples/Tests/pass_mpi_communicator/inputs_test_2d_pass_mpi_comm_picmi.py new file mode 100755 index 00000000000..c87a7a0045c --- /dev/null +++ b/Examples/Tests/pass_mpi_communicator/inputs_test_2d_pass_mpi_comm_picmi.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# +# --- Test if MPI_communicator is correctly passed from Python to CPP +# --- This test should be run with MPI enabled +# --- Inputs taken from langmuir_2d. Runs 10 steps, and will check +# --- if the correct amount of processors are initialized in AMReX. + +from mpi4py import MPI + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# MPI communicator setup +########################## + +# split processor 0 into separate communicator from others +comm_world = MPI.COMM_WORLD +rank = comm_world.Get_rank() +if rank == 0: + color = 0 +else: + color = 1 +new_comm = comm_world.Split(color) + +########################## +# physics parameters +########################## + +# different communicators will be passed different plasma density +plasma_density = [1.0e18, 1.0e19] +plasma_xmin = 0.0 +plasma_x_velocity = 0.1 * constants.c + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 10 +diagnostic_intervals = "::10" + +# --- Grid +nx = 64 +ny = 64 + +xmin = -20.0e-6 +ymin = -20.0e-6 +xmax = +20.0e-6 +ymax = +20.0e-6 + +number_per_cell_each_dim = [2, 2] + +########################## +# physics components +########################## + +uniform_plasma = picmi.UniformDistribution( + density=plasma_density[color], + upper_bound=[0.0, None, None], + directed_velocity=[0.1 * constants.c, 0.0, 0.0], +) + +electrons = picmi.Species( + particle_type="electron", name="electrons", initial_distribution=uniform_plasma +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic"], + moving_window_velocity=[0.0, 0.0, 0.0], + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.0) + +########################## +# diagnostics +########################## + +field_diag = picmi.FieldDiagnostic( + name=f"diag{color + 1}", + grid=grid, + period=diagnostic_intervals, + data_list=["Ex", "Jx"], + warpx_file_prefix=f"Python_pass_mpi_comm_plt{color + 1}_", +) + +part_diag = picmi.ParticleDiagnostic( + name=f"diag{color + 1}", + period=diagnostic_intervals, + species=[electrons], + data_list=["weighting", "ux"], +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + warpx_current_deposition_algo="direct", +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout( + n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid + ), +) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +########################## +# simulation run +########################## + +# TODO: Enable in pyAMReX, then enable lines in inputs_test_2d_pass_mpi_comm_picmi.py again +# https://github.com/AMReX-Codes/pyamrex/issues/163 + +# sim.step(max_steps, mpi_comm=new_comm) + +########################## +# test +########################## + +# NOTE: Some tests are done in this input PICMI file. These tests +# check that amrex initialized the correct amount of procs and +# that the procs ranks in amrex are correct. +# If any of these tests fail, the terminal will print that the +# program crashed. + +# TODO: Enable in pyAMReX, then enable lines in inputs_test_2d_pass_mpi_comm_picmi.py again +# https://github.com/AMReX-Codes/pyamrex/issues/163 +# comm_world_size = comm_world.size +# new_comm_size = new_comm.size + +# if color == 0: +# # verify that communicator contains correct number of procs (1) +# assert sim.extension.getNProcs() == comm_world_size - 1 +# assert sim.extension.getNProcs() == new_comm_size + +# else: +# # verify that amrex initialized with 1 fewer proc than comm world +# assert sim.extension.getNProcs() == comm_world_size - 1 +# assert sim.extension.getNProcs() == new_comm_size + +# verify that amrex proc ranks are offset by -1 from +# world comm proc ranks +# assert libwarpx.amr.ParallelDescriptor.MyProc() == rank - 1 diff --git a/Examples/Tests/pec/CMakeLists.txt b/Examples/Tests/pec/CMakeLists.txt new file mode 100644 index 00000000000..15aa17c2d5f --- /dev/null +++ b/Examples/Tests/pec/CMakeLists.txt @@ -0,0 +1,72 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_pec_field # name + 3 # dims + 2 # nprocs + inputs_test_3d_pec_field # inputs + "analysis_pec.py diags/diag1000125" # analysis + "analysis_default_regression.py --path diags/diag1000125" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_pec_field_mr # name + 3 # dims + 2 # nprocs + inputs_test_3d_pec_field_mr # inputs + "analysis_pec_mr.py diags/diag1000125" # analysis + "analysis_default_regression.py --path diags/diag1000125" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_pec_particle # name + 3 # dims + 2 # nprocs + inputs_test_3d_pec_particle # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_pec_field_insulator # name + 2 # dims + 2 # nprocs + inputs_test_2d_pec_field_insulator # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_pmc_field # name + 3 # dims + 2 # nprocs + inputs_test_3d_pmc_field # inputs + "analysis_pec.py diags/diag1000134" # analysis + "analysis_default_regression.py --path diags/diag1000134" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_pec_field_insulator_implicit # name + 2 # dims + 2 # nprocs + inputs_test_2d_pec_field_insulator_implicit # inputs + "analysis_pec_insulator_implicit.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_pec_field_insulator_implicit_restart # name + 2 # dims + 2 # nprocs + inputs_test_2d_pec_field_insulator_implicit_restart # inputs + "analysis_pec_insulator_implicit.py diags/diag1000020" # analysis + "analysis_default_regression.py --path diags/diag1000020" # checksum + test_2d_pec_field_insulator_implicit # dependency +) diff --git a/Examples/Tests/pec/analysis_default_regression.py b/Examples/Tests/pec/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/pec/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/pec/analysis_pec.py b/Examples/Tests/pec/analysis_pec.py index 841cc06ae22..251a17e017c 100755 --- a/Examples/Tests/pec/analysis_pec.py +++ b/Examples/Tests/pec/analysis_pec.py @@ -11,13 +11,11 @@ # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. -import os -import re import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -25,50 +23,58 @@ import numpy as np -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] # Parameters (these parameters must match the parameters in `inputs.multi.rt`) -Ey_in = 1.e5 -E_th = 2.0*Ey_in +Ey_in = 1.0e5 +E_th = 2.0 * Ey_in # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) -Ey_array = data[('mesh','Ey')].to_ndarray() +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Ey_array = data[("mesh", "Ey")].to_ndarray() max_Ey_sim = Ey_array.max() min_Ey_sim = Ey_array.min() -max_Ey_error = abs(max_Ey_sim-E_th)/abs(E_th) -min_Ey_error = abs(min_Ey_sim - (-E_th))/abs(E_th) -print('Ey_max is %s: Max Eth is %s : Max error for Ey_max: %.2e' %(max_Ey_sim,E_th,max_Ey_error)) -print('Ey_min is %s: Min Eth is %s : Max error for Ey_min: %.2e' %(min_Ey_sim,(-E_th), min_Ey_error)) -error_rel = 0. -max_Ey_error_rel = max( error_rel, max_Ey_error ) -min_Ey_error_rel = max( error_rel, min_Ey_error ) +max_Ey_error = abs(max_Ey_sim - E_th) / abs(E_th) +min_Ey_error = abs(min_Ey_sim - (-E_th)) / abs(E_th) +print( + "Ey_max is %s: Max Eth is %s : Max error for Ey_max: %.2e" + % (max_Ey_sim, E_th, max_Ey_error) +) +print( + "Ey_min is %s: Min Eth is %s : Max error for Ey_min: %.2e" + % (min_Ey_sim, (-E_th), min_Ey_error) +) +error_rel = 0.0 +max_Ey_error_rel = max(error_rel, max_Ey_error) +min_Ey_error_rel = max(error_rel, min_Ey_error) # Plot the last field from the loop (Ez at iteration 40) -field = 'Ey' +field = "Ey" xCell = ds.domain_dimensions[0] yCell = ds.domain_dimensions[1] zCell = ds.domain_dimensions[2] -z_array = data['z'].to_ndarray() - -plt.figure(figsize=(8,4)) -plt.plot(z_array[int(xCell/2),int(yCell/2),:]-z_array[int(xCell/2),int(yCell/2),int(zCell/2)],Ey_array[int(xCell/2),int(yCell/2),:]) +z_array = data["z"].to_ndarray() + +plt.figure(figsize=(8, 4)) +plt.plot( + z_array[int(xCell / 2), int(yCell / 2), :] + - z_array[int(xCell / 2), int(yCell / 2), int(zCell / 2)], + Ey_array[int(xCell / 2), int(yCell / 2), :], +) plt.ylim(-1.2e-3, 1.2e-3) plt.ylim(-2.2e5, 2.2e5) plt.xlim(-4.0e-6, 4.000001e-6) -plt.xticks(np.arange(-4.e-6,4.000001e-6, step=1e-6)) -plt.xlabel('z (m)') -plt.ylabel(field +' (V/m)') +plt.xticks(np.arange(-4.0e-6, 4.000001e-6, step=1e-6)) +plt.xlabel("z (m)") +plt.ylabel(field + " (V/m)") plt.tight_layout() -plt.savefig('Ey_pec_analysis.png') +plt.savefig("Ey_pec_analysis.png") tolerance_rel = 0.01 @@ -77,12 +83,5 @@ print("max_Ey_error_rel : " + str(max_Ey_error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( max_Ey_error_rel < tolerance_rel ) -assert( min_Ey_error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] - -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +assert max_Ey_error_rel < tolerance_rel +assert min_Ey_error_rel < tolerance_rel diff --git a/Examples/Tests/pec/analysis_pec_insulator_implicit.py b/Examples/Tests/pec/analysis_pec_insulator_implicit.py new file mode 100755 index 00000000000..1fdbc2261a8 --- /dev/null +++ b/Examples/Tests/pec/analysis_pec_insulator_implicit.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the scripts `inputs_test_2d_pec_field_insulator_implicit` and +# `inputs_test_2d_pec_field_insulator_implicit_restart`. +# The scripts model an insulator boundary condition on part of the +# upper x boundary that pushes B field into the domain. The implicit +# solver is used, converging to machine tolerance. The energy accounting +# should be exact to machine precision, so that the energy is the system +# should be the same as the amount of energy pushed in from the boundary. +# This is checked using the FieldEnergy and FieldPoyntingFlux reduced +# diagnostics. +import sys + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +# this will be the name of the plot file +fn = sys.argv[1] + +EE = np.loadtxt(f"{fn}/../reducedfiles/fieldenergy.txt", skiprows=1) +SS = np.loadtxt(f"{fn}/../reducedfiles/poyntingflux.txt", skiprows=1) +SSsum = SS[:, 2:6].sum(1) +EEloss = SS[:, 7:].sum(1) + +dt = EE[1, 1] + +fig, ax = plt.subplots() +ax.plot(EE[:, 0], EE[:, 2], label="field energy") +ax.plot(SS[:, 0], -EEloss, label="-flux*dt") +ax.legend() +ax.set_xlabel("time (s)") +ax.set_ylabel("energy (J)") +fig.savefig("energy_history.png") + +fig, ax = plt.subplots() +ax.plot(EE[:, 0], (EE[:, 2] + EEloss) / EE[:, 2].max()) +ax.set_xlabel("time (s)") +ax.set_ylabel("energy difference/max energy (1)") +fig.savefig("energy_difference.png") + +tolerance_rel = 1.0e-13 + +energy_difference_fraction = np.abs((EE[:, 2] + EEloss) / EE[:, 2].max()).max() +print(f"energy accounting error = {energy_difference_fraction}") +print(f"tolerance_rel = {tolerance_rel}") + +assert energy_difference_fraction < tolerance_rel diff --git a/Examples/Tests/pec/analysis_pec_mr.py b/Examples/Tests/pec/analysis_pec_mr.py index e8aab4dcd6f..a99c4b0bafb 100755 --- a/Examples/Tests/pec/analysis_pec_mr.py +++ b/Examples/Tests/pec/analysis_pec_mr.py @@ -11,13 +11,11 @@ # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. -import os -import re import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import yt @@ -25,51 +23,59 @@ import numpy as np -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # this will be the name of the plot file fn = sys.argv[1] # Parameters (these parameters must match the parameters in `inputs.multi.rt`) -Ey_in = 1.e5 -E_th = 2.0*Ey_in +Ey_in = 1.0e5 +E_th = 2.0 * Ey_in # Read the file ds = yt.load(fn) t0 = ds.current_time.to_value() -data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) -Ey_array = data[('mesh','Ey')].to_ndarray() +data = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Ey_array = data[("mesh", "Ey")].to_ndarray() max_Ey_sim = Ey_array.max() min_Ey_sim = Ey_array.min() -max_Ey_error = abs(max_Ey_sim-E_th)/abs(E_th) -min_Ey_error = abs(min_Ey_sim - (-E_th))/abs(E_th) -print('Ey_max is %s: Max Eth is %s : Max error for Ey_max: %.2e' %(max_Ey_sim,E_th,max_Ey_error)) -print('Ey_min is %s: Min Eth is %s : Max error for Ey_min: %.2e' %(min_Ey_sim,(-E_th), min_Ey_error)) -error_rel = 0. -max_Ey_error_rel = max( error_rel, max_Ey_error ) -min_Ey_error_rel = max( error_rel, min_Ey_error ) +max_Ey_error = abs(max_Ey_sim - E_th) / abs(E_th) +min_Ey_error = abs(min_Ey_sim - (-E_th)) / abs(E_th) +print( + "Ey_max is %s: Max Eth is %s : Max error for Ey_max: %.2e" + % (max_Ey_sim, E_th, max_Ey_error) +) +print( + "Ey_min is %s: Min Eth is %s : Max error for Ey_min: %.2e" + % (min_Ey_sim, (-E_th), min_Ey_error) +) +error_rel = 0.0 +max_Ey_error_rel = max(error_rel, max_Ey_error) +min_Ey_error_rel = max(error_rel, min_Ey_error) # Plot the last field from the loop (Ez at iteration 40) # Plot the last field from the loop (Ez at iteration 40) -field = 'Ey' +field = "Ey" xCell = ds.domain_dimensions[0] yCell = ds.domain_dimensions[1] zCell = ds.domain_dimensions[2] -z_array = data['z'].to_ndarray() - -plt.figure(figsize=(8,4)) -plt.plot(z_array[int(xCell/2),int(yCell/2),:]-z_array[int(xCell/2),int(yCell/2),int(zCell/2)],Ey_array[int(xCell/2),int(yCell/2),:]) +z_array = data["z"].to_ndarray() + +plt.figure(figsize=(8, 4)) +plt.plot( + z_array[int(xCell / 2), int(yCell / 2), :] + - z_array[int(xCell / 2), int(yCell / 2), int(zCell / 2)], + Ey_array[int(xCell / 2), int(yCell / 2), :], +) plt.ylim(-1.2e-3, 1.2e-3) plt.ylim(-2.2e5, 2.2e5) plt.xlim(-4.0e-6, 4.000001e-6) -plt.xticks(np.arange(-4.e-6,4.000001e-6, step=1e-6)) -plt.xlabel('z (m)') -plt.ylabel(field +' (V/m)') +plt.xticks(np.arange(-4.0e-6, 4.000001e-6, step=1e-6)) +plt.xlabel("z (m)") +plt.ylabel(field + " (V/m)") plt.tight_layout() -plt.savefig('Ey_pec_analysis_mr.png') +plt.savefig("Ey_pec_analysis_mr.png") tolerance_rel = 0.05 @@ -77,12 +83,5 @@ print("max_Ey_error_rel : " + str(max_Ey_error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( max_Ey_error_rel < tolerance_rel ) -assert( min_Ey_error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] - -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) -else: - checksumAPI.evaluate_checksum(test_name, fn) +assert max_Ey_error_rel < tolerance_rel +assert min_Ey_error_rel < tolerance_rel diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator new file mode 100644 index 00000000000..912b77efcf6 --- /dev/null +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator @@ -0,0 +1,34 @@ +# Maximum number of time steps +max_step = 10 + +# number of grid points +amr.n_cell = 32 32 +amr.blocking_factor = 16 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0. 2.e-2 # physical domain +geometry.prob_hi = 1.e-2 3.e-2 + +# Boundary condition +boundary.field_lo = neumann periodic +boundary.field_hi = pec_insulator periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# CFL +warpx.cfl = 1.0 + +insulator.area_x_hi(y,z) = (2.25e-2 <= z and z <= 2.75e-2) +insulator.By_x_hi(y,z,t) = min(t/1.0e-12,1)*1.e1*3.3e-4 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 10 +diag1.diag_type = Full diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit new file mode 100644 index 00000000000..ec61e3f8605 --- /dev/null +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit @@ -0,0 +1,73 @@ +# Maximum number of time steps +max_step = 20 + +# number of grid points +amr.n_cell = 32 32 +amr.blocking_factor = 16 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0. 2.e-2 # physical domain +geometry.prob_hi = 1.e-2 3.e-2 + +# Boundary condition +boundary.field_lo = neumann periodic +boundary.field_hi = pec_insulator periodic + +insulator.area_x_hi(y,z) = (2.25e-2 <= z and z <= 2.75e-2) +insulator.By_x_hi(y,z,t) = min(t/1.0e-12,1)*1.e1*3.3e-4 + +warpx.serialize_initial_conditions = 1 + +# Implicit setup +# Note that this is the CFL step size for the explicit simulation, over 2. +# This value allows quick convergence of the Picard solver. +warpx.const_dt = 7.37079480234276e-13/2. + +algo.maxwell_solver = Yee +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +#implicit_evolve.max_particle_iterations = 21 +#implicit_evolve.particle_tolerance = 1.0e-12 + +implicit_evolve.nonlinear_solver = "picard" +picard.verbose = true +picard.max_iterations = 25 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +#implicit_evolve.nonlinear_solver = "newton" +#newton.verbose = true +#newton.max_iterations = 20 +#newton.relative_tolerance = 1.0e-20 +#newton.absolute_tolerance = 0.0 +#newton.require_convergence = false + +#gmres.verbose_int = 2 +#gmres.max_iterations = 1000 +#gmres.relative_tolerance = 1.0e-20 +#gmres.absolute_tolerance = 0.0 + +# Verbosity +warpx.verbose = 1 + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 20 +diag1.diag_type = Full + +chk.intervals = 10 +chk.diag_type = Full +chk.format = checkpoint + +warpx.reduced_diags_names = fieldenergy poyntingflux +poyntingflux.type = FieldPoyntingFlux +poyntingflux.intervals = 1 +fieldenergy.type = FieldEnergy +fieldenergy.intervals = 1 diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit_restart b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit_restart new file mode 100644 index 00000000000..35b78d01acd --- /dev/null +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator_implicit_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_2d_pec_field_insulator_implicit + +# test input parameters +amr.restart = "../test_2d_pec_field_insulator_implicit/diags/chk000010" diff --git a/Examples/Tests/pec/inputs_field_PEC_3d b/Examples/Tests/pec/inputs_test_3d_pec_field similarity index 100% rename from Examples/Tests/pec/inputs_field_PEC_3d rename to Examples/Tests/pec/inputs_test_3d_pec_field diff --git a/Examples/Tests/pec/inputs_field_PEC_mr_3d b/Examples/Tests/pec/inputs_test_3d_pec_field_mr similarity index 100% rename from Examples/Tests/pec/inputs_field_PEC_mr_3d rename to Examples/Tests/pec/inputs_test_3d_pec_field_mr diff --git a/Examples/Tests/pec/inputs_particle_PEC_3d b/Examples/Tests/pec/inputs_test_3d_pec_particle similarity index 100% rename from Examples/Tests/pec/inputs_particle_PEC_3d rename to Examples/Tests/pec/inputs_test_3d_pec_particle diff --git a/Examples/Tests/pec/inputs_test_3d_pmc_field b/Examples/Tests/pec/inputs_test_3d_pmc_field new file mode 100644 index 00000000000..2fc1cb9e5ab --- /dev/null +++ b/Examples/Tests/pec/inputs_test_3d_pmc_field @@ -0,0 +1,54 @@ +# Set-up to test the PMC Boundary condition for the fields +# Constructive interference between the incident and reflected wave result in a +# standing wave. + +# max step +max_step = 134 + +# number of grid points +amr.n_cell = 32 32 256 + +# Maximum allowable size of each subdomain +amr.max_grid_size = 1024 +amr.blocking_factor = 32 + +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -8.e-6 -8.e-6 -4.e-6 +geometry.prob_hi = 8.e-6 8.e-6 4.e-6 + +# Boundary condition +boundary.field_lo = periodic periodic pmc +boundary.field_hi = periodic periodic pmc + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +# CFL +warpx.cfl = 0.9 + + +my_constants.z1 = -2.e-6 +my_constants.z2 = 2.e-6 +my_constants.wavelength = 1.e-6 +warpx.E_ext_grid_init_style = parse_E_ext_grid_function +warpx.Ez_external_grid_function(x,y,z) = "0." +warpx.Ex_external_grid_function(x,y,z) = "0." +warpx.Ey_external_grid_function(x,y,z) = "((1.e5*sin(2*pi*(z)/wavelength)) * (zz1))" + +warpx.B_ext_grid_init_style = parse_B_ext_grid_function +warpx.Bx_external_grid_function(x,y,z)= "(((-1.e5*sin(2*pi*(z)/wavelength))/clight))*(zz1) " +warpx.By_external_grid_function(x,y,z)= "0." +warpx.Bz_external_grid_function(x,y,z) = "0." + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 134 +diag1.diag_type = Full +diag1.fields_to_plot = Ey Bx diff --git a/Examples/Tests/performance_tests/automated_test_1_uniform_rest_32ppc b/Examples/Tests/performance_tests/automated_test_1_uniform_rest_32ppc deleted file mode 120000 index 169abb79fde..00000000000 --- a/Examples/Tests/performance_tests/automated_test_1_uniform_rest_32ppc +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_1_uniform_rest_32ppc \ No newline at end of file diff --git a/Examples/Tests/performance_tests/automated_test_2_uniform_rest_1ppc b/Examples/Tests/performance_tests/automated_test_2_uniform_rest_1ppc deleted file mode 120000 index 5f6c6d2ed5d..00000000000 --- a/Examples/Tests/performance_tests/automated_test_2_uniform_rest_1ppc +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_2_uniform_rest_1ppc \ No newline at end of file diff --git a/Examples/Tests/performance_tests/automated_test_3_uniform_drift_4ppc b/Examples/Tests/performance_tests/automated_test_3_uniform_drift_4ppc deleted file mode 120000 index 0fe17ba03cb..00000000000 --- a/Examples/Tests/performance_tests/automated_test_3_uniform_drift_4ppc +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_3_uniform_drift_4ppc \ No newline at end of file diff --git a/Examples/Tests/performance_tests/automated_test_4_labdiags_2ppc b/Examples/Tests/performance_tests/automated_test_4_labdiags_2ppc deleted file mode 120000 index 1e67353c400..00000000000 --- a/Examples/Tests/performance_tests/automated_test_4_labdiags_2ppc +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_4_labdiags_2ppc \ No newline at end of file diff --git a/Examples/Tests/performance_tests/automated_test_5_loadimbalance b/Examples/Tests/performance_tests/automated_test_5_loadimbalance deleted file mode 120000 index 40734b2fe9b..00000000000 --- a/Examples/Tests/performance_tests/automated_test_5_loadimbalance +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_5_loadimbalance \ No newline at end of file diff --git a/Examples/Tests/performance_tests/automated_test_6_output_2ppc b/Examples/Tests/performance_tests/automated_test_6_output_2ppc deleted file mode 120000 index 4be041eb91e..00000000000 --- a/Examples/Tests/performance_tests/automated_test_6_output_2ppc +++ /dev/null @@ -1 +0,0 @@ -../../../Tools/PerformanceTests/automated_test_6_output_2ppc \ No newline at end of file diff --git a/Examples/Tests/photon_pusher/CMakeLists.txt b/Examples/Tests/photon_pusher/CMakeLists.txt new file mode 100644 index 00000000000..78bc1d0b416 --- /dev/null +++ b/Examples/Tests/photon_pusher/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_photon_pusher # name + 3 # dims + 2 # nprocs + inputs_test_3d_photon_pusher # inputs + "analysis.py diags/diag1000050" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency +) diff --git a/Examples/Tests/photon_pusher/analysis.py b/Examples/Tests/photon_pusher/analysis.py new file mode 100755 index 00000000000..e2ccfc42656 --- /dev/null +++ b/Examples/Tests/photon_pusher/analysis.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Luca Fedeli, Maxence Thevenet, Weiqun Zhang +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +import sys + +import numpy as np +import yt + +# This script checks if photons initialized with different momenta and +# different initial directions propagate along straight lines at the speed of +# light. The plotfile to be analyzed is passed as a command line argument. + +# If the script is run without a command line argument, it regenerates a new +# inputfile according to the initial conditions listed below. + + +# Physical constants +c = 299792458.0 +m_e = 9.1093837015e-31 +# ________________________________________ + +# Test cases +spec_names = [ + "p_xp_1", + "p_xn_1", + "p_yp_1", + "p_yn_1", + "p_zp_1", + "p_zn_1", + "p_dp_1", + "p_dn_1", + "p_xp_10", + "p_xn_10", + "p_yp_10", + "p_yn_10", + "p_zp_10", + "p_zn_10", + "p_dp_10", + "p_dn_10", +] +# photon momenta are in units of m_e c +mxp1 = np.array([1, 0.0, 0.0]) +mxn1 = np.array([-1, 0.0, 0.0]) +myp1 = np.array([0.0, 1, 0.0]) +myn1 = np.array([0.0, -1, 0.0]) +mzp1 = np.array([0.0, 0.0, 1]) +mzn1 = np.array([0.0, 0.0, -1]) +mdp1 = np.array([1, 1, 1]) +mdn1 = np.array([-1, -1, -1]) +mxp10 = np.array([10, 0.0, 0.0]) +mxn10 = np.array([-10, 0.0, 0.0]) +myp10 = np.array([0.0, 10, 0.0]) +myn10 = np.array([0.0, -10, 0.0]) +mzp10 = np.array([0.0, 0.0, 10]) +mzn10 = np.array([0.0, 0.0, -10]) +mdp10 = np.array([10, 10, 10]) +mdn10 = np.array([-10, -10, -10]) +gamma_beta_list = np.array( + [ + mxp1, + mxn1, + myp1, + myn1, + mzp1, + mzn1, + mdp1, + mdn1, + mxp10, + mxn10, + myp10, + myn10, + mzp10, + mzn10, + mdp10, + mdn10, + ] +) +init_pos = np.array([0.0, 0.0, 0.0]) +# ________________________________________ + +# Tolerance +tol_pos = 1.0e-14 +tol_mom = 0.0 # momentum should be conserved exactly +# ________________________________________ + +# Input filename +inputname = "inputs" +# ________________________________________ + + +# This function reads the WarpX plotfile given as the first command-line +# argument, and check if the position of each photon agrees with theory. +def check(): + filename = sys.argv[1] + data_set_end = yt.load(filename) + + sim_time = data_set_end.current_time.to_value() + + # expected positions list + ll = sim_time * c + answ_pos = init_pos + ll * gamma_beta_list / np.linalg.norm( + gamma_beta_list, axis=1, keepdims=True + ) + + # expected momenta list + answ_mom = m_e * c * gamma_beta_list # momenta don't change + + # simulation results + all_data = data_set_end.all_data() + res_pos = [ + np.array( + [ + all_data[sp, "particle_position_x"].v[0], + all_data[sp, "particle_position_y"].v[0], + all_data[sp, "particle_position_z"].v[0], + ] + ) + for sp in spec_names + ] + res_mom = [ + np.array( + [ + all_data[sp, "particle_momentum_x"].v[0], + all_data[sp, "particle_momentum_y"].v[0], + all_data[sp, "particle_momentum_z"].v[0], + ] + ) + for sp in spec_names + ] + + # check discrepancies + disc_pos = [ + np.linalg.norm(a - b) / np.linalg.norm(b) for a, b in zip(res_pos, answ_pos) + ] + disc_mom = [ + np.linalg.norm(a - b) / np.linalg.norm(b) for a, b in zip(res_mom, answ_mom) + ] + + print("max(disc_pos) = %s" % max(disc_pos)) + print("tol_pos = %s" % tol_pos) + print("max(disc_mom) = %s" % max(disc_mom)) + print("tol_mom = %s" % tol_mom) + + assert (max(disc_pos) <= tol_pos) and (max(disc_mom) <= tol_mom) + + +# This function generates the input file to test the photon pusher. +def generate(): + with open(inputname, "w") as f: + f.write("#Automatically generated inputfile\n") + f.write("#Run check.py without arguments to regenerate\n") + f.write("#\n\n") + f.write("max_step = 50\n") + f.write("amr.n_cell = 64 64 64\n") + f.write("amr.max_level = 0\n") + f.write("amr.blocking_factor = 8\n") + f.write("amr.max_grid_size = 8\n") + f.write("amr.plot_int = 1\n") + f.write("geometry.dims = 3\n") + f.write("boundary.field_lo = periodic periodic periodic\n") + f.write("boundary.field_hi = periodic periodic periodic\n") + f.write("geometry.prob_lo = -0.5e-6 -0.5e-6 -0.5e-6\n") + f.write("geometry.prob_hi = 0.5e-6 0.5e-6 0.5e-6\n") + f.write("algo.charge_deposition = standard\n") + f.write("algo.field_gathering = energy-conserving\n") + f.write("warpx.cfl = 1.0\n") + + f.write("particles.species_names = {}\n".format(" ".join(spec_names))) + f.write("particles.photon_species = {}\n".format(" ".join(spec_names))) + + f.write("\namr.plot_int = 50\n\n") + + for name in spec_names: + f.write("diag1.{}.variables = ux uy uz\n".format(name)) + + f.write("\n") + + data = zip(spec_names, gamma_beta_list) + for case in data: + name = case[0] + velx, vely, velz = case[1] + f.write("{}.species_type = photon\n".format(name)) + f.write('{}.injection_style = "SingleParticle"\n'.format(name)) + f.write( + "{}.single_particle_pos = {} {} {}\n".format( + name, init_pos[0], init_pos[1], init_pos[2] + ) + ) + f.write("{}.single_particle_u = {} {} {}\n".format(name, velx, vely, velz)) + f.write("{}.single_particle_weight = 1.0\n".format(name)) + f.write("\n") + + +def main(): + if len(sys.argv) < 2: + generate() + else: + check() + + +if __name__ == "__main__": + main() diff --git a/Examples/Tests/photon_pusher/analysis_default_regression.py b/Examples/Tests/photon_pusher/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/photon_pusher/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/photon_pusher/analysis_photon_pusher.py b/Examples/Tests/photon_pusher/analysis_photon_pusher.py deleted file mode 100755 index 7518bd5adb0..00000000000 --- a/Examples/Tests/photon_pusher/analysis_photon_pusher.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Luca Fedeli, Maxence Thevenet, Weiqun Zhang -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import os -import sys - -import numpy as np -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -#This script checks if photons initialized with different momenta and -#different initial directions propagate along straight lines at the speed of -#light. The plotfile to be analyzed is passed as a command line argument. - -#If the script is run without a command line argument, it regenerates a new -#inputfile according to the initial conditions listed below. - - -#Physical constants -c = 299792458. -m_e = 9.1093837015e-31 -#________________________________________ - -#Test cases -spec_names = ["p_xp_1", "p_xn_1", "p_yp_1", "p_yn_1", - "p_zp_1", "p_zn_1","p_dp_1", "p_dn_1", - "p_xp_10", "p_xn_10", "p_yp_10", "p_yn_10", - "p_zp_10", "p_zn_10", "p_dp_10", "p_dn_10"] -#photon momenta are in units of m_e c -mxp1 = np.array([1, 0.0, 0.0]) -mxn1 = np.array([-1, 0.0, 0.0]) -myp1 = np.array([0.0, 1, 0.0]) -myn1 = np.array([0.0, -1, 0.0]) -mzp1 = np.array([0.0, 0.0, 1]) -mzn1 = np.array([0.0, 0.0, -1]) -mdp1 = np.array([1, 1, 1]) -mdn1 = np.array([-1, -1, -1]) -mxp10 = np.array([10, 0.0, 0.0]) -mxn10 = np.array([-10, 0.0, 0.0]) -myp10 = np.array([0.0, 10, 0.0]) -myn10 = np.array([0.0, -10, 0.0]) -mzp10 = np.array([0.0, 0.0, 10]) -mzn10 = np.array([0.0, 0.0, -10]) -mdp10 = np.array([10, 10, 10]) -mdn10 = np.array([-10,-10, -10]) -gamma_beta_list = np.array([mxp1, mxn1, myp1, myn1, mzp1, mzn1, mdp1, mdn1, - mxp10, mxn10, myp10, myn10, mzp10, mzn10, mdp10, mdn10]) -init_pos = np.array([0.0, 0.0, 0.0]) -#________________________________________ - -#Tolerance -tol_pos = 1.0e-14; -tol_mom = 0.0; #momentum should be conserved exactly -#________________________________________ - -#Input filename -inputname = "inputs" -#________________________________________ - -# This function reads the WarpX plotfile given as the first command-line -# argument, and check if the position of each photon agrees with theory. -def check(): - filename = sys.argv[1] - data_set_end = yt.load(filename) - - sim_time = data_set_end.current_time.to_value() - - #expected positions list - ll = sim_time*c - answ_pos = init_pos + \ - ll*gamma_beta_list/np.linalg.norm(gamma_beta_list,axis=1, keepdims=True) - - #expected momenta list - answ_mom = m_e * c *gamma_beta_list #momenta don't change - - #simulation results - all_data = data_set_end.all_data() - res_pos = [np.array([ - all_data[sp, 'particle_position_x'].v[0], - all_data[sp, 'particle_position_y'].v[0], - all_data[sp, 'particle_position_z'].v[0]]) - for sp in spec_names] - res_mom = [np.array([ - all_data[sp, 'particle_momentum_x'].v[0], - all_data[sp, 'particle_momentum_y'].v[0], - all_data[sp, 'particle_momentum_z'].v[0]]) - for sp in spec_names] - - #check discrepancies - disc_pos = [np.linalg.norm(a-b)/np.linalg.norm(b) - for a,b in zip(res_pos, answ_pos)] - disc_mom = [np.linalg.norm(a-b)/np.linalg.norm(b) - for a,b in zip(res_mom, answ_mom)] - - print("max(disc_pos) = %s" %max(disc_pos)) - print("tol_pos = %s" %tol_pos) - print("max(disc_mom) = %s" %max(disc_mom)) - print("tol_mom = %s" %tol_mom) - - assert ((max(disc_pos) <= tol_pos) and (max(disc_mom) <= tol_mom)) - - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename) - -# This function generates the input file to test the photon pusher. -def generate(): - with open(inputname,'w') as f: - f.write("#Automatically generated inputfile\n") - f.write("#Run check.py without arguments to regenerate\n") - f.write("#\n\n") - f.write("max_step = 50\n") - f.write("amr.n_cell = 64 64 64\n") - f.write("amr.max_level = 0\n") - f.write("amr.blocking_factor = 8\n") - f.write("amr.max_grid_size = 8\n") - f.write("amr.plot_int = 1\n") - f.write("geometry.dims = 3\n") - f.write("boundary.field_lo = periodic periodic periodic\n") - f.write("boundary.field_hi = periodic periodic periodic\n") - f.write("geometry.prob_lo = -0.5e-6 -0.5e-6 -0.5e-6\n") - f.write("geometry.prob_hi = 0.5e-6 0.5e-6 0.5e-6\n") - f.write("algo.charge_deposition = standard\n") - f.write("algo.field_gathering = energy-conserving\n") - f.write("warpx.cfl = 1.0\n") - - f.write("particles.species_names = {}\n".format(' '.join(spec_names))) - f.write("particles.photon_species = {}\n".format(' '.join(spec_names))) - - f.write("\namr.plot_int = 50\n\n") - - for name in spec_names: - f.write("diag1.{}.variables = ux uy uz\n".format(name)) - - f.write("\n") - - data = zip(spec_names, gamma_beta_list) - for case in data: - name = case[0] - velx, vely ,velz = case[1] - f.write("{}.species_type = photon\n".format(name)) - f.write('{}.injection_style = "SingleParticle"\n'.format(name)) - f.write("{}.single_particle_pos = {} {} {}\n". - format(name, init_pos[0], init_pos[1], init_pos[2])) - f.write("{}.single_particle_u = {} {} {}\n". - format(name, velx, vely, velz)) - f.write("{}.single_particle_weight = 1.0\n".format(name)) - f.write("\n".format(name)) - -def main(): - if (len(sys.argv) < 2): - generate() - else: - check() - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/photon_pusher/inputs_3d b/Examples/Tests/photon_pusher/inputs_test_3d_photon_pusher similarity index 100% rename from Examples/Tests/photon_pusher/inputs_3d rename to Examples/Tests/photon_pusher/inputs_test_3d_photon_pusher diff --git a/Examples/Tests/plasma_lens/CMakeLists.txt b/Examples/Tests/plasma_lens/CMakeLists.txt new file mode 100644 index 00000000000..f6d6ea6daeb --- /dev/null +++ b/Examples/Tests/plasma_lens/CMakeLists.txt @@ -0,0 +1,52 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_plasma_lens # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_lens # inputs + "analysis.py diags/diag1000084" # analysis + "analysis_default_regression.py --path diags/diag1000084" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_lens_boosted # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_lens_boosted # inputs + "analysis.py diags/diag1000084" # analysis + "analysis_default_regression.py --path diags/diag1000084" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_lens_hard_edged # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_lens_hard_edged # inputs + "analysis.py diags/diag1000084" # analysis + "analysis_default_regression.py --path diags/diag1000084" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_lens_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_lens_picmi.py # inputs + "analysis.py diags/diag1000084" # analysis + "analysis_default_regression.py --path diags/diag1000084" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_plasma_lens_short # name + 3 # dims + 2 # nprocs + inputs_test_3d_plasma_lens_short # inputs + "analysis.py diags/diag1000084" # analysis + "analysis_default_regression.py --path diags/diag1000084" # checksum + OFF # dependency +) diff --git a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py deleted file mode 100644 index 50d222bbf36..00000000000 --- a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Physical constants -c = picmi.constants.c -q_e = picmi.constants.q_e - -# Number of time steps -max_steps = 84 - -# Number of cells -nx = 16 -ny = 16 -nz = 16 - -# Physical domain -xmin = -1. -xmax = 1. -ymin = -1. -ymax = 1. -zmin = 0. -zmax = 2. - -# Create grid -grid = picmi.Cartesian3DGrid(number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions = ['dirichlet', 'dirichlet', 'dirichlet'], - upper_boundary_conditions = ['dirichlet', 'dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing'], - upper_boundary_conditions_particles = ['absorbing', 'absorbing', 'absorbing']) - -# Particles -vel_z = 0.5*c -offset_x_particle = picmi.ParticleListDistribution(x = [0.05], - y = [0.], - z = [0.05], - ux = [0.], - uy = [0.], - uz = [vel_z], - weight = [1.]) - -offset_y_particle = picmi.ParticleListDistribution(x = [0.], - y = [0.04], - z = [0.05], - ux = [0.], - uy = [0.], - uz = [vel_z], - weight = [1.]) - -electrons = picmi.Species(particle_type = 'electron', - name = 'electrons', - initial_distribution = [offset_x_particle, offset_y_particle]) - -# Plasma lenses -plasma_lenses = picmi.PlasmaLens(period = 0.5, - starts = [0.1, 0.11, 0.12, 0.13], - lengths = [0.1, 0.11, 0.12, 0.13], - strengths_E = [600000., 800000., 600000., 200000.], - strengths_B = [0.0, 0.0, 0.0, 0.0]) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver(grid = grid, - method = 'Yee', - cfl = 0.7) - -# Diagnostics -part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', - period = max_steps, - species = [electrons], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z'], - write_dir = '.', - warpx_file_prefix = 'Python_plasma_lens_plt') - -field_diag1 = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = max_steps, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir = '.', - warpx_file_prefix = 'Python_plasma_lens_plt') -# Set up simulation -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'linear', - warpx_serialize_initial_conditions = 1, - warpx_do_dynamic_scheduling = 0) - -# Add plasma electrons -sim.add_species(electrons, layout = None) - -# Add the plasma lenses -sim.add_applied_field(plasma_lenses) - -# Add diagnostics -sim.add_diagnostic(part_diag1) -sim.add_diagnostic(field_diag1) - -# Write input file that can be used to run with the compiled version -#sim.write_input_file(file_name = 'inputs_3d_picmi') - -# Advance simulation until last time step -sim.step(max_steps) diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index 212e71087f9..ed7ee653af6 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -15,7 +15,6 @@ The motion is slow enough that relativistic effects are ignored. """ -import os import sys import numpy as np @@ -23,93 +22,110 @@ from scipy.constants import c, e, m_e yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) ad = ds.all_data() # Get final position of the particles. # There are two particles, one moves in x, the other in y. # The particles may not be in order, so determine which is which # by looking at their max positions in the respective planes. -i0 = np.argmax(np.abs(ad['electrons', 'particle_position_x'].v)) -i1 = np.argmax(np.abs(ad['electrons', 'particle_position_y'].v)) +i0 = np.argmax(np.abs(ad["electrons", "particle_position_x"].v)) +i1 = np.argmax(np.abs(ad["electrons", "particle_position_y"].v)) -xx_sim = ad['electrons', 'particle_position_x'].v[i0] -yy_sim = ad['electrons', 'particle_position_y'].v[i1] -zz_sim0 = ad['electrons', 'particle_position_z'].v[i0] -zz_sim1 = ad['electrons', 'particle_position_z'].v[i1] +xx_sim = ad["electrons", "particle_position_x"].v[i0] +yy_sim = ad["electrons", "particle_position_y"].v[i1] +zz_sim0 = ad["electrons", "particle_position_z"].v[i0] +zz_sim1 = ad["electrons", "particle_position_z"].v[i1] -ux_sim = ad['electrons', 'particle_momentum_x'].v[i0]/m_e -uy_sim = ad['electrons', 'particle_momentum_y'].v[i1]/m_e +ux_sim = ad["electrons", "particle_momentum_x"].v[i0] / m_e +uy_sim = ad["electrons", "particle_momentum_y"].v[i1] / m_e -if 'warpx.gamma_boost' in ds.parameters: - gamma_boost = float(ds.parameters.get('warpx.gamma_boost')) - uz_boost = np.sqrt(gamma_boost*gamma_boost - 1.)*c +if "warpx.gamma_boost" in ds.parameters: + gamma_boost = float(ds.parameters.get("warpx.gamma_boost")) + uz_boost = np.sqrt(gamma_boost * gamma_boost - 1.0) * c time = ds.current_time.to_value() - zz_sim0 = gamma_boost*zz_sim0 + uz_boost*time - zz_sim1 = gamma_boost*zz_sim1 + uz_boost*time + zz_sim0 = gamma_boost * zz_sim0 + uz_boost * time + zz_sim1 = gamma_boost * zz_sim1 + uz_boost * time def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): - kb0 = np.sqrt(e/(m_e*gamma*vz0**2)*lens_strength) - x1 = x0*np.cos(kb0*lens_length) + (vx0/vz0)/kb0*np.sin(kb0*lens_length) - vx1 = vz0*(-kb0*x0*np.sin(kb0*lens_length) + (vx0/vz0)*np.cos(kb0*lens_length)) + kb0 = np.sqrt(e / (m_e * gamma * vz0**2) * lens_strength) + x1 = x0 * np.cos(kb0 * lens_length) + (vx0 / vz0) / kb0 * np.sin(kb0 * lens_length) + vx1 = vz0 * ( + -kb0 * x0 * np.sin(kb0 * lens_length) + (vx0 / vz0) * np.cos(kb0 * lens_length) + ) return x1, vx1 + clight = c try: - vel_z = eval(ds.parameters.get('my_constants.vel_z')) + vel_z = eval(ds.parameters.get("my_constants.vel_z")) except TypeError: # vel_z is not saved in my_constants with the PICMI version - vel_z = 0.5*c - -if 'particles.repeated_plasma_lens_period' in ds.parameters: - plasma_lens_period = float(ds.parameters.get('particles.repeated_plasma_lens_period')) - plasma_lens_starts = [float(x) for x in ds.parameters.get('particles.repeated_plasma_lens_starts').split()] - plasma_lens_lengths = [float(x) for x in ds.parameters.get('particles.repeated_plasma_lens_lengths').split()] - plasma_lens_strengths_E = [eval(x) for x in ds.parameters.get('particles.repeated_plasma_lens_strengths_E').split()] - plasma_lens_strengths_B = [eval(x) for x in ds.parameters.get('particles.repeated_plasma_lens_strengths_B').split()] -elif 'lattice.elements' in ds.parameters: - lattice_elements = ds.parameters.get('lattice.elements').split() + vel_z = 0.5 * c + +if "particles.repeated_plasma_lens_period" in ds.parameters: + plasma_lens_period = float( + ds.parameters.get("particles.repeated_plasma_lens_period") + ) + plasma_lens_starts = [ + float(x) + for x in ds.parameters.get("particles.repeated_plasma_lens_starts").split() + ] + plasma_lens_lengths = [ + float(x) + for x in ds.parameters.get("particles.repeated_plasma_lens_lengths").split() + ] + plasma_lens_strengths_E = [ + eval(x) + for x in ds.parameters.get("particles.repeated_plasma_lens_strengths_E").split() + ] + plasma_lens_strengths_B = [ + eval(x) + for x in ds.parameters.get("particles.repeated_plasma_lens_strengths_B").split() + ] +elif "lattice.elements" in ds.parameters: + lattice_elements = ds.parameters.get("lattice.elements").split() plasma_lens_zstarts = [] plasma_lens_lengths = [] plasma_lens_strengths_E = [] - z_location = 0. + z_location = 0.0 for element in lattice_elements: - element_type = ds.parameters.get(f'{element}.type') - length = float(ds.parameters.get(f'{element}.ds')) - if element_type == 'plasmalens': + element_type = ds.parameters.get(f"{element}.type") + length = float(ds.parameters.get(f"{element}.ds")) + if element_type == "plasmalens": plasma_lens_zstarts.append(z_location) plasma_lens_lengths.append(length) - plasma_lens_strengths_E.append(float(ds.parameters.get(f'{element}.dEdx'))) + plasma_lens_strengths_E.append(float(ds.parameters.get(f"{element}.dEdx"))) z_location += length plasma_lens_period = 0.5 - plasma_lens_starts = plasma_lens_zstarts - plasma_lens_period*np.arange(len(plasma_lens_zstarts)) + plasma_lens_starts = plasma_lens_zstarts - plasma_lens_period * np.arange( + len(plasma_lens_zstarts) + ) plasma_lens_strengths_B = np.zeros(len(plasma_lens_zstarts)) try: # The picmi version - x0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_x')) - y0 = float(ds.parameters.get('electrons.dist1.multiple_particles_pos_y')) - z0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_z')) - ux0 = float(ds.parameters.get('electrons.dist0.multiple_particles_ux'))*c - uy0 = float(ds.parameters.get('electrons.dist1.multiple_particles_uy'))*c - uz0 = eval(ds.parameters.get('electrons.dist0.multiple_particles_uz'))*c + x0 = float(ds.parameters.get("electrons.dist0.multiple_particles_pos_x")) + y0 = float(ds.parameters.get("electrons.dist1.multiple_particles_pos_y")) + z0 = float(ds.parameters.get("electrons.dist0.multiple_particles_pos_z")) + ux0 = float(ds.parameters.get("electrons.dist0.multiple_particles_ux")) * c + uy0 = float(ds.parameters.get("electrons.dist1.multiple_particles_uy")) * c + uz0 = eval(ds.parameters.get("electrons.dist0.multiple_particles_uz")) * c except TypeError: # The inputs version - x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) - y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) - z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) - ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c - uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c - uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c - -tt = 0. + x0 = float(ds.parameters.get("electrons.multiple_particles_pos_x").split()[0]) + y0 = float(ds.parameters.get("electrons.multiple_particles_pos_y").split()[1]) + z0 = float(ds.parameters.get("electrons.multiple_particles_pos_z").split()[0]) + ux0 = float(ds.parameters.get("electrons.multiple_particles_ux").split()[0]) * c + uy0 = float(ds.parameters.get("electrons.multiple_particles_uy").split()[1]) * c + uz0 = eval(ds.parameters.get("electrons.multiple_particles_uz").split()[0]) * c + +tt = 0.0 xx = x0 yy = y0 zz = z0 @@ -117,37 +133,45 @@ def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): uy = uy0 uz = uz0 -gamma = np.sqrt(uz0**2/c**2 + 1.) -vz = uz/gamma +gamma = np.sqrt(uz0**2 / c**2 + 1.0) +vz = uz / gamma for i in range(len(plasma_lens_starts)): - z_lens = i*plasma_lens_period + plasma_lens_starts[i] - vx = ux/gamma - vy = uy/gamma - dt = (z_lens - zz)/vz + z_lens = i * plasma_lens_period + plasma_lens_starts[i] + vx = ux / gamma + vy = uy / gamma + dt = (z_lens - zz) / vz tt = tt + dt - xx = xx + dt*vx - yy = yy + dt*vy - lens_strength = plasma_lens_strengths_E[i] + plasma_lens_strengths_B[i]*vel_z + xx = xx + dt * vx + yy = yy + dt * vy + lens_strength = plasma_lens_strengths_E[i] + plasma_lens_strengths_B[i] * vel_z xx, vx = applylens(xx, vx, vz, gamma, plasma_lens_lengths[i], lens_strength) yy, vy = applylens(yy, vy, vz, gamma, plasma_lens_lengths[i], lens_strength) - dt = plasma_lens_lengths[i]/vz + dt = plasma_lens_lengths[i] / vz tt = tt + dt - ux = gamma*vx - uy = gamma*vy + ux = gamma * vx + uy = gamma * vy zz = z_lens + plasma_lens_lengths[i] -dt0 = (zz_sim0 - zz)/vz -dt1 = (zz_sim1 - zz)/vz -vx = ux/gamma -vy = uy/gamma -xx = xx + dt0*vx -yy = yy + dt1*vy - -print(f'Error in x position is {abs(np.abs((xx - xx_sim)/xx))}, which should be < 0.02') -print(f'Error in y position is {abs(np.abs((yy - yy_sim)/yy))}, which should be < 0.02') -print(f'Error in x velocity is {abs(np.abs((ux - ux_sim)/ux))}, which should be < 0.002') -print(f'Error in y velocity is {abs(np.abs((uy - uy_sim)/uy))}, which should be < 0.002') +dt0 = (zz_sim0 - zz) / vz +dt1 = (zz_sim1 - zz) / vz +vx = ux / gamma +vy = uy / gamma +xx = xx + dt0 * vx +yy = yy + dt1 * vy + +print( + f"Error in x position is {abs(np.abs((xx - xx_sim) / xx))}, which should be < 0.02" +) +print( + f"Error in y position is {abs(np.abs((yy - yy_sim) / yy))}, which should be < 0.02" +) +print( + f"Error in x velocity is {abs(np.abs((ux - ux_sim) / ux))}, which should be < 0.002" +) +print( + f"Error in y velocity is {abs(np.abs((uy - uy_sim) / uy))}, which should be < 0.002" +) if plasma_lens_lengths[0] < 0.01: # The shorter lens requires a larger tolerance since @@ -158,15 +182,15 @@ def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): position_tolerance = 0.02 velocity_tolerance = 0.002 -assert abs(np.abs((xx - xx_sim)/xx)) < position_tolerance, Exception('error in x particle position') -assert abs(np.abs((yy - yy_sim)/yy)) < position_tolerance, Exception('error in y particle position') -assert abs(np.abs((ux - ux_sim)/ux)) < velocity_tolerance, Exception('error in x particle velocity') -assert abs(np.abs((uy - uy_sim)/uy)) < velocity_tolerance, Exception('error in y particle velocity') - -test_name = os.path.split(os.getcwd())[1] -# The PICMI and native input versions of `inputs_3d` run the same test, so -# their results are compared to the same benchmark file. -if test_name == "Python_plasma_lens": - test_name = "Plasma_lens" - -checksumAPI.evaluate_checksum(test_name, filename) +assert abs(np.abs((xx - xx_sim) / xx)) < position_tolerance, Exception( + "error in x particle position" +) +assert abs(np.abs((yy - yy_sim) / yy)) < position_tolerance, Exception( + "error in y particle position" +) +assert abs(np.abs((ux - ux_sim) / ux)) < velocity_tolerance, Exception( + "error in x particle velocity" +) +assert abs(np.abs((uy - uy_sim) / uy)) < velocity_tolerance, Exception( + "error in y particle velocity" +) diff --git a/Examples/Tests/plasma_lens/analysis_default_regression.py b/Examples/Tests/plasma_lens/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/plasma_lens/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/plasma_lens/inputs_boosted_3d b/Examples/Tests/plasma_lens/inputs_boosted_3d deleted file mode 100644 index fa18ac439c4..00000000000 --- a/Examples/Tests/plasma_lens/inputs_boosted_3d +++ /dev/null @@ -1,53 +0,0 @@ -# Maximum number of time steps -max_step = 84 - -# number of grid points -amr.n_cell = 16 16 16 - -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -1.0 -1.0 -1.0 # physical domain -geometry.prob_hi = 1.0 1.0 2.0 - -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -boundary.particle_lo = absorbing absorbing absorbing -boundary.particle_hi = absorbing absorbing absorbing - -# Algorithms -algo.particle_shape = 1 -warpx.gamma_boost = 2. -warpx.boost_direction = z -warpx.cfl = 0.7 - -my_constants.vel_z = 0.5*clight - -# particles -particles.species_names = electrons - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "MultipleParticles" -electrons.multiple_particles_pos_x = 0.05 0. -electrons.multiple_particles_pos_y = 0. 0.04 -electrons.multiple_particles_pos_z = 0.05 0.05 -electrons.multiple_particles_ux = 0. 0. -electrons.multiple_particles_uy = 0. 0. -electrons.multiple_particles_uz = vel_z/clight vel_z/clight -electrons.multiple_particles_weight = 1. 1. - -particles.E_ext_particle_init_style = repeated_plasma_lens -particles.B_ext_particle_init_style = repeated_plasma_lens -particles.repeated_plasma_lens_period = 0.5 -particles.repeated_plasma_lens_starts = 0.1 0.11 0.12 0.13 -particles.repeated_plasma_lens_lengths = 0.1 0.11 0.12 0.13 -particles.repeated_plasma_lens_strengths_E = 600000. 800000. 600000. 200000. -particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 84 -diag1.diag_type = Full -diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_3d b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens similarity index 100% rename from Examples/Tests/plasma_lens/inputs_3d rename to Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens diff --git a/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_boosted b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_boosted new file mode 100644 index 00000000000..b00779bae65 --- /dev/null +++ b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_boosted @@ -0,0 +1,53 @@ +# Maximum number of time steps +max_step = 84 + +# number of grid points +amr.n_cell = 16 16 16 + +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -1.0 -1.0 -1.866 # physical domain +geometry.prob_hi = 1.0 1.0 3.732 + +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +boundary.particle_lo = absorbing absorbing absorbing +boundary.particle_hi = absorbing absorbing absorbing + +# Algorithms +algo.particle_shape = 1 +warpx.gamma_boost = 2. +warpx.boost_direction = z +warpx.cfl = 0.7 + +my_constants.vel_z = 0.5*clight + +# particles +particles.species_names = electrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "MultipleParticles" +electrons.multiple_particles_pos_x = 0.05 0. +electrons.multiple_particles_pos_y = 0. 0.04 +electrons.multiple_particles_pos_z = 0.05 0.05 +electrons.multiple_particles_ux = 0. 0. +electrons.multiple_particles_uy = 0. 0. +electrons.multiple_particles_uz = vel_z/clight vel_z/clight +electrons.multiple_particles_weight = 1. 1. + +particles.E_ext_particle_init_style = repeated_plasma_lens +particles.B_ext_particle_init_style = repeated_plasma_lens +particles.repeated_plasma_lens_period = 0.5 +particles.repeated_plasma_lens_starts = 0.1 0.11 0.12 0.13 +particles.repeated_plasma_lens_lengths = 0.1 0.11 0.12 0.13 +particles.repeated_plasma_lens_strengths_E = 600000. 800000. 600000. 200000. +particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 84 +diag1.diag_type = Full +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_lattice_3d b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_hard_edged similarity index 100% rename from Examples/Tests/plasma_lens/inputs_lattice_3d rename to Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_hard_edged diff --git a/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_picmi.py b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_picmi.py new file mode 100644 index 00000000000..b9672e17e1a --- /dev/null +++ b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_picmi.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# Number of time steps +max_steps = 84 + +# Number of cells +nx = 16 +ny = 16 +nz = 16 + +# Physical domain +xmin = -1.0 +xmax = 1.0 +ymin = -1.0 +ymax = 1.0 +zmin = 0.0 +zmax = 2.0 + +# Create grid +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], +) + +# Particles +vel_z = 0.5 * c +offset_x_particle = picmi.ParticleListDistribution( + x=[0.05], y=[0.0], z=[0.05], ux=[0.0], uy=[0.0], uz=[vel_z], weight=[1.0] +) + +offset_y_particle = picmi.ParticleListDistribution( + x=[0.0], y=[0.04], z=[0.05], ux=[0.0], uy=[0.0], uz=[vel_z], weight=[1.0] +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=[offset_x_particle, offset_y_particle], +) + +# Plasma lenses +plasma_lenses = picmi.PlasmaLens( + period=0.5, + starts=[0.1, 0.11, 0.12, 0.13], + lengths=[0.1, 0.11, 0.12, 0.13], + strengths_E=[600000.0, 800000.0, 600000.0, 200000.0], + strengths_B=[0.0, 0.0, 0.0, 0.0], +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=0.7) + +# Diagnostics +part_diag1 = picmi.ParticleDiagnostic( + name="diag1", + period=max_steps, + species=[electrons], + data_list=["ux", "uy", "uz", "x", "y", "z"], +) + +field_diag1 = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=max_steps, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], +) +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="linear", + warpx_serialize_initial_conditions=1, + warpx_do_dynamic_scheduling=0, +) + +# Add plasma electrons +sim.add_species(electrons, layout=None) + +# Add the plasma lenses +sim.add_applied_field(plasma_lenses) + +# Add diagnostics +sim.add_diagnostic(part_diag1) +sim.add_diagnostic(field_diag1) + +# Write input file that can be used to run with the compiled version +# sim.write_input_file(file_name = 'inputs_3d_picmi') + +# Advance simulation until last time step +sim.step(max_steps) diff --git a/Examples/Tests/plasma_lens/inputs_short_3d b/Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_short similarity index 100% rename from Examples/Tests/plasma_lens/inputs_short_3d rename to Examples/Tests/plasma_lens/inputs_test_3d_plasma_lens_short diff --git a/Examples/Tests/pml/CMakeLists.txt b/Examples/Tests/pml/CMakeLists.txt new file mode 100644 index 00000000000..8ba70f77aef --- /dev/null +++ b/Examples/Tests/pml/CMakeLists.txt @@ -0,0 +1,92 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_pml_x_ckc # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_ckc # inputs + "analysis_pml_ckc.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_2d_pml_x_galilean # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_galilean # inputs + "analysis_pml_psatd.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_pml_x_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_psatd # inputs + "analysis_pml_psatd.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_2d_pml_x_psatd_restart # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_psatd_restart # inputs + "analysis_default_restart.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + test_2d_pml_x_psatd # dependency + ) +endif() + +add_warpx_test( + test_2d_pml_x_yee # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_yee # inputs + "analysis_pml_yee.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_pml_x_yee_restart # name + 2 # dims + 2 # nprocs + inputs_test_2d_pml_x_yee_restart # inputs + "analysis_default_restart.py diags/diag1000300" # analysis + "analysis_default_regression.py --path diags/diag1000300" # checksum + test_2d_pml_x_yee # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_3d_pml_psatd_dive_divb_cleaning # name + 3 # dims + 2 # nprocs + inputs_test_3d_pml_psatd_dive_divb_cleaning # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_rz_pml_psatd # name + RZ # dims + 2 # nprocs + inputs_test_rz_pml_psatd # inputs + "analysis_pml_psatd_rz.py diags/diag1000500" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/pml/analysis_default_regression.py b/Examples/Tests/pml/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/pml/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/pml/analysis_default_restart.py b/Examples/Tests/pml/analysis_default_restart.py new file mode 120000 index 00000000000..0459986eebc --- /dev/null +++ b/Examples/Tests/pml/analysis_default_restart.py @@ -0,0 +1 @@ +../../analysis_default_restart.py \ No newline at end of file diff --git a/Examples/Tests/pml/analysis_pml_ckc.py b/Examples/Tests/pml/analysis_pml_ckc.py index c4b9d54647d..b50cbb867e6 100755 --- a/Examples/Tests/pml/analysis_pml_ckc.py +++ b/Examples/Tests/pml/analysis_pml_ckc.py @@ -8,15 +8,13 @@ # License: BSD-3-Clause-LBNL -import os import sys import numpy as np import scipy.constants as scc +import yt -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(0) filename = sys.argv[1] @@ -28,31 +26,30 @@ ########################## ### FINAL LASER ENERGY ### ########################## -ds = yt.load( filename ) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() -By = all_data_level_0['boxlib', 'By'].v.squeeze() -Bz = all_data_level_0['boxlib', 'Bz'].v.squeeze() -Ex = all_data_level_0['boxlib', 'Ex'].v.squeeze() -Ey = all_data_level_0['boxlib', 'Ey'].v.squeeze() -Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() -energyE = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) -energyB = np.sum(1./scc.mu_0/2*(Bx**2+By**2+Bz**2)) +ds = yt.load(filename) +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Bx = all_data_level_0["boxlib", "Bx"].v.squeeze() +By = all_data_level_0["boxlib", "By"].v.squeeze() +Bz = all_data_level_0["boxlib", "Bz"].v.squeeze() +Ex = all_data_level_0["boxlib", "Ex"].v.squeeze() +Ey = all_data_level_0["boxlib", "Ey"].v.squeeze() +Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() +energyE = np.sum(scc.epsilon_0 / 2 * (Ex**2 + Ey**2 + Ez**2)) +energyB = np.sum(1.0 / scc.mu_0 / 2 * (Bx**2 + By**2 + Bz**2)) energy_end = energyE + energyB -Reflectivity = energy_end/energy_start +Reflectivity = energy_end / energy_start Reflectivity_theory = 1.8015e-06 -print("Reflectivity: %s" %Reflectivity) -print("Reflectivity_theory: %s" %Reflectivity_theory) +print("Reflectivity: %s" % Reflectivity) +print("Reflectivity_theory: %s" % Reflectivity_theory) -error_rel = abs(Reflectivity-Reflectivity_theory) / Reflectivity_theory -tolerance_rel = 5./100 +error_rel = abs(Reflectivity - Reflectivity_theory) / Reflectivity_theory +tolerance_rel = 5.0 / 100 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/pml/analysis_pml_psatd.py b/Examples/Tests/pml/analysis_pml_psatd.py index 50d0b2ac1c1..156f58362ce 100755 --- a/Examples/Tests/pml/analysis_pml_psatd.py +++ b/Examples/Tests/pml/analysis_pml_psatd.py @@ -13,32 +13,33 @@ import numpy as np import scipy.constants as scc +import yt -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(0) filename = sys.argv[1] -galilean = True if re.search("galilean", filename) else False +cwd = os.getcwd() +filename_init = os.path.join(cwd, "diags/diag1000050") +galilean = True if re.search("galilean", cwd) else False # Initial laser energy (at iteration 50) if galilean: - filename_init = 'pml_x_galilean_plt000050' energy_start = 4.439376199524034e-08 else: - filename_init = 'pml_x_psatd_plt000050' energy_start = 7.282940107273505e-08 # Check consistency of field energy diagnostics with initial energy above ds = yt.load(filename_init) -all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() -By = all_data_level_0['boxlib', 'By'].v.squeeze() -Bz = all_data_level_0['boxlib', 'Bz'].v.squeeze() -Ex = all_data_level_0['boxlib', 'Ex'].v.squeeze() -Ey = all_data_level_0['boxlib', 'Ey'].v.squeeze() -Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Bx = all_data_level_0["boxlib", "Bx"].v.squeeze() +By = all_data_level_0["boxlib", "By"].v.squeeze() +Bz = all_data_level_0["boxlib", "Bz"].v.squeeze() +Ex = all_data_level_0["boxlib", "Ex"].v.squeeze() +Ey = all_data_level_0["boxlib", "Ey"].v.squeeze() +Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() energyE = np.sum(0.5 * scc.epsilon_0 * (Ex**2 + Ey**2 + Ez**2)) energyB = np.sum(0.5 / scc.mu_0 * (Bx**2 + By**2 + Bz**2)) energy_start_diags = energyE + energyB @@ -47,17 +48,19 @@ print("energy_start expected = " + str(energy_start)) print("energy_start_diags = " + str(energy_start_diags)) print("relative error = " + str(error)) -assert (error < tolerance) +assert error < tolerance # Final laser energy ds = yt.load(filename) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() -By = all_data_level_0['boxlib', 'By'].v.squeeze() -Bz = all_data_level_0['boxlib', 'Bz'].v.squeeze() -Ex = all_data_level_0['boxlib', 'Ex'].v.squeeze() -Ey = all_data_level_0['boxlib', 'Ey'].v.squeeze() -Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Bx = all_data_level_0["boxlib", "Bx"].v.squeeze() +By = all_data_level_0["boxlib", "By"].v.squeeze() +Bz = all_data_level_0["boxlib", "Bz"].v.squeeze() +Ex = all_data_level_0["boxlib", "Ex"].v.squeeze() +Ey = all_data_level_0["boxlib", "Ey"].v.squeeze() +Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() energyE = np.sum(0.5 * scc.epsilon_0 * (Ex**2 + Ey**2 + Ez**2)) energyB = np.sum(0.5 / scc.mu_0 * (Bx**2 + By**2 + Bz**2)) energy_end = energyE + energyB @@ -68,14 +71,4 @@ print("reflectivity = " + str(reflectivity)) print("reflectivity_max = " + str(reflectivity_max)) -assert(reflectivity < reflectivity_max) - -# Check restart data v. original data -sys.path.insert(0, '../../../../warpx/Examples/') -from analysis_default_restart import check_restart - -if not galilean: - check_restart(filename) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +assert reflectivity < reflectivity_max diff --git a/Examples/Tests/pml/analysis_pml_psatd_rz.py b/Examples/Tests/pml/analysis_pml_psatd_rz.py index d4f1ff42ae6..f06b7a52c6c 100755 --- a/Examples/Tests/pml/analysis_pml_psatd_rz.py +++ b/Examples/Tests/pml/analysis_pml_psatd_rz.py @@ -15,41 +15,39 @@ most of the pulse escapes the radial boundary. If the PML fails, the pulse will remain with in the domain. """ -import os + import sys import numpy as np import yt yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # Open plotfile specified in command line filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. -if 'force_periodicity' in dir(ds): ds.force_periodicity() +if "force_periodicity" in dir(ds): + ds.force_periodicity() # Check that the field is low enough -ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Ex_array = ad0['boxlib', 'Er'].to_ndarray() -Ez_array = ad0['boxlib', 'Ez'].to_ndarray() +ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Ex_array = ad0["boxlib", "Er"].to_ndarray() +Ez_array = ad0["boxlib", "Ez"].to_ndarray() max_Ex = np.abs(Ex_array).max() max_Ez = np.abs(Ez_array).max() -print( f'max Ex = {max_Ex}' ) -print( f'max Ez = {max_Ez}' ) +print(f"max Ex = {max_Ex}") +print(f"max Ez = {max_Ez}") max_Efield = max(max_Ex, max_Ez) # This tolerance was obtained empirically. As the simulation progresses, the field energy is leaking # out through PML so that the max field diminishes with time. When the PML is working properly, # the field level falls below 2 at the end of the simulation. -tolerance_abs = 2. -print('tolerance_abs: ' + str(tolerance_abs)) +tolerance_abs = 2.0 +print("tolerance_abs: " + str(tolerance_abs)) assert max_Efield < tolerance_abs - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/pml/analysis_pml_yee.py b/Examples/Tests/pml/analysis_pml_yee.py index f10b281c544..b6aad4ccb7e 100755 --- a/Examples/Tests/pml/analysis_pml_yee.py +++ b/Examples/Tests/pml/analysis_pml_yee.py @@ -8,15 +8,13 @@ # License: BSD-3-Clause-LBNL -import os import sys import numpy as np import scipy.constants as scc +import yt -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +yt.funcs.mylog.setLevel(0) filename = sys.argv[1] @@ -28,37 +26,30 @@ ########################## ### FINAL LASER ENERGY ### ########################## -ds = yt.load( filename ) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() -By = all_data_level_0['boxlib', 'By'].v.squeeze() -Bz = all_data_level_0['boxlib', 'Bz'].v.squeeze() -Ex = all_data_level_0['boxlib', 'Ex'].v.squeeze() -Ey = all_data_level_0['boxlib', 'Ey'].v.squeeze() -Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() -energyE = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) -energyB = np.sum(1./scc.mu_0/2*(Bx**2+By**2+Bz**2)) +ds = yt.load(filename) +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Bx = all_data_level_0["boxlib", "Bx"].v.squeeze() +By = all_data_level_0["boxlib", "By"].v.squeeze() +Bz = all_data_level_0["boxlib", "Bz"].v.squeeze() +Ex = all_data_level_0["boxlib", "Ex"].v.squeeze() +Ey = all_data_level_0["boxlib", "Ey"].v.squeeze() +Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() +energyE = np.sum(scc.epsilon_0 / 2 * (Ex**2 + Ey**2 + Ez**2)) +energyB = np.sum(1.0 / scc.mu_0 / 2 * (Bx**2 + By**2 + Bz**2)) energy_end = energyE + energyB -Reflectivity = energy_end/energy_start +Reflectivity = energy_end / energy_start Reflectivity_theory = 5.683000058954201e-07 -print("Reflectivity: %s" %Reflectivity) -print("Reflectivity_theory: %s" %Reflectivity_theory) +print("Reflectivity: %s" % Reflectivity) +print("Reflectivity_theory: %s" % Reflectivity_theory) -error_rel = abs(Reflectivity-Reflectivity_theory) / Reflectivity_theory -tolerance_rel = 5./100 +error_rel = abs(Reflectivity - Reflectivity_theory) / Reflectivity_theory +tolerance_rel = 5.0 / 100 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) -assert( error_rel < tolerance_rel ) - -# Check restart data v. original data -sys.path.insert(0, '../../../../warpx/Examples/') -from analysis_default_restart import check_restart - -check_restart(filename) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) +assert error_rel < tolerance_rel diff --git a/Examples/Tests/pml/inputs_3d b/Examples/Tests/pml/inputs_3d deleted file mode 100644 index e152afc7cc7..00000000000 --- a/Examples/Tests/pml/inputs_3d +++ /dev/null @@ -1,88 +0,0 @@ -max_step = 100 - -# Cell size and mesh refinement -amr.n_cell = 32 32 32 -amr.max_level = 0 - -# Domain and boundary conditions -geometry.dims = 3 -geometry.prob_lo = -7.5e-06 -7.5e-06 -7.5e-06 -geometry.prob_hi = 7.5e-06 7.5e-06 7.5e-06 - -# Boundary condition -boundary.field_lo = pml pml pml -boundary.field_hi = pml pml pml - -# Numerical parameters -warpx.cfl = 1.0 -warpx.grid_type = staggered -warpx.do_dive_cleaning = 1 -warpx.do_divb_cleaning = 1 -warpx.do_pml_dive_cleaning = 1 -warpx.do_pml_divb_cleaning = 1 -warpx.use_filter = 1 -warpx.verbose = 1 - -# Order of particle shape factors -algo.particle_shape = 1 - -# PSATD algorithm -algo.maxwell_solver = psatd -psatd.nox = 8 -psatd.noy = 8 -psatd.noz = 8 -psatd.update_with_rho = 1 - -# Lasers -lasers.names = laser1 laser2 laser3 -# -laser1.profile = Gaussian -laser2.profile = Gaussian -laser3.profile = Gaussian -# -laser1.wavelength = 1e-06 -laser2.wavelength = 1e-06 -laser3.wavelength = 1e-06 -# -laser1.e_max = 10 -laser2.e_max = 10 -laser3.e_max = 10 -# -laser1.polarization = 0 -1 1 -laser2.polarization = 1 0 -1 -laser3.polarization = -1 1 0 -# -laser1.profile_waist = 2e-06 -laser2.profile_waist = 2e-06 -laser3.profile_waist = 2e-06 -# -laser1.profile_duration = 7.5e-15 -laser2.profile_duration = 7.5e-15 -laser3.profile_duration = 7.5e-15 -# -laser1.do_continuous_injection = 0 -laser2.do_continuous_injection = 0 -laser3.do_continuous_injection = 0 -# -laser1.position = 0 0 0 -laser2.position = 0 0 0 -laser3.position = 0 0 0 -# -laser1.direction = 1 0 0 -laser2.direction = 0 1 0 -laser3.direction = 0 0 1 -# -laser1.profile_focal_distance = 5e-07 -laser2.profile_focal_distance = 5e-07 -laser3.profile_focal_distance = 5e-07 -# -laser1.profile_t_peak = 1.5e-14 -laser2.profile_t_peak = 1.5e-14 -laser3.profile_t_peak = 1.5e-14 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.diag_type = Full -diag1.format = plotfile -diag1.intervals = 100 -diag1.fields_to_plot = Bx By Bz Ex Ey Ez rho diff --git a/Examples/Tests/pml/inputs_2d b/Examples/Tests/pml/inputs_base_2d similarity index 100% rename from Examples/Tests/pml/inputs_2d rename to Examples/Tests/pml/inputs_base_2d diff --git a/Examples/Tests/pml/inputs_rz b/Examples/Tests/pml/inputs_rz deleted file mode 100644 index f5e23fe0399..00000000000 --- a/Examples/Tests/pml/inputs_rz +++ /dev/null @@ -1,49 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 500 -amr.n_cell = 32 128 -amr.max_level = 0 - -geometry.dims = RZ -geometry.prob_lo = 0. -100.e-6 -geometry.prob_hi = 50.e-6 +100.e-6 - -warpx.n_rz_azimuthal_modes = 2 - -################################# -######## Boundary condition ##### -################################# -boundary.field_lo = none periodic -boundary.field_hi = pml periodic - -# PML -warpx.pml_ncell = 10 -warpx.do_pml_in_domain = 0 - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = psatd -warpx.use_filter = 0 -algo.particle_shape = 1 - -################################# -############ PARTICLE ########### -################################# -particles.species_names = electron - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "singleparticle" -electron.single_particle_pos = 0. 0. 0. -electron.single_particle_u = 10. 0. 0. -electron.single_particle_weight = 1. - -################################# -########## DIAGNOSTICS ########## -################################# -diagnostics.diags_names = diag1 -diag1.intervals = 500 -diag1.fields_to_plot = Bt Er Ez jr jt jz rho -diag1.diag_type = Full diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_ckc b/Examples/Tests/pml/inputs_test_2d_pml_x_ckc new file mode 100644 index 00000000000..f686674ae14 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_ckc @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = ckc diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_galilean b/Examples/Tests/pml/inputs_test_2d_pml_x_galilean new file mode 100644 index 00000000000..34a9081a181 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_galilean @@ -0,0 +1,14 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +psatd.current_correction = 0 +psatd.update_with_rho = 1 +psatd.v_galilean = 0. 0. 0.99 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 +warpx.do_pml_divb_cleaning = 1 +warpx.do_pml_dive_cleaning = 1 +warpx.grid_type = collocated diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_psatd b/Examples/Tests/pml/inputs_test_2d_pml_x_psatd new file mode 100644 index 00000000000..191d5774530 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_psatd @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = psatd +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7071067811865475 +warpx.do_pml_divb_cleaning = 0 +warpx.do_pml_dive_cleaning = 0 +psatd.current_correction = 0 +psatd.update_with_rho = 1 diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_psatd_restart b/Examples/Tests/pml/inputs_test_2d_pml_x_psatd_restart new file mode 100644 index 00000000000..44b9be4494a --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_psatd_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_2d_pml_x_psatd + +# test input parameters +amr.restart = "../test_2d_pml_x_psatd/diags/chk000150" diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_yee b/Examples/Tests/pml/inputs_test_2d_pml_x_yee new file mode 100644 index 00000000000..390cf079c16 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_yee @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.maxwell_solver = yee diff --git a/Examples/Tests/pml/inputs_test_2d_pml_x_yee_restart b/Examples/Tests/pml/inputs_test_2d_pml_x_yee_restart new file mode 100644 index 00000000000..b626e3aa662 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_2d_pml_x_yee_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_2d_pml_x_yee + +# test input parameters +amr.restart = "../test_2d_pml_x_yee/diags/chk000150" diff --git a/Examples/Tests/pml/inputs_test_3d_pml_psatd_dive_divb_cleaning b/Examples/Tests/pml/inputs_test_3d_pml_psatd_dive_divb_cleaning new file mode 100644 index 00000000000..4e8c3c78329 --- /dev/null +++ b/Examples/Tests/pml/inputs_test_3d_pml_psatd_dive_divb_cleaning @@ -0,0 +1,91 @@ +max_step = 100 + +# Cell size and mesh refinement +amr.n_cell = 32 32 32 +amr.max_level = 0 + +# Domain and boundary conditions +geometry.dims = 3 +geometry.prob_lo = -7.5e-06 -7.5e-06 -7.5e-06 +geometry.prob_hi = 7.5e-06 7.5e-06 7.5e-06 + +# Boundary condition +boundary.field_lo = pml pml pml +boundary.field_hi = pml pml pml + +# Numerical parameters +ablastr.fillboundary_always_sync = 1 +warpx.abort_on_warning_threshold = medium +warpx.cfl = 1.0 +warpx.grid_type = staggered +warpx.do_dive_cleaning = 1 +warpx.do_divb_cleaning = 1 +warpx.do_pml_dive_cleaning = 1 +warpx.do_pml_divb_cleaning = 1 +warpx.do_similar_dm_pml = 0 +warpx.use_filter = 1 +warpx.verbose = 1 + +# Order of particle shape factors +algo.particle_shape = 1 + +# PSATD algorithm +algo.maxwell_solver = psatd +psatd.nox = 8 +psatd.noy = 8 +psatd.noz = 8 +psatd.update_with_rho = 1 + +# Lasers +lasers.names = laser1 laser2 laser3 +# +laser1.profile = Gaussian +laser2.profile = Gaussian +laser3.profile = Gaussian +# +laser1.wavelength = 1e-06 +laser2.wavelength = 1e-06 +laser3.wavelength = 1e-06 +# +laser1.e_max = 10 +laser2.e_max = 10 +laser3.e_max = 10 +# +laser1.polarization = 0 -1 1 +laser2.polarization = 1 0 -1 +laser3.polarization = -1 1 0 +# +laser1.profile_waist = 2e-06 +laser2.profile_waist = 2e-06 +laser3.profile_waist = 2e-06 +# +laser1.profile_duration = 7.5e-15 +laser2.profile_duration = 7.5e-15 +laser3.profile_duration = 7.5e-15 +# +laser1.do_continuous_injection = 0 +laser2.do_continuous_injection = 0 +laser3.do_continuous_injection = 0 +# +laser1.position = 0 0 0 +laser2.position = 0 0 0 +laser3.position = 0 0 0 +# +laser1.direction = 1 0 0 +laser2.direction = 0 1 0 +laser3.direction = 0 0 1 +# +laser1.profile_focal_distance = 5e-07 +laser2.profile_focal_distance = 5e-07 +laser3.profile_focal_distance = 5e-07 +# +laser1.profile_t_peak = 1.5e-14 +laser2.profile_t_peak = 1.5e-14 +laser3.profile_t_peak = 1.5e-14 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.diag_type = Full +diag1.format = plotfile +diag1.intervals = 100 +diag1.fields_to_plot = Bx By Bz Ex Ey Ez rho diff --git a/Examples/Tests/pml/inputs_test_rz_pml_psatd b/Examples/Tests/pml/inputs_test_rz_pml_psatd new file mode 100644 index 00000000000..87b4d7a5b3f --- /dev/null +++ b/Examples/Tests/pml/inputs_test_rz_pml_psatd @@ -0,0 +1,52 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 500 +amr.n_cell = 32 128 +amr.max_level = 0 + +geometry.dims = RZ +geometry.prob_lo = 0. -100.e-6 +geometry.prob_hi = 50.e-6 +100.e-6 + +warpx.n_rz_azimuthal_modes = 2 + +################################# +######## Boundary condition ##### +################################# +boundary.field_lo = none periodic +boundary.field_hi = pml periodic + +# PML +warpx.pml_ncell = 10 +warpx.do_pml_in_domain = 0 + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = psatd +warpx.abort_on_warning_threshold = medium +warpx.cfl = 0.7 +warpx.use_filter = 0 +algo.particle_shape = 1 +psatd.current_correction = 0 + +################################# +############ PARTICLE ########### +################################# +particles.species_names = electron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "singleparticle" +electron.single_particle_pos = 0. 0. 0. +electron.single_particle_u = 10. 0. 0. +electron.single_particle_weight = 1. + +################################# +########## DIAGNOSTICS ########## +################################# +diagnostics.diags_names = diag1 +diag1.intervals = 500 +diag1.fields_to_plot = Bt Er Ez jr jt jz rho +diag1.diag_type = Full diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py deleted file mode 100755 index 042fc811a62..00000000000 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -""" -This script tests the coordinates of the point of contact of an electron hitting a sphere in 3D. -It compares the numerical results with the analytical solutions. -The sphere is centered on O and has a radius of 0.2 (EB) -The electron is initially at: (-0.25,0,0) and moves with a normalized momentum: (1,0.5,0) -An input file PICMI_inputs_3d.py is used. -""" -import os -import sys - -import numpy as np -import yt -from openpmd_viewer import OpenPMDTimeSeries - -yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Open plotfile specified in command line -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') - -ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') - -it=ts_scraping.iterations -step_scraped, delta, x, y, z, nx, ny, nz=ts_scraping.get_particle( ['stepScraped','deltaTimeScraped','x','y','z', 'nx', 'ny', 'nz'], species='electron', iteration=it ) -delta_reduced=delta[0]*1e10 - -# Analytical results calculated -x_analytic=-0.1983 -y_analytic=0.02584 -z_analytic=0.0000 -nx_analytic=-0.99 -ny_analytic=0.13 -nz_analytic=0.0 - -#result obtained by analysis of simulations -step_ref=3 -delta_reduced_ref=0.59 - -print('NUMERICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_scraped[0],delta_reduced,x[0], y[0], z[0], nx[0], ny[0], nz[0])) -print('\n') -print('ANALYTICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_ref, delta_reduced_ref, x_analytic, y_analytic, z_analytic, nx_analytic, ny_analytic, nz_analytic)) - -tolerance=0.001 -tolerance_t=0.01 -tolerance_n=0.01 -print("tolerance = "+ str(tolerance *100) + '%') -print("tolerance for the time = "+ str(tolerance_t *100) + '%') -print("tolerance for the normal components = "+ str(tolerance_n *100) + '%') - -diff_step=np.abs((step_scraped[0]-step_ref)/step_ref) -diff_delta=np.abs((delta_reduced-delta_reduced_ref)/delta_reduced_ref) -diff_x=np.abs((x[0]-x_analytic)/x_analytic) -diff_y=np.abs((y[0]-y_analytic)/y_analytic) -diff_nx=np.abs((nx[0]-nx_analytic)/nx_analytic) -diff_ny=np.abs((ny[0]-ny_analytic)/ny_analytic) - -print("percentage error for x = %5.4f %%" %(diff_x *100)) -print("percentage error for y = %5.4f %%" %(diff_y *100)) -print("percentage error for nx = %5.2f %%" %(diff_nx *100)) -print("percentage error for ny = %5.2f %%" %(diff_ny *100)) -print("nz = %5.2f " %(nz[0])) - -assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_delta < tolerance_t) and (diff_nx < tolerance_n) and (diff_ny < tolerance_n) and (np.abs(nz) < 1e-8) , 'Test point_of_contact did not pass' diff --git a/Examples/Tests/point_of_contact_eb/CMakeLists.txt b/Examples/Tests/point_of_contact_eb/CMakeLists.txt new file mode 100644 index 00000000000..700eba6f92f --- /dev/null +++ b/Examples/Tests/point_of_contact_eb/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_point_of_contact_eb # name + 3 # dims + 2 # nprocs + inputs_test_3d_point_of_contact_eb # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_point_of_contact_eb # name + RZ # dims + 2 # nprocs + inputs_test_rz_point_of_contact_eb # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/point_of_contact_eb/analysis.py b/Examples/Tests/point_of_contact_eb/analysis.py new file mode 100755 index 00000000000..55a65f2cee3 --- /dev/null +++ b/Examples/Tests/point_of_contact_eb/analysis.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +""" +This script tests the coordinates of the point of contact of an electron hitting a sphere in 3D. +It compares the numerical results with the analytical solutions. +The sphere is centered on O and has a radius of 0.2 (EB) +The electron is initially at: (-0.25,0,0) and moves with a normalized momentum: (1,0.5,0) +An input file inputs_test_3d_point_of_contact_eb is used. +""" + +import numpy as np +import yt +from openpmd_viewer import OpenPMDTimeSeries + +yt.funcs.mylog.setLevel(0) + +ts_scraping = OpenPMDTimeSeries("./diags/diag2/particles_at_eb/") + +it = ts_scraping.iterations +step_scraped, delta, x, y, z, nx, ny, nz = ts_scraping.get_particle( + ["stepScraped", "deltaTimeScraped", "x", "y", "z", "nx", "ny", "nz"], + species="electron", + iteration=it, +) +delta_reduced = delta[0] * 1e10 + +# Analytical results calculated +x_analytic = -0.1983 +y_analytic = 0.02584 +z_analytic = 0.0000 +nx_analytic = -0.99 +ny_analytic = 0.13 +nz_analytic = 0.0 + +# result obtained by analysis of simulations +step_ref = 3 +delta_reduced_ref = 0.59 + +print("NUMERICAL coordinates of the point of contact:") +print( + "step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f" + % (step_scraped[0], delta_reduced, x[0], y[0], z[0], nx[0], ny[0], nz[0]) +) +print("\n") +print("ANALYTICAL coordinates of the point of contact:") +print( + "step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f" + % ( + step_ref, + delta_reduced_ref, + x_analytic, + y_analytic, + z_analytic, + nx_analytic, + ny_analytic, + nz_analytic, + ) +) + +tolerance = 0.001 +tolerance_t = 0.01 +tolerance_n = 0.01 +print("tolerance = " + str(tolerance * 100) + "%") +print("tolerance for the time = " + str(tolerance_t * 100) + "%") +print("tolerance for the normal components = " + str(tolerance_n * 100) + "%") + +diff_step = np.abs((step_scraped[0] - step_ref) / step_ref) +diff_delta = np.abs((delta_reduced - delta_reduced_ref) / delta_reduced_ref) +diff_x = np.abs((x[0] - x_analytic) / x_analytic) +diff_y = np.abs((y[0] - y_analytic) / y_analytic) +diff_nx = np.abs((nx[0] - nx_analytic) / nx_analytic) +diff_ny = np.abs((ny[0] - ny_analytic) / ny_analytic) + +print("percentage error for x = %5.4f %%" % (diff_x * 100)) +print("percentage error for y = %5.4f %%" % (diff_y * 100)) +print("percentage error for nx = %5.2f %%" % (diff_nx * 100)) +print("percentage error for ny = %5.2f %%" % (diff_ny * 100)) +print("nz = %5.2f " % (nz[0])) + +assert ( + (diff_x < tolerance) + and (diff_y < tolerance) + and (np.abs(z[0]) < 1e-8) + and (diff_step < 1e-8) + and (diff_delta < tolerance_t) + and (diff_nx < tolerance_n) + and (diff_ny < tolerance_n) + and (np.abs(nz) < 1e-8) +), "Test point_of_contact did not pass" diff --git a/Examples/Tests/point_of_contact_eb/analysis_default_regression.py b/Examples/Tests/point_of_contact_eb/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/point_of_contact_eb/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_eb/inputs_test_3d_point_of_contact_eb similarity index 100% rename from Examples/Tests/point_of_contact_EB/inputs_3d rename to Examples/Tests/point_of_contact_eb/inputs_test_3d_point_of_contact_eb diff --git a/Examples/Tests/point_of_contact_EB/inputs_rz b/Examples/Tests/point_of_contact_eb/inputs_test_rz_point_of_contact_eb similarity index 100% rename from Examples/Tests/point_of_contact_EB/inputs_rz rename to Examples/Tests/point_of_contact_eb/inputs_test_rz_point_of_contact_eb diff --git a/Examples/Tests/projection_divb_cleaner/CMakeLists.txt b/Examples/Tests/projection_divb_cleaner/CMakeLists.txt new file mode 100644 index 00000000000..40b84bd0397 --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_projection_divb_cleaner_callback_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_projection_divb_cleaner_callback_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_projection_divb_cleaner_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_projection_divb_cleaner_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_projection_divb_cleaner # name + RZ # dims + 1 # nprocs + inputs_test_rz_projection_divb_cleaner # inputs + "analysis.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/projection_divb_cleaner/analysis.py b/Examples/Tests/projection_divb_cleaner/analysis.py new file mode 100755 index 00000000000..5db145eba7e --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/analysis.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +# Copyright 2022 Yinjian Zhao +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This test tests the external field loading feature. +# A magnetic mirror field is loaded, and a single particle +# in the mirror will be reflected by the magnetic mirror effect. +# At the end of the simulation, the position of the particle +# is compared with known correct results. + +# Possible errors: 6.235230443866285e-9 +# tolerance: 1.0e-8 +# Possible running time: 0.327827743 s + +import sys + +import numpy as np +import yt + +tolerance = 4e-3 + +filename = sys.argv[1] + +ds = yt.load(filename) +grid0 = ds.index.grids[0] + + +r_min = grid0.LeftEdge.v[0] +r_max = grid0.RightEdge.v[0] +nr = grid0.shape[0] +dri = 1 // grid0.dds.v[0] +ir = np.arange(nr) + +ru = 1.0 + 0.5 / (r_min * dri + ir + 0.5) +rd = 1.0 - 0.5 / (r_min * dri + ir + 0.5) + +z_min = grid0.LeftEdge.v[1] +z_max = grid0.RightEdge.v[1] +nz = grid0.shape[1] +dzi = 1.0 / grid0.dds.v[1] + +RU, ZU = np.meshgrid(ru, np.arange(nz), indexing="ij") +RD, ZD = np.meshgrid(rd, np.arange(nz), indexing="ij") + +dBrdr = ( + RU * grid0["raw", "Bx_aux"].v[:, :, 0, 1] + - RD * grid0["raw", "Bx_aux"].v[:, :, 0, 0] +) * dri +dBzdz = ( + grid0["raw", "Bz_aux"].v[:, :, 0, 1] - grid0["raw", "Bz_aux"].v[:, :, 0, 0] +) * dzi + +divB = dBrdr + dBzdz + +import matplotlib.pyplot as plt + +plt.imshow(np.log10(np.abs(divB[1:-1, 1:-1]))) +plt.title("log10(|div(B)|)") +plt.colorbar() +plt.savefig("divb.png") + +error = np.sqrt((divB[1:-1, 1:-1] ** 2).sum()) + +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/projection_divb_cleaner/analysis_default_regression.py b/Examples/Tests/projection_divb_cleaner/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_callback_picmi.py b/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_callback_picmi.py new file mode 100644 index 00000000000..e6eb7ecf904 --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_callback_picmi.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from Python callback + +import numpy as np +import scipy.constants as con +from mpi4py import MPI as mpi +from scipy.special import ellipe, ellipk + +from pywarpx import fields, picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(verbose=True) + + +def augment_vector(v): + d = v[1] - v[0] + vp = (v[1:] + v[:-1]) / 2.0 + vp = np.insert(vp, 0, vp[0] - d) + vp = np.append(vp, vp[-1] + d) + return vp + + +class CurrentLoop(object): + def __init__(self, **kw): + self.radius = kw.pop("radius") # (m) + self.z0 = kw.pop("z0", 0.0) # (m) Defines the middle of the current loop + self.B0 = kw.pop("B0", 1.0) # (T) Strength of field in middle of loop + + # Compute loop current to normalize the B field of the center of the current loop + # from the following on axis equation for Bz + + self.I = 2.0 * self.radius * self.B0 / con.mu_0 + + def psi(self, r, z): + # Convert to spherical coordinates + rho = np.sqrt(r**2 + (z - self.z0) ** 2) + theta = np.arctan2(r, z - self.z0) + + coeff = con.mu_0 * self.I / (4.0 * np.pi) + denom = self.radius**2 + rho**2 + 2.0 * self.radius * rho * np.sin(theta) + + k2 = 4.0 * self.radius * rho * np.sin(theta) / denom + 1e-12 + + term1 = 4.0 * self.radius / np.sqrt(denom) + term2 = ((2.0 - k2) * ellipk(k2) - 2.0 * ellipe(k2)) / k2 + + return coeff * term1 * term2 * r + + def __call__(self, xv, yv, zv, coord="x"): + # Generate B-field mesh + XMB, YMB, ZMB = np.meshgrid(xv, yv, zv, indexing="ij") + RMB = np.sqrt(XMB**2 + YMB**2) + + dx = xv[1] - xv[0] + dy = yv[1] - yv[0] + dz = zv[1] - zv[0] + + if coord == "x": + # Gradient is along z direction + zvp = augment_vector(zv) + + # A mesh, which will be reduced later after gradients taken + XMP, YMP, ZMP = np.meshgrid(xv, yv, zvp, indexing="ij") + RMP = np.sqrt(XMP**2 + YMP**2) + psi = self.psi(RMP, ZMP) + grad_psi_z = (psi[:, :, 1:] - psi[:, :, :-1]) / dz + + return -XMB * grad_psi_z / RMB**2 + elif coord == "y": + # Gradient is along z direction + zvp = augment_vector(zv) + + # Psi mesh, which will be reduced later after gradients taken + XMP, YMP, ZMP = np.meshgrid(xv, yv, zvp, indexing="ij") + RMP = np.sqrt(XMP**2 + YMP**2) + psi = self.psi(RMP, ZMP) + grad_psi_z = (psi[:, :, 1:] - psi[:, :, :-1]) / dz + + return -YMB * grad_psi_z / RMB**2 + elif coord == "z": + # Gradient is along x,y directions + xvp = augment_vector(xv) + + # Psi mesh, which will be reduced later after gradients taken + XMP, YMP, ZMP = np.meshgrid(xvp, yv, zv, indexing="ij") + RMP = np.sqrt(XMP**2 + YMP**2) + psi = self.psi(RMP, ZMP) + grad_psi_x = (psi[1:, :, :] - psi[:-1, :, :]) / dx + + yvp = augment_vector(yv) + + # Psi mesh, which will be reduced later after gradients taken + XMP, YMP, ZMP = np.meshgrid(xv, yvp, zv, indexing="ij") + RMP = np.sqrt(XMP**2 + YMP**2) + psi = self.psi(RMP, ZMP) + grad_psi_y = (psi[:, 1:, :] - psi[:, :-1, :]) / dy + + return (XMB * grad_psi_x + YMB * grad_psi_y) / RMB**2 + else: + error("coord must be x/y/z") + return None + + +class ProjectionDivCleanerTest(object): + # Spatial domain + Nx = 40 # number of cells in x direction + Ny = 40 # number of cells in y direction + Nz = 80 # number of cells in z direction + + MAX_GRID = 40 + BLOCKING_FACTOR = 8 + + # Numerical parameters + DT = 1e-9 # Time step + + def __init__(self): + """Get input parameters for the specific case desired.""" + + # output diagnostics 5 times per cyclotron period + self.diag_steps = 1 + self.total_steps = 1 + + self.Lx = 1.0 + self.Ly = 1.0 + self.Lz = 2.0 + + self.DX = self.Lx / self.Nx + self.DY = self.Ly / self.Ny + self.DZ = self.Lz / self.Nz + + self.dt = self.DT + + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + self.grid = picmi.Cartesian3DGrid( + number_of_cells=[self.Nx, self.Ny, self.Nz], + warpx_max_grid_size_x=self.MAX_GRID, + warpx_max_grid_size_y=self.MAX_GRID, + warpx_max_grid_size_z=self.MAX_GRID, + warpx_blocking_factor=self.BLOCKING_FACTOR, + lower_bound=[-self.Lx / 2.0, -self.Ly / 2.0, -self.Lz / 2], + upper_bound=[self.Lx / 2.0, self.Ly / 2.0, self.Lz / 2.0], + lower_boundary_conditions=["periodic", "periodic", "neumann"], + upper_boundary_conditions=["periodic", "periodic", "neumann"], + lower_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + upper_boundary_conditions_particles=["periodic", "periodic", "absorbing"], + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.ElectrostaticSolver(grid=self.grid) + simulation.solver = self.solver + + ####################################################################### + # Install Callbacks # + ####################################################################### + init_field = picmi.LoadInitialFieldFromPython( + load_from_python=load_current_ring, + warpx_do_divb_cleaning_external=True, + load_E=False, + ) + simulation.add_applied_field(init_field) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=self.grid, + period=self.diag_steps, + data_list=["B"], + warpx_format="plotfile", + ) + simulation.add_diagnostic(field_diag) + + comm.Barrier() + + # Initialize inputs and WarpX instance + simulation.initialize_inputs() + simulation.initialize_warpx() + + +def load_current_ring(): + curr_loop = CurrentLoop(radius=0.75) + + Bx = fields.BxFPExternalWrapper(include_ghosts=True) + By = fields.ByFPExternalWrapper(include_ghosts=True) + Bz = fields.BzFPExternalWrapper(include_ghosts=True) + + Bx[:, :, :] = curr_loop(Bx.mesh("x"), Bx.mesh("y"), Bx.mesh("z"), coord="x") + + By[:, :, :] = curr_loop(By.mesh("x"), By.mesh("y"), By.mesh("z"), coord="y") + + Bz[:, :, :] = curr_loop(Bz.mesh("x"), Bz.mesh("y"), Bz.mesh("z"), coord="z") + + comm.Barrier() + + +run = ProjectionDivCleanerTest() +simulation.step() + +############################################## +# Post load image generation and error check # +############################################## +Bxg = fields.BxWrapper(include_ghosts=True) +Byg = fields.ByWrapper(include_ghosts=True) +Bzg = fields.BzWrapper(include_ghosts=True) + +Bx_local = Bxg[:, :, :] +By_local = Byg[:, :, :] +Bz_local = Bzg[:, :, :] + +dBxdx = (Bx_local[1:, :, :] - Bx_local[:-1, :, :]) / run.DX +dBydy = (By_local[:, 1:, :] - By_local[:, :-1, :]) / run.DY +dBzdz = (Bz_local[:, :, 1:] - Bz_local[:, :, :-1]) / run.DZ + +divB = dBxdx + dBydy + dBzdz + +import matplotlib.pyplot as plt + +plt.imshow(np.log10(np.abs(divB[:, 24, :]))) +plt.title("log10(|div(B)|)") +plt.colorbar() +plt.savefig("divb.png") + +error = np.sqrt((divB[2:-2, 2:-2, 2:-2] ** 2).sum()) +tolerance = 1e-12 + +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_picmi.py b/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_picmi.py new file mode 100644 index 00000000000..f4347f30e56 --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/inputs_test_3d_projection_divb_cleaner_picmi.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +import numpy as np +import yt + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 48 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +################################# +######## INITIAL FIELD ########## +################################# + +initial_field = picmi.LoadInitialField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False, +) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], + upper_boundary_conditions_particles=["absorbing", "absorbing", "absorbing"], +) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=1, + data_list=["Bx", "By", "Bz"], + warpx_plot_raw_fields=True, + warpx_plot_raw_fields_guards=True, +) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, +) + +sim.add_applied_field(initial_field) + +sim.add_diagnostic(field_diag) + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +################################# +##### SIMULATION EXECUTION ###### +################################# + +sim.step(1) + +################################# +##### SIMULATION ANALYSIS ###### +################################# + +filename = "diags/diag1000001" + +ds = yt.load(filename) +grid0 = ds.index.grids[0] + +dBxdx = ( + grid0["raw", "Bx_aux"].v[:, :, :, 1] - grid0["raw", "Bx_aux"].v[:, :, :, 0] +) / grid0.dds.v[0] +dBydy = ( + grid0["raw", "By_aux"].v[:, :, :, 1] - grid0["raw", "By_aux"].v[:, :, :, 0] +) / grid0.dds.v[1] +dBzdz = ( + grid0["raw", "Bz_aux"].v[:, :, :, 1] - grid0["raw", "Bz_aux"].v[:, :, :, 0] +) / grid0.dds.v[2] + +divB = dBxdx + dBydy + dBzdz + +import matplotlib.pyplot as plt + +plt.imshow(np.log10(np.abs(divB[:, 24, :]))) +plt.title("log10(|div(B)|)") +plt.colorbar() +plt.savefig("divb.png") + +error = np.sqrt((divB[2:-2, 2:-2, 2:-2] ** 2).sum()) +tolerance = 1e-12 + +print("error = ", error) +print("tolerance = ", tolerance) +assert error < tolerance diff --git a/Examples/Tests/projection_divb_cleaner/inputs_test_rz_projection_divb_cleaner b/Examples/Tests/projection_divb_cleaner/inputs_test_rz_projection_divb_cleaner new file mode 100644 index 00000000000..3e8f69ee411 --- /dev/null +++ b/Examples/Tests/projection_divb_cleaner/inputs_test_rz_projection_divb_cleaner @@ -0,0 +1,50 @@ +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +warpx.B_ext_grid_init_style = "read_from_file" +warpx.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 1 +amr.n_cell = 32 64 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 +warpx.do_divb_cleaning_external = true + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Br Bt Bz +diag1.plot_raw_fields = true +diag1.plot_raw_fields_guards = true +diag1.format = plotfile diff --git a/Examples/Tests/python_wrappers/CMakeLists.txt b/Examples/Tests/python_wrappers/CMakeLists.txt new file mode 100644 index 00000000000..060cdd7c183 --- /dev/null +++ b/Examples/Tests/python_wrappers/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_FFT) + add_warpx_test( + test_2d_python_wrappers_picmi # name + 2 # dims + 2 # nprocs + inputs_test_2d_python_wrappers_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000100" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py deleted file mode 100755 index db1cc7dcad8..00000000000 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 - -import matplotlib.pyplot as plt -import numpy as np -from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable - -from pywarpx import picmi - -# Number of time steps -max_steps = 100 - -# Grid -nx = 128 -nz = 128 - -# Domain -xmin = 0.e-6 -zmin = 0.e-6 -xmax = 50.e-6 -zmax = 50.e-6 - -# Cell size -dx = (xmax - xmin) / nx -dz = (zmax - zmin) / nz - -# Domain decomposition -max_grid_size_x = 64 -max_grid_size_z = 64 - -# PML -nxpml = 10 -nzpml = 10 -field_boundary = ['open', 'open'] - -# Spectral order -nox = 8 -noz = 8 - -# Guard cells -nxg = 8 -nzg = 8 - -# Initialize grid -grid = picmi.Cartesian2DGrid(number_of_cells = [nx,nz], - lower_bound = [xmin,zmin], - upper_bound = [xmax,zmax], - lower_boundary_conditions = field_boundary, - upper_boundary_conditions = field_boundary, - guard_cells = [nxg,nzg], - moving_window_velocity = [0.,0.,0], - warpx_max_grid_size_x = max_grid_size_x, - warpx_max_grid_size_y = max_grid_size_z) - -# Initialize field solver -solver = picmi.ElectromagneticSolver(grid=grid, cfl=0.95, method='PSATD', - stencil_order = [nox,noz], - divE_cleaning = 1, - divB_cleaning = 1, - pml_divE_cleaning = 1, - pml_divB_cleaning = 1, - warpx_psatd_update_with_rho = True) - -# Initialize diagnostics -diag_field_list = ["E", "B"] -particle_diag = picmi.ParticleDiagnostic(name = 'diag1', - period = 10, - write_dir = '.', - warpx_file_prefix = 'Python_wrappers_plt', - data_list = diag_field_list) -field_diag = picmi.FieldDiagnostic(name = 'diag1', - grid = grid, - period = 10, - write_dir = '.', - warpx_file_prefix = 'Python_wrappers_plt', - data_list = diag_field_list) - -# Initialize simulation -sim = picmi.Simulation(solver = solver, - max_steps = max_steps, - verbose = 1, - particle_shape = 'cubic', - warpx_current_deposition_algo = 'direct', - warpx_particle_pusher_algo = 'boris', - warpx_field_gathering_algo = 'energy-conserving', - warpx_use_filter = 1) - -# Add diagnostics to simulation -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Write input file to run with compiled version -sim.write_input_file(file_name = 'inputs_2d') - -# Whether to include guard cells in data returned by Python wrappers -include_ghosts = 1 - -# Compute min and max of fields data -def compute_minmax(data): - vmax = np.abs(data).max() - vmin = -vmax - return vmin, vmax - -# Plot fields data either in valid domain or in PML -def plot_data(data, pml, title, name): - fig, ax = plt.subplots(nrows = 1, ncols = 1, gridspec_kw = dict(wspace = 0.5), figsize = [6,5]) - cax = make_axes_locatable(ax).append_axes('right', size='5%', pad='5%') - lw = 0.8 - ls = '--' - if pml: - # Draw PMLs and ghost regions - ax.axvline(x = 0 , linewidth = lw, linestyle = ls) - ax.axvline(x = 0+nxg , linewidth = lw, linestyle = ls) - ax.axvline(x = -nxpml , linewidth = lw, linestyle = ls) - ax.axvline(x = nx , linewidth = lw, linestyle = ls) - ax.axvline(x = nx-nxg , linewidth = lw, linestyle = ls) - ax.axvline(x = nx+nxpml, linewidth = lw, linestyle = ls) - ax.axhline(y = 0 , linewidth = lw, linestyle = ls) - ax.axhline(y = 0+nzg , linewidth = lw, linestyle = ls) - ax.axhline(y = -nzpml , linewidth = lw, linestyle = ls) - ax.axhline(y = nz , linewidth = lw, linestyle = ls) - ax.axhline(y = nz-nzg , linewidth = lw, linestyle = ls) - ax.axhline(y = nz+nzpml, linewidth = lw, linestyle = ls) - # Annotations - ax.annotate('PML', xy = (-nxpml//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML', xy = (nx+nxpml//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML', xy = (nx//2,-nzpml//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('PML', xy = (nx//2,nz+nzpml//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (-nxpml-nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx-nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx+nxpml+nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx//2,nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx//2,-nzpml-nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx//2,nz-nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('PML ghost', xy = (nx//2,nz+nzpml+nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - # Set extent and sliced data - extent = np.array([-nxg-nxpml, nx+nxpml+nxg, -nzg-nzpml, nz+nzpml+nzg]) - else: - # Draw ghost regions - ax.axvline(x = 0 , linewidth = lw, linestyle = ls) - ax.axvline(x = nx, linewidth = lw, linestyle = ls) - ax.axhline(y = 0 , linewidth = lw, linestyle = ls) - ax.axhline(y = nz, linewidth = lw, linestyle = ls) - # Annotations - ax.annotate('ghost', xy = (-nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('ghost', xy = (nx+nxg//2,nz//2), rotation = 'vertical', ha = 'center', va = 'center') - ax.annotate('ghost', xy = (nx//2,-nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - ax.annotate('ghost', xy = (nx//2,nz+nzg//2), rotation = 'horizontal', ha = 'center', va = 'center') - # Set extent and sliced data - extent = np.array([-nxg, nx+nxg, -nzg, nz+nzg]) - X = data[:,:].transpose() - # Min and max for colorbar - vmin, vmax = compute_minmax(X) - # Display data as image - im = ax.imshow(X = X, origin = 'lower', extent = extent, vmin = vmin, vmax = vmax, cmap = 'seismic') - # Add colorbar to plot - fig.colorbar(im, cax = cax) - # Set label for x- and y-axis, set title - ax.set_xlabel('x') - ax.set_ylabel('z') - ax.set_title(title) - # Set plot title - suptitle = 'PML in (x,z), 4 grids 64 x 64' - plt.suptitle(suptitle) - # Save figure - figname = 'figure_' + name + '.png' - fig.savefig(figname, dpi = 100) - -# Initialize fields data (unit pulse) and apply smoothing -def init_data(data): - impulse_1d = np.array([1./4., 1./2., 1./4.]) - impulse = np.outer(impulse_1d, impulse_1d) - data[nx//2-1:nx//2+2,nz//2-1:nz//2+2] = impulse - -# Initialize inputs and WarpX instance -sim.initialize_inputs() -sim.initialize_warpx() - -# Get fields data using Python wrappers -import pywarpx.fields as pwxf - -Ex = pwxf.ExFPWrapper(include_ghosts = include_ghosts) -Ey = pwxf.EyFPWrapper(include_ghosts = include_ghosts) -Ez = pwxf.EzFPWrapper(include_ghosts = include_ghosts) -Bx = pwxf.BxFPWrapper(include_ghosts = include_ghosts) -By = pwxf.ByFPWrapper(include_ghosts = include_ghosts) -Bz = pwxf.BzFPWrapper(include_ghosts = include_ghosts) -F = pwxf.FFPWrapper(include_ghosts = include_ghosts) -G = pwxf.GFPWrapper(include_ghosts = include_ghosts) -Expml = pwxf.ExFPPMLWrapper(include_ghosts = include_ghosts) -Eypml = pwxf.EyFPPMLWrapper(include_ghosts = include_ghosts) -Ezpml = pwxf.EzFPPMLWrapper(include_ghosts = include_ghosts) -Bxpml = pwxf.BxFPPMLWrapper(include_ghosts = include_ghosts) -Bypml = pwxf.ByFPPMLWrapper(include_ghosts = include_ghosts) -Bzpml = pwxf.BzFPPMLWrapper(include_ghosts = include_ghosts) -Fpml = pwxf.FFPPMLWrapper(include_ghosts = include_ghosts) -Gpml = pwxf.GFPPMLWrapper(include_ghosts = include_ghosts) - -# Initialize fields data in valid domain -init_data(Ex) -init_data(Ey) -init_data(Ez) -init_data(Bx) -init_data(By) -init_data(Bz) -init_data(F) -init_data(G) - -# Advance simulation until last time step -sim.step(max_steps) - -# Plot E -plot_data(Ex, pml = False, title = 'Ex', name = 'Ex') -plot_data(Ey, pml = False, title = 'Ey', name = 'Ey') -plot_data(Ez, pml = False, title = 'Ez', name = 'Ez') - -# Plot B -plot_data(Bx, pml = False, title = 'Bx', name = 'Bx') -plot_data(By, pml = False, title = 'By', name = 'By') -plot_data(Bz, pml = False, title = 'Bz', name = 'Bz') - -# F and G -plot_data(F, pml = False, title = 'F', name = 'F') -plot_data(G, pml = False, title = 'G', name = 'G') - -# Plot E in PML -plot_data(Expml[:,:,0], pml = True, title = 'Exy in PML', name = 'Exy') -plot_data(Expml[:,:,1], pml = True, title = 'Exz in PML', name = 'Exz') -plot_data(Expml[:,:,2], pml = True, title = 'Exx in PML', name = 'Exx') -plot_data(Eypml[:,:,0], pml = True, title = 'Eyz in PML', name = 'Eyz') -plot_data(Eypml[:,:,1], pml = True, title = 'Eyx in PML', name = 'Eyx') -plot_data(Eypml[:,:,2], pml = True, title = 'Eyy in PML', name = 'Eyy') # zero -plot_data(Ezpml[:,:,0], pml = True, title = 'Ezx in PML', name = 'Ezx') -plot_data(Ezpml[:,:,1], pml = True, title = 'Ezy in PML', name = 'Ezy') # zero -plot_data(Ezpml[:,:,2], pml = True, title = 'Ezz in PML', name = 'Ezz') - -# Plot B in PML -plot_data(Bxpml[:,:,0], pml = True, title = 'Bxy in PML', name = 'Bxy') -plot_data(Bxpml[:,:,1], pml = True, title = 'Bxz in PML', name = 'Bxz') -plot_data(Bxpml[:,:,2], pml = True, title = 'Bxx in PML', name = 'Bxx') -plot_data(Bypml[:,:,0], pml = True, title = 'Byz in PML', name = 'Byz') -plot_data(Bypml[:,:,1], pml = True, title = 'Byx in PML', name = 'Byx') -plot_data(Bypml[:,:,2], pml = True, title = 'Byy in PML', name = 'Byy') # zero -plot_data(Bzpml[:,:,0], pml = True, title = 'Bzx in PML', name = 'Bzx') -plot_data(Bzpml[:,:,1], pml = True, title = 'Bzy in PML', name = 'Bzy') # zero -plot_data(Bzpml[:,:,2], pml = True, title = 'Bzz in PML', name = 'Bzz') - -# Plot F and G in PML -plot_data(Fpml[:,:,0], pml = True, title = 'Fx in PML', name = 'Fx') -plot_data(Fpml[:,:,1], pml = True, title = 'Fy in PML', name = 'Fy') -plot_data(Fpml[:,:,2], pml = True, title = 'Fz in PML', name = 'Fz') -plot_data(Gpml[:,:,0], pml = True, title = 'Gx in PML', name = 'Gx') -plot_data(Gpml[:,:,1], pml = True, title = 'Gy in PML', name = 'Gy') -plot_data(Gpml[:,:,2], pml = True, title = 'Gz in PML', name = 'Gz') - -# Check values with benchmarks (precomputed from the same Python arrays) -def check_values(benchmark, data, rtol, atol): - passed = np.allclose(benchmark, np.sum(np.abs(data[:,:])), rtol = rtol, atol = atol) - assert(passed) - -rtol = 5e-08 -atol = 1e-12 - -# E -check_values(1013263608.6369569, Ex[:,:], rtol, atol) -check_values(717278256.7957529 , Ey[:,:], rtol, atol) -check_values(717866566.5718911 , Ez[:,:], rtol, atol) -# B -check_values(3.0214509313437636, Bx[:,:], rtol, atol) -check_values(3.0242765102729985, By[:,:], rtol, atol) -check_values(3.0214509326970465, Bz[:,:], rtol, atol) -# F and G -check_values(3.0188584528062377, F[:,:], rtol, atol) -check_values(1013672631.8764204, G[:,:], rtol, atol) -# E in PML -check_values(364287936.1526477 , Expml[:,:,0], rtol, atol) -check_values(183582352.20753333, Expml[:,:,1], rtol, atol) -check_values(190065766.41491824, Expml[:,:,2], rtol, atol) -check_values(440581907.0828975 , Eypml[:,:,0], rtol, atol) -check_values(178117294.05871135, Eypml[:,:,1], rtol, atol) -check_values(0.0 , Eypml[:,:,2], rtol, atol) -check_values(430277101.26568377, Ezpml[:,:,0], rtol, atol) -check_values(0.0 , Ezpml[:,:,1], rtol, atol) -check_values(190919663.2167449 , Ezpml[:,:,2], rtol, atol) -# B in PML -check_values(1.0565189315366146 , Bxpml[:,:,0], rtol, atol) -check_values(0.46181913800643065, Bxpml[:,:,1], rtol, atol) -check_values(0.6849858305343736 , Bxpml[:,:,2], rtol, atol) -check_values(1.7228584190213505 , Bypml[:,:,0], rtol, atol) -check_values(0.47697332248020935, Bypml[:,:,1], rtol, atol) -check_values(0.0 , Bypml[:,:,2], rtol, atol) -check_values(1.518338068658267 , Bzpml[:,:,0], rtol, atol) -check_values(0.0 , Bzpml[:,:,1], rtol, atol) -check_values(0.6849858291863835 , Bzpml[:,:,2], rtol, atol) -# F and G in PML -check_values(1.7808748509425263, Fpml[:,:,0], rtol, atol) -check_values(0.0 , Fpml[:,:,1], rtol, atol) -check_values(0.4307845604625681, Fpml[:,:,2], rtol, atol) -check_values(536552745.42701197, Gpml[:,:,0], rtol, atol) -check_values(0.0 , Gpml[:,:,1], rtol, atol) -check_values(196016270.97767758, Gpml[:,:,2], rtol, atol) diff --git a/Examples/Tests/python_wrappers/analysis_default_regression.py b/Examples/Tests/python_wrappers/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/python_wrappers/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/python_wrappers/inputs_test_2d_python_wrappers_picmi.py b/Examples/Tests/python_wrappers/inputs_test_2d_python_wrappers_picmi.py new file mode 100755 index 00000000000..66917b4146b --- /dev/null +++ b/Examples/Tests/python_wrappers/inputs_test_2d_python_wrappers_picmi.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 + +import matplotlib.pyplot as plt +import numpy as np +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable + +from pywarpx import picmi + +# Number of time steps +max_steps = 100 + +# Grid +nx = 128 +nz = 128 + +# Domain +xmin = 0.0e-6 +zmin = 0.0e-6 +xmax = 50.0e-6 +zmax = 50.0e-6 + +# Cell size +dx = (xmax - xmin) / nx +dz = (zmax - zmin) / nz + +# Domain decomposition +max_grid_size_x = 64 +max_grid_size_z = 64 + +# PML +nxpml = 10 +nzpml = 10 +field_boundary = ["open", "open"] + +# Spectral order +nox = 8 +noz = 8 + +# Guard cells +nxg = 8 +nzg = 8 + +# Initialize grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=field_boundary, + upper_boundary_conditions=field_boundary, + guard_cells=[nxg, nzg], + moving_window_velocity=[0.0, 0.0, 0], + warpx_max_grid_size_x=max_grid_size_x, + warpx_max_grid_size_y=max_grid_size_z, +) + +# Initialize field solver +solver = picmi.ElectromagneticSolver( + grid=grid, + cfl=0.95, + method="PSATD", + stencil_order=[nox, noz], + divE_cleaning=1, + divB_cleaning=1, + pml_divE_cleaning=1, + pml_divB_cleaning=1, + warpx_psatd_update_with_rho=True, +) + +# Initialize diagnostics +diag_field_list = ["E", "B"] +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, + data_list=diag_field_list, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=diag_field_list, +) + +# Initialize simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape="cubic", + warpx_current_deposition_algo="direct", + warpx_particle_pusher_algo="boris", + warpx_field_gathering_algo="energy-conserving", + warpx_use_filter=1, +) + +# Add diagnostics to simulation +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Write input file to run with compiled version +sim.write_input_file(file_name="inputs_2d") + +# Whether to include guard cells in data returned by Python wrappers +include_ghosts = 1 + + +# Compute min and max of fields data +def compute_minmax(data): + vmax = np.abs(data).max() + vmin = -vmax + return vmin, vmax + + +# Plot fields data either in valid domain or in PML +def plot_data(data, pml, title, name): + fig, ax = plt.subplots( + nrows=1, ncols=1, gridspec_kw=dict(wspace=0.5), figsize=[6, 5] + ) + cax = make_axes_locatable(ax).append_axes("right", size="5%", pad="5%") + lw = 0.8 + ls = "--" + if pml: + # Draw PMLs and ghost regions + ax.axvline(x=0, linewidth=lw, linestyle=ls) + ax.axvline(x=0 + nxg, linewidth=lw, linestyle=ls) + ax.axvline(x=-nxpml, linewidth=lw, linestyle=ls) + ax.axvline(x=nx, linewidth=lw, linestyle=ls) + ax.axvline(x=nx - nxg, linewidth=lw, linestyle=ls) + ax.axvline(x=nx + nxpml, linewidth=lw, linestyle=ls) + ax.axhline(y=0, linewidth=lw, linestyle=ls) + ax.axhline(y=0 + nzg, linewidth=lw, linestyle=ls) + ax.axhline(y=-nzpml, linewidth=lw, linestyle=ls) + ax.axhline(y=nz, linewidth=lw, linestyle=ls) + ax.axhline(y=nz - nzg, linewidth=lw, linestyle=ls) + ax.axhline(y=nz + nzpml, linewidth=lw, linestyle=ls) + # Annotations + ax.annotate( + "PML", + xy=(-nxpml // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML", + xy=(nx + nxpml // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML", + xy=(nx // 2, -nzpml // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "PML", + xy=(nx // 2, nz + nzpml // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(-nxpml - nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx - nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx + nxpml + nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx // 2, nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx // 2, -nzpml - nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx // 2, nz - nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "PML ghost", + xy=(nx // 2, nz + nzpml + nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + # Set extent and sliced data + extent = np.array( + [-nxg - nxpml, nx + nxpml + nxg, -nzg - nzpml, nz + nzpml + nzg] + ) + else: + # Draw ghost regions + ax.axvline(x=0, linewidth=lw, linestyle=ls) + ax.axvline(x=nx, linewidth=lw, linestyle=ls) + ax.axhline(y=0, linewidth=lw, linestyle=ls) + ax.axhline(y=nz, linewidth=lw, linestyle=ls) + # Annotations + ax.annotate( + "ghost", + xy=(-nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "ghost", + xy=(nx + nxg // 2, nz // 2), + rotation="vertical", + ha="center", + va="center", + ) + ax.annotate( + "ghost", + xy=(nx // 2, -nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + ax.annotate( + "ghost", + xy=(nx // 2, nz + nzg // 2), + rotation="horizontal", + ha="center", + va="center", + ) + # Set extent and sliced data + extent = np.array([-nxg, nx + nxg, -nzg, nz + nzg]) + X = data[:, :].transpose() + # Min and max for colorbar + vmin, vmax = compute_minmax(X) + # Display data as image + im = ax.imshow( + X=X, origin="lower", extent=extent, vmin=vmin, vmax=vmax, cmap="seismic" + ) + # Add colorbar to plot + fig.colorbar(im, cax=cax) + # Set label for x- and y-axis, set title + ax.set_xlabel("x") + ax.set_ylabel("z") + ax.set_title(title) + # Set plot title + suptitle = "PML in (x,z), 4 grids 64 x 64" + plt.suptitle(suptitle) + # Save figure + figname = "figure_" + name + ".png" + fig.savefig(figname, dpi=100) + + +# Initialize fields data (unit pulse) and apply smoothing +def init_data(data): + impulse_1d = np.array([1.0 / 4.0, 1.0 / 2.0, 1.0 / 4.0]) + impulse = np.outer(impulse_1d, impulse_1d) + data[nx // 2 - 1 : nx // 2 + 2, nz // 2 - 1 : nz // 2 + 2] = impulse + + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Get fields data using Python wrappers +import pywarpx.fields as pwxf + +Ex = pwxf.ExFPWrapper(include_ghosts=include_ghosts) +Ey = pwxf.EyFPWrapper(include_ghosts=include_ghosts) +Ez = pwxf.EzFPWrapper(include_ghosts=include_ghosts) +Bx = pwxf.BxFPWrapper(include_ghosts=include_ghosts) +By = pwxf.ByFPWrapper(include_ghosts=include_ghosts) +Bz = pwxf.BzFPWrapper(include_ghosts=include_ghosts) +F = pwxf.FFPWrapper(include_ghosts=include_ghosts) +G = pwxf.GFPWrapper(include_ghosts=include_ghosts) +Expml = pwxf.ExFPPMLWrapper(include_ghosts=include_ghosts) +Eypml = pwxf.EyFPPMLWrapper(include_ghosts=include_ghosts) +Ezpml = pwxf.EzFPPMLWrapper(include_ghosts=include_ghosts) +Bxpml = pwxf.BxFPPMLWrapper(include_ghosts=include_ghosts) +Bypml = pwxf.ByFPPMLWrapper(include_ghosts=include_ghosts) +Bzpml = pwxf.BzFPPMLWrapper(include_ghosts=include_ghosts) +Fpml = pwxf.FFPPMLWrapper(include_ghosts=include_ghosts) +Gpml = pwxf.GFPPMLWrapper(include_ghosts=include_ghosts) + +# Initialize fields data in valid domain +init_data(Ex) +init_data(Ey) +init_data(Ez) +init_data(Bx) +init_data(By) +init_data(Bz) +init_data(F) +init_data(G) + +# Advance simulation until last time step +sim.step(max_steps) + +# Plot E +plot_data(Ex, pml=False, title="Ex", name="Ex") +plot_data(Ey, pml=False, title="Ey", name="Ey") +plot_data(Ez, pml=False, title="Ez", name="Ez") + +# Plot B +plot_data(Bx, pml=False, title="Bx", name="Bx") +plot_data(By, pml=False, title="By", name="By") +plot_data(Bz, pml=False, title="Bz", name="Bz") + +# F and G +plot_data(F, pml=False, title="F", name="F") +plot_data(G, pml=False, title="G", name="G") + +# Plot E in PML +plot_data(Expml[:, :, 0], pml=True, title="Exy in PML", name="Exy") +plot_data(Expml[:, :, 1], pml=True, title="Exz in PML", name="Exz") +plot_data(Expml[:, :, 2], pml=True, title="Exx in PML", name="Exx") +plot_data(Eypml[:, :, 0], pml=True, title="Eyz in PML", name="Eyz") +plot_data(Eypml[:, :, 1], pml=True, title="Eyx in PML", name="Eyx") +plot_data(Eypml[:, :, 2], pml=True, title="Eyy in PML", name="Eyy") # zero +plot_data(Ezpml[:, :, 0], pml=True, title="Ezx in PML", name="Ezx") +plot_data(Ezpml[:, :, 1], pml=True, title="Ezy in PML", name="Ezy") # zero +plot_data(Ezpml[:, :, 2], pml=True, title="Ezz in PML", name="Ezz") + +# Plot B in PML +plot_data(Bxpml[:, :, 0], pml=True, title="Bxy in PML", name="Bxy") +plot_data(Bxpml[:, :, 1], pml=True, title="Bxz in PML", name="Bxz") +plot_data(Bxpml[:, :, 2], pml=True, title="Bxx in PML", name="Bxx") +plot_data(Bypml[:, :, 0], pml=True, title="Byz in PML", name="Byz") +plot_data(Bypml[:, :, 1], pml=True, title="Byx in PML", name="Byx") +plot_data(Bypml[:, :, 2], pml=True, title="Byy in PML", name="Byy") # zero +plot_data(Bzpml[:, :, 0], pml=True, title="Bzx in PML", name="Bzx") +plot_data(Bzpml[:, :, 1], pml=True, title="Bzy in PML", name="Bzy") # zero +plot_data(Bzpml[:, :, 2], pml=True, title="Bzz in PML", name="Bzz") + +# Plot F and G in PML +plot_data(Fpml[:, :, 0], pml=True, title="Fx in PML", name="Fx") +plot_data(Fpml[:, :, 1], pml=True, title="Fy in PML", name="Fy") +plot_data(Fpml[:, :, 2], pml=True, title="Fz in PML", name="Fz") +plot_data(Gpml[:, :, 0], pml=True, title="Gx in PML", name="Gx") +plot_data(Gpml[:, :, 1], pml=True, title="Gy in PML", name="Gy") +plot_data(Gpml[:, :, 2], pml=True, title="Gz in PML", name="Gz") + + +# Check values with benchmarks (precomputed from the same Python arrays) +def check_values(benchmark, data, rtol, atol): + passed = np.allclose(benchmark, np.sum(np.abs(data[:, :])), rtol=rtol, atol=atol) + assert passed + + +rtol = 5e-08 +atol = 1e-12 + +# E +check_values(1013263608.6369569, Ex[:, :], rtol, atol) +check_values(717278256.7957529, Ey[:, :], rtol, atol) +check_values(717866566.5718911, Ez[:, :], rtol, atol) +# B +check_values(3.0214509313437636, Bx[:, :], rtol, atol) +check_values(3.0242765102729985, By[:, :], rtol, atol) +check_values(3.0214509326970465, Bz[:, :], rtol, atol) +# F and G +check_values(3.0188584528062377, F[:, :], rtol, atol) +check_values(1013672631.8764204, G[:, :], rtol, atol) +# E in PML +check_values(364287936.1526477, Expml[:, :, 0], rtol, atol) +check_values(183582352.20753333, Expml[:, :, 1], rtol, atol) +check_values(190065766.41491824, Expml[:, :, 2], rtol, atol) +check_values(440581907.0828975, Eypml[:, :, 0], rtol, atol) +check_values(178117294.05871135, Eypml[:, :, 1], rtol, atol) +check_values(0.0, Eypml[:, :, 2], rtol, atol) +check_values(430277101.26568377, Ezpml[:, :, 0], rtol, atol) +check_values(0.0, Ezpml[:, :, 1], rtol, atol) +check_values(190919663.2167449, Ezpml[:, :, 2], rtol, atol) +# B in PML +check_values(1.0565189315366146, Bxpml[:, :, 0], rtol, atol) +check_values(0.46181913800643065, Bxpml[:, :, 1], rtol, atol) +check_values(0.6849858305343736, Bxpml[:, :, 2], rtol, atol) +check_values(1.7228584190213505, Bypml[:, :, 0], rtol, atol) +check_values(0.47697332248020935, Bypml[:, :, 1], rtol, atol) +check_values(0.0, Bypml[:, :, 2], rtol, atol) +check_values(1.518338068658267, Bzpml[:, :, 0], rtol, atol) +check_values(0.0, Bzpml[:, :, 1], rtol, atol) +check_values(0.6849858291863835, Bzpml[:, :, 2], rtol, atol) +# F and G in PML +check_values(1.7808748509425263, Fpml[:, :, 0], rtol, atol) +check_values(0.0, Fpml[:, :, 1], rtol, atol) +check_values(0.4307845604625681, Fpml[:, :, 2], rtol, atol) +check_values(536552745.42701197, Gpml[:, :, 0], rtol, atol) +check_values(0.0, Gpml[:, :, 1], rtol, atol) +check_values(196016270.97767758, Gpml[:, :, 2], rtol, atol) diff --git a/Examples/Tests/qed/CMakeLists.txt b/Examples/Tests/qed/CMakeLists.txt new file mode 100644 index 00000000000..d38c2e4fc69 --- /dev/null +++ b/Examples/Tests/qed/CMakeLists.txt @@ -0,0 +1,102 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_qed_breit_wheeler # name + 2 # dims + 2 # nprocs + inputs_test_2d_qed_breit_wheeler # inputs + "analysis_breit_wheeler_yt.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_qed_breit_wheeler_opmd # name + 2 # dims + 2 # nprocs + inputs_test_2d_qed_breit_wheeler_opmd # inputs + "analysis_breit_wheeler_opmd.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_qed_quantum_sync # name + 2 # dims + 2 # nprocs + inputs_test_2d_qed_quantum_sync # inputs + "analysis_quantum_sync.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_breit_wheeler # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_breit_wheeler # inputs + "analysis_breit_wheeler_yt.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_breit_wheeler_opmd # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_breit_wheeler_opmd # inputs + "analysis_breit_wheeler_opmd.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_quantum_sync # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_quantum_sync # inputs + "analysis_quantum_sync.py diags/diag1000002" # analysis + "analysis_default_regression.py --path diags/diag1000002" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_schwinger_1 # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_schwinger_1 # inputs + "analysis_schwinger.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_schwinger_2 # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_schwinger_2 # inputs + "analysis_schwinger.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_schwinger_3 # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_schwinger_3 # inputs + "analysis_schwinger.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_qed_schwinger_4 # name + 3 # dims + 2 # nprocs + inputs_test_3d_qed_schwinger_4 # inputs + "analysis_schwinger.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/qed/analysis_breit_wheeler_core.py b/Examples/Tests/qed/analysis_breit_wheeler_core.py new file mode 100755 index 00000000000..6f5441355e8 --- /dev/null +++ b/Examples/Tests/qed/analysis_breit_wheeler_core.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2019 Luca Fedeli, Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +import matplotlib.pyplot as plt +import numpy as np +import scipy.integrate as integ +import scipy.special as spe +import scipy.stats as st + +# This script performs detailed checks of the Breit-Wheeler pair production process. +# Four populations of photons are initialized with different momenta in different +# directions in a background EM field (with non-zero components along each direction). +# Specifically the script checks that: +# +# - The expected number of generated pairs n_pairs is in agreement with theory +# (the maximum tolerated error is 5*sqrt(n_pairs) +# - The weight of the generated particles is equal to the weight of the photon +# - Momenta of the residual photons are still equal to the original momentum +# - The generated particles are emitted in the right direction +# - Total energy is conserved in each event +# - The energy distribution of the generated particles is in agreement with theory +# - The optical depths of the product species are correctly initialized (QED effects are +# enabled for product species too). +# +# More details on the theoretical formulas used in this script can be found in +# the Jupyter notebook picsar/src/multi_physics/QED_tests/validation/validation.ipynb +# +# References: +# 1) R. Duclous et al 2011 Plasma Phys. Control. Fusion 53 015009 +# 2) A. Gonoskov et al. 2015 Phys. Rev. E 92, 023305 +# 3) M. Lobet. PhD thesis "Effets radiatifs et d'electrodynamique +# quantique dans l'interaction laser-matiere ultra-relativiste" +# URL: https://tel.archives-ouvertes.fr/tel-01314224 + + +# Tolerances +tol = 1.0e-8 +tol_red = 2.0e-2 + +# Physical constants (from CODATA 2018, see: https://physics.nist.gov/cuu/Constants/index.html ) +me = 9.1093837015e-31 # electron mass +c = 299792458 # speed of light +hbar = 6.62607015e-34 / (2 * np.pi) # reduced Plank constant +fine_structure = 7.2973525693e-3 # fine structure constant +qe = 1.602176634e-19 # elementary charge +E_s = (me**2 * c**3) / (qe * hbar) # Schwinger E field +B_s = E_s / c # Schwinger B field + +mec = me * c +mec2 = mec * c +# ______________ + +# Initial parameters +spec_names_phot = ["p1", "p2", "p3", "p4"] +spec_names_ele = ["ele1", "ele2", "ele3", "ele4"] +spec_names_pos = ["pos1", "pos2", "pos3", "pos4"] +initial_momenta = [ + np.array([2000.0, 0, 0]) * mec, + np.array([0.0, 5000.0, 0.0]) * mec, + np.array([0.0, 0.0, 10000.0]) * mec, + np.array([57735.02691896, 57735.02691896, 57735.02691896]) * mec, +] +initial_particle_number = 1048576 + +E_f = np.array([-2433321316961438.0, 973328526784575.0, 1459992790176863.0]) +B_f = np.array([2857142.85714286, 4285714.28571428, 8571428.57142857]) + +NNS = [128, 128, 128, 128] # bins for energy distribution comparison. +# ______________ + + +# Returns all the species names and if they are photon species or not +def get_all_species_names_and_types(): + names = spec_names_phot + spec_names_ele + spec_names_pos + types = [True] * len(spec_names_phot) + [False] * ( + len(spec_names_ele) + len(spec_names_pos) + ) + return names, types + + +def calc_chi_gamma(p, E, B): + pnorm = np.linalg.norm(p) + v = c * (p / pnorm) + EpvvecB = E + np.cross(v, B) + vdotEoverc = np.dot(v, E) / c + ff = np.sqrt(np.dot(EpvvecB, EpvvecB) - np.dot(vdotEoverc, vdotEoverc)) + gamma_phot = pnorm / mec + return gamma_phot * ff / E_s + + +# Auxiliary functions +@np.vectorize +def BW_inner(x): + return integ.quad( + lambda s: np.sqrt(s) * spe.kv(1.0 / 3.0, 2.0 / 3.0 * s ** (3.0 / 2.0)), + x, + np.inf, + )[0] + + +def BW_X(chi_phot, chi_ele): + div = chi_ele * (chi_phot - chi_ele) + div = np.where(np.logical_and(chi_phot > chi_ele, chi_ele != 0), div, 1.0) + res = np.where( + np.logical_and(chi_phot > chi_ele, chi_ele != 0), + np.power(chi_phot / div, 2.0 / 3.0), + np.inf, + ) + return res + + +def BW_F(chi_phot, chi_ele): + X = BW_X(chi_phot, chi_ele) + res = np.where( + np.logical_or(chi_phot == chi_ele, chi_ele == 0), + 0.0, + BW_inner(X) + - (2.0 - chi_phot * X ** (3.0 / 2.0)) + * spe.kv(2.0 / 3.0, 2.0 / 3.0 * X ** (3.0 / 2.0)), + ) + return res + + +@np.vectorize +def BW_T(chi_phot): + coeff = 1.0 / (np.pi * np.sqrt(3.0) * (chi_phot**2)) + return coeff * integ.quad(lambda chi_ele: BW_F(chi_phot, chi_ele), 0, chi_phot)[0] + + +def small_diff(vv, val): + if val != 0.0: + return np.max(np.abs((vv - val) / val)) < tol + else: + return np.max(np.abs(vv)) < tol + + +# __________________ + + +# Breit-Wheeler total and differential cross sections +def BW_dN_dt(chi_phot, gamma_phot): + coeff_BW = fine_structure * me * c**2 / hbar + return coeff_BW * BW_T(chi_phot) * (chi_phot / gamma_phot) + + +def BW_d2N_dt_dchi(chi_phot, gamma_phot, chi_ele): + coeff_BW = fine_structure * me * c**2 / hbar + return coeff_BW * BW_F(chi_phot, chi_ele) * (gamma_phot / gamma_phot) + + +# __________________ + +# Individual tests + + +def check_number_of_pairs( + particle_data, + phot_name, + ele_name, + pos_name, + chi_phot, + gamma_phot, + dt, + particle_number, +): + dNBW_dt_theo = BW_dN_dt(chi_phot, gamma_phot) + expected_pairs = (1.0 - np.exp(-dNBW_dt_theo * dt)) * particle_number + expected_pairs_tolerance = 5.0 * np.sqrt(expected_pairs) + n_ele = len(particle_data[ele_name]["w"]) + n_pos = len(particle_data[pos_name]["w"]) + n_phot = len(particle_data[phot_name]["w"]) + n_lost = initial_particle_number - n_phot + assert (n_ele == n_pos) and (n_ele == n_lost) + assert np.abs(n_ele - expected_pairs) < expected_pairs_tolerance + print(" [OK] generated pair number is within expectations") + return n_lost + + +def check_weights(phot_data, ele_data, pos_data): + assert np.all(phot_data["w"] == phot_data["w"][0]) + assert np.all(ele_data["w"] == phot_data["w"][0]) + assert np.all(pos_data["w"] == phot_data["w"][0]) + print(" [OK] particles weights are the expected ones") + + +def check_momenta(phot_data, ele_data, pos_data, p0, p_ele, p_pos): + assert small_diff(phot_data["px"], p0[0]) + assert small_diff(phot_data["py"], p0[1]) + assert small_diff(phot_data["pz"], p0[2]) + print(" [OK] residual photons still have initial momentum") + + pdir = p0 / np.linalg.norm(p0) + assert small_diff(ele_data["px"] / p_ele, pdir[0]) + assert small_diff(ele_data["py"] / p_ele, pdir[1]) + assert small_diff(ele_data["pz"] / p_ele, pdir[2]) + assert small_diff(pos_data["px"] / p_pos, pdir[0]) + assert small_diff(pos_data["py"] / p_pos, pdir[1]) + assert small_diff(pos_data["pz"] / p_pos, pdir[2]) + print(" [OK] pairs move along the initial photon direction") + + +def check_energy(energy_phot, energy_ele, energy_pos): + # Sorting the arrays is required because electrons and positrons are not + # necessarily dumped in the same order. + s_energy_ele = np.sort(energy_ele) + is_energy_pos = np.sort(energy_pos)[::-1] + product_energy = s_energy_ele + is_energy_pos + assert small_diff(product_energy, energy_phot) + print(" [OK] energy is conserved in each event") + + +def check_opt_depths(phot_data, ele_data, pos_data): + data = (phot_data, ele_data, pos_data) + for dd in data: + # Remove the negative optical depths that correspond to photons that will decay into pairs + # at the beginning of the next timestep + loc, scale = st.expon.fit(dd["opt"][dd["opt"] > 0]) + assert np.abs(loc - 0) < tol_red + assert np.abs(scale - 1) < tol_red + print(" [OK] optical depth distributions are still exponential") + + +def check_energy_distrib( + energy_ele, energy_pos, gamma_phot, chi_phot, n_lost, NN, idx, do_plot=False +): + gamma_min = 1.0001 + gamma_max = gamma_phot - 1.0001 + h_gamma_ele, c_gamma = np.histogram( + energy_ele / mec2, bins=NN, range=[gamma_min, gamma_max] + ) + h_gamma_pos, _ = np.histogram( + energy_pos / mec2, bins=NN, range=[gamma_min, gamma_max] + ) + + cchi_part_min = chi_phot * (gamma_min - 1) / (gamma_phot - 2) + cchi_part_max = chi_phot * (gamma_max - 1) / (gamma_phot - 2) + + # Rudimentary integration over npoints for each bin + npoints = 20 + aux_chi = np.linspace(cchi_part_min, cchi_part_max, NN * npoints) + distrib = BW_d2N_dt_dchi(chi_phot, gamma_phot, aux_chi) + distrib = np.sum(distrib.reshape(-1, npoints), 1) + distrib = n_lost * distrib / np.sum(distrib) + + if do_plot: + # Visual comparison of distributions + c_gamma_centered = 0.5 * (c_gamma[1:] + c_gamma[:-1]) + plt.clf() + plt.xlabel("γ_particle") + plt.ylabel("N") + plt.title("χ_photon = {:f}".format(chi_phot)) + plt.plot(c_gamma_centered, distrib, label="theory") + plt.plot(c_gamma_centered, h_gamma_ele, label="BW electrons") + plt.plot(c_gamma_centered, h_gamma_pos, label="BW positrons") + plt.legend() + plt.savefig("case_{:d}".format(idx + 1)) + + discr_ele = np.abs(h_gamma_ele - distrib) + discr_pos = np.abs(h_gamma_pos - distrib) + max_discr = 5.0 * np.sqrt(distrib) + assert np.all(discr_ele < max_discr) + assert np.all(discr_pos < max_discr) + print(" [OK] energy distribution is within expectations") + + +# __________________ + + +def check(dt, particle_data): + for idx in range(4): + phot_name = spec_names_phot[idx] + ele_name = spec_names_ele[idx] + pos_name = spec_names_pos[idx] + p0 = initial_momenta[idx] + + p2_phot = p0[0] ** 2 + p0[1] ** 2 + p0[2] ** 2 + p_phot = np.sqrt(p2_phot) + energy_phot = p_phot * c + chi_phot = calc_chi_gamma(p0, E_f, B_f) + gamma_phot = np.linalg.norm(p0) / mec + + print("** Case {:d} **".format(idx + 1)) + print(" initial momentum: ", p0) + print(" quantum parameter: {:f}".format(chi_phot)) + print(" normalized photon energy: {:f}".format(gamma_phot)) + print(" timestep: {:f} fs".format(dt * 1e15)) + + phot_data = particle_data[phot_name] + ele_data = particle_data[ele_name] + pos_data = particle_data[pos_name] + + p2_ele = ele_data["px"] ** 2 + ele_data["py"] ** 2 + ele_data["pz"] ** 2 + p_ele = np.sqrt(p2_ele) + energy_ele = np.sqrt(1.0 + p2_ele / mec**2) * mec2 + p2_pos = pos_data["px"] ** 2 + pos_data["py"] ** 2 + pos_data["pz"] ** 2 + p_pos = np.sqrt(p2_pos) + energy_pos = np.sqrt(1.0 + p2_pos / mec**2) * mec2 + + n_lost = check_number_of_pairs( + particle_data, + phot_name, + ele_name, + pos_name, + chi_phot, + gamma_phot, + dt, + initial_particle_number, + ) + + check_weights(phot_data, ele_data, pos_data) + + check_momenta(phot_data, ele_data, pos_data, p0, p_ele, p_pos) + + check_energy(energy_phot, energy_ele, energy_pos) + + check_energy_distrib( + energy_ele, energy_pos, gamma_phot, chi_phot, n_lost, NNS[idx], idx + ) + + check_opt_depths(phot_data, ele_data, pos_data) + + print("*************\n") diff --git a/Examples/Tests/qed/analysis_breit_wheeler_opmd.py b/Examples/Tests/qed/analysis_breit_wheeler_opmd.py new file mode 100755 index 00000000000..1803305f008 --- /dev/null +++ b/Examples/Tests/qed/analysis_breit_wheeler_opmd.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Copyright 2019 Luca Fedeli +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# -*- coding: utf-8 -*- + +import sys + +import analysis_breit_wheeler_core as ac +import openpmd_api as io + +# This script is a frontend for the analysis routines +# in analysis_breit_wheeler_core.py (please refer to this file for +# a full description). It reads output files in openPMD +# format and extracts the data needed for +# the analysis routines. + + +def main(): + print("Opening openPMD output") + prefix = sys.argv[1] + series = io.Series(prefix + "/openpmd_%T.h5", io.Access.read_only) + data_set_end = series.iterations[2] + + # get simulation time + sim_time = data_set_end.time + # no particles can be created on the first timestep so we have 2 timesteps in the test case, + # with only the second one resulting in particle creation + dt = sim_time / 2.0 + + # get particle data + particle_data = {} + + names, types = ac.get_all_species_names_and_types() + for spec_name_type in zip(names, types): + spec_name = spec_name_type[0] + is_photon = spec_name_type[1] + data = {} + + px = data_set_end.particles[spec_name]["momentum"]["x"][:] + py = data_set_end.particles[spec_name]["momentum"]["y"][:] + pz = data_set_end.particles[spec_name]["momentum"]["z"][:] + w = data_set_end.particles[spec_name]["weighting"][ + io.Mesh_Record_Component.SCALAR + ][:] + + if is_photon: + opt = data_set_end.particles[spec_name]["opticalDepthBW"][ + io.Mesh_Record_Component.SCALAR + ][:] + else: + opt = data_set_end.particles[spec_name]["opticalDepthQSR"][ + io.Mesh_Record_Component.SCALAR + ][:] + + series.flush() + + data["px"] = px + data["py"] = py + data["pz"] = pz + data["w"] = w + data["opt"] = opt + + particle_data[spec_name] = data + + ac.check(dt, particle_data) + + +if __name__ == "__main__": + main() diff --git a/Examples/Tests/qed/analysis_breit_wheeler_yt.py b/Examples/Tests/qed/analysis_breit_wheeler_yt.py new file mode 100755 index 00000000000..bd8f4454723 --- /dev/null +++ b/Examples/Tests/qed/analysis_breit_wheeler_yt.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright 2019 Luca Fedeli +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# -*- coding: utf-8 -*- + +import sys + +import analysis_breit_wheeler_core as ac +import yt + +# This script is a frontend for the analysis routines +# in analysis_breit_wheeler_core.py (please refer to this file for +# a full description). It reads output files in yt +# format and extracts the data needed for +# the analysis routines. +yt + + +def main(): + print("Opening yt output") + filename_end = sys.argv[1] + data_set_end = yt.load(filename_end) + + # get simulation time + sim_time = data_set_end.current_time.to_value() + # no particles can be created on the first timestep so we have 2 timesteps in the test case, + # with only the second one resulting in particle creation + dt = sim_time / 2.0 + + # get particle data + all_data_end = data_set_end.all_data() + particle_data = {} + + names, types = ac.get_all_species_names_and_types() + for spec_name_type in zip(names, types): + spec_name = spec_name_type[0] + is_photon = spec_name_type[1] + data = {} + data["px"] = all_data_end[spec_name, "particle_momentum_x"].v + data["py"] = all_data_end[spec_name, "particle_momentum_y"].v + data["pz"] = all_data_end[spec_name, "particle_momentum_z"].v + data["w"] = all_data_end[spec_name, "particle_weighting"].v + + if is_photon: + data["opt"] = all_data_end[spec_name, "particle_opticalDepthBW"].v + else: + data["opt"] = all_data_end[spec_name, "particle_opticalDepthQSR"].v + + particle_data[spec_name] = data + + ac.check(dt, particle_data) + + +if __name__ == "__main__": + main() diff --git a/Examples/Tests/qed/analysis_default_regression.py b/Examples/Tests/qed/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/qed/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/qed/analysis_quantum_sync.py b/Examples/Tests/qed/analysis_quantum_sync.py new file mode 100755 index 00000000000..e4ede19260c --- /dev/null +++ b/Examples/Tests/qed/analysis_quantum_sync.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Luca Fedeli, Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# -*- coding: utf-8 -*- + +import sys + +import matplotlib.pyplot as plt +import numpy as np +import scipy.integrate as integ +import scipy.special as spe +import scipy.stats as st +import yt + +# This script performs detailed checks of the Quantum Synchrotron photon emission process. +# Two electron populations and two positron populations are initialized with different momenta in different +# directions in a background EM field (with non-zero components along each direction). +# Specifically the script checks that: +# +# - The expected number of generated photons n_phot is in agreement with theory. +# The maximum tolerated error is 5*sqrt(n_phot), except for the last 8 points, +# for which a higher tolerance is used (this is due to the fact that the resolution +# of the builtin table is quite limited). +# - The weight of the generated particles is equal to the weight of the photon +# - The generated particles are emitted in the right direction +# - The energy distribution of the generated particles is in agreement with theory +# - The optical depths of the product species are correctly initialized (QED effects are +# enabled for product species too). +# +# More details on the theoretical formulas used in this script can be found in +# the Jupyter notebook picsar/src/multi_physics/QED_tests/validation/validation.ipynb +# +# References: +# 1) R. Duclous et al 2011 Plasma Phys. Control. Fusion 53 015009 +# 2) A. Gonoskov et al. 2015 Phys. Rev. E 92, 023305 +# 3) M. Lobet. PhD thesis "Effets radiatifs et d'electrodynamique +# quantique dans l'interaction laser-matiere ultra-relativiste" +# URL: https://tel.archives-ouvertes.fr/tel-01314224 + + +# Tolerances +tol = 1.0e-8 +tol_red = 1.0e-2 + +# Physical constants (from CODATA 2018, see: https://physics.nist.gov/cuu/Constants/index.html ) +me = 9.1093837015e-31 # electron mass +c = 299792458 # speed of light +hbar = 6.62607015e-34 / (2 * np.pi) # reduced Plank constant +fine_structure = 7.2973525693e-3 # fine structure constant +qe = 1.602176634e-19 # elementary charge +E_s = (me**2 * c**3) / (qe * hbar) # Schwinger E field +B_s = E_s / c # Schwinger B field + +mec = me * c +mec2 = mec * c +# ______________ + +# Initial parameters +spec_names = ["p1", "p2", "p3", "p4"] +spec_names_phot = ["qsp_1", "qsp_2", "qsp_3", "qsp_4"] +initial_momenta = [ + np.array([10.0, 0, 0]) * mec, + np.array([0.0, 100.0, 0.0]) * mec, + np.array([0.0, 0.0, 1000.0]) * mec, + np.array([5773.502691896, 5773.502691896, 5773.502691896]) * mec, +] +csign = [-1, -1, 1, 1] +initial_particle_number = 1048576 + +E_f = np.array([-2433321316961438.0, 973328526784575.0, 1459992790176863.0]) +B_f = np.array([2857142.85714286, 4285714.28571428, 8571428.57142857]) + +NNS = [64, 64, 64, 64] # bins for energy distribution comparison. +# ______________ + + +def calc_chi_part(p, E, B): + gamma_part = np.sqrt(1.0 + np.dot(p, p) / mec**2) + v = p / (gamma_part * me) + EpvvecB = E + np.cross(v, B) + vdotEoverc = np.dot(v, E) / c + ff = np.sqrt(np.dot(EpvvecB, EpvvecB) - np.dot(vdotEoverc, vdotEoverc)) + return gamma_part * ff / E_s + + +# Auxiliary functions +@np.vectorize +def IC_inner_alternative(y): + def ff(x): + return ( + np.exp(-y * (1 + 4 * x**2 / 3) * np.sqrt(1 + x * x / 3)) + * (9 + 36 * x**2 + 16 * x**4) + / (3 + 4 * x**2) + / np.sqrt(1 + x**2 / 3) + ) + + # This integration may not converge in some cases, in which case a python warning message can + # be issued. This is probably not a significant issue for this test case and these warnings can + # be ignored. + return integ.quad(ff, 0, np.inf)[0] / np.sqrt(3) + + +def IC_Y(chi_ele, xi): + div = chi_ele * (1 - xi) + div = np.where(np.logical_and(xi < 1, chi_ele != 0), div, 1.0) + res = (2 / 3) * np.where(np.logical_and(xi < 1, chi_ele != 0), xi / div, np.inf) + return res + + +def IC_S(chi_ele, xi): + Y = IC_Y(chi_ele, xi) + coeff = np.sqrt(3) / 2.0 / np.pi + first = IC_inner_alternative(Y) + div = np.where(xi == 1, 1.0, 1.0 / (1 - xi)) + res = np.where( + np.logical_or(xi == 1, xi == 0), + 0.0, + coeff * xi * (first + (xi**2 * spe.kv(2.0 / 3.0, Y) * div)), + ) + return res + + +def IC_SXI(chi_ele, xi): + div = np.where(xi != 0, xi, 1.0) + return np.where(xi != 0, IC_S(chi_ele, xi) / div, np.inf) + + +@np.vectorize +def IC_G(chi_ele): + return integ.quad(lambda xi: IC_SXI(chi_ele, xi), 0, 1)[0] + + +def small_diff(vv, val): + if val != 0.0: + return np.max(np.abs((vv - val) / val)) < tol + else: + return np.max(np.abs(vv)) < tol + + +def boris(pp, dt, charge_sign): + econst = 0.5 * qe * dt * charge_sign / me + u = pp / (me) + u += econst * E_f + inv_gamma = 1 / np.sqrt(1 + np.dot(u, u) / c**2) + t = econst * B_f * inv_gamma + s = 2 * t / (1 + np.dot(t, t)) + u_p = u + np.cross(u, t) + u += np.cross(u_p, s) + u += econst * E_f + return u * me + + +# __________________ + + +# Quantum Synchrotron total and differential cross sections +def QS_dN_dt(chi_ele, gamma_ele): + coeff_IC = (2.0 / 3.0) * fine_structure * me * c**2 / hbar + return coeff_IC * IC_G(chi_ele) / gamma_ele + + +def QS_d2N_dt_dchi(chi, gamma_ele, chi_phot): + coeff_IC = (2.0 / 3.0) * fine_structure * me * c**2 / hbar + return coeff_IC * IC_S(chi, chi_phot / chi) / chi_phot / gamma_ele + + +# __________________ + + +# Get data for a species +def get_spec(ytdata, specname, is_photon): + px = ytdata[specname, "particle_momentum_x"].v + pz = ytdata[specname, "particle_momentum_z"].v + py = ytdata[specname, "particle_momentum_y"].v + + w = ytdata[specname, "particle_weighting"].v + + if is_photon: + opt = ytdata[specname, "particle_opticalDepthBW"].v + else: + opt = ytdata[specname, "particle_opticalDepthQSR"].v + + return {"px": px, "py": py, "pz": pz, "w": w, "opt": opt} + + +# Individual tests +def check_number_of_photons( + ytdataset, part_name, phot_name, chi_part, gamma_part, dt, particle_number +): + dNQS_dt_theo = QS_dN_dt(chi_part, gamma_part) + expected_photons = (1.0 - np.exp(-dNQS_dt_theo * dt)) * particle_number + expected_photons_tolerance = 5.0 * np.sqrt(expected_photons) + n_phot = ytdataset.particle_type_counts[phot_name] + assert np.abs(n_phot - expected_photons) < expected_photons_tolerance + print(" [OK] generated photons number is within expectations") + return n_phot + + +def check_weights(part_data, phot_data): + assert np.all(part_data["w"] == part_data["w"][0]) + assert np.all(phot_data["w"] == part_data["w"][0]) + print(" [OK] particles weights are the expected ones") + + +def check_momenta(phot_data, p_phot, p0): + pdir = p0 / np.linalg.norm(p0) + assert small_diff(phot_data["px"] / p_phot, pdir[0]) + assert small_diff(phot_data["py"] / p_phot, pdir[1]) + assert small_diff(phot_data["pz"] / p_phot, pdir[2]) + print(" [OK] photons move along the initial particle direction") + + +def check_opt_depths(part_data, phot_data): + data = (part_data, phot_data) + for dd in data: + # Remove the negative optical depths that will be + # reset at the beginning of the next timestep + loc, scale = st.expon.fit(dd["opt"][dd["opt"] > 0]) + assert np.abs(loc - 0) < tol_red + assert np.abs(scale - 1) < tol_red + print(" [OK] optical depth distributions are still exponential") + + +def check_energy_distrib( + gamma_phot, chi_part, gamma_part, n_phot, NN, idx, do_plot=False +): + gamma_phot_min = 1e-12 * gamma_part + gamma_phot_max = gamma_part + h_log_gamma_phot, c_gamma_phot = np.histogram( + gamma_phot, + bins=np.logspace(np.log10(gamma_phot_min), np.log10(gamma_phot_max), NN + 1), + ) + + cchi_phot_min = chi_part * (gamma_phot_min) / (gamma_part - 1) + cchi_phot_max = chi_part * (gamma_phot_max) / (gamma_part - 1) + + # Rudimentary integration over npoints for each bin + npoints = 20 + aux_chi = np.logspace( + np.log10(cchi_phot_min), np.log10(cchi_phot_max), NN * npoints + ) + distrib = QS_d2N_dt_dchi(chi_part, gamma_part, aux_chi) * aux_chi + distrib = np.sum(distrib.reshape(-1, npoints), 1) + distrib = n_phot * distrib / np.sum(distrib) + + if do_plot: + # Visual comparison of distributions + c_gamma_phot = np.exp( + 0.5 * (np.log(c_gamma_phot[1:]) + np.log(c_gamma_phot[:-1])) + ) + plt.clf() + + fig, (ax1, ax2) = plt.subplots(1, 2) + fig.suptitle("χ_particle = {:f}".format(chi_part)) + ax1.plot(c_gamma_phot, distrib, label="theory") + ax1.loglog(c_gamma_phot, h_log_gamma_phot, label="QSR photons") + ax1.set_xlim(1e-12 * (gamma_part - 1), gamma_part - 1) + ax1.set_ylim(1, 1e5) + ax2.plot(c_gamma_phot, distrib, label="theory") + ax2.semilogy(c_gamma_phot, h_log_gamma_phot, label="QSR photons") + ax2.set_ylim(1, 1e5) + ax2.set_xlim(1e-12 * (gamma_part - 1), gamma_part - 1) + ax1.set_xlabel("γ_photon") + ax1.set_xlabel("N") + ax2.set_xlabel("γ_photon") + ax2.set_xlabel("N") + plt.legend() + plt.savefig("case_{:d}".format(idx + 1)) + + discr = np.abs(h_log_gamma_phot - distrib) + + max_discr = np.sqrt(distrib) * 5.0 + # Use a higer tolerance for the last 8 points (this is due to limitations + # of the builtin table) + max_discr[-8:] *= 2.0 + assert np.all(discr < max_discr) + + print(" [OK] energy distribution is within expectations") + + +# __________________ + + +def check(): + filename_end = sys.argv[1] + data_set_end = yt.load(filename_end) + + sim_time = data_set_end.current_time.to_value() + # no particles can be created on the first timestep so we have 2 timesteps in the test case, + # with only the second one resulting in particle creation + dt = sim_time / 2.0 + + all_data_end = data_set_end.all_data() + + for idx in range(4): + part_name = spec_names[idx] + phot_name = spec_names_phot[idx] + t_pi = initial_momenta[idx] + pm = boris(t_pi, -dt * 0.5, csign[idx]) + p0 = boris(pm, dt * 1.0, csign[idx]) + + p2_part = p0[0] ** 2 + p0[1] ** 2 + p0[2] ** 2 + energy_part = np.sqrt(mec2**2 + p2_part * c**2) + gamma_part = energy_part / mec2 + chi_part = calc_chi_part(p0, E_f, B_f) + + print("** Case {:d} **".format(idx + 1)) + print(" initial momentum: ", t_pi) + print(" quantum parameter: {:f}".format(chi_part)) + print(" normalized particle energy: {:f}".format(gamma_part)) + print(" timestep: {:f} fs".format(dt * 1e15)) + + part_data_final = get_spec(all_data_end, part_name, is_photon=False) + phot_data = get_spec(all_data_end, phot_name, is_photon=True) + + p_phot = np.sqrt( + phot_data["px"] ** 2 + phot_data["py"] ** 2 + phot_data["pz"] ** 2 + ) + energy_phot = p_phot * c + gamma_phot = energy_phot / mec2 + + n_phot = check_number_of_photons( + data_set_end, + part_name, + phot_name, + chi_part, + gamma_part, + dt, + initial_particle_number, + ) + + check_weights(part_data_final, phot_data) + + check_momenta(phot_data, p_phot, p0) + + check_energy_distrib(gamma_phot, chi_part, gamma_part, n_phot, NNS[idx], idx) + + check_opt_depths(part_data_final, phot_data) + + print("*************\n") + + +def main(): + check() + + +if __name__ == "__main__": + main() diff --git a/Examples/Tests/qed/analysis_schwinger.py b/Examples/Tests/qed/analysis_schwinger.py new file mode 100755 index 00000000000..5d1c5485ba3 --- /dev/null +++ b/Examples/Tests/qed/analysis_schwinger.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Luca Fedeli, Neil Zaim +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +## This module is imported by each of the scripts used to analyze the Schwinger tests. + +## The pair production rate is calculated using the formula described in +## Bulanov, S. S., et al. Physical review letters 104.22 (2010): 220404. + +import os +import re +import sys + +import numpy as np +import yt + +# define some parameters + +c = 299792458.0 +m_e = 9.1093837015e-31 +e = 1.602176634e-19 +hbar = 1.054571817e-34 +E_S = m_e**2 * c**3 / e / hbar # Schwinger field + +dV = (1.0e-6) ** 3 # total simulation volume +dt = 9.827726403e-17 +filename = sys.argv[1] + +Ex_test = 0.0 +Ey_test = 0.0 +Ez_test = 0.0 +Bx_test = 0.0 +By_test = 0.0 +Bz_test = 0.0 + +# Find which test we are doing +test_name = os.path.split(os.getcwd())[1] +test_number = re.search("qed_schwinger_([1234])", test_name).group(1) +if test_number == "1": + # First Schwinger test with "weak" EM field. No pair should be created. + Ex_test = 1.0e16 + Bx_test = 16792888.570516706 + By_test = 5256650.141557486 + Bz_test = 18363530.799561853 +elif test_number == "2": + # Second Schwinger test with stronger EM field. Many pairs are created and a Gaussian + # distribution is used to get the weights of the particles. This is the most sensitive test + # because the relative std is extremely low. + Ex_test = 1.0e18 + Bx_test = 1679288857.0516706 + By_test = 525665014.1557486 + Bz_test = 1836353079.9561853 + dV = dV / 2.0 # Schwinger is only activated in part of the simulation domain +elif test_number == "3": + # Third Schwinger test with intermediate electric field such that average created pair per cell + # is 1. A Poisson distribution is used to obtain the weights of the particles. + Ey_test = 1.090934525450495e17 +elif test_number == "4": + # Fourth Schwinger test with extremely strong EM field but with E and B perpendicular and nearly + # equal so that the pair production rate is fairly low. A Gaussian distribution is used in this + # case. + Ez_test = 2.5e20 + By_test = 833910140000.0 + dV = ( + dV * (3.0 / 4.0) ** 2.0 + ) # Schwinger is only activated in part of the simulation domain +else: + assert False + + +def calculate_rate(Ex, Ey, Ez, Bx, By, Bz): + ## Calculate theoretical pair production rate from EM field value + + E_squared = Ex**2 + Ey**2 + Ez**2 + H_squared = c**2 * (Bx**2 + By**2 + Bz**2) + + F = (E_squared - H_squared) / 2.0 + G = c * (Ex * Bx + Ey * By + Ez * Bz) + + epsilon = np.sqrt(np.sqrt(F**2 + G**2) + F) / E_S + eta = np.sqrt(np.sqrt(F**2 + G**2) - F) / E_S + + if epsilon != 0.0 and eta != 0.0: + return ( + e**2 + * E_S**2 + / 4.0 + / np.pi**2 + / c + / hbar**2 + * epsilon + * eta + / np.tanh(np.pi * eta / epsilon) + * np.exp(-np.pi / epsilon) + ) + elif epsilon == 0.0: + return 0.0 + else: + return ( + e**2 + * E_S**2 + / 4.0 + / np.pi**2 + / c + / hbar**2 + * epsilon**2 + / np.pi + * np.exp(-np.pi / epsilon) + ) + + +def do_analysis(Ex, Ey, Ez, Bx, By, Bz): + data_set = yt.load(filename) + + expected_total_physical_pairs_created = ( + dV * dt * calculate_rate(Ex, Ey, Ez, Bx, By, Bz) + ) + if expected_total_physical_pairs_created < 0.01: + np_ele = ( + data_set.particle_type_counts["ele_schwinger"] + if "ele_schwinger" in data_set.particle_type_counts.keys() + else 0 + ) + np_pos = ( + data_set.particle_type_counts["pos_schwinger"] + if "pos_schwinger" in data_set.particle_type_counts.keys() + else 0 + ) + assert np_ele == 0 and np_pos == 0 + ## Assert whether pairs are created or not. + + else: + all_data = data_set.all_data() + + ele_data = all_data["ele_schwinger", "particle_weight"] + pos_data = all_data["pos_schwinger", "particle_weight"] + + std_total_physical_pairs_created = np.sqrt( + expected_total_physical_pairs_created + ) + + # Sorting the arrays is required because electrons and positrons are not necessarily + # dumped in the same order. + assert np.array_equal(np.sort(ele_data), np.sort(pos_data)) + # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions + error = np.abs(np.sum(ele_data) - expected_total_physical_pairs_created) + print( + "difference between expected and actual number of pairs created: " + + str(error) + ) + print("tolerance: " + str(5 * std_total_physical_pairs_created)) + assert error < 5 * std_total_physical_pairs_created + + +do_analysis(Ex_test, Ey_test, Ez_test, Bx_test, By_test, Bz_test) diff --git a/Examples/Tests/qed/breit_wheeler/analysis_core.py b/Examples/Tests/qed/breit_wheeler/analysis_core.py deleted file mode 100755 index 61d928bf2cc..00000000000 --- a/Examples/Tests/qed/breit_wheeler/analysis_core.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2019 Luca Fedeli, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import matplotlib.pyplot as plt -import numpy as np -import scipy.integrate as integ -import scipy.special as spe -import scipy.stats as st - -# This script performs detailed checks of the Breit-Wheeler pair production process. -# Four populations of photons are initialized with different momenta in different -# directions in a background EM field (with non-zero components along each direction). -# Specifically the script checks that: -# -# - The expected number of generated pairs n_pairs is in agreement with theory -# (the maximum tolerated error is 5*sqrt(n_pairs) -# - The weight of the generated particles is equal to the weight of the photon -# - Momenta of the residual photons are still equal to the original momentum -# - The generated particles are emitted in the right direction -# - Total energy is conserved in each event -# - The energy distribution of the generated particles is in agreement with theory -# - The optical depths of the product species are correctly initialized (QED effects are -# enabled for product species too). -# -# More details on the theoretical formulas used in this script can be found in -# the Jupyter notebook picsar/src/multi_physics/QED_tests/validation/validation.ipynb -# -# References: -# 1) R. Duclous et al 2011 Plasma Phys. Control. Fusion 53 015009 -# 2) A. Gonoskov et al. 2015 Phys. Rev. E 92, 023305 -# 3) M. Lobet. PhD thesis "Effets radiatifs et d'electrodynamique -# quantique dans l'interaction laser-matiere ultra-relativiste" -# URL: https://tel.archives-ouvertes.fr/tel-01314224 - - -# Tolerances -tol = 1.e-8 -tol_red = 2.e-2 - -# Physical constants (from CODATA 2018, see: https://physics.nist.gov/cuu/Constants/index.html ) -me = 9.1093837015e-31 #electron mass -c = 299792458 #speed of light -hbar = 6.62607015e-34/(2*np.pi) #reduced Plank constant -fine_structure = 7.2973525693e-3 #fine structure constant -qe = 1.602176634e-19#elementary charge -E_s = (me**2 * c**3)/(qe * hbar) #Schwinger E field -B_s = E_s/c #Schwinger B field - -mec = me*c -mec2 = mec*c -#______________ - -# Initial parameters -spec_names_phot = ["p1", "p2", "p3", "p4"] -spec_names_ele = ["ele1", "ele2", "ele3", "ele4"] -spec_names_pos = ["pos1", "pos2", "pos3", "pos4"] -initial_momenta = [ - np.array([2000.0,0,0])*mec, - np.array([0.0,5000.0,0.0])*mec, - np.array([0.0,0.0,10000.0])*mec, - np.array([57735.02691896, 57735.02691896, 57735.02691896])*mec -] -initial_particle_number = 1048576 - -E_f = np.array([-2433321316961438., 973328526784575., 1459992790176863.]) -B_f = np.array([2857142.85714286, 4285714.28571428, 8571428.57142857]) - -NNS = [128,128,128,128] #bins for energy distribution comparison. -#______________ - -#Returns all the species names and if they are photon species or not -def get_all_species_names_and_types(): - names = spec_names_phot + spec_names_ele + spec_names_pos - types = [True]*len(spec_names_phot) + [False]*(len(spec_names_ele)+len(spec_names_pos)) - return names, types - -def calc_chi_gamma(p, E, B): - pnorm = np.linalg.norm(p) - v = c*(p/pnorm) - EpvvecB = E + np.cross(v,B) - vdotEoverc = np.dot(v,E)/c - ff = np.sqrt(np.dot(EpvvecB,EpvvecB) - np.dot(vdotEoverc,vdotEoverc)) - gamma_phot = pnorm/mec - return gamma_phot*ff/E_s - -#Auxiliary functions -@np.vectorize -def BW_inner(x): - return integ.quad(lambda s: np.sqrt(s)*spe.kv(1./3., 2./3. * s**(3./2.)), x, np.inf)[0] - -def BW_X(chi_phot, chi_ele): - div = (chi_ele*(chi_phot-chi_ele)) - div = np.where(np.logical_and(chi_phot > chi_ele, chi_ele != 0), div, 1.0); - res = np.where(np.logical_and(chi_phot > chi_ele, chi_ele != 0), np.power(chi_phot/div, 2./3.), np.inf) - return res - -def BW_F(chi_phot, chi_ele): - X = BW_X(chi_phot, chi_ele) - res = np.where(np.logical_or(chi_phot == chi_ele, chi_ele == 0), 0.0, - BW_inner(X) - (2.0 - chi_phot* X**(3./2.))*spe.kv(2./3., 2./3. * X**(3./2.)) ) - return res - -@np.vectorize -def BW_T(chi_phot): - coeff = 1./(np.pi * np.sqrt(3.) * (chi_phot**2)) - return coeff*integ.quad(lambda chi_ele: BW_F(chi_phot, chi_ele), 0, chi_phot)[0] - -def small_diff(vv, val): - if(val != 0.0): - return np.max(np.abs((vv - val)/val)) < tol - else: - return np.max(np.abs(vv)) < tol -#__________________ - -# Breit-Wheeler total and differential cross sections -def BW_dN_dt(chi_phot, gamma_phot): - coeff_BW = fine_structure * me*c**2/hbar - return coeff_BW*BW_T(chi_phot)*(chi_phot/gamma_phot) - -def BW_d2N_dt_dchi(chi_phot, gamma_phot, chi_ele): - coeff_BW = fine_structure * me*c**2/hbar - return coeff_BW*BW_F(chi_phot, chi_ele)*(gamma_phot/gamma_phot) -#__________________ - -# Individual tests - -def check_number_of_pairs(particle_data, phot_name, ele_name, pos_name, chi_phot, gamma_phot, dt, particle_number): - dNBW_dt_theo = BW_dN_dt(chi_phot, gamma_phot) - expected_pairs = (1.-np.exp(-dNBW_dt_theo*dt))*particle_number - expected_pairs_tolerance = 5.0*np.sqrt(expected_pairs) - n_ele = len(particle_data[ele_name]["w"]) - n_pos = len(particle_data[pos_name]["w"]) - n_phot = len(particle_data[phot_name]["w"]) - n_lost = initial_particle_number-n_phot - assert((n_ele == n_pos) and (n_ele == n_lost)) - assert( np.abs(n_ele-expected_pairs) < expected_pairs_tolerance) - print(" [OK] generated pair number is within expectations") - return n_lost - -def check_weights(phot_data, ele_data, pos_data): - assert(np.all(phot_data["w"] == phot_data["w"][0])) - assert(np.all(ele_data["w"] == phot_data["w"][0])) - assert(np.all(pos_data["w"] == phot_data["w"][0])) - print(" [OK] particles weights are the expected ones") - -def check_momenta(phot_data, ele_data, pos_data, p0, p_ele, p_pos): - assert(small_diff(phot_data["px"], p0[0])) - assert(small_diff(phot_data["py"], p0[1])) - assert(small_diff(phot_data["pz"], p0[2])) - print(" [OK] residual photons still have initial momentum") - - pdir = p0/np.linalg.norm(p0) - assert(small_diff(ele_data["px"]/p_ele, pdir[0])) - assert(small_diff(ele_data["py"]/p_ele, pdir[1])) - assert(small_diff(ele_data["pz"]/p_ele, pdir[2])) - assert(small_diff(pos_data["px"]/p_pos, pdir[0])) - assert(small_diff(pos_data["py"]/p_pos, pdir[1])) - assert(small_diff(pos_data["pz"]/p_pos, pdir[2])) - print(" [OK] pairs move along the initial photon direction") - -def check_energy(energy_phot, energy_ele, energy_pos): - # Sorting the arrays is required because electrons and positrons are not - # necessarily dumped in the same order. - s_energy_ele = np.sort(energy_ele) - is_energy_pos = np.sort(energy_pos)[::-1] - product_energy = s_energy_ele + is_energy_pos - assert(small_diff(product_energy, energy_phot)) - print(" [OK] energy is conserved in each event") - -def check_opt_depths(phot_data, ele_data, pos_data): - data = (phot_data, ele_data, pos_data) - for dd in data: - # Remove the negative optical depths that correspond to photons that will decay into pairs - # at the beginning of the next timestep - loc, scale = st.expon.fit(dd["opt"][dd["opt"] > 0]) - assert( np.abs(loc - 0) < tol_red ) - assert( np.abs(scale - 1) < tol_red ) - print(" [OK] optical depth distributions are still exponential") - -def check_energy_distrib(energy_ele, energy_pos, gamma_phot, - chi_phot, n_lost, NN, idx, do_plot=False): - gamma_min = 1.0001 - gamma_max = gamma_phot-1.0001 - h_gamma_ele, c_gamma = np.histogram(energy_ele/mec2, bins=NN, range=[gamma_min,gamma_max]) - h_gamma_pos, _ = np.histogram(energy_pos/mec2, bins=NN, range=[gamma_min,gamma_max]) - - cchi_part_min = chi_phot*(gamma_min - 1)/(gamma_phot - 2) - cchi_part_max = chi_phot*(gamma_max - 1)/(gamma_phot - 2) - - #Rudimentary integration over npoints for each bin - npoints= 20 - aux_chi = np.linspace(cchi_part_min, cchi_part_max, NN*npoints) - distrib = BW_d2N_dt_dchi(chi_phot, gamma_phot, aux_chi) - distrib = np.sum(distrib.reshape(-1, npoints),1) - distrib = n_lost*distrib/np.sum(distrib) - - if do_plot : - # Visual comparison of distributions - c_gamma_centered = 0.5*(c_gamma[1:]+c_gamma[:-1]) - plt.clf() - plt.xlabel("γ_particle") - plt.ylabel("N") - plt.title("χ_photon = {:f}".format(chi_phot)) - plt.plot(c_gamma_centered, distrib,label="theory") - plt.plot(c_gamma_centered, h_gamma_ele,label="BW electrons") - plt.plot(c_gamma_centered, h_gamma_pos,label="BW positrons") - plt.legend() - plt.savefig("case_{:d}".format(idx+1)) - - discr_ele = np.abs(h_gamma_ele-distrib) - discr_pos = np.abs(h_gamma_pos-distrib) - max_discr = 5.0 * np.sqrt(distrib) - assert(np.all(discr_ele < max_discr)) - assert(np.all(discr_pos < max_discr)) - print(" [OK] energy distribution is within expectations") - -#__________________ - -def check(dt, particle_data): - - for idx in range(4): - phot_name = spec_names_phot[idx] - ele_name = spec_names_ele[idx] - pos_name = spec_names_pos[idx] - p0 = initial_momenta[idx] - - p2_phot = p0[0]**2 + p0[1]**2 + p0[2]**2 - p_phot = np.sqrt(p2_phot) - energy_phot = p_phot*c - chi_phot = calc_chi_gamma(p0, E_f, B_f) - gamma_phot = np.linalg.norm(p0)/mec - - print("** Case {:d} **".format(idx+1)) - print(" initial momentum: ", p0) - print(" quantum parameter: {:f}".format(chi_phot)) - print(" normalized photon energy: {:f}".format(gamma_phot)) - print(" timestep: {:f} fs".format(dt*1e15)) - - phot_data = particle_data[phot_name] - ele_data = particle_data[ele_name] - pos_data = particle_data[pos_name] - - p2_ele = ele_data["px"]**2 + ele_data["py"]**2 + ele_data["pz"]**2 - p_ele = np.sqrt(p2_ele) - energy_ele = np.sqrt(1.0 + p2_ele/mec**2 )*mec2 - p2_pos = pos_data["px"]**2 + pos_data["py"]**2 + pos_data["pz"]**2 - p_pos = np.sqrt(p2_pos) - energy_pos = np.sqrt(1.0 + p2_pos/mec**2 )*mec2 - - n_lost = check_number_of_pairs(particle_data, - phot_name, ele_name, pos_name, - chi_phot, gamma_phot, dt, - initial_particle_number) - - check_weights(phot_data, ele_data, pos_data) - - check_momenta(phot_data, ele_data, pos_data, p0, p_ele, p_pos) - - check_energy(energy_phot, energy_ele, energy_pos) - - check_energy_distrib(energy_ele, energy_pos, gamma_phot, chi_phot, n_lost, NNS[idx], idx) - - check_opt_depths(phot_data, ele_data, pos_data) - - print("*************\n") diff --git a/Examples/Tests/qed/breit_wheeler/analysis_opmd.py b/Examples/Tests/qed/breit_wheeler/analysis_opmd.py deleted file mode 100755 index 2b1fbc7038b..00000000000 --- a/Examples/Tests/qed/breit_wheeler/analysis_opmd.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Luca Fedeli -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# -*- coding: utf-8 -*- - -import sys - -import analysis_core as ac -import openpmd_api as io - -#sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -#import checksumAPI - - -# This script is a frontend for the analysis routines -# in analysis_core.py (please refer to this file for -# a full description). It reads output files in openPMD -# format and extracts the data needed for -# the analysis routines. - -def main(): - print("Opening openPMD output") - prefix = sys.argv[1] - series = io.Series(prefix+"/openpmd_%T.h5", io.Access.read_only) - data_set_end = series.iterations[2] - - # get simulation time - sim_time = data_set_end.time - # no particles can be created on the first timestep so we have 2 timesteps in the test case, - # with only the second one resulting in particle creation - dt = sim_time/2. - - # get particle data - particle_data = {} - - names, types = ac.get_all_species_names_and_types() - for spec_name_type in zip(names, types): - spec_name = spec_name_type[0] - is_photon = spec_name_type[1] - data = {} - - px = data_set_end.particles[spec_name]["momentum"]["x"][:] - py = data_set_end.particles[spec_name]["momentum"]["y"][:] - pz = data_set_end.particles[spec_name]["momentum"]["z"][:] - w = data_set_end.particles[spec_name]["weighting"][io.Mesh_Record_Component.SCALAR][:] - - if is_photon : - opt = data_set_end.particles[spec_name]["opticalDepthBW"][io.Mesh_Record_Component.SCALAR][:] - else: - opt = data_set_end.particles[spec_name]["opticalDepthQSR"][io.Mesh_Record_Component.SCALAR][:] - - series.flush() - - data["px"] = px - data["py"] = py - data["pz"] = pz - data["w"] = w - data["opt"] = opt - - particle_data[spec_name] = data - - ac.check(dt, particle_data) - - #test_name = os.path.split(os.getcwd())[1] - #checksumAPI.evaluate_checksum(test_name, filename_end) - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/qed/breit_wheeler/analysis_yt.py b/Examples/Tests/qed/breit_wheeler/analysis_yt.py deleted file mode 100755 index e8950419f25..00000000000 --- a/Examples/Tests/qed/breit_wheeler/analysis_yt.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Luca Fedeli -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# -*- coding: utf-8 -*- - -import os -import sys - -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import analysis_core as ac -import checksumAPI - -# This script is a frontend for the analysis routines -# in analysis_core.py (please refer to this file for -# a full description). It reads output files in yt -# format and extracts the data needed for -# the analysis routines. -yt -def main(): - print("Opening yt output") - filename_end = sys.argv[1] - data_set_end = yt.load(filename_end) - - # get simulation time - sim_time = data_set_end.current_time.to_value() - # no particles can be created on the first timestep so we have 2 timesteps in the test case, - # with only the second one resulting in particle creation - dt = sim_time/2. - - # get particle data - all_data_end = data_set_end.all_data() - particle_data = {} - - names, types = ac.get_all_species_names_and_types() - for spec_name_type in zip(names, types): - spec_name = spec_name_type[0] - is_photon = spec_name_type[1] - data = {} - data["px"] = all_data_end[spec_name,"particle_momentum_x"].v - data["py"] = all_data_end[spec_name,"particle_momentum_y"].v - data["pz"] = all_data_end[spec_name,"particle_momentum_z"].v - data["w"] = all_data_end[spec_name,"particle_weighting"].v - - if is_photon : - data["opt"] = all_data_end[spec_name, "particle_opticalDepthBW"].v - else: - data["opt"] = all_data_end[spec_name, "particle_opticalDepthQSR"].v - - particle_data[spec_name] = data - - ac.check(dt, particle_data) - - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/qed/breit_wheeler/inputs_2d b/Examples/Tests/qed/breit_wheeler/inputs_2d deleted file mode 100644 index 857b3243ac6..00000000000 --- a/Examples/Tests/qed/breit_wheeler/inputs_2d +++ /dev/null @@ -1,219 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 2 -amr.n_cell = 32 32 -amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain -amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain -geometry.dims = 2 -geometry.prob_lo = -0.5e-6 -0.5e-6 # physical domain -geometry.prob_hi = 0.5e-6 0.5e-6 -amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -algo.current_deposition = esirkepov -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -algo.particle_pusher = boris -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 1 -warpx.cfl = 1. # if 1., the time step is set to its CFL limit -warpx.serialize_initial_conditions = 1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ PLASMA ############# -################################# -particles.species_names = p1 p2 p3 p4 ele1 ele2 ele3 ele4 pos1 pos2 pos3 pos4 dummy_phot -particles.photon_species = p1 p2 p3 p4 dummy_phot -################################# - -p1.species_type = "photon" -p1.injection_style = "NUniformPerCell" -p1.profile = "constant" -p1.num_particles_per_cell_each_dim = 32 32 -p1.density = 1e19 -p1.momentum_distribution_type = "constant" -p1.ux = 2000.0 -##########QED#################### -p1.do_qed_breit_wheeler = 1 -p1.qed_breit_wheeler_ele_product_species = ele1 -p1.qed_breit_wheeler_pos_product_species = pos1 -################################# - -p2.species_type = "photon" -p2.injection_style = "NUniformPerCell" -p2.profile = "constant" -p2.num_particles_per_cell_each_dim = 32 32 -p2.density = 1e19 -p2.momentum_distribution_type = "constant" -p2.uy = 5000.0 -##########QED#################### -p2.do_qed_breit_wheeler = 1 -p2.qed_breit_wheeler_ele_product_species = ele2 -p2.qed_breit_wheeler_pos_product_species = pos2 -################################# - -p3.species_type = "photon" -p3.injection_style = "NUniformPerCell" -p3.profile = "constant" -p3.num_particles_per_cell_each_dim = 32 32 -p3.density = 1e19 -p3.momentum_distribution_type = "constant" -p3.uz = 10000.0 -##########QED#################### -p3.do_qed_breit_wheeler = 1 -p3.qed_breit_wheeler_ele_product_species = ele3 -p3.qed_breit_wheeler_pos_product_species = pos3 -################################# - -p4.species_type = "photon" -p4.injection_style = "NUniformPerCell" -p4.profile = "constant" -p4.num_particles_per_cell_each_dim = 32 32 -p4.density = 1e19 -p4.momentum_distribution_type = "constant" -p4.ux = 57735.02691896 -p4.uy = 57735.02691896 -p4.uz = 57735.02691896 -##########QED#################### -p4.do_qed_breit_wheeler = 1 -p4.qed_breit_wheeler_ele_product_species = ele4 -p4.qed_breit_wheeler_pos_product_species = pos4 -################################# - -### PRODUCT SPECIES ### -ele1.species_type = "electron" -ele1.injection_style = "none" -ele1.do_not_push = 1 -ele1.do_qed_quantum_sync = 1 -ele1.qed_quantum_sync_phot_product_species = dummy_phot - -ele2.species_type = "electron" -ele2.injection_style = "none" -ele2.do_not_push = 1 -ele2.do_qed_quantum_sync = 1 -ele2.qed_quantum_sync_phot_product_species = dummy_phot - -ele3.species_type = "electron" -ele3.injection_style = "none" -ele3.do_not_push = 1 -ele3.do_qed_quantum_sync = 1 -ele3.qed_quantum_sync_phot_product_species = dummy_phot - -ele4.species_type = "electron" -ele4.injection_style = "none" -ele4.do_not_push = 1 -ele4.do_qed_quantum_sync = 1 -ele4.qed_quantum_sync_phot_product_species = dummy_phot - -pos1.species_type = "positron" -pos1.injection_style = "none" -pos1.do_not_push = 1 -pos1.do_qed_quantum_sync = 1 -pos1.qed_quantum_sync_phot_product_species = dummy_phot - -pos2.species_type = "positron" -pos2.injection_style = "none" -pos2.do_not_push = 1 -pos2.do_qed_quantum_sync = 1 -pos2.qed_quantum_sync_phot_product_species = dummy_phot - -pos3.species_type = "positron" -pos3.injection_style = "none" -pos3.do_not_push = 1 -pos3.do_qed_quantum_sync = 1 -pos3.qed_quantum_sync_phot_product_species = dummy_phot - -pos4.species_type = "positron" -pos4.injection_style = "none" -pos4.do_not_push = 1 -pos4.do_qed_quantum_sync = 1 -pos4.qed_quantum_sync_phot_product_species = dummy_phot - -dummy_phot.species_type = "photon" -dummy_phot.injection_style = "none" - -################################# - -##########QED TABLES#################### -qed_bw.chi_min = 0.001 - -qed_bw.lookup_table_mode = "builtin" - -#qed_bw.lookup_table_mode = "generate" -#qed_bw.tab_dndt_chi_min = 0.01 -#qed_bw.tab_dndt_chi_max = 1000.0 -#qed_bw.tab_dndt_how_many = 256 -#qed_bw.tab_pair_chi_min = 0.01 -#qed_bw.tab_pair_chi_max = 1000.0 -#qed_bw.tab_pair_chi_how_many = 256 -#qed_bw.tab_pair_frac_how_many = 256 -#qed_bw.save_table_in = "bw_table" - -#qed_bw.lookup_table_mode = "load" -#qed_bw.load_table_from = "bw_table" - -qed_qs.chi_min = 0.001 - -qed_qs.lookup_table_mode = "builtin" - -qed_qs.photon_creation_energy_threshold = 2 - -#qed_qs.lookup_table_mode = "generate" -#qed_qs.tab_dndt_chi_min = 0.001 -#qed_qs.tab_dndt_chi_max = 1000.0 -#qed_qs.tab_dndt_how_many = 256 -#qed_qs.tab_em_chi_min = 0.001 -#qed_qs.tab_em_frac_min = 1.0e-12 -#qed_qs.tab_em_chi_max = 1000.0 -#qed_qs.tab_em_chi_how_many = 256 -#qed_qs.tab_em_frac_how_many = 256 -#qed_qs.save_table_in = "qs_table" - -#qed_qs.lookup_table_mode = "load" -#qed_qs.load_table_from = "qs_table" -################################# - -### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) -particles.E_ext_particle_init_style = "constant" -particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 -#### - -### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) -particles.B_ext_particle_init_style = "constant" -particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 -#### - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 2 -diag1.diag_type = Full -diag1.fields_to_plot = Ex -diag1.p1.variables = x z ux uy uz w opticalDepthBW -diag1.p2.variables = x z ux uy uz w opticalDepthBW -diag1.p3.variables = x z ux uy uz w opticalDepthBW -diag1.p4.variables = x z ux uy uz w opticalDepthBW - -diag1.ele1.variables = x z ux uy uz w opticalDepthQSR -diag1.ele2.variables = x z ux uy uz w opticalDepthQSR -diag1.ele3.variables = x z ux uy uz w opticalDepthQSR -diag1.ele4.variables = x z ux uy uz w opticalDepthQSR - -diag1.pos1.variables = x z ux uy uz w opticalDepthQSR -diag1.pos2.variables = x z ux uy uz w opticalDepthQSR -diag1.pos3.variables = x z ux uy uz w opticalDepthQSR -diag1.pos4.variables = x z ux uy uz w opticalDepthQSR - -diag1.format = plotfile diff --git a/Examples/Tests/qed/inputs_base_2d_breit_wheeler b/Examples/Tests/qed/inputs_base_2d_breit_wheeler new file mode 100644 index 00000000000..201520966c2 --- /dev/null +++ b/Examples/Tests/qed/inputs_base_2d_breit_wheeler @@ -0,0 +1,220 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 2 +amr.n_cell = 32 32 +amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 2 +geometry.prob_lo = -0.5e-6 -0.5e-6 # physical domain +geometry.prob_hi = 0.5e-6 0.5e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.particle_pusher = boris +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 1 +warpx.cfl = 1. # if 1., the time step is set to its CFL limit +warpx.serialize_initial_conditions = 1 +warpx.abort_on_warning_threshold = high + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ PLASMA ############# +################################# +particles.species_names = p1 p2 p3 p4 ele1 ele2 ele3 ele4 pos1 pos2 pos3 pos4 dummy_phot +particles.photon_species = p1 p2 p3 p4 dummy_phot +################################# + +p1.species_type = "photon" +p1.injection_style = "NUniformPerCell" +p1.profile = "constant" +p1.num_particles_per_cell_each_dim = 32 32 +p1.density = 1e19 +p1.momentum_distribution_type = "constant" +p1.ux = 2000.0 +##########QED#################### +p1.do_qed_breit_wheeler = 1 +p1.qed_breit_wheeler_ele_product_species = ele1 +p1.qed_breit_wheeler_pos_product_species = pos1 +################################# + +p2.species_type = "photon" +p2.injection_style = "NUniformPerCell" +p2.profile = "constant" +p2.num_particles_per_cell_each_dim = 32 32 +p2.density = 1e19 +p2.momentum_distribution_type = "constant" +p2.uy = 5000.0 +##########QED#################### +p2.do_qed_breit_wheeler = 1 +p2.qed_breit_wheeler_ele_product_species = ele2 +p2.qed_breit_wheeler_pos_product_species = pos2 +################################# + +p3.species_type = "photon" +p3.injection_style = "NUniformPerCell" +p3.profile = "constant" +p3.num_particles_per_cell_each_dim = 32 32 +p3.density = 1e19 +p3.momentum_distribution_type = "constant" +p3.uz = 10000.0 +##########QED#################### +p3.do_qed_breit_wheeler = 1 +p3.qed_breit_wheeler_ele_product_species = ele3 +p3.qed_breit_wheeler_pos_product_species = pos3 +################################# + +p4.species_type = "photon" +p4.injection_style = "NUniformPerCell" +p4.profile = "constant" +p4.num_particles_per_cell_each_dim = 32 32 +p4.density = 1e19 +p4.momentum_distribution_type = "constant" +p4.ux = 57735.02691896 +p4.uy = 57735.02691896 +p4.uz = 57735.02691896 +##########QED#################### +p4.do_qed_breit_wheeler = 1 +p4.qed_breit_wheeler_ele_product_species = ele4 +p4.qed_breit_wheeler_pos_product_species = pos4 +################################# + +### PRODUCT SPECIES ### +ele1.species_type = "electron" +ele1.injection_style = "none" +ele1.do_not_push = 1 +ele1.do_qed_quantum_sync = 1 +ele1.qed_quantum_sync_phot_product_species = dummy_phot + +ele2.species_type = "electron" +ele2.injection_style = "none" +ele2.do_not_push = 1 +ele2.do_qed_quantum_sync = 1 +ele2.qed_quantum_sync_phot_product_species = dummy_phot + +ele3.species_type = "electron" +ele3.injection_style = "none" +ele3.do_not_push = 1 +ele3.do_qed_quantum_sync = 1 +ele3.qed_quantum_sync_phot_product_species = dummy_phot + +ele4.species_type = "electron" +ele4.injection_style = "none" +ele4.do_not_push = 1 +ele4.do_qed_quantum_sync = 1 +ele4.qed_quantum_sync_phot_product_species = dummy_phot + +pos1.species_type = "positron" +pos1.injection_style = "none" +pos1.do_not_push = 1 +pos1.do_qed_quantum_sync = 1 +pos1.qed_quantum_sync_phot_product_species = dummy_phot + +pos2.species_type = "positron" +pos2.injection_style = "none" +pos2.do_not_push = 1 +pos2.do_qed_quantum_sync = 1 +pos2.qed_quantum_sync_phot_product_species = dummy_phot + +pos3.species_type = "positron" +pos3.injection_style = "none" +pos3.do_not_push = 1 +pos3.do_qed_quantum_sync = 1 +pos3.qed_quantum_sync_phot_product_species = dummy_phot + +pos4.species_type = "positron" +pos4.injection_style = "none" +pos4.do_not_push = 1 +pos4.do_qed_quantum_sync = 1 +pos4.qed_quantum_sync_phot_product_species = dummy_phot + +dummy_phot.species_type = "photon" +dummy_phot.injection_style = "none" + +################################# + +##########QED TABLES#################### +qed_bw.chi_min = 0.001 + +qed_bw.lookup_table_mode = "builtin" + +#qed_bw.lookup_table_mode = "generate" +#qed_bw.tab_dndt_chi_min = 0.01 +#qed_bw.tab_dndt_chi_max = 1000.0 +#qed_bw.tab_dndt_how_many = 256 +#qed_bw.tab_pair_chi_min = 0.01 +#qed_bw.tab_pair_chi_max = 1000.0 +#qed_bw.tab_pair_chi_how_many = 256 +#qed_bw.tab_pair_frac_how_many = 256 +#qed_bw.save_table_in = "bw_table" + +#qed_bw.lookup_table_mode = "load" +#qed_bw.load_table_from = "bw_table" + +qed_qs.chi_min = 0.001 + +qed_qs.lookup_table_mode = "builtin" + +qed_qs.photon_creation_energy_threshold = 2 + +#qed_qs.lookup_table_mode = "generate" +#qed_qs.tab_dndt_chi_min = 0.001 +#qed_qs.tab_dndt_chi_max = 1000.0 +#qed_qs.tab_dndt_how_many = 256 +#qed_qs.tab_em_chi_min = 0.001 +#qed_qs.tab_em_frac_min = 1.0e-12 +#qed_qs.tab_em_chi_max = 1000.0 +#qed_qs.tab_em_chi_how_many = 256 +#qed_qs.tab_em_frac_how_many = 256 +#qed_qs.save_table_in = "qs_table" + +#qed_qs.lookup_table_mode = "load" +#qed_qs.load_table_from = "qs_table" +################################# + +### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) +particles.E_ext_particle_init_style = "constant" +particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 +#### + +### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) +particles.B_ext_particle_init_style = "constant" +particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 +#### + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 2 +diag1.diag_type = Full +diag1.fields_to_plot = Ex +diag1.p1.variables = x z ux uy uz w opticalDepthBW +diag1.p2.variables = x z ux uy uz w opticalDepthBW +diag1.p3.variables = x z ux uy uz w opticalDepthBW +diag1.p4.variables = x z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x z ux uy uz w opticalDepthQSR + +diag1.format = plotfile diff --git a/Examples/Tests/qed/breit_wheeler/inputs_3d b/Examples/Tests/qed/inputs_base_3d_breit_wheeler similarity index 100% rename from Examples/Tests/qed/breit_wheeler/inputs_3d rename to Examples/Tests/qed/inputs_base_3d_breit_wheeler diff --git a/Examples/Tests/qed/schwinger/inputs_3d_schwinger b/Examples/Tests/qed/inputs_base_3d_schwinger similarity index 100% rename from Examples/Tests/qed/schwinger/inputs_3d_schwinger rename to Examples/Tests/qed/inputs_base_3d_schwinger diff --git a/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler b/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler new file mode 100644 index 00000000000..53d3cf9e97c --- /dev/null +++ b/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_2d_breit_wheeler diff --git a/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler_opmd b/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler_opmd new file mode 100644 index 00000000000..7edecbcd0a3 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_2d_qed_breit_wheeler_opmd @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d_breit_wheeler + +# test input parameters +diag1.format = openpmd +diag1.openpmd_backend = h5 diff --git a/Examples/Tests/qed/inputs_test_2d_qed_quantum_sync b/Examples/Tests/qed/inputs_test_2d_qed_quantum_sync new file mode 100644 index 00000000000..83d0cee16aa --- /dev/null +++ b/Examples/Tests/qed/inputs_test_2d_qed_quantum_sync @@ -0,0 +1,190 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 2 +amr.n_cell = 32 32 +amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 2 +geometry.prob_lo = -0.25e-6 -0.25e-6 # physical domain +geometry.prob_hi = 0.25e-6 0.25e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.particle_pusher = boris +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 1 +warpx.cfl = 1. # if 1., the time step is set to its CFL limit +warpx.serialize_initial_conditions = 1 +warpx.abort_on_warning_threshold = high + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ PLASMA ############# +################################# +particles.species_names = p1 p2 p3 p4 qsp_1 qsp_2 qsp_3 qsp_4 dummy_ele dummy_pos +particles.photon_species = qsp_1 qsp_2 qsp_3 qsp_4 +################################# + +p1.species_type = "electron" +p1.injection_style = "NUniformPerCell" +p1.profile = "constant" +p1.num_particles_per_cell_each_dim = 32 32 +p1.density = 1e1 +p1.momentum_distribution_type = "constant" +p1.ux = 10.0 +##########QED#################### +p1.do_qed_quantum_sync = 1 +p1.qed_quantum_sync_phot_product_species = qsp_1 +################################# + +p2.species_type = "electron" +p2.injection_style = "NUniformPerCell" +p2.profile = "constant" +p2.num_particles_per_cell_each_dim = 32 32 +p2.density = 1e1 +p2.momentum_distribution_type = "constant" +p2.uy = 100.0 +##########QED#################### +p2.do_qed_quantum_sync = 1 +p2.qed_quantum_sync_phot_product_species = qsp_2 +################################# + +p3.species_type = "positron" +p3.injection_style = "NUniformPerCell" +p3.profile = "constant" +p3.num_particles_per_cell_each_dim = 32 32 +p3.density = 1e1 +p3.momentum_distribution_type = "constant" +p3.uz = 1000.0 +##########QED#################### +p3.do_qed_quantum_sync = 1 +p3.qed_quantum_sync_phot_product_species = qsp_3 +################################# + +p4.species_type = "positron" +p4.injection_style = "NUniformPerCell" +p4.profile = "constant" +p4.num_particles_per_cell_each_dim = 32 32 +p4.density = 1e1 +p4.momentum_distribution_type = "constant" +p4.ux = 5773.502691896 +p4.uy = 5773.502691896 +p4.uz = 5773.502691896 +##########QED#################### +p4.do_qed_quantum_sync = 1 +p4.qed_quantum_sync_phot_product_species = qsp_4 +################################# + +### PRODUCT SPECIES ### +qsp_1.species_type = "photon" +qsp_1.injection_style = "none" +qsp_1.do_qed_breit_wheeler = 1 +qsp_1.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_1.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_2.species_type = "photon" +qsp_2.injection_style = "none" +qsp_2.do_qed_breit_wheeler = 1 +qsp_2.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_2.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_3.species_type = "photon" +qsp_3.injection_style = "none" +qsp_3.do_qed_breit_wheeler = 1 +qsp_3.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_3.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_4.species_type = "photon" +qsp_4.injection_style = "none" +qsp_4.do_qed_breit_wheeler = 1 +qsp_4.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_4.qed_breit_wheeler_pos_product_species = dummy_pos + +################################# + +dummy_ele.species_type = "electron" +dummy_ele.injection_style = "none" + +dummy_pos.species_type = "positron" +dummy_pos.injection_style = "none" + +################################# + +##########QED TABLES#################### +qed_bw.chi_min = 0.001 + +qed_bw.lookup_table_mode = "builtin" + +#qed_bw.lookup_table_mode = "generate" +#qed_bw.tab_dndt_chi_min = 0.01 +#qed_bw.tab_dndt_chi_max = 1000.0 +#qed_bw.tab_dndt_how_many = 256 +#qed_bw.tab_pair_chi_min = 0.01 +#qed_bw.tab_pair_chi_max = 1000.0 +#qed_bw.tab_pair_chi_how_many = 256 +#qed_bw.tab_pair_frac_how_many = 256 +#qed_bw.save_table_in = "bw_table" + +#qed_bw.lookup_table_mode = "load" +#qed_bw.load_table_from = "bw_table" + +qed_qs.chi_min = 0.001 + +qed_qs.lookup_table_mode = "builtin" + +qed_qs.photon_creation_energy_threshold = 0.0 + +#qed_qs.lookup_table_mode = "generate" +#qed_qs.tab_dndt_chi_min = 0.001 +#qed_qs.tab_dndt_chi_max = 1000.0 +#qed_qs.tab_dndt_how_many = 256 +#qed_qs.tab_em_chi_min = 0.001 +#qed_qs.tab_em_frac_min = 1.0e-12 +#qed_qs.tab_em_chi_max = 1000.0 +#qed_qs.tab_em_chi_how_many = 256 +#qed_qs.tab_em_frac_how_many = 256 +#qed_qs.save_table_in = "qs_table" + +#qed_qs.lookup_table_mode = "load" +#qed_qs.load_table_from = "qs_table" +################################# + +### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) +particles.E_ext_particle_init_style = "constant" +particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 +#### + +### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) +particles.B_ext_particle_init_style = "constant" +particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 +#### + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 2 +diag1.diag_type = Full +diag1.fields_to_plot = Ex +diag1.p1.variables = x z ux uy uz w opticalDepthQSR +diag1.p2.variables = x z ux uy uz w opticalDepthQSR +diag1.p3.variables = x z ux uy uz w opticalDepthQSR +diag1.p4.variables = x z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler b/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler new file mode 100644 index 00000000000..2058dccb493 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_3d_breit_wheeler diff --git a/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler_opmd b/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler_opmd new file mode 100644 index 00000000000..78847d0a0d4 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_breit_wheeler_opmd @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d_breit_wheeler + +# test input parameters +diag1.format = openpmd +diag1.openpmd_backend = h5 diff --git a/Examples/Tests/qed/inputs_test_3d_qed_quantum_sync b/Examples/Tests/qed/inputs_test_3d_qed_quantum_sync new file mode 100644 index 00000000000..87ffd746ec8 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_quantum_sync @@ -0,0 +1,190 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 2 +amr.n_cell = 16 16 16 +amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 3 +geometry.prob_lo = -0.25e-6 -0.25e-6 -0.25e-6 # physical domain +geometry.prob_hi = 0.25e-6 0.25e-6 0.25e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic + +################################# +############ NUMERICS ########### +################################# +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.particle_pusher = boris +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 1 +warpx.cfl = 1. # if 1., the time step is set to its CFL limit +warpx.serialize_initial_conditions = 1 +warpx.abort_on_warning_threshold = high + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ PLASMA ############# +################################# +particles.species_names = p1 p2 p3 p4 qsp_1 qsp_2 qsp_3 qsp_4 dummy_ele dummy_pos +particles.photon_species = qsp_1 qsp_2 qsp_3 qsp_4 +################################# + +p1.species_type = "electron" +p1.injection_style = "NUniformPerCell" +p1.profile = "constant" +p1.num_particles_per_cell_each_dim = 8 8 4 +p1.density = 1e1 +p1.momentum_distribution_type = "constant" +p1.ux = 10.0 +##########QED#################### +p1.do_qed_quantum_sync = 1 +p1.qed_quantum_sync_phot_product_species = qsp_1 +################################# + +p2.species_type = "electron" +p2.injection_style = "NUniformPerCell" +p2.profile = "constant" +p2.num_particles_per_cell_each_dim = 8 8 4 +p2.density = 1e1 +p2.momentum_distribution_type = "constant" +p2.uy = 100.0 +##########QED#################### +p2.do_qed_quantum_sync = 1 +p2.qed_quantum_sync_phot_product_species = qsp_2 +################################# + +p3.species_type = "positron" +p3.injection_style = "NUniformPerCell" +p3.profile = "constant" +p3.num_particles_per_cell_each_dim = 8 8 4 +p3.density = 1e1 +p3.momentum_distribution_type = "constant" +p3.uz = 1000.0 +##########QED#################### +p3.do_qed_quantum_sync = 1 +p3.qed_quantum_sync_phot_product_species = qsp_3 +################################# + +p4.species_type = "positron" +p4.injection_style = "NUniformPerCell" +p4.profile = "constant" +p4.num_particles_per_cell_each_dim = 8 8 4 +p4.density = 1e1 +p4.momentum_distribution_type = "constant" +p4.ux = 5773.502691896 +p4.uy = 5773.502691896 +p4.uz = 5773.502691896 +##########QED#################### +p4.do_qed_quantum_sync = 1 +p4.qed_quantum_sync_phot_product_species = qsp_4 +################################# + +### PRODUCT SPECIES ### +qsp_1.species_type = "photon" +qsp_1.injection_style = "none" +qsp_1.do_qed_breit_wheeler = 1 +qsp_1.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_1.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_2.species_type = "photon" +qsp_2.injection_style = "none" +qsp_2.do_qed_breit_wheeler = 1 +qsp_2.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_2.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_3.species_type = "photon" +qsp_3.injection_style = "none" +qsp_3.do_qed_breit_wheeler = 1 +qsp_3.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_3.qed_breit_wheeler_pos_product_species = dummy_pos + +qsp_4.species_type = "photon" +qsp_4.injection_style = "none" +qsp_4.do_qed_breit_wheeler = 1 +qsp_4.qed_breit_wheeler_ele_product_species = dummy_ele +qsp_4.qed_breit_wheeler_pos_product_species = dummy_pos + +################################# + +dummy_ele.species_type = "electron" +dummy_ele.injection_style = "none" + +dummy_pos.species_type = "positron" +dummy_pos.injection_style = "none" + +################################# + +##########QED TABLES#################### +qed_bw.chi_min = 0.001 + +qed_bw.lookup_table_mode = "builtin" + +#qed_bw.lookup_table_mode = "generate" +#qed_bw.tab_dndt_chi_min = 0.01 +#qed_bw.tab_dndt_chi_max = 1000.0 +#qed_bw.tab_dndt_how_many = 256 +#qed_bw.tab_pair_chi_min = 0.01 +#qed_bw.tab_pair_chi_max = 1000.0 +#qed_bw.tab_pair_chi_how_many = 256 +#qed_bw.tab_pair_frac_how_many = 256 +#qed_bw.save_table_in = "bw_table" + +#qed_bw.lookup_table_mode = "load" +#qed_bw.load_table_from = "bw_table" + +qed_qs.chi_min = 0.001 + +qed_qs.lookup_table_mode = "builtin" + +qed_qs.photon_creation_energy_threshold = 0.0 + +#qed_qs.lookup_table_mode = "generate" +#qed_qs.tab_dndt_chi_min = 0.001 +#qed_qs.tab_dndt_chi_max = 1000.0 +#qed_qs.tab_dndt_how_many = 256 +#qed_qs.tab_em_chi_min = 0.001 +#qed_qs.tab_em_frac_min = 1.0e-12 +#qed_qs.tab_em_chi_max = 1000.0 +#qed_qs.tab_em_chi_how_many = 256 +#qed_qs.tab_em_frac_how_many = 256 +#qed_qs.save_table_in = "qs_table" + +#qed_qs.lookup_table_mode = "load" +#qed_qs.load_table_from = "qs_table" +################################# + +### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) +particles.E_ext_particle_init_style = "constant" +particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 +#### + +### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) +particles.B_ext_particle_init_style = "constant" +particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 +#### + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 2 +diag1.diag_type = Full +diag1.fields_to_plot = Ex +diag1.p1.variables = x y z ux uy uz w opticalDepthQSR +diag1.p2.variables = x y z ux uy uz w opticalDepthQSR +diag1.p3.variables = x y z ux uy uz w opticalDepthQSR +diag1.p4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/inputs_test_3d_qed_schwinger_1 b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_1 new file mode 100644 index 00000000000..cfa0ca80845 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_1 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d_schwinger + +# test input parameters +warpx.B_external_grid = 16792888.570516706 5256650.141557486 18363530.799561853 +warpx.E_external_grid = 1.e16 0 0 diff --git a/Examples/Tests/qed/inputs_test_3d_qed_schwinger_2 b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_2 new file mode 100644 index 00000000000..420e6bce31f --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_2 @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_3d_schwinger + +# test input parameters +warpx.B_external_grid = 1679288857.0516706 525665014.1557486 1836353079.9561853 +warpx.E_external_grid = 1.e18 0 0 +qed_schwinger.xmin = -2.5e-7 +qed_schwinger.xmax = 2.49e-7 diff --git a/Examples/Tests/qed/inputs_test_3d_qed_schwinger_3 b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_3 new file mode 100644 index 00000000000..e77ce567f32 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_3 @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d_schwinger + +# test input parameters +warpx.E_external_grid = 0 1.090934525450495e+17 0 diff --git a/Examples/Tests/qed/inputs_test_3d_qed_schwinger_4 b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_4 new file mode 100644 index 00000000000..78454e8bb75 --- /dev/null +++ b/Examples/Tests/qed/inputs_test_3d_qed_schwinger_4 @@ -0,0 +1,8 @@ +# base input parameters +FILE = inputs_base_3d_schwinger + +# test input parameters +warpx.B_external_grid = 0 833910140000. 0 +warpx.E_external_grid = 0 0 2.5e+20 +qed_schwinger.ymin = -2.5e-7 +qed_schwinger.zmax = 2.49e-7 diff --git a/Examples/Tests/qed/quantum_synchrotron/analysis.py b/Examples/Tests/qed/quantum_synchrotron/analysis.py deleted file mode 100755 index ee6139f75c8..00000000000 --- a/Examples/Tests/qed/quantum_synchrotron/analysis.py +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Luca Fedeli, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# -*- coding: utf-8 -*- - -import os -import sys - -import numpy as np -import scipy.integrate as integ -import scipy.special as spe -import scipy.stats as st -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI -import matplotlib.pyplot as plt - -# This script performs detailed checks of the Quantum Synchrotron photon emission process. -# Two electron populations and two positron populations are initialized with different momenta in different -# directions in a background EM field (with non-zero components along each direction). -# Specifically the script checks that: -# -# - The expected number of generated photons n_phot is in agreement with theory. -# The maximum tolerated error is 5*sqrt(n_phot), except for the last 8 points, -# for which a higher tolerance is used (this is due to the fact that the resolution -# of the builtin table is quite limited). -# - The weight of the generated particles is equal to the weight of the photon -# - The generated particles are emitted in the right direction -# - The energy distribution of the generated particles is in agreement with theory -# - The optical depths of the product species are correctly initialized (QED effects are -# enabled for product species too). -# -# More details on the theoretical formulas used in this script can be found in -# the Jupyter notebook picsar/src/multi_physics/QED_tests/validation/validation.ipynb -# -# References: -# 1) R. Duclous et al 2011 Plasma Phys. Control. Fusion 53 015009 -# 2) A. Gonoskov et al. 2015 Phys. Rev. E 92, 023305 -# 3) M. Lobet. PhD thesis "Effets radiatifs et d'electrodynamique -# quantique dans l'interaction laser-matiere ultra-relativiste" -# URL: https://tel.archives-ouvertes.fr/tel-01314224 - - -# Tolerances -tol = 1.e-8 -tol_red = 1.e-2 - -# Physical constants (from CODATA 2018, see: https://physics.nist.gov/cuu/Constants/index.html ) -me = 9.1093837015e-31 #electron mass -c = 299792458 #speed of light -hbar = 6.62607015e-34/(2*np.pi) #reduced Plank constant -fine_structure = 7.2973525693e-3 #fine structure constant -qe = 1.602176634e-19#elementary charge -E_s = (me**2 * c**3)/(qe * hbar) #Schwinger E field -B_s = E_s/c #Schwinger B field - -mec = me*c -mec2 = mec*c -#______________ - -# Initial parameters -spec_names = ["p1", "p2", "p3", "p4"] -spec_names_phot = ["qsp_1", "qsp_2", "qsp_3", "qsp_4"] -initial_momenta = [ - np.array([10.0,0,0])*mec, - np.array([0.0,100.0,0.0])*mec, - np.array([0.0,0.0,1000.0])*mec, - np.array([5773.502691896, 5773.502691896, 5773.502691896])*mec -] -csign = [-1,-1,1,1] -initial_particle_number = 1048576 - -E_f = np.array([-2433321316961438., 973328526784575., 1459992790176863.]) -B_f = np.array([2857142.85714286, 4285714.28571428, 8571428.57142857]) - -NNS = [64,64,64,64] #bins for energy distribution comparison. -#______________ - -def calc_chi_part(p, E, B): - gamma_part = np.sqrt(1.0 + np.dot(p,p)/mec**2) - v = p/(gamma_part*me) - EpvvecB = E + np.cross(v,B) - vdotEoverc = np.dot(v,E)/c - ff = np.sqrt(np.dot(EpvvecB,EpvvecB) - np.dot(vdotEoverc,vdotEoverc)) - return gamma_part*ff/E_s - -#Auxiliary functions -@np.vectorize -def IC_inner_alternative(y): - ff = lambda x : np.exp(-y*(1+(4*x**2)/3)*np.sqrt(1+x*x/3))*(9+36*x**2 + 16*x**4)/(3 + 4*x**2)/np.sqrt(1+(x**2)/3) - # This integration may not converge in some cases, in which case a python warning message can - # be issued. This is probably not a significant issue for this test case and these warnings can - # be ignored. - return integ.quad(ff, 0, np.inf)[0]/np.sqrt(3) - -def IC_Y(chi_ele, xi): - div = (chi_ele*(1-xi)) - div = np.where(np.logical_and(xi < 1, chi_ele != 0), div, 1.0) - res = (2/3)*np.where(np.logical_and(xi < 1, chi_ele != 0), xi/div, np.inf) - return res - -def IC_S(chi_ele, xi): - Y = IC_Y(chi_ele, xi) - coeff = np.sqrt(3)/2.0/np.pi - first = IC_inner_alternative(Y) - div = np.where(xi == 1, 1.0, 1.0/(1-xi) ) - res = np.where(np.logical_or(xi == 1, xi == 0), 0.0, - coeff*xi*( first + (xi**2 * spe.kv(2./3.,Y)*div ) ) ) - return res - -def IC_SXI(chi_ele, xi): - div = np.where(xi != 0, xi, 1.0) - return np.where(xi != 0, IC_S(chi_ele, xi)/div, np.inf) - -@np.vectorize -def IC_G(chi_ele): - return integ.quad(lambda xi: IC_SXI(chi_ele, xi), 0, 1)[0] - -def small_diff(vv, val): - if(val != 0.0): - return np.max(np.abs((vv - val)/val)) < tol - else: - return np.max(np.abs(vv)) < tol - -def boris(pp, dt, charge_sign): - econst = 0.5*qe*dt*charge_sign/me - u = pp/(me) - u += econst*E_f - inv_gamma = 1/np.sqrt(1 + np.dot(u,u)/c**2) - t = econst*B_f*inv_gamma - s = 2*t/(1 + np.dot(t,t)) - u_p = u + np.cross(u,t) - u += np.cross(u_p, s) - u += econst*E_f - return u *me -#__________________ - -# Quantum Synchrotron total and differential cross sections -def QS_dN_dt(chi_ele, gamma_ele): - coeff_IC = (2./3.) * fine_structure * me*c**2/hbar - return coeff_IC*IC_G(chi_ele)/gamma_ele - -def QS_d2N_dt_dchi(chi, gamma_ele, chi_phot): - coeff_IC = (2./3.) * fine_structure * me*c**2/hbar - return coeff_IC*IC_S(chi, chi_phot/chi)/chi_phot/gamma_ele -#__________________ - -# Get data for a species -def get_spec(ytdata, specname, is_photon): - px = ytdata[specname,"particle_momentum_x"].v - pz = ytdata[specname,"particle_momentum_z"].v - py = ytdata[specname,"particle_momentum_y"].v - - w = ytdata[specname,"particle_weighting"].v - - if (is_photon): - opt = ytdata[specname,"particle_opticalDepthBW"].v - else: - opt = ytdata[specname,"particle_opticalDepthQSR"].v - - return {"px" : px, "py" : py, "pz" : pz, "w" : w, "opt" : opt} - -# Individual tests -def check_number_of_photons(ytdataset, part_name, phot_name, chi_part, gamma_part, dt, particle_number): - dNQS_dt_theo = QS_dN_dt(chi_part, gamma_part) - expected_photons = (1.-np.exp(-dNQS_dt_theo*dt))*particle_number - expected_photons_tolerance = 5.0*np.sqrt(expected_photons) - n_phot = ytdataset.particle_type_counts[phot_name] - assert( np.abs(n_phot-expected_photons) < expected_photons_tolerance) - print(" [OK] generated photons number is within expectations") - return n_phot - -def check_weights(part_data, phot_data): - assert(np.all(part_data["w"] == part_data["w"][0])) - assert(np.all(phot_data["w"] == part_data["w"][0])) - print(" [OK] particles weights are the expected ones") - -def check_momenta(phot_data, p_phot, p0): - pdir = p0/np.linalg.norm(p0) - assert(small_diff(phot_data["px"]/p_phot, pdir[0])) - assert(small_diff(phot_data["py"]/p_phot, pdir[1])) - assert(small_diff(phot_data["pz"]/p_phot, pdir[2])) - print(" [OK] photons move along the initial particle direction") - -def check_opt_depths(part_data, phot_data): - data = (part_data, phot_data) - for dd in data: - # Remove the negative optical depths that will be - # reset at the beginning of the next timestep - loc, scale = st.expon.fit(dd["opt"][dd["opt"] > 0]) - assert( np.abs(loc - 0) < tol_red ) - assert( np.abs(scale - 1) < tol_red ) - print(" [OK] optical depth distributions are still exponential") - -def check_energy_distrib(gamma_phot, chi_part, - gamma_part, n_phot, NN, idx, do_plot=False): - gamma_phot_min = 1e-12*gamma_part - gamma_phot_max = gamma_part - h_log_gamma_phot, c_gamma_phot = np.histogram(gamma_phot, bins=np.logspace(np.log10(gamma_phot_min),np.log10(gamma_phot_max),NN+1)) - - cchi_phot_min = chi_part*(gamma_phot_min)/(gamma_part-1) - cchi_phot_max = chi_part*(gamma_phot_max)/(gamma_part-1) - - #Rudimentary integration over npoints for each bin - npoints= 20 - aux_chi = np.logspace(np.log10(cchi_phot_min),np.log10(cchi_phot_max), NN*npoints) - distrib = QS_d2N_dt_dchi(chi_part, gamma_part, aux_chi)*aux_chi - distrib = np.sum(distrib.reshape(-1, npoints),1) - distrib = n_phot*distrib/np.sum(distrib) - - if do_plot : - # Visual comparison of distributions - c_gamma_phot = np.exp(0.5*(np.log(c_gamma_phot[1:])+np.log(c_gamma_phot[:-1]))) - plt.clf() - - fig, (ax1, ax2) = plt.subplots(1, 2) - fig.suptitle("χ_particle = {:f}".format(chi_part)) - ax1.plot(c_gamma_phot, distrib,label="theory") - ax1.loglog(c_gamma_phot, h_log_gamma_phot,label="QSR photons") - ax1.set_xlim(1e-12*(gamma_part-1),gamma_part-1) - ax1.set_ylim(1,1e5) - ax2.plot(c_gamma_phot, distrib,label="theory") - ax2.semilogy(c_gamma_phot, h_log_gamma_phot,label="QSR photons") - ax2.set_ylim(1,1e5) - ax2.set_xlim(1e-12*(gamma_part-1),gamma_part-1) - ax1.set_xlabel("γ_photon") - ax1.set_xlabel("N") - ax2.set_xlabel("γ_photon") - ax2.set_xlabel("N") - plt.legend() - plt.savefig("case_{:d}".format(idx+1)) - - discr = np.abs(h_log_gamma_phot-distrib) - - max_discr = np.sqrt(distrib)*5.0 - # Use a higer tolerance for the last 8 points (this is due to limitations - # of the builtin table) - max_discr[-8:] *= 2.0 - assert(np.all( discr < max_discr )) - - print(" [OK] energy distribution is within expectations") - -#__________________ - -def check(): - filename_end = sys.argv[1] - data_set_end = yt.load(filename_end) - - sim_time = data_set_end.current_time.to_value() - # no particles can be created on the first timestep so we have 2 timesteps in the test case, - # with only the second one resulting in particle creation - dt = sim_time/2. - - all_data_end = data_set_end.all_data() - - for idx in range(4): - part_name = spec_names[idx] - phot_name = spec_names_phot[idx] - t_pi = initial_momenta[idx] - pm = boris(t_pi,-dt*0.5,csign[idx]) - p0 = boris(pm,dt*1.0,csign[idx]) - - p2_part = p0[0]**2 + p0[1]**2 + p0[2]**2 - energy_part = np.sqrt(mec2**2 + p2_part*c**2) - gamma_part = energy_part/mec2 - chi_part = calc_chi_part(p0, E_f, B_f) - - print("** Case {:d} **".format(idx+1)) - print(" initial momentum: ", t_pi) - print(" quantum parameter: {:f}".format(chi_part)) - print(" normalized particle energy: {:f}".format(gamma_part)) - print(" timestep: {:f} fs".format(dt*1e15)) - - part_data_final = get_spec(all_data_end, part_name, is_photon=False) - phot_data = get_spec(all_data_end, phot_name, is_photon=True) - - p_phot = np.sqrt(phot_data["px"]**2 + phot_data["py"]**2 + phot_data["pz"]**2) - energy_phot = p_phot*c - gamma_phot = energy_phot/mec2 - - n_phot = check_number_of_photons(data_set_end, - part_name, phot_name, - chi_part, gamma_part, dt, - initial_particle_number) - - check_weights(part_data_final, phot_data) - - check_momenta(phot_data, p_phot, p0) - - check_energy_distrib(gamma_phot, chi_part, gamma_part, n_phot, NNS[idx], idx) - - check_opt_depths(part_data_final, phot_data) - - print("*************\n") - - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename_end) - -def main(): - check() - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_2d b/Examples/Tests/qed/quantum_synchrotron/inputs_2d deleted file mode 100644 index 2ac2c782ccd..00000000000 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_2d +++ /dev/null @@ -1,189 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 2 -amr.n_cell = 32 32 -amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain -amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain -geometry.dims = 2 -geometry.prob_lo = -0.25e-6 -0.25e-6 # physical domain -geometry.prob_hi = 0.25e-6 0.25e-6 -amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic -boundary.field_hi = periodic periodic - -################################# -############ NUMERICS ########### -################################# -algo.current_deposition = esirkepov -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -algo.particle_pusher = boris -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 1 -warpx.cfl = 1. # if 1., the time step is set to its CFL limit -warpx.serialize_initial_conditions = 1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ PLASMA ############# -################################# -particles.species_names = p1 p2 p3 p4 qsp_1 qsp_2 qsp_3 qsp_4 dummy_ele dummy_pos -particles.photon_species = qsp_1 qsp_2 qsp_3 qsp_4 -################################# - -p1.species_type = "electron" -p1.injection_style = "NUniformPerCell" -p1.profile = "constant" -p1.num_particles_per_cell_each_dim = 32 32 -p1.density = 1e1 -p1.momentum_distribution_type = "constant" -p1.ux = 10.0 -##########QED#################### -p1.do_qed_quantum_sync = 1 -p1.qed_quantum_sync_phot_product_species = qsp_1 -################################# - -p2.species_type = "electron" -p2.injection_style = "NUniformPerCell" -p2.profile = "constant" -p2.num_particles_per_cell_each_dim = 32 32 -p2.density = 1e1 -p2.momentum_distribution_type = "constant" -p2.uy = 100.0 -##########QED#################### -p2.do_qed_quantum_sync = 1 -p2.qed_quantum_sync_phot_product_species = qsp_2 -################################# - -p3.species_type = "positron" -p3.injection_style = "NUniformPerCell" -p3.profile = "constant" -p3.num_particles_per_cell_each_dim = 32 32 -p3.density = 1e1 -p3.momentum_distribution_type = "constant" -p3.uz = 1000.0 -##########QED#################### -p3.do_qed_quantum_sync = 1 -p3.qed_quantum_sync_phot_product_species = qsp_3 -################################# - -p4.species_type = "positron" -p4.injection_style = "NUniformPerCell" -p4.profile = "constant" -p4.num_particles_per_cell_each_dim = 32 32 -p4.density = 1e1 -p4.momentum_distribution_type = "constant" -p4.ux = 5773.502691896 -p4.uy = 5773.502691896 -p4.uz = 5773.502691896 -##########QED#################### -p4.do_qed_quantum_sync = 1 -p4.qed_quantum_sync_phot_product_species = qsp_4 -################################# - -### PRODUCT SPECIES ### -qsp_1.species_type = "photon" -qsp_1.injection_style = "none" -qsp_1.do_qed_breit_wheeler = 1 -qsp_1.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_1.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_2.species_type = "photon" -qsp_2.injection_style = "none" -qsp_2.do_qed_breit_wheeler = 1 -qsp_2.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_2.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_3.species_type = "photon" -qsp_3.injection_style = "none" -qsp_3.do_qed_breit_wheeler = 1 -qsp_3.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_3.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_4.species_type = "photon" -qsp_4.injection_style = "none" -qsp_4.do_qed_breit_wheeler = 1 -qsp_4.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_4.qed_breit_wheeler_pos_product_species = dummy_pos - -################################# - -dummy_ele.species_type = "electron" -dummy_ele.injection_style = "none" - -dummy_pos.species_type = "positron" -dummy_pos.injection_style = "none" - -################################# - -##########QED TABLES#################### -qed_bw.chi_min = 0.001 - -qed_bw.lookup_table_mode = "builtin" - -#qed_bw.lookup_table_mode = "generate" -#qed_bw.tab_dndt_chi_min = 0.01 -#qed_bw.tab_dndt_chi_max = 1000.0 -#qed_bw.tab_dndt_how_many = 256 -#qed_bw.tab_pair_chi_min = 0.01 -#qed_bw.tab_pair_chi_max = 1000.0 -#qed_bw.tab_pair_chi_how_many = 256 -#qed_bw.tab_pair_frac_how_many = 256 -#qed_bw.save_table_in = "bw_table" - -#qed_bw.lookup_table_mode = "load" -#qed_bw.load_table_from = "bw_table" - -qed_qs.chi_min = 0.001 - -qed_qs.lookup_table_mode = "builtin" - -qed_qs.photon_creation_energy_threshold = 0.0 - -#qed_qs.lookup_table_mode = "generate" -#qed_qs.tab_dndt_chi_min = 0.001 -#qed_qs.tab_dndt_chi_max = 1000.0 -#qed_qs.tab_dndt_how_many = 256 -#qed_qs.tab_em_chi_min = 0.001 -#qed_qs.tab_em_frac_min = 1.0e-12 -#qed_qs.tab_em_chi_max = 1000.0 -#qed_qs.tab_em_chi_how_many = 256 -#qed_qs.tab_em_frac_how_many = 256 -#qed_qs.save_table_in = "qs_table" - -#qed_qs.lookup_table_mode = "load" -#qed_qs.load_table_from = "qs_table" -################################# - -### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) -particles.E_ext_particle_init_style = "constant" -particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 -#### - -### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) -particles.B_ext_particle_init_style = "constant" -particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 -#### - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 2 -diag1.diag_type = Full -diag1.fields_to_plot = Ex -diag1.p1.variables = x z ux uy uz w opticalDepthQSR -diag1.p2.variables = x z ux uy uz w opticalDepthQSR -diag1.p3.variables = x z ux uy uz w opticalDepthQSR -diag1.p4.variables = x z ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW -diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW -diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW -diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_3d b/Examples/Tests/qed/quantum_synchrotron/inputs_3d deleted file mode 100644 index 429666ef938..00000000000 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_3d +++ /dev/null @@ -1,189 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -max_step = 2 -amr.n_cell = 16 16 16 -amr.max_grid_size = 16 # maximum size of each AMReX box, used to decompose the domain -amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain -geometry.dims = 3 -geometry.prob_lo = -0.25e-6 -0.25e-6 -0.25e-6 # physical domain -geometry.prob_hi = 0.25e-6 0.25e-6 0.25e-6 -amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic periodic -boundary.field_hi = periodic periodic periodic - -################################# -############ NUMERICS ########### -################################# -algo.current_deposition = esirkepov -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -algo.particle_pusher = boris -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 1 -warpx.cfl = 1. # if 1., the time step is set to its CFL limit -warpx.serialize_initial_conditions = 1 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -############ PLASMA ############# -################################# -particles.species_names = p1 p2 p3 p4 qsp_1 qsp_2 qsp_3 qsp_4 dummy_ele dummy_pos -particles.photon_species = qsp_1 qsp_2 qsp_3 qsp_4 -################################# - -p1.species_type = "electron" -p1.injection_style = "NUniformPerCell" -p1.profile = "constant" -p1.num_particles_per_cell_each_dim = 8 8 4 -p1.density = 1e1 -p1.momentum_distribution_type = "constant" -p1.ux = 10.0 -##########QED#################### -p1.do_qed_quantum_sync = 1 -p1.qed_quantum_sync_phot_product_species = qsp_1 -################################# - -p2.species_type = "electron" -p2.injection_style = "NUniformPerCell" -p2.profile = "constant" -p2.num_particles_per_cell_each_dim = 8 8 4 -p2.density = 1e1 -p2.momentum_distribution_type = "constant" -p2.uy = 100.0 -##########QED#################### -p2.do_qed_quantum_sync = 1 -p2.qed_quantum_sync_phot_product_species = qsp_2 -################################# - -p3.species_type = "positron" -p3.injection_style = "NUniformPerCell" -p3.profile = "constant" -p3.num_particles_per_cell_each_dim = 8 8 4 -p3.density = 1e1 -p3.momentum_distribution_type = "constant" -p3.uz = 1000.0 -##########QED#################### -p3.do_qed_quantum_sync = 1 -p3.qed_quantum_sync_phot_product_species = qsp_3 -################################# - -p4.species_type = "positron" -p4.injection_style = "NUniformPerCell" -p4.profile = "constant" -p4.num_particles_per_cell_each_dim = 8 8 4 -p4.density = 1e1 -p4.momentum_distribution_type = "constant" -p4.ux = 5773.502691896 -p4.uy = 5773.502691896 -p4.uz = 5773.502691896 -##########QED#################### -p4.do_qed_quantum_sync = 1 -p4.qed_quantum_sync_phot_product_species = qsp_4 -################################# - -### PRODUCT SPECIES ### -qsp_1.species_type = "photon" -qsp_1.injection_style = "none" -qsp_1.do_qed_breit_wheeler = 1 -qsp_1.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_1.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_2.species_type = "photon" -qsp_2.injection_style = "none" -qsp_2.do_qed_breit_wheeler = 1 -qsp_2.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_2.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_3.species_type = "photon" -qsp_3.injection_style = "none" -qsp_3.do_qed_breit_wheeler = 1 -qsp_3.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_3.qed_breit_wheeler_pos_product_species = dummy_pos - -qsp_4.species_type = "photon" -qsp_4.injection_style = "none" -qsp_4.do_qed_breit_wheeler = 1 -qsp_4.qed_breit_wheeler_ele_product_species = dummy_ele -qsp_4.qed_breit_wheeler_pos_product_species = dummy_pos - -################################# - -dummy_ele.species_type = "electron" -dummy_ele.injection_style = "none" - -dummy_pos.species_type = "positron" -dummy_pos.injection_style = "none" - -################################# - -##########QED TABLES#################### -qed_bw.chi_min = 0.001 - -qed_bw.lookup_table_mode = "builtin" - -#qed_bw.lookup_table_mode = "generate" -#qed_bw.tab_dndt_chi_min = 0.01 -#qed_bw.tab_dndt_chi_max = 1000.0 -#qed_bw.tab_dndt_how_many = 256 -#qed_bw.tab_pair_chi_min = 0.01 -#qed_bw.tab_pair_chi_max = 1000.0 -#qed_bw.tab_pair_chi_how_many = 256 -#qed_bw.tab_pair_frac_how_many = 256 -#qed_bw.save_table_in = "bw_table" - -#qed_bw.lookup_table_mode = "load" -#qed_bw.load_table_from = "bw_table" - -qed_qs.chi_min = 0.001 - -qed_qs.lookup_table_mode = "builtin" - -qed_qs.photon_creation_energy_threshold = 0.0 - -#qed_qs.lookup_table_mode = "generate" -#qed_qs.tab_dndt_chi_min = 0.001 -#qed_qs.tab_dndt_chi_max = 1000.0 -#qed_qs.tab_dndt_how_many = 256 -#qed_qs.tab_em_chi_min = 0.001 -#qed_qs.tab_em_frac_min = 1.0e-12 -#qed_qs.tab_em_chi_max = 1000.0 -#qed_qs.tab_em_chi_how_many = 256 -#qed_qs.tab_em_frac_how_many = 256 -#qed_qs.save_table_in = "qs_table" - -#qed_qs.lookup_table_mode = "load" -#qed_qs.load_table_from = "qs_table" -################################# - -### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) -particles.E_ext_particle_init_style = "constant" -particles.E_external_particle = -2433321316961438 973328526784575 1459992790176863 -#### - -### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) -particles.B_ext_particle_init_style = "constant" -particles.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 -#### - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 2 -diag1.diag_type = Full -diag1.fields_to_plot = Ex -diag1.p1.variables = x y z ux uy uz w opticalDepthQSR -diag1.p2.variables = x y z ux uy uz w opticalDepthQSR -diag1.p3.variables = x y z ux uy uz w opticalDepthQSR -diag1.p4.variables = x y z ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW -diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW -diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW -diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/schwinger/analysis_schwinger.py b/Examples/Tests/qed/schwinger/analysis_schwinger.py deleted file mode 100755 index c93c920216f..00000000000 --- a/Examples/Tests/qed/schwinger/analysis_schwinger.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Luca Fedeli, Neil Zaim -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -## This module is imported by each of the scripts used to analyze the Schwinger tests. - -## The pair production rate is calculated using the formula described in -## Bulanov, S. S., et al. Physical review letters 104.22 (2010): 220404. - -import os -import re -import sys - -import numpy as np -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# define some parameters - -c = 299792458. -m_e = 9.1093837015e-31 -e =1.602176634e-19 -hbar = 1.054571817e-34 -E_S = m_e**2*c**3/e/hbar # Schwinger field - -dV = (1.e-6)**3 # total simulation volume -dt = 9.827726403e-17 -filename = sys.argv[1] - -Ex_test = 0. -Ey_test = 0. -Ez_test = 0. -Bx_test = 0. -By_test = 0. -Bz_test = 0. - -# Find which test we are doing -test_number = re.search( 'qed_schwinger([1234])', filename ).group(1) -if test_number == '1': - # First Schwinger test with "weak" EM field. No pair should be created. - Ex_test = 1.e16 - Bx_test = 16792888.570516706 - By_test = 5256650.141557486 - Bz_test = 18363530.799561853 -elif test_number == '2': - # Second Schwinger test with stronger EM field. Many pairs are created and a Gaussian - # distribution is used to get the weights of the particles. This is the most sensitive test - # because the relative std is extremely low. - Ex_test = 1.e18 - Bx_test = 1679288857.0516706 - By_test = 525665014.1557486 - Bz_test = 1836353079.9561853 - dV = dV/2. # Schwinger is only activated in part of the simulation domain -elif test_number == '3': - # Third Schwinger test with intermediate electric field such that average created pair per cell - # is 1. A Poisson distribution is used to obtain the weights of the particles. - Ey_test = 1.090934525450495e+17 -elif test_number == '4': - # Fourth Schwinger test with extremely strong EM field but with E and B perpendicular and nearly - # equal so that the pair production rate is fairly low. A Gaussian distribution is used in this - # case. - Ez_test = 2.5e+20 - By_test = 833910140000. - dV = dV*(3./4.)**2. # Schwinger is only activated in part of the simulation domain -else: - assert(False) - -def calculate_rate(Ex,Ey,Ez,Bx,By,Bz): -## Calculate theoretical pair production rate from EM field value - - E_squared = Ex**2 + Ey**2 + Ez**2 - H_squared = c**2*(Bx**2 + By**2 + Bz**2) - - F = (E_squared - H_squared)/2. - G = c*(Ex*Bx + Ey*By + Ez*Bz) - - epsilon = np.sqrt(np.sqrt(F**2+G**2)+F)/E_S - eta = np.sqrt(np.sqrt(F**2+G**2)-F)/E_S - - if(epsilon != 0. and eta != 0.): - return e**2*E_S**2/4./np.pi**2/c/hbar**2*epsilon*eta/np.tanh(np.pi*eta/epsilon)*np.exp(-np.pi/epsilon) - elif (epsilon == 0.): - return 0. - else: - return e**2*E_S**2/4./np.pi**2/c/hbar**2*epsilon**2/np.pi*np.exp(-np.pi/epsilon) - - -def do_analysis(Ex,Ey,Ez,Bx,By,Bz): - - data_set = yt.load(filename) - - expected_total_physical_pairs_created = dV*dt*calculate_rate(Ex,Ey,Ez,Bx,By,Bz) - if expected_total_physical_pairs_created < 0.01: - np_ele = data_set.particle_type_counts["ele_schwinger"] if \ - "ele_schwinger" in data_set.particle_type_counts.keys() else 0 - np_pos = data_set.particle_type_counts["pos_schwinger"] if \ - "pos_schwinger" in data_set.particle_type_counts.keys() else 0 - assert(np_ele == 0 and np_pos == 0) - ## Assert whether pairs are created or not. - - else: - all_data = data_set.all_data() - - ele_data = all_data["ele_schwinger",'particle_weight'] - pos_data = all_data["pos_schwinger",'particle_weight'] - - std_total_physical_pairs_created = np.sqrt(expected_total_physical_pairs_created) - - # Sorting the arrays is required because electrons and positrons are not necessarily - # dumped in the same order. - assert(np.array_equal(np.sort(ele_data),np.sort(pos_data))) - # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions - error = np.abs(np.sum(ele_data)-expected_total_physical_pairs_created) - print("difference between expected and actual number of pairs created: " + str(error)) - print("tolerance: " + str(5*std_total_physical_pairs_created)) - assert(error<5*std_total_physical_pairs_created) - -do_analysis(Ex_test, Ey_test, Ez_test, Bx_test, By_test, Bz_test) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/radiation_reaction/CMakeLists.txt b/Examples/Tests/radiation_reaction/CMakeLists.txt new file mode 100644 index 00000000000..3286f4efd93 --- /dev/null +++ b/Examples/Tests/radiation_reaction/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_radiation_reaction # name + 3 # dims + 2 # nprocs + inputs_test_3d_radiation_reaction # inputs + "analysis.py diags/diag1000064" # analysis + "analysis_default_regression.py --path diags/diag1000064" # checksum + OFF # dependency +) diff --git a/Examples/Tests/radiation_reaction/analysis.py b/Examples/Tests/radiation_reaction/analysis.py new file mode 100755 index 00000000000..0d4fcf12e8e --- /dev/null +++ b/Examples/Tests/radiation_reaction/analysis.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Luca Fedeli, Maxence Thevenet, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +# This script contains few simple tests for the radiation reaction pusher +# It initializes an electron or a positron with normalized momentum in different +# directions, propagating in a static magnetic field (along [2/7,3/7,6/7]). +# If the initial momentum is perpendicular to the the field, there is a very +# simple analytical formula for the gamma factor: +# gamma(t) = coth(t/tc - C) +# where tc = 1./(t0 * omega_c * omega_c) +# and omega_c = qB/m , t0 = (2/3) * (r_e / c) +# and r_e = classical electron radius +# C is chosen so that gamma(0) = initial_gamma +# If the initial momentum is parallel to the field, it should not change. +# This test checks these predictions with a tolerance of 5%. +# If the script is run without a command line argument, it regenerates a new +# inputfile according to the initial conditions listed below. +# For detailed references see: +# 1) M. Tamburini. PhD Thesis. University of Pisa (2011) +# https://etd.adm.unipi.it/t/etd-11232011-111059/ +# 2) H. Spohn Europhysics Letters 50 287 (2000) +# https://arxiv.org/abs/physics/9911027 +# 3) H. Spohn, Dynamics of charged particles and their radiation field +# (Cambridge University Press, Cambridge, 2004) + +import sys + +import numpy as np +import yt + +# Input filename +inputname = "inputs" +# ________________________________________ + +# Physical constants +c = 299792458.0 +m_e = 9.1093837015e-31 +q_0 = 1.602176634e-19 +classical_electron_radius = 2.81794e-15 +reference_length = 1.0e-6 +very_small_dot_product = 1.0e-4 +very_small_weight = 1.0e-8 +# ________________________________________ + +# Sim box data +sim_size = 0.8e-6 +resolution = 64 +steps = 64 +init_pos = np.array([0.0, 0.0, 0.0]) +# ________________________________________ + +# Momentum vals +p_aux_0 = np.array([2.0, 3.0, 6.0]) +p_aux_1 = np.array([1, 0, 0]) +p_aux_2 = np.array([0, 1, 0]) +Q, _ = np.linalg.qr(np.column_stack([p_aux_0, p_aux_1, p_aux_2])) # Gram-Schmidt +p_0 = -Q[:, 0] +p_1 = -Q[:, 1] +p_2 = -Q[:, 2] + +p_vals = [50, 200, 1000] +# ________________________________________ + +# Field val +B_val_norm = 300 +B_val = B_val_norm * m_e * 2.0 * np.pi * c / q_0 / reference_length +B = p_0 * B_val +# ________________________________________ + +# Tolerance +tolerance_rel = 0.05 +# ________________________________________ + +# tau_c +omega_c = q_0 * B_val / m_e +t0 = (2.0 / 3.0) * classical_electron_radius / c +tau_c = 1.0 / omega_c / omega_c / t0 +# ________________________________________ + + +# Simulation case struct +class sim_case: + def __init__(self, _name, _init_mom, _type): + self.name = _name + self.init_mom = _init_mom + self.type = _type + + +# ________________________________________ + +# All cases +cases = [ + sim_case("ele_para0", p_0 * p_vals[2], "-q_e"), + sim_case("ele_perp0", p_1 * p_vals[0], "-q_e"), + sim_case("ele_perp1", p_2 * p_vals[1], "-q_e"), + sim_case("ele_perp2", p_1 * p_vals[2], "-q_e"), + sim_case("pos_perp2", p_1 * p_vals[2], "q_e"), +] +# ________________________________________ + + +# Auxiliary functions +def gamma(p): + return np.sqrt(1.0 + np.dot(p, p)) + + +def exp_res(cc, time): + if np.all(np.linalg.norm(np.cross(cc.init_mom, B)) < very_small_dot_product): + return gamma(cc.init_mom) + else: + tt = time / tau_c + g0 = gamma(cc.init_mom) + C = -0.5 * np.log((g0 + 1) / (g0 - 1)) + return 1.0 / np.tanh(tt - C) + + +# ________________________________________ + + +def check(): + filename = sys.argv[1] + data_set_end = yt.load(filename) + + sim_time = data_set_end.current_time.to_value() + + # simulation results + all_data = data_set_end.all_data() + spec_names = [cc.name for cc in cases] + + # All momenta + res_mom = np.array( + [ + np.array( + [ + all_data[sp, "particle_momentum_x"].v[0], + all_data[sp, "particle_momentum_y"].v[0], + all_data[sp, "particle_momentum_z"].v[0], + ] + ) + for sp in spec_names + ] + ) + + for cc in zip(cases, res_mom): + end_gamma = gamma(cc[1] / m_e / c) + exp_gamma = exp_res(cc[0], sim_time) + + error_rel = np.abs(end_gamma - exp_gamma) / exp_gamma + + print("error_rel : " + str(error_rel)) + print("tolerance_rel: " + str(tolerance_rel)) + + assert error_rel < tolerance_rel + + +def generate(): + with open(inputname, "w") as f: + f.write("#Automatically generated inputfile\n") + f.write("#Run check.py without arguments to regenerate\n") + f.write("#\n\n") + f.write("max_step = {}\n".format(steps)) + f.write("amr.n_cell = {} {} {}\n".format(resolution, resolution, resolution)) + f.write("amr.max_level = 0\n") + f.write("amr.blocking_factor = 32\n") + f.write("amr.max_grid_size = 64\n") + f.write("geometry.dims = 3\n") + f.write("boundary.field_lo = periodic periodic periodic\n") + f.write("boundary.field_hi = periodic periodic periodic\n") + f.write("geometry.prob_lo = {} {} {}\n".format(-sim_size, -sim_size, -sim_size)) + f.write("geometry.prob_hi = {} {} {}\n".format(sim_size, sim_size, sim_size)) + f.write("algo.charge_deposition = standard\n") + f.write("algo.field_gathering = energy-conserving\n") + f.write("warpx.cfl = 1.0\n") + f.write("warpx.serialize_initial_conditions = 1\n") + + f.write("particles.species_names = ") + for cc in cases: + f.write(" {}".format(cc.name)) + f.write("\n") + + f.write("\namr.plot_int = {}\n\n".format(steps)) + + f.write("warpx.fields_to_plot = rho\n\n") + + for cc in cases: + f.write("{}.charge = {}\n".format(cc.name, cc.type)) + f.write("{}.mass = m_e\n".format(cc.name)) + f.write('{}.injection_style = "SingleParticle"\n'.format(cc.name)) + f.write( + "{}.single_particle_pos = {} {} {}\n".format( + cc.name, init_pos[0], init_pos[1], init_pos[2] + ) + ) + f.write( + "{}.single_particle_u = {} {} {}\n".format( + cc.name, cc.init_mom[0], cc.init_mom[1], cc.init_mom[2] + ) + ) + f.write( + "{}.single_particle_weight = {}\n".format(cc.name, very_small_weight) + ) + f.write("{}.do_classical_radiation_reaction = 1\n".format(cc.name)) + f.write("\n") + + f.write("warpx.B_external_particle = {} {} {}\n".format(*B)) + + +def main(): + if len(sys.argv) < 2: + generate() + else: + check() + + +if __name__ == "__main__": + main() diff --git a/Examples/Tests/radiation_reaction/analysis_default_regression.py b/Examples/Tests/radiation_reaction/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/radiation_reaction/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/radiation_reaction/test_const_B_analytical/inputs_3d b/Examples/Tests/radiation_reaction/inputs_test_3d_radiation_reaction similarity index 100% rename from Examples/Tests/radiation_reaction/test_const_B_analytical/inputs_3d rename to Examples/Tests/radiation_reaction/inputs_test_3d_radiation_reaction diff --git a/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py b/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py deleted file mode 100755 index e123562fbe1..00000000000 --- a/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Luca Fedeli, Maxence Thevenet, Remi Lehe -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -# This script contains few simple tests for the radiation reaction pusher -# It initializes an electron or a positron with normalized momentum in different -# directions, propagating in a static magnetic field (along [2/7,3/7,6/7]). -# If the initial momentum is perpendicular to the the field, there is a very -# simple analytical formula for the gamma factor: -# gamma(t) = coth(t/tc - C) -# where tc = 1./(t0 * omega_c * omega_c) -# and omega_c = qB/m , t0 = (2/3) * (r_e / c) -# and r_e = classical electron radius -# C is chosen so that gamma(0) = initial_gamma -# If the initial momentum is parallel to the field, it should not change. -# This test checks these predictions with a tolerance of 5%. -# If the script is run without a command line argument, it regenerates a new -# inputfile according to the initial conditions listed below. -# For detailed references see: -# 1) M. Tamburini. PhD Thesis. University of Pisa (2011) -# https://etd.adm.unipi.it/t/etd-11232011-111059/ -# 2) H. Spohn Europhysics Letters 50 287 (2000) -# https://arxiv.org/abs/physics/9911027 -# 3) H. Spohn, Dynamics of charged particles and their radiation field -# (Cambridge University Press, Cambridge, 2004) - -import os -import sys - -import numpy as np -import yt - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -#Input filename -inputname = "inputs" -#________________________________________ - -#Physical constants -c = 299792458. -m_e = 9.1093837015e-31 -q_0 = 1.602176634e-19 -classical_electron_radius = 2.81794e-15 -reference_length = 1.0e-6 -very_small_dot_product = 1.0e-4 -very_small_weight = 1.0e-8 -#________________________________________ - -#Sim box data -sim_size = 0.8e-6 -resolution = 64 -steps = 64 -init_pos = np.array([0.,0.,0.]) -#________________________________________ - -#Momentum vals -p_aux_0 = np.array([2.,3.,6.]) -p_aux_1 = np.array([1,0,0]) -p_aux_2 = np.array([0,1,0]) -Q, _ = np.linalg.qr(np.column_stack( [p_aux_0, p_aux_1, p_aux_2] )) #Gram-Schmidt -p_0 = -Q[:,0] -p_1 = -Q[:,1] -p_2 = -Q[:,2] - -p_vals = [50,200,1000] -#________________________________________ - -#Field val -B_val_norm = 300 -B_val = B_val_norm*m_e * 2.0 * np.pi * c /q_0/reference_length -B = p_0 * B_val -#________________________________________ - -#Tolerance -tolerance_rel = 0.05 -#________________________________________ - -#tau_c -omega_c = q_0*B_val/m_e -t0 = (2./3.)*classical_electron_radius/c -tau_c = 1.0/omega_c/omega_c/t0 -#________________________________________ - - -#Simulation case struct -class sim_case: - def __init__(self, _name, _init_mom, _type): - self.name = _name - self.init_mom = _init_mom - self.type = _type -#________________________________________ - -#All cases -cases = [ - sim_case("ele_para0", p_0 * p_vals[2], "-q_e"), - sim_case("ele_perp0", p_1 * p_vals[0], "-q_e"), - sim_case("ele_perp1", p_2 * p_vals[1], "-q_e"), - sim_case("ele_perp2", p_1 * p_vals[2], "-q_e"), - sim_case("pos_perp2", p_1 * p_vals[2], "q_e"), -] -#________________________________________ - -#Auxiliary functions -def gamma(p) : - return np.sqrt(1.0 + np.dot(p,p)) - -def exp_res(cc, time): - if np.all(np.linalg.norm(np.cross(cc.init_mom, B)) < very_small_dot_product): - return gamma(cc.init_mom) - else : - tt = time/tau_c - g0 = gamma(cc.init_mom) - C = -0.5 * np.log((g0+1)/(g0-1)) - return 1.0/np.tanh(tt - C) -#________________________________________ - - -def check(): - filename = sys.argv[1] - data_set_end = yt.load(filename) - - sim_time = data_set_end.current_time.to_value() - - #simulation results - all_data = data_set_end.all_data() - spec_names = [cc.name for cc in cases] - - #All momenta - res_mom = np.array([np.array([ - all_data[sp, 'particle_momentum_x'].v[0], - all_data[sp, 'particle_momentum_y'].v[0], - all_data[sp, 'particle_momentum_z'].v[0]]) - for sp in spec_names]) - - for cc in zip(cases, res_mom): - init_gamma = gamma(cc[0].init_mom) - end_gamma = gamma(cc[1]/m_e/c) - exp_gamma = exp_res(cc[0], sim_time) - - error_rel = np.abs(end_gamma-exp_gamma)/exp_gamma - - print("error_rel : " + str(error_rel)) - print("tolerance_rel: " + str(tolerance_rel)) - - assert( error_rel < tolerance_rel ) - - test_name = os.path.split(os.getcwd())[1] - checksumAPI.evaluate_checksum(test_name, filename) - -def generate(): - - with open(inputname,'w') as f: - f.write("#Automatically generated inputfile\n") - f.write("#Run check.py without arguments to regenerate\n") - f.write("#\n\n") - f.write("max_step = {}\n".format(steps)) - f.write("amr.n_cell = {} {} {}\n".format(resolution, resolution, resolution)) - f.write("amr.max_level = 0\n") - f.write("amr.blocking_factor = 32\n") - f.write("amr.max_grid_size = 64\n") - f.write("geometry.dims = 3\n") - f.write("boundary.field_lo = periodic periodic periodic\n") - f.write("boundary.field_hi = periodic periodic periodic\n") - f.write("geometry.prob_lo = {} {} {}\n".format(-sim_size, -sim_size, -sim_size)) - f.write("geometry.prob_hi = {} {} {}\n".format(sim_size, sim_size, sim_size)) - f.write("algo.charge_deposition = standard\n") - f.write("algo.field_gathering = energy-conserving\n") - f.write("warpx.cfl = 1.0\n") - f.write("warpx.serialize_initial_conditions = 1\n") - - f.write("particles.species_names = ") - for cc in cases: - f.write(" {}".format(cc.name)) - f.write("\n") - - f.write("\namr.plot_int = {}\n\n".format(steps)) - - f.write("warpx.fields_to_plot = rho\n\n") - - for cc in cases: - f.write("{}.charge = {}\n".format(cc.name, cc.type)) - f.write("{}.mass = m_e\n".format(cc.name)) - f.write('{}.injection_style = "SingleParticle"\n'.format(cc.name)) - f.write("{}.single_particle_pos = {} {} {}\n". - format(cc.name, init_pos[0], init_pos[1], init_pos[2])) - f.write("{}.single_particle_u = {} {} {}\n". - format(cc.name, cc.init_mom[0], cc.init_mom[1], cc.init_mom[2])) - f.write("{}.single_particle_weight = {}\n".format(cc.name, very_small_weight)) - f.write("{}.do_classical_radiation_reaction = 1\n".format(cc.name)) - f.write("\n") - - f.write("warpx.B_external_particle = {} {} {}\n".format(*B)) - -def main(): - if (len(sys.argv) < 2): - generate() - else: - check() - -if __name__ == "__main__": - main() diff --git a/Examples/Tests/reduced_diags/CMakeLists.txt b/Examples/Tests/reduced_diags/CMakeLists.txt new file mode 100644 index 00000000000..743afa79df5 --- /dev/null +++ b/Examples/Tests/reduced_diags/CMakeLists.txt @@ -0,0 +1,54 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_reduced_diags # name + 3 # dims + 2 # nprocs + inputs_test_3d_reduced_diags # inputs + "analysis_reduced_diags.py diags/diag1000200" # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_reduced_diags_load_balance_costs_heuristic # name + 3 # dims + 2 # nprocs + inputs_test_3d_reduced_diags_load_balance_costs_heuristic # inputs + "analysis_reduced_diags_load_balance_costs.py diags/diag1000003" # analysis + "analysis_default_regression.py --path diags/diag1000003" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_reduced_diags_load_balance_costs_timers # name + 3 # dims + 2 # nprocs + inputs_test_3d_reduced_diags_load_balance_costs_timers # inputs + "analysis_reduced_diags_load_balance_costs.py diags/diag1000003" # analysis + "analysis_default_regression.py --path diags/diag1000003" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_reduced_diags_load_balance_costs_timers_picmi # name + 3 # dims + 2 # nprocs + inputs_test_3d_reduced_diags_load_balance_costs_timers_picmi.py # inputs + "analysis_reduced_diags_load_balance_costs.py diags/diag1000003" # analysis + "analysis_default_regression.py --path diags/diag1000003" # checksum + OFF # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_3d_reduced_diags_load_balance_costs_timers_psatd # name + 3 # dims + 2 # nprocs + inputs_test_3d_reduced_diags_load_balance_costs_timers_psatd # inputs + "analysis_reduced_diags_load_balance_costs.py diags/diag1000003" # analysis + "analysis_default_regression.py --path diags/diag1000003" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py b/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py deleted file mode 100644 index 0583a6fe1d0..00000000000 --- a/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 - -from pywarpx import picmi - -# Number of time steps -max_steps = 3 - -# Number of cells -nx = 128 -ny = 32 -nz = 128 - -# Physical domain -xmin = 0. -xmax = 4. -ymin = 0. -ymax = 4. -zmin = 0. -zmax = 4. - -# Create grid -grid = picmi.Cartesian3DGrid( - number_of_cells=[nx, ny, nz], - warpx_max_grid_size=32, - lower_bound=[xmin, ymin, zmin], - upper_bound=[xmax, ymax, zmax], - lower_boundary_conditions=['periodic', 'periodic', 'periodic'], - upper_boundary_conditions=['periodic', 'periodic', 'periodic'] -) - -# Electromagnetic solver -solver = picmi.ElectromagneticSolver( - grid=grid, - method='Yee', - cfl=0.99999 -) - -# Particles -electrons = picmi.Species( - particle_type='electron', - name='electrons', - initial_distribution=picmi.UniformDistribution( - density=1e14, - rms_velocity=[0]*3, - upper_bound=[xmax, ymax, 1.0] - ) -) -layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[1, 1, 1], grid=grid -) - -# Reduced diagnostic -reduced_diags = [] -reduced_diags.append(picmi.ReducedDiagnostic( - diag_type='LoadBalanceCosts', - period=1, - name='LBC' -)) - -reduced_diags.append(picmi.ReducedDiagnostic( - diag_type='FieldReduction', - period=1, - name='FR', - reduction_type='Maximum', - reduced_function = 'Ez' -)) - -reduced_diags.append(picmi.ReducedDiagnostic( - diag_type='ParticleHistogram', - period=1, - name='PH', - species = electrons, - bin_number = 10, - bin_min=0., - bin_max = xmax, - normalization = 'unity_particle_weight', - histogram_function = 'x' -)) - -# Diagnostic -particle_diag = picmi.ParticleDiagnostic( - name='diag1', - period=3, - species=[electrons], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], - write_dir='.', - warpx_file_prefix='Python_reduced_diags_loadbalancecosts_timers_plt' -) -field_diag = picmi.FieldDiagnostic( - name='diag1', - grid=grid, - period=3, - data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], - write_dir='.', - warpx_file_prefix='Python_reduced_diags_loadbalancecosts_timers_plt' -) - -# Set up simulation -sim = picmi.Simulation( - solver=solver, - max_steps=max_steps, - verbose=1, - particle_shape=1, - warpx_current_deposition_algo='esirkepov', - warpx_field_gathering_algo='energy-conserving', - warpx_load_balance_intervals=2 -) - -# Add species -sim.add_species(electrons, layout=layout) - -# Add reduced diagnostics -for reduced_diag in reduced_diags: - sim.add_diagnostic(reduced_diag) - -# Add diagnostics -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) - -# Advance simulation until last time step -# sim.write_input_file("test_input") -sim.step(max_steps) diff --git a/Examples/Tests/reduced_diags/analysis_default_regression.py b/Examples/Tests/reduced_diags/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/reduced_diags/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags.py b/Examples/Tests/reduced_diags/analysis_reduced_diags.py index 1885800376b..2972324e2ac 100755 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags.py +++ b/Examples/Tests/reduced_diags/analysis_reduced_diags.py @@ -13,4 +13,4 @@ import analysis_reduced_diags_impl as an -an.do_analysis(single_precision = False) +an.do_analysis(single_precision=False) diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py index 21359ed8171..e0c1fe1d1b3 100755 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py +++ b/Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py @@ -11,288 +11,349 @@ # Various particle and field quantities are written to file using the reduced diagnostics # and compared with the corresponding quantities computed from the data in the plotfiles. -import os import sys import numpy as np import yt -from scipy.constants import c +from scipy.constants import c, m_e, m_p from scipy.constants import epsilon_0 as eps0 -from scipy.constants import m_e, m_p from scipy.constants import mu_0 as mu0 -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # gamma threshold to switch between the relativistic expression of # the kinetic energy and its Taylor expansion. gamma_relativistic_threshold = 1.005 -def do_analysis(single_precision = False): + +def do_analysis(single_precision=False): fn = sys.argv[1] ds = yt.load(fn) ad = ds.all_data() - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Part 1: get results from plotfiles (label '_yt') - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Quantities computed from plotfiles values_yt = dict() # Electrons - px = ad['electrons', 'particle_momentum_x'].to_ndarray() - py = ad['electrons', 'particle_momentum_y'].to_ndarray() - pz = ad['electrons', 'particle_momentum_z'].to_ndarray() - w = ad['electrons', 'particle_weight'].to_ndarray() + px = ad["electrons", "particle_momentum_x"].to_ndarray() + py = ad["electrons", "particle_momentum_y"].to_ndarray() + pz = ad["electrons", "particle_momentum_z"].to_ndarray() + w = ad["electrons", "particle_weight"].to_ndarray() p2 = px**2 + py**2 + pz**2 # Accumulate particle energy, store number of particles and sum of weights - e_u2 = p2/(m_e**2 * c**2) + e_u2 = p2 / (m_e**2 * c**2) e_gamma = np.sqrt(1 + e_u2) - e_energy = (m_e * c**2)*np.where(e_gamma > gamma_relativistic_threshold, - e_gamma-1, - (e_u2)/2 - (e_u2**2)/8 + (e_u2**3)/16 - (e_u2**4)*(5/128) + (e_u2**5)*(7/256)) - values_yt['electrons: particle energy'] = np.sum(e_energy * w) - values_yt['electrons: particle momentum in x'] = np.sum(px * w) - values_yt['electrons: particle momentum in y'] = np.sum(py * w) - values_yt['electrons: particle momentum in z'] = np.sum(pz * w) - values_yt['electrons: number of particles'] = w.shape[0] - values_yt['electrons: sum of weights'] = np.sum(w) + e_energy = (m_e * c**2) * np.where( + e_gamma > gamma_relativistic_threshold, + e_gamma - 1, + (e_u2) / 2 + - (e_u2**2) / 8 + + (e_u2**3) / 16 + - (e_u2**4) * (5 / 128) + + (e_u2**5) * (7 / 256), + ) + values_yt["electrons: particle energy"] = np.sum(e_energy * w) + values_yt["electrons: particle momentum in x"] = np.sum(px * w) + values_yt["electrons: particle momentum in y"] = np.sum(py * w) + values_yt["electrons: particle momentum in z"] = np.sum(pz * w) + values_yt["electrons: number of particles"] = w.shape[0] + values_yt["electrons: sum of weights"] = np.sum(w) # Protons - px = ad['protons', 'particle_momentum_x'].to_ndarray() - py = ad['protons', 'particle_momentum_y'].to_ndarray() - pz = ad['protons', 'particle_momentum_z'].to_ndarray() - w = ad['protons', 'particle_weight'].to_ndarray() + px = ad["protons", "particle_momentum_x"].to_ndarray() + py = ad["protons", "particle_momentum_y"].to_ndarray() + pz = ad["protons", "particle_momentum_z"].to_ndarray() + w = ad["protons", "particle_weight"].to_ndarray() p2 = px**2 + py**2 + pz**2 # Accumulate particle energy, store number of particles and sum of weights - p_u2 = p2/(m_p**2 * c**2) + p_u2 = p2 / (m_p**2 * c**2) p_gamma = np.sqrt(1 + p_u2) - p_energy = (m_p * c**2)*np.where(p_gamma > gamma_relativistic_threshold, - p_gamma-1, - (p_u2)/2 - (p_u2**2)/8 + (p_u2**3)/16 - (p_u2**4)*(5/128) + (p_u2**5)*(7/256)) - values_yt['protons: particle energy'] = np.sum(p_energy * w) - values_yt['protons: particle momentum in x'] = np.sum(px * w) - values_yt['protons: particle momentum in y'] = np.sum(py * w) - values_yt['protons: particle momentum in z'] = np.sum(pz * w) - values_yt['protons: number of particles'] = w.shape[0] - values_yt['protons: sum of weights'] = np.sum(w) + p_energy = (m_p * c**2) * np.where( + p_gamma > gamma_relativistic_threshold, + p_gamma - 1, + (p_u2) / 2 + - (p_u2**2) / 8 + + (p_u2**3) / 16 + - (p_u2**4) * (5 / 128) + + (p_u2**5) * (7 / 256), + ) + values_yt["protons: particle energy"] = np.sum(p_energy * w) + values_yt["protons: particle momentum in x"] = np.sum(px * w) + values_yt["protons: particle momentum in y"] = np.sum(py * w) + values_yt["protons: particle momentum in z"] = np.sum(pz * w) + values_yt["protons: number of particles"] = w.shape[0] + values_yt["protons: sum of weights"] = np.sum(w) # Photons - px = ad['photons', 'particle_momentum_x'].to_ndarray() - py = ad['photons', 'particle_momentum_y'].to_ndarray() - pz = ad['photons', 'particle_momentum_z'].to_ndarray() - w = ad['photons', 'particle_weight'].to_ndarray() + px = ad["photons", "particle_momentum_x"].to_ndarray() + py = ad["photons", "particle_momentum_y"].to_ndarray() + pz = ad["photons", "particle_momentum_z"].to_ndarray() + w = ad["photons", "particle_weight"].to_ndarray() p2 = px**2 + py**2 + pz**2 # Accumulate particle energy, store number of particles and sum of weights - values_yt['photons: particle energy'] = np.sum(np.sqrt(p2 * c**2) * w) - values_yt['photons: particle momentum in x'] = np.sum(px * w) - values_yt['photons: particle momentum in y'] = np.sum(py * w) - values_yt['photons: particle momentum in z'] = np.sum(pz * w) - values_yt['photons: number of particles'] = w.shape[0] - values_yt['photons: sum of weights'] = np.sum(w) + values_yt["photons: particle energy"] = np.sum(np.sqrt(p2 * c**2) * w) + values_yt["photons: particle momentum in x"] = np.sum(px * w) + values_yt["photons: particle momentum in y"] = np.sum(py * w) + values_yt["photons: particle momentum in z"] = np.sum(pz * w) + values_yt["photons: number of particles"] = w.shape[0] + values_yt["photons: sum of weights"] = np.sum(w) # Accumulate total particle diagnostics - values_yt['particle energy'] = values_yt['electrons: particle energy'] \ - + values_yt['protons: particle energy'] \ - + values_yt['photons: particle energy'] - - values_yt['particle momentum in x'] = values_yt['electrons: particle momentum in x'] \ - + values_yt['protons: particle momentum in x'] \ - + values_yt['photons: particle momentum in x'] - - values_yt['particle momentum in y'] = values_yt['electrons: particle momentum in y'] \ - + values_yt['protons: particle momentum in y'] \ - + values_yt['photons: particle momentum in y'] - - values_yt['particle momentum in z'] = values_yt['electrons: particle momentum in z'] \ - + values_yt['protons: particle momentum in z'] \ - + values_yt['photons: particle momentum in z'] - - values_yt['number of particles'] = values_yt['electrons: number of particles'] \ - + values_yt['protons: number of particles'] \ - + values_yt['photons: number of particles'] - - values_yt['sum of weights'] = values_yt['electrons: sum of weights'] \ - + values_yt['protons: sum of weights'] \ - + values_yt['photons: sum of weights'] - - values_yt['mean particle energy'] = values_yt['particle energy'] / values_yt['sum of weights'] - - values_yt['mean particle momentum in x'] = values_yt['particle momentum in x'] / values_yt['sum of weights'] - values_yt['mean particle momentum in y'] = values_yt['particle momentum in y'] / values_yt['sum of weights'] - values_yt['mean particle momentum in z'] = values_yt['particle momentum in z'] / values_yt['sum of weights'] - - values_yt['electrons: mean particle energy'] = values_yt['electrons: particle energy'] \ - / values_yt['electrons: sum of weights'] - - values_yt['electrons: mean particle momentum in x'] = values_yt['electrons: particle momentum in x'] \ - / values_yt['electrons: sum of weights'] - values_yt['electrons: mean particle momentum in y'] = values_yt['electrons: particle momentum in y'] \ - / values_yt['electrons: sum of weights'] - values_yt['electrons: mean particle momentum in z'] = values_yt['electrons: particle momentum in z'] \ - / values_yt['electrons: sum of weights'] - - values_yt['protons: mean particle energy'] = values_yt['protons: particle energy'] \ - / values_yt['protons: sum of weights'] - - values_yt['protons: mean particle momentum in x'] = values_yt['protons: particle momentum in x'] \ - / values_yt['protons: sum of weights'] - values_yt['protons: mean particle momentum in y'] = values_yt['protons: particle momentum in y'] \ - / values_yt['protons: sum of weights'] - values_yt['protons: mean particle momentum in z'] = values_yt['protons: particle momentum in z'] \ - / values_yt['protons: sum of weights'] - - values_yt['photons: mean particle energy'] = values_yt['photons: particle energy'] \ - / values_yt['photons: sum of weights'] - - values_yt['photons: mean particle momentum in x'] = values_yt['photons: particle momentum in x'] \ - / values_yt['photons: sum of weights'] - values_yt['photons: mean particle momentum in y'] = values_yt['photons: particle momentum in y'] \ - / values_yt['photons: sum of weights'] - values_yt['photons: mean particle momentum in z'] = values_yt['photons: particle momentum in z'] \ - / values_yt['photons: sum of weights'] + values_yt["particle energy"] = ( + values_yt["electrons: particle energy"] + + values_yt["protons: particle energy"] + + values_yt["photons: particle energy"] + ) + + values_yt["particle momentum in x"] = ( + values_yt["electrons: particle momentum in x"] + + values_yt["protons: particle momentum in x"] + + values_yt["photons: particle momentum in x"] + ) + + values_yt["particle momentum in y"] = ( + values_yt["electrons: particle momentum in y"] + + values_yt["protons: particle momentum in y"] + + values_yt["photons: particle momentum in y"] + ) + + values_yt["particle momentum in z"] = ( + values_yt["electrons: particle momentum in z"] + + values_yt["protons: particle momentum in z"] + + values_yt["photons: particle momentum in z"] + ) + + values_yt["number of particles"] = ( + values_yt["electrons: number of particles"] + + values_yt["protons: number of particles"] + + values_yt["photons: number of particles"] + ) + + values_yt["sum of weights"] = ( + values_yt["electrons: sum of weights"] + + values_yt["protons: sum of weights"] + + values_yt["photons: sum of weights"] + ) + + values_yt["mean particle energy"] = ( + values_yt["particle energy"] / values_yt["sum of weights"] + ) + + values_yt["mean particle momentum in x"] = ( + values_yt["particle momentum in x"] / values_yt["sum of weights"] + ) + values_yt["mean particle momentum in y"] = ( + values_yt["particle momentum in y"] / values_yt["sum of weights"] + ) + values_yt["mean particle momentum in z"] = ( + values_yt["particle momentum in z"] / values_yt["sum of weights"] + ) + + values_yt["electrons: mean particle energy"] = ( + values_yt["electrons: particle energy"] / values_yt["electrons: sum of weights"] + ) + + values_yt["electrons: mean particle momentum in x"] = ( + values_yt["electrons: particle momentum in x"] + / values_yt["electrons: sum of weights"] + ) + values_yt["electrons: mean particle momentum in y"] = ( + values_yt["electrons: particle momentum in y"] + / values_yt["electrons: sum of weights"] + ) + values_yt["electrons: mean particle momentum in z"] = ( + values_yt["electrons: particle momentum in z"] + / values_yt["electrons: sum of weights"] + ) + + values_yt["protons: mean particle energy"] = ( + values_yt["protons: particle energy"] / values_yt["protons: sum of weights"] + ) + + values_yt["protons: mean particle momentum in x"] = ( + values_yt["protons: particle momentum in x"] + / values_yt["protons: sum of weights"] + ) + values_yt["protons: mean particle momentum in y"] = ( + values_yt["protons: particle momentum in y"] + / values_yt["protons: sum of weights"] + ) + values_yt["protons: mean particle momentum in z"] = ( + values_yt["protons: particle momentum in z"] + / values_yt["protons: sum of weights"] + ) + + values_yt["photons: mean particle energy"] = ( + values_yt["photons: particle energy"] / values_yt["photons: sum of weights"] + ) + + values_yt["photons: mean particle momentum in x"] = ( + values_yt["photons: particle momentum in x"] + / values_yt["photons: sum of weights"] + ) + values_yt["photons: mean particle momentum in y"] = ( + values_yt["photons: particle momentum in y"] + / values_yt["photons: sum of weights"] + ) + values_yt["photons: mean particle momentum in z"] = ( + values_yt["photons: particle momentum in z"] + / values_yt["photons: sum of weights"] + ) # Load 3D data from plotfiles - ad = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) - Ex = ad[('mesh','Ex')].to_ndarray() - Ey = ad[('mesh','Ey')].to_ndarray() - Ez = ad[('mesh','Ez')].to_ndarray() - Bx = ad[('mesh','Bx')].to_ndarray() - By = ad[('mesh','By')].to_ndarray() - Bz = ad[('mesh','Bz')].to_ndarray() - jx = ad[('mesh','jx')].to_ndarray() - jy = ad[('mesh','jy')].to_ndarray() - jz = ad[('mesh','jz')].to_ndarray() - rho = ad[('boxlib','rho')].to_ndarray() - rho_electrons = ad[('boxlib','rho_electrons')].to_ndarray() - rho_protons = ad[('boxlib','rho_protons')].to_ndarray() - x = ad[('boxlib','x')].to_ndarray() - y = ad[('boxlib','y')].to_ndarray() - z = ad[('boxlib','z')].to_ndarray() + ad = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions + ) + Ex = ad[("mesh", "Ex")].to_ndarray() + Ey = ad[("mesh", "Ey")].to_ndarray() + Ez = ad[("mesh", "Ez")].to_ndarray() + Bx = ad[("mesh", "Bx")].to_ndarray() + By = ad[("mesh", "By")].to_ndarray() + Bz = ad[("mesh", "Bz")].to_ndarray() + jx = ad[("mesh", "jx")].to_ndarray() + jy = ad[("mesh", "jy")].to_ndarray() + jz = ad[("mesh", "jz")].to_ndarray() + rho = ad[("boxlib", "rho")].to_ndarray() + rho_electrons = ad[("boxlib", "rho_electrons")].to_ndarray() + rho_protons = ad[("boxlib", "rho_protons")].to_ndarray() + x = ad[("boxlib", "x")].to_ndarray() + y = ad[("boxlib", "y")].to_ndarray() + z = ad[("boxlib", "z")].to_ndarray() # Field energy E2 = np.sum(Ex**2) + np.sum(Ey**2) + np.sum(Ez**2) B2 = np.sum(Bx**2) + np.sum(By**2) + np.sum(Bz**2) - N = np.array(ds.domain_width / ds.domain_dimensions) + N = np.array(ds.domain_width / ds.domain_dimensions) dV = N[0] * N[1] * N[2] - values_yt['field energy'] = 0.5 * dV * (E2 * eps0 + B2 / mu0) - values_yt['field momentum in x'] = eps0 * np.sum(Ey * Bz - Ez * By) * dV - values_yt['field momentum in y'] = eps0 * np.sum(Ez * Bx - Ex * Bz) * dV - values_yt['field momentum in z'] = eps0 * np.sum(Ex * By - Ey * Bx) * dV + values_yt["field energy"] = 0.5 * dV * (E2 * eps0 + B2 / mu0) + values_yt["field momentum in x"] = eps0 * np.sum(Ey * Bz - Ez * By) * dV + values_yt["field momentum in y"] = eps0 * np.sum(Ez * Bx - Ex * Bz) * dV + values_yt["field momentum in z"] = eps0 * np.sum(Ex * By - Ey * Bx) * dV # Field energy in quarter of simulation domain - E2 = np.sum((Ex**2 + Ey**2 + Ez**2)*(y > 0)*(z < 0)) - B2 = np.sum((Bx**2 + By**2 + Bz**2)*(y > 0)*(z < 0)) - values_yt['field energy in quarter of simulation domain'] = 0.5 * dV * (E2 * eps0 + B2 / mu0) + E2 = np.sum((Ex**2 + Ey**2 + Ez**2) * (y > 0) * (z < 0)) + B2 = np.sum((Bx**2 + By**2 + Bz**2) * (y > 0) * (z < 0)) + values_yt["field energy in quarter of simulation domain"] = ( + 0.5 * dV * (E2 * eps0 + B2 / mu0) + ) # Max/min values of various grid quantities - values_yt['maximum of |Ex|'] = np.amax(np.abs(Ex)) - values_yt['maximum of |Ey|'] = np.amax(np.abs(Ey)) - values_yt['maximum of |Ez|'] = np.amax(np.abs(Ez)) - values_yt['maximum of |Bx|'] = np.amax(np.abs(Bx)) - values_yt['maximum of |By|'] = np.amax(np.abs(By)) - values_yt['maximum of |Bz|'] = np.amax(np.abs(Bz)) - values_yt['maximum of |E|'] = np.amax(np.sqrt(Ex**2 + Ey**2 + Ez**2)) - values_yt['maximum of |B|'] = np.amax(np.sqrt(Bx**2 + By**2 + Bz**2)) - values_yt['maximum of rho'] = np.amax(rho) - values_yt['minimum of rho'] = np.amin(rho) - values_yt['electrons: maximum of |rho|'] = np.amax(np.abs(rho_electrons)) - values_yt['protons: maximum of |rho|'] = np.amax(np.abs(rho_protons)) - values_yt['maximum of |B| from generic field reduction'] = np.amax(np.sqrt(Bx**2 + By**2 + Bz**2)) - values_yt['minimum of x*Ey*Bz'] = np.amin(x*Ey*Bz) - values_yt['maximum of Edotj'] = np.amax(Ex*jx + Ey*jy + Ez*jz) - - #-------------------------------------------------------------------------------------------------- + values_yt["maximum of |Ex|"] = np.amax(np.abs(Ex)) + values_yt["maximum of |Ey|"] = np.amax(np.abs(Ey)) + values_yt["maximum of |Ez|"] = np.amax(np.abs(Ez)) + values_yt["maximum of |Bx|"] = np.amax(np.abs(Bx)) + values_yt["maximum of |By|"] = np.amax(np.abs(By)) + values_yt["maximum of |Bz|"] = np.amax(np.abs(Bz)) + values_yt["maximum of |E|"] = np.amax(np.sqrt(Ex**2 + Ey**2 + Ez**2)) + values_yt["maximum of |B|"] = np.amax(np.sqrt(Bx**2 + By**2 + Bz**2)) + values_yt["maximum of rho"] = np.amax(rho) + values_yt["minimum of rho"] = np.amin(rho) + values_yt["electrons: maximum of |rho|"] = np.amax(np.abs(rho_electrons)) + values_yt["protons: maximum of |rho|"] = np.amax(np.abs(rho_protons)) + values_yt["maximum of |B| from generic field reduction"] = np.amax( + np.sqrt(Bx**2 + By**2 + Bz**2) + ) + values_yt["minimum of x*Ey*Bz"] = np.amin(x * Ey * Bz) + values_yt["maximum of Edotj"] = np.amax(Ex * jx + Ey * jy + Ez * jz) + + # -------------------------------------------------------------------------------------------------- # Part 2: get results from reduced diagnostics (label '_rd') - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- # Quantities computed from reduced diagnostics values_rd = dict() # Load data from output files - EFdata = np.genfromtxt('./diags/reducedfiles/EF.txt') # Field energy - EPdata = np.genfromtxt('./diags/reducedfiles/EP.txt') # Particle energy - PFdata = np.genfromtxt('./diags/reducedfiles/PF.txt') # Field momentum - PPdata = np.genfromtxt('./diags/reducedfiles/PP.txt') # Particle momentum - MFdata = np.genfromtxt('./diags/reducedfiles/MF.txt') # Field maximum - MRdata = np.genfromtxt('./diags/reducedfiles/MR.txt') # Rho maximum - NPdata = np.genfromtxt('./diags/reducedfiles/NP.txt') # Particle number - FR_Maxdata = np.genfromtxt('./diags/reducedfiles/FR_Max.txt') # Field Reduction using maximum - FR_Mindata = np.genfromtxt('./diags/reducedfiles/FR_Min.txt') # Field Reduction using minimum - FR_Integraldata = np.genfromtxt('./diags/reducedfiles/FR_Integral.txt') # Field Reduction using integral - Edotjdata = np.genfromtxt('./diags/reducedfiles/Edotj.txt') # E dot j maximum + EFdata = np.genfromtxt("./diags/reducedfiles/EF.txt") # Field energy + EPdata = np.genfromtxt("./diags/reducedfiles/EP.txt") # Particle energy + PFdata = np.genfromtxt("./diags/reducedfiles/PF.txt") # Field momentum + PPdata = np.genfromtxt("./diags/reducedfiles/PP.txt") # Particle momentum + MFdata = np.genfromtxt("./diags/reducedfiles/MF.txt") # Field maximum + MRdata = np.genfromtxt("./diags/reducedfiles/MR.txt") # Rho maximum + NPdata = np.genfromtxt("./diags/reducedfiles/NP.txt") # Particle number + FR_Maxdata = np.genfromtxt( + "./diags/reducedfiles/FR_Max.txt" + ) # Field Reduction using maximum + FR_Mindata = np.genfromtxt( + "./diags/reducedfiles/FR_Min.txt" + ) # Field Reduction using minimum + FR_Integraldata = np.genfromtxt( + "./diags/reducedfiles/FR_Integral.txt" + ) # Field Reduction using integral + Edotjdata = np.genfromtxt("./diags/reducedfiles/Edotj.txt") # E dot j maximum # First index "1" points to the values written at the last time step - values_rd['field energy'] = EFdata[1][2] - values_rd['field energy in quarter of simulation domain'] = FR_Integraldata[1][2] - values_rd['particle energy'] = EPdata[1][2] - values_rd['electrons: particle energy'] = EPdata[1][3] - values_rd['protons: particle energy'] = EPdata[1][4] - values_rd['photons: particle energy'] = EPdata[1][5] - values_rd['mean particle energy'] = EPdata[1][6] - values_rd['electrons: mean particle energy'] = EPdata[1][7] - values_rd['protons: mean particle energy'] = EPdata[1][8] - values_rd['photons: mean particle energy'] = EPdata[1][9] - values_rd['field momentum in x'] = PFdata[1][2] - values_rd['field momentum in y'] = PFdata[1][3] - values_rd['field momentum in z'] = PFdata[1][4] - values_rd['particle momentum in x'] = PPdata[1][2] - values_rd['particle momentum in y'] = PPdata[1][3] - values_rd['particle momentum in z'] = PPdata[1][4] - values_rd['electrons: particle momentum in x'] = PPdata[1][5] - values_rd['electrons: particle momentum in y'] = PPdata[1][6] - values_rd['electrons: particle momentum in z'] = PPdata[1][7] - values_rd['protons: particle momentum in x'] = PPdata[1][8] - values_rd['protons: particle momentum in y'] = PPdata[1][9] - values_rd['protons: particle momentum in z'] = PPdata[1][10] - values_rd['photons: particle momentum in x'] = PPdata[1][11] - values_rd['photons: particle momentum in y'] = PPdata[1][12] - values_rd['photons: particle momentum in z'] = PPdata[1][13] - values_rd['mean particle momentum in x'] = PPdata[1][14] - values_rd['mean particle momentum in y'] = PPdata[1][15] - values_rd['mean particle momentum in z'] = PPdata[1][16] - values_rd['electrons: mean particle momentum in x'] = PPdata[1][17] - values_rd['electrons: mean particle momentum in y'] = PPdata[1][18] - values_rd['electrons: mean particle momentum in z'] = PPdata[1][19] - values_rd['protons: mean particle momentum in x'] = PPdata[1][20] - values_rd['protons: mean particle momentum in y'] = PPdata[1][21] - values_rd['protons: mean particle momentum in z'] = PPdata[1][22] - values_rd['photons: mean particle momentum in x'] = PPdata[1][23] - values_rd['photons: mean particle momentum in y'] = PPdata[1][24] - values_rd['photons: mean particle momentum in z'] = PPdata[1][25] - values_rd['maximum of |Ex|'] = MFdata[1][2] - values_rd['maximum of |Ey|'] = MFdata[1][3] - values_rd['maximum of |Ez|'] = MFdata[1][4] - values_rd['maximum of |E|'] = MFdata[1][5] - values_rd['maximum of |Bx|'] = MFdata[1][6] - values_rd['maximum of |By|'] = MFdata[1][7] - values_rd['maximum of |Bz|'] = MFdata[1][8] - values_rd['maximum of |B|'] = MFdata[1][9] - values_rd['maximum of rho'] = MRdata[1][2] - values_rd['minimum of rho'] = MRdata[1][3] - values_rd['electrons: maximum of |rho|'] = MRdata[1][4] - values_rd['protons: maximum of |rho|'] = MRdata[1][5] - values_rd['number of particles'] = NPdata[1][2] - values_rd['electrons: number of particles'] = NPdata[1][3] - values_rd['protons: number of particles'] = NPdata[1][4] - values_rd['photons: number of particles'] = NPdata[1][5] - values_rd['sum of weights'] = NPdata[1][6] - values_rd['electrons: sum of weights'] = NPdata[1][7] - values_rd['protons: sum of weights'] = NPdata[1][8] - values_rd['photons: sum of weights'] = NPdata[1][9] - values_rd['maximum of |B| from generic field reduction'] = FR_Maxdata[1][2] - values_rd['minimum of x*Ey*Bz'] = FR_Mindata[1][2] - values_rd['maximum of Edotj'] = Edotjdata[1][2] - - #-------------------------------------------------------------------------------------------------- + values_rd["field energy"] = EFdata[1][2] + values_rd["field energy in quarter of simulation domain"] = FR_Integraldata[1][2] + values_rd["particle energy"] = EPdata[1][2] + values_rd["electrons: particle energy"] = EPdata[1][3] + values_rd["protons: particle energy"] = EPdata[1][4] + values_rd["photons: particle energy"] = EPdata[1][5] + values_rd["mean particle energy"] = EPdata[1][6] + values_rd["electrons: mean particle energy"] = EPdata[1][7] + values_rd["protons: mean particle energy"] = EPdata[1][8] + values_rd["photons: mean particle energy"] = EPdata[1][9] + values_rd["field momentum in x"] = PFdata[1][2] + values_rd["field momentum in y"] = PFdata[1][3] + values_rd["field momentum in z"] = PFdata[1][4] + values_rd["particle momentum in x"] = PPdata[1][2] + values_rd["particle momentum in y"] = PPdata[1][3] + values_rd["particle momentum in z"] = PPdata[1][4] + values_rd["electrons: particle momentum in x"] = PPdata[1][5] + values_rd["electrons: particle momentum in y"] = PPdata[1][6] + values_rd["electrons: particle momentum in z"] = PPdata[1][7] + values_rd["protons: particle momentum in x"] = PPdata[1][8] + values_rd["protons: particle momentum in y"] = PPdata[1][9] + values_rd["protons: particle momentum in z"] = PPdata[1][10] + values_rd["photons: particle momentum in x"] = PPdata[1][11] + values_rd["photons: particle momentum in y"] = PPdata[1][12] + values_rd["photons: particle momentum in z"] = PPdata[1][13] + values_rd["mean particle momentum in x"] = PPdata[1][14] + values_rd["mean particle momentum in y"] = PPdata[1][15] + values_rd["mean particle momentum in z"] = PPdata[1][16] + values_rd["electrons: mean particle momentum in x"] = PPdata[1][17] + values_rd["electrons: mean particle momentum in y"] = PPdata[1][18] + values_rd["electrons: mean particle momentum in z"] = PPdata[1][19] + values_rd["protons: mean particle momentum in x"] = PPdata[1][20] + values_rd["protons: mean particle momentum in y"] = PPdata[1][21] + values_rd["protons: mean particle momentum in z"] = PPdata[1][22] + values_rd["photons: mean particle momentum in x"] = PPdata[1][23] + values_rd["photons: mean particle momentum in y"] = PPdata[1][24] + values_rd["photons: mean particle momentum in z"] = PPdata[1][25] + values_rd["maximum of |Ex|"] = MFdata[1][2] + values_rd["maximum of |Ey|"] = MFdata[1][3] + values_rd["maximum of |Ez|"] = MFdata[1][4] + values_rd["maximum of |E|"] = MFdata[1][5] + values_rd["maximum of |Bx|"] = MFdata[1][6] + values_rd["maximum of |By|"] = MFdata[1][7] + values_rd["maximum of |Bz|"] = MFdata[1][8] + values_rd["maximum of |B|"] = MFdata[1][9] + values_rd["maximum of rho"] = MRdata[1][2] + values_rd["minimum of rho"] = MRdata[1][3] + values_rd["electrons: maximum of |rho|"] = MRdata[1][4] + values_rd["protons: maximum of |rho|"] = MRdata[1][5] + values_rd["number of particles"] = NPdata[1][2] + values_rd["electrons: number of particles"] = NPdata[1][3] + values_rd["protons: number of particles"] = NPdata[1][4] + values_rd["photons: number of particles"] = NPdata[1][5] + values_rd["sum of weights"] = NPdata[1][6] + values_rd["electrons: sum of weights"] = NPdata[1][7] + values_rd["protons: sum of weights"] = NPdata[1][8] + values_rd["photons: sum of weights"] = NPdata[1][9] + values_rd["maximum of |B| from generic field reduction"] = FR_Maxdata[1][2] + values_rd["minimum of x*Ey*Bz"] = FR_Mindata[1][2] + values_rd["maximum of Edotj"] = Edotjdata[1][2] + + # -------------------------------------------------------------------------------------------------- # Part 3: compare values from plotfiles and reduced diagnostics and print output - #-------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- error = dict() tolerance = 5e-3 if single_precision else 1e-12 @@ -303,15 +364,10 @@ def do_analysis(single_precision = False): # while the field energy from the reduced diagnostics is computed from (Yee) staggered data. for k in values_yt.keys(): print() - print('values_yt[' + k + '] = ', values_yt[k]) - print('values_rd[' + k + '] = ', values_rd[k]) + print("values_yt[" + k + "] = ", values_yt[k]) + print("values_rd[" + k + "] = ", values_rd[k]) error[k] = abs(values_yt[k] - values_rd[k]) / abs(values_yt[k]) - print('relative error = ', error[k]) - tol = field_energy_tolerance if (k == 'field energy') else tolerance - assert(error[k] < tol) + print("relative error = ", error[k]) + tol = field_energy_tolerance if (k == "field energy") else tolerance + assert error[k] < tol print() - - test_name = os.path.split(os.getcwd())[1] - - checksum_rtol = 2e-9 if single_precision else 1e-9 - checksumAPI.evaluate_checksum(test_name, fn, rtol=checksum_rtol) diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_load_balance_costs.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_load_balance_costs.py new file mode 100755 index 00000000000..978b1fcd4ec --- /dev/null +++ b/Examples/Tests/reduced_diags/analysis_reduced_diags_load_balance_costs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Michael Rowan +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the reduced diagnostics `LoadBalanceCosts`. +# The setup is a uniform plasma with electrons. +# A measure of cost (based on timers or heuristic measure from cell and particles) +# diagnostic is output in the reduced diagnostic. The efficiency (mean of cost +# per rank, normalized to the maximum cost over all ranks) extracted from the +# reduced diagnostic is compared before and after the load balance step; the test +# ensures that efficiency, measured via the reduced diagnostic, improves after +# the load balance step. + +# Possible running time: ~ 1 s + +import sys + +import numpy as np + +# Command line argument +fn = sys.argv[1] + +# Load costs data +data = np.genfromtxt("./diags/reducedfiles/LBC.txt") +data = data[:, 2:] + +# Compute the number of datafields saved per box +n_data_fields = 0 +with open("./diags/reducedfiles/LBC.txt") as f: + h = f.readlines()[0] + unique_headers = ["".join([ln for ln in w if not ln.isdigit()]) for w in h.split()][ + 2:: + ] + n_data_fields = len(set(unique_headers)) + f.close() + +# From data header, data layout is: +# [step, time, +# cost_box_0, proc_box_0, lev_box_0, i_low_box_0, j_low_box_0, k_low_box_0(, gpu_ID_box_0 if GPU run), hostname_box_0, +# cost_box_1, proc_box_1, lev_box_1, i_low_box_1, j_low_box_1, k_low_box_1(, gpu_ID_box_1 if GPU run), hostname_box_1, +# ... +# cost_box_n, proc_box_n, lev_box_n, i_low_box_n, j_low_box_n, k_low_box_n(, gpu_ID_box_n if GPU run), hostname_box_n] + + +# Function to get efficiency at an iteration i +def get_efficiency(i): + # First get the unique ranks + costs, ranks = data[i, 0::n_data_fields], data[i, 1::n_data_fields].astype(int) + rank_to_cost_map = {r: 0.0 for r in set(ranks)} + + # Compute efficiency before/after load balance and check it is improved + for c, r in zip(costs, ranks): + rank_to_cost_map[r] += c + + # Normalize the costs + efficiencies = np.array(list(rank_to_cost_map.values())) + efficiencies /= efficiencies.max() + + return efficiencies.mean() + + +# The iteration i=2 is load balanced; examine before/after load balance +efficiency_before, efficiency_after = get_efficiency(1), get_efficiency(2) +print("load balance efficiency (before load balance): ", efficiency_before) +print("load balance efficiency (after load balance): ", efficiency_after) + +# The load balanced case is expected to be more efficient +# than non-load balanced case +assert efficiency_before < efficiency_after diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py deleted file mode 100755 index a706aace1f6..00000000000 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Michael Rowan -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This script tests the reduced diagnostics `LoadBalanceCosts`. -# The setup is a uniform plasma with electrons. -# A measure of cost (based on timers or heuristic measure from cell and particles) -# diagnostic is output in the reduced diagnostic. The efficiency (mean of cost -# per rank, normalized to the maximum cost over all ranks) extracted from the -# reduced diagnostic is compared before and after the load balance step; the test -# ensures that efficiency, measured via the reduced diagnostic, improves after -# the load balance step. - -# Possible running time: ~ 1 s - -import sys - -import numpy as np - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Command line argument -fn = sys.argv[1] - -# Load costs data -data = np.genfromtxt("./diags/reducedfiles/LBC.txt") -data = data[:,2:] - -# Compute the number of datafields saved per box -n_data_fields = 0 -with open("./diags/reducedfiles/LBC.txt") as f: - h = f.readlines()[0] - unique_headers=[''.join([l for l in w if not l.isdigit()]) for w in h.split()][2::] - n_data_fields = len(set(unique_headers)) - f.close() - -# From data header, data layout is: -# [step, time, -# cost_box_0, proc_box_0, lev_box_0, i_low_box_0, j_low_box_0, k_low_box_0(, gpu_ID_box_0 if GPU run), hostname_box_0, -# cost_box_1, proc_box_1, lev_box_1, i_low_box_1, j_low_box_1, k_low_box_1(, gpu_ID_box_1 if GPU run), hostname_box_1, -# ... -# cost_box_n, proc_box_n, lev_box_n, i_low_box_n, j_low_box_n, k_low_box_n(, gpu_ID_box_n if GPU run), hostname_box_n] - -# Function to get efficiency at an iteration i -def get_efficiency(i): - # First get the unique ranks - costs, ranks = data[i,0::n_data_fields], data[i,1::n_data_fields].astype(int) - rank_to_cost_map = {r:0. for r in set(ranks)} - - # Compute efficiency before/after load balance and check it is improved - for c, r in zip(costs, ranks): - rank_to_cost_map[r] += c - - # Normalize the costs - efficiencies = np.array(list(rank_to_cost_map.values())) - efficiencies /= efficiencies.max() - - return efficiencies.mean() - -# The iteration i=2 is load balanced; examine before/after load balance -efficiency_before, efficiency_after = get_efficiency(1), get_efficiency(2) -print('load balance efficiency (before load balance): ', efficiency_before) -print('load balance efficiency (after load balance): ', efficiency_after) - -# The load balanced case is expected to be more efficient -# than non-load balanced case -assert(efficiency_before < efficiency_after) - -test_name = 'reduced_diags_loadbalancecosts_timers' -checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/reduced_diags/analysis_reduced_diags_single.py b/Examples/Tests/reduced_diags/analysis_reduced_diags_single.py deleted file mode 100755 index 0bd83854c4d..00000000000 --- a/Examples/Tests/reduced_diags/analysis_reduced_diags_single.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2021 Luca Fedeli, Yinjian Zhao -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This script tests the reduced diagnostics. -# The setup is a uniform plasma with electrons, protons and photons. -# Various particle and field quantities are written to file using the reduced diagnostics -# and compared with the corresponding quantities computed from the data in the plotfiles. - -import analysis_reduced_diags_impl as an - -an.do_analysis(single_precision = True) diff --git a/Examples/Tests/reduced_diags/inputs b/Examples/Tests/reduced_diags/inputs deleted file mode 100644 index dc0c57264ba..00000000000 --- a/Examples/Tests/reduced_diags/inputs +++ /dev/null @@ -1,145 +0,0 @@ -# Maximum number of time steps -max_step = 200 - -# number of grid points -amr.n_cell = 32 32 32 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 32 - -# Maximum level in hierarchy -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -1. -1. -1. # physical domain -geometry.prob_hi = 1. 1. 1. - -# Boundary condition -boundary.field_lo = periodic periodic periodic -boundary.field_hi = periodic periodic periodic - -# Algorithms -algo.current_deposition = esirkepov -algo.field_gathering = energy-conserving # or momentum-conserving -warpx.use_filter = 1 -algo.maxwell_solver = yee # or ckc - -# Order of particle shape factors -algo.particle_shape = 1 - -# CFL -warpx.cfl = 0.99999 - -# Particles -particles.species_names = electrons protons photons -particles.photon_species = photons - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 1 1 1 -electrons.profile = constant -electrons.density = 1.e14 # number of electrons per m^3 -electrons.momentum_distribution_type = gaussian -electrons.ux_th = 0.035 -electrons.uy_th = 0.035 -electrons.uz_th = 0.035 - -protons.charge = q_e -protons.mass = m_p -protons.injection_style = "NUniformPerCell" -protons.num_particles_per_cell_each_dim = 1 1 1 -protons.profile = constant -protons.density = 1.e14 # number of protons per m^3 -protons.momentum_distribution_type = at_rest - -photons.species_type = "photon" -photons.injection_style = "NUniformPerCell" -photons.num_particles_per_cell_each_dim = 1 1 1 -photons.profile = constant -photons.density = 1.e14 # number of protons per m^3 -photons.momentum_distribution_type = gaussian -photons.ux_th = 0.2 -photons.uy_th = 0.2 -photons.uz_th = 0.2 - -################################# -###### REDUCED DIAGS ############ -################################# -warpx.reduced_diags_names = EP NP EF PP PF MF MR FP FP_integrate FP_line FP_plane FR_Max FR_Min FR_Integral Edotj -EP.type = ParticleEnergy -EP.intervals = 200 -EF.type = FieldEnergy -EF.intervals = 200 -PP.type = ParticleMomentum -PP.intervals = 200 -PF.type = FieldMomentum -PF.intervals = 200 -MF.type = FieldMaximum -MF.intervals = 200 -FP.type = FieldProbe -FP.intervals = 200 -#The probe is placed at a cell center to match the value in the plotfile -FP.x_probe = 0.53125 -FP.y_probe = 0.53125 -FP.z_probe = 0.53125 -FP_integrate.type = FieldProbe -FP_integrate.intervals = 20 -FP_integrate.probe_geometry = Point -FP_integrate.x_probe = 0.53125 -FP_integrate.y_probe = 0.53125 -FP_integrate.z_probe = 0.53125 -FP_integrate.integrate = 1 -FP_line.type = FieldProbe -FP_line.intervals = 200 -FP_line.probe_geometry = Line -FP_line.x_probe = 0.53125 -FP_line.y_probe = 0.53125 -FP_line.z_probe = 0.53125 -FP_line.x1_probe = 0.70225 -FP_line.y1_probe = 0.70225 -FP_line.z1_probe = 0.70225 -FP_line.resolution = 100 -FP_plane.type = FieldProbe -FP_plane.intervals = 200 -FP_plane.probe_geometry = Plane -FP_plane.x_probe = 0.5 -FP_plane.y_probe = 0.5 -FP_plane.z_probe = 0.5 -FP_plane.target_normal_x = 0 -FP_plane.target_normal_y = 0 -FP_plane.target_normal_z = 1 -FP_plane.target_up_x = 0 -FP_plane.target_up_y = 1 -FP_plane.target_up_z = 0 -FP_plane.detector_radius = 0.25 -FP_plane.resolution = 10 -MR.type = RhoMaximum -MR.intervals = 200 -NP.type = ParticleNumber -NP.intervals = 200 -FR_Max.type = FieldReduction -FR_Max.intervals = 200 -FR_Max.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = sqrt(Bx**2 + By**2 + Bz**2) -FR_Max.reduction_type = Maximum -FR_Min.type = FieldReduction -FR_Min.intervals = 200 -FR_Min.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = x*Ey*Bz -FR_Min.reduction_type = Minimum -FR_Integral.type = FieldReduction -FR_Integral.intervals = 200 -FR_Integral.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = "if(y > 0 and z < 0, - 0.5*((Ex**2 + Ey**2 + Ez**2)*epsilon0+(Bx**2 + By**2 + Bz**2)/mu0), 0)" -FR_Integral.reduction_type = Integral -Edotj.type = FieldReduction -Edotj.intervals = 200 -Edotj.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = Ex*jx + Ey*jy + Ez*jz -Edotj.reduction_type = Maximum - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 200 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_protons diff --git a/Examples/Tests/reduced_diags/inputs_loadbalancecosts b/Examples/Tests/reduced_diags/inputs_base_3d similarity index 100% rename from Examples/Tests/reduced_diags/inputs_loadbalancecosts rename to Examples/Tests/reduced_diags/inputs_base_3d diff --git a/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags new file mode 100644 index 00000000000..cc1b658c27f --- /dev/null +++ b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags @@ -0,0 +1,147 @@ +# Maximum number of time steps +max_step = 200 + +# number of grid points +amr.n_cell = 32 32 32 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 32 + +# Maximum level in hierarchy +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -1. -1. -1. # physical domain +geometry.prob_hi = 1. 1. 1. + +# Boundary condition +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic + +# Algorithms +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving # or momentum-conserving +warpx.use_filter = 1 +algo.maxwell_solver = yee # or ckc + +# Order of particle shape factors +algo.particle_shape = 1 + +# CFL +warpx.cfl = 0.99999 + +# Particles +particles.species_names = electrons protons photons +particles.photon_species = photons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 1 1 1 +electrons.profile = constant +electrons.density = 1.e14 # number of electrons per m^3 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = 0.035 +electrons.uy_th = 0.035 +electrons.uz_th = 0.035 + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = 1 1 1 +protons.profile = constant +protons.density = 1.e14 # number of protons per m^3 +protons.momentum_distribution_type = at_rest + +photons.species_type = "photon" +photons.injection_style = "NUniformPerCell" +photons.num_particles_per_cell_each_dim = 1 1 1 +photons.profile = constant +photons.density = 1.e14 # number of protons per m^3 +photons.momentum_distribution_type = gaussian +photons.ux_th = 0.2 +photons.uy_th = 0.2 +photons.uz_th = 0.2 + +################################# +###### REDUCED DIAGS ############ +################################# +warpx.reduced_diags_names = EP NP EF PP PF MF PX MR FP FP_integrate FP_line FP_plane FR_Max FR_Min FR_Integral Edotj +EP.type = ParticleEnergy +EP.intervals = 200 +EF.type = FieldEnergy +EF.intervals = 200 +PP.type = ParticleMomentum +PP.intervals = 200 +PF.type = FieldMomentum +PF.intervals = 200 +MF.type = FieldMaximum +MF.intervals = 200 +PX.type = FieldPoyntingFlux +PX.intervals = 200 +FP.type = FieldProbe +FP.intervals = 200 +#The probe is placed at a cell center to match the value in the plotfile +FP.x_probe = 0.53125 +FP.y_probe = 0.53125 +FP.z_probe = 0.53125 +FP_integrate.type = FieldProbe +FP_integrate.intervals = 20 +FP_integrate.probe_geometry = Point +FP_integrate.x_probe = 0.53125 +FP_integrate.y_probe = 0.53125 +FP_integrate.z_probe = 0.53125 +FP_integrate.integrate = 1 +FP_line.type = FieldProbe +FP_line.intervals = 200 +FP_line.probe_geometry = Line +FP_line.x_probe = 0.53125 +FP_line.y_probe = 0.53125 +FP_line.z_probe = 0.53125 +FP_line.x1_probe = 0.70225 +FP_line.y1_probe = 0.70225 +FP_line.z1_probe = 0.70225 +FP_line.resolution = 100 +FP_plane.type = FieldProbe +FP_plane.intervals = 200 +FP_plane.probe_geometry = Plane +FP_plane.x_probe = 0.5 +FP_plane.y_probe = 0.5 +FP_plane.z_probe = 0.5 +FP_plane.target_normal_x = 0 +FP_plane.target_normal_y = 0 +FP_plane.target_normal_z = 1 +FP_plane.target_up_x = 0 +FP_plane.target_up_y = 1 +FP_plane.target_up_z = 0 +FP_plane.detector_radius = 0.25 +FP_plane.resolution = 10 +MR.type = RhoMaximum +MR.intervals = 200 +NP.type = ParticleNumber +NP.intervals = 200 +FR_Max.type = FieldReduction +FR_Max.intervals = 200 +FR_Max.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = sqrt(Bx**2 + By**2 + Bz**2) +FR_Max.reduction_type = Maximum +FR_Min.type = FieldReduction +FR_Min.intervals = 200 +FR_Min.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = x*Ey*Bz +FR_Min.reduction_type = Minimum +FR_Integral.type = FieldReduction +FR_Integral.intervals = 200 +FR_Integral.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = "if(y > 0 and z < 0, + 0.5*((Ex**2 + Ey**2 + Ez**2)*epsilon0+(Bx**2 + By**2 + Bz**2)/mu0), 0)" +FR_Integral.reduction_type = Integral +Edotj.type = FieldReduction +Edotj.intervals = 200 +Edotj.reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz) = Ex*jx + Ey*jy + Ez*jz +Edotj.reduction_type = Maximum + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 200 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_protons diff --git a/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_heuristic b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_heuristic new file mode 100644 index 00000000000..18777d5a1fa --- /dev/null +++ b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_heuristic @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.load_balance_costs_update = Heuristic diff --git a/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers new file mode 100644 index 00000000000..7d8586cd913 --- /dev/null +++ b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.load_balance_costs_update = Timers diff --git a/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_picmi.py b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_picmi.py new file mode 100644 index 00000000000..d1dc6935bb7 --- /dev/null +++ b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_picmi.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Number of time steps +max_steps = 3 + +# Number of cells +nx = 128 +ny = 32 +nz = 128 + +# Physical domain +xmin = 0.0 +xmax = 4.0 +ymin = 0.0 +ymax = 4.0 +zmin = 0.0 +zmax = 4.0 + +# Create grid +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=32, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["periodic", "periodic", "periodic"], + upper_boundary_conditions=["periodic", "periodic", "periodic"], +) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver(grid=grid, method="Yee", cfl=0.99999) + +# Particles +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=picmi.UniformDistribution( + density=1e14, rms_velocity=[0] * 3, upper_bound=[xmax, ymax, 1.0] + ), +) +layout = picmi.GriddedLayout(n_macroparticle_per_cell=[1, 1, 1], grid=grid) + +# Reduced diagnostic +reduced_diags = [] +reduced_diags.append( + picmi.ReducedDiagnostic(diag_type="LoadBalanceCosts", period=1, name="LBC") +) + +reduced_diags.append( + picmi.ReducedDiagnostic( + diag_type="FieldReduction", + period=1, + name="FR", + reduction_type="Maximum", + reduced_function="Ez", + ) +) + +reduced_diags.append( + picmi.ReducedDiagnostic( + diag_type="ParticleHistogram", + period=1, + name="PH", + species=electrons, + bin_number=10, + bin_min=0.0, + bin_max=xmax, + normalization="unity_particle_weight", + histogram_function="x", + ) +) + +# Diagnostic +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=3, + species=[electrons], + data_list=["ux", "uy", "uz", "x", "y", "z", "weighting"], +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=3, + data_list=["Bx", "By", "Bz", "Ex", "Ey", "Ez", "Jx", "Jy", "Jz"], +) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=1, + particle_shape=1, + warpx_current_deposition_algo="esirkepov", + warpx_field_gathering_algo="energy-conserving", + warpx_load_balance_intervals=2, + warpx_load_balance_costs_update="timers", +) + +# Add species +sim.add_species(electrons, layout=layout) + +# Add reduced diagnostics +for reduced_diag in reduced_diags: + sim.add_diagnostic(reduced_diag) + +# Add diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) + +# Advance simulation until last time step +# sim.write_input_file("test_input") +sim.step(max_steps) diff --git a/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_psatd b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_psatd new file mode 100644 index 00000000000..7d8586cd913 --- /dev/null +++ b/Examples/Tests/reduced_diags/inputs_test_3d_reduced_diags_load_balance_costs_timers_psatd @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.load_balance_costs_update = Timers diff --git a/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt b/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt new file mode 100644 index 00000000000..df73a264429 --- /dev/null +++ b/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_relativistic_space_charge_initialization # name + 3 # dims + 2 # nprocs + inputs_test_3d_relativistic_space_charge_initialization # inputs + "analysis.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles" # checksum + OFF # dependency +) diff --git a/Examples/Tests/relativistic_space_charge_initialization/analysis.py b/Examples/Tests/relativistic_space_charge_initialization/analysis.py index 569a7d2678a..beb2b889ed7 100755 --- a/Examples/Tests/relativistic_space_charge_initialization/analysis.py +++ b/Examples/Tests/relativistic_space_charge_initialization/analysis.py @@ -11,82 +11,83 @@ verifying that the space-charge field of a Gaussian beam corresponds to the expected theoretical field. """ -import os + import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc import yt yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # Parameters from the Simulation -Qtot = -1.e-20 -r0 = 2.e-6 +Qtot = -1.0e-20 +r0 = 2.0e-6 # Open data file filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) # Extract data -ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -Ex_array = ad0[('mesh','Ex')].to_ndarray().squeeze() -By_array = ad0[('mesh','By')].to_ndarray() +ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +Ex_array = ad0[("mesh", "Ex")].to_ndarray().squeeze() +By_array = ad0[("mesh", "By")].to_ndarray() # Extract grid coordinates -Nx, Ny, Nz = ds.domain_dimensions +Nx, Ny, Nz = ds.domain_dimensions xmin, ymin, zmin = ds.domain_left_edge.v Lx, Ly, Lz = ds.domain_width.v -x = xmin + Lx/Nx*(0.5+np.arange(Nx)) -y = ymin + Ly/Ny*(0.5+np.arange(Ny)) -z = zmin + Lz/Nz*(0.5+np.arange(Nz)) +x = xmin + Lx / Nx * (0.5 + np.arange(Nx)) +y = ymin + Ly / Ny * (0.5 + np.arange(Ny)) +z = zmin + Lz / Nz * (0.5 + np.arange(Nz)) # Compute theoretical field -x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') +x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing="ij") r2 = x_2d**2 + y_2d**2 -factor = Qtot/scc.epsilon_0/(2*np.pi*r2) * (1-np.exp(-r2/(2*r0**2))) -factor_z = 1./(2*np.pi)**.5/r0 * np.exp(-z_2d**2/(2*r0**2)) -Ex_th = factor*factor_z*x_2d -Ey_th = factor*factor_z*y_2d +factor = Qtot / scc.epsilon_0 / (2 * np.pi * r2) * (1 - np.exp(-r2 / (2 * r0**2))) +factor_z = 1.0 / (2 * np.pi) ** 0.5 / r0 * np.exp(-(z_2d**2) / (2 * r0**2)) +Ex_th = factor * factor_z * x_2d +Ey_th = factor * factor_z * y_2d + # Plot theory and data def make_2d(arr): if arr.ndim == 3: - return arr[:,Ny//2,:] + return arr[:, Ny // 2, :] else: return arr -plt.figure(figsize=(10,10)) + + +plt.figure(figsize=(10, 10)) plt.subplot(221) -plt.title('Ex: Theory') +plt.title("Ex: Theory") plt.imshow(make_2d(Ex_th)) plt.colorbar() plt.subplot(222) -plt.title('Ex: Simulation') +plt.title("Ex: Simulation") plt.imshow(make_2d(Ex_array)) plt.colorbar() plt.subplot(223) -plt.title('By: Theory') -plt.imshow(make_2d(Ex_th/scc.c)) +plt.title("By: Theory") +plt.imshow(make_2d(Ex_th / scc.c)) plt.colorbar() plt.subplot(224) -plt.title('By: Simulation') +plt.title("By: Simulation") plt.imshow(make_2d(By_array)) plt.colorbar() -plt.savefig('Comparison.png') +plt.savefig("Comparison.png") + # Automatically check the results def check(E, E_th, label): - print( 'Relative error in %s: %.3f'%( - label, abs(E-E_th).max()/E_th.max())) - assert np.allclose( E, E_th, atol=0.175*E_th.max() ) + print("Relative error in %s: %.3f" % (label, abs(E - E_th).max() / E_th.max())) + assert np.allclose(E, E_th, atol=0.175 * E_th.max()) -check( Ex_array, Ex_th, 'Ex' ) -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, do_particles=False) +check(Ex_array, Ex_th, "Ex") diff --git a/Examples/Tests/relativistic_space_charge_initialization/analysis_default_regression.py b/Examples/Tests/relativistic_space_charge_initialization/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/relativistic_space_charge_initialization/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d b/Examples/Tests/relativistic_space_charge_initialization/inputs_test_3d_relativistic_space_charge_initialization similarity index 100% rename from Examples/Tests/relativistic_space_charge_initialization/inputs_3d rename to Examples/Tests/relativistic_space_charge_initialization/inputs_test_3d_relativistic_space_charge_initialization diff --git a/Examples/Tests/repelling_particles/CMakeLists.txt b/Examples/Tests/repelling_particles/CMakeLists.txt new file mode 100644 index 00000000000..e5b64cb9166 --- /dev/null +++ b/Examples/Tests/repelling_particles/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_repelling_particles # name + 2 # dims + 2 # nprocs + inputs_test_2d_repelling_particles # inputs + "analysis.py diags/diag1000200" # analysis + "analysis_default_regression.py --path diags/diag1000200" # checksum + OFF # dependency +) diff --git a/Examples/Tests/repelling_particles/analysis.py b/Examples/Tests/repelling_particles/analysis.py new file mode 100755 index 00000000000..5f052361fc7 --- /dev/null +++ b/Examples/Tests/repelling_particles/analysis.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the interaction between 2 electron macroparticles. + +The macroparticles are moving towards each other and slow down due to the +repelling force between charges of the same sign. The particles should +be moving slow enough for relativistic effects to be negligible. In +this case the conservation of energy gives: + +beta(t)^2 = beta(0)^2 + 2 * w * re * log(d(0)/d(t)) + +where: +re is the classical electron radius +w is the weight of the individual macroparticles +d is the distance between them +beta is the velocity normalized by the speed of light +""" + +import glob +import re +import sys + +import matplotlib.pyplot as plt +import numpy as np +import yt +from scipy.constants import c, m_e, physical_constants + +yt.funcs.mylog.setLevel(0) + +# Check plotfile name specified in command line +last_filename = sys.argv[1] +filename_radical = re.findall(r"(.*?)\d+/*$", last_filename)[0] + +# Loop through files, and extract the position and velocity of both particles +x1 = [] +x2 = [] +beta1 = [] +beta2 = [] +for filename in sorted(glob.glob(filename_radical + "*")): + print(filename) + ds = yt.load(filename) + ad = ds.all_data() + + x1.append(float(ad[("electron1", "particle_position_x")][0])) + x2.append(float(ad[("electron2", "particle_position_x")][0])) + beta1.append(float(ad[("electron1", "particle_momentum_x")][0]) / (m_e * c)) + beta2.append(float(ad[("electron2", "particle_momentum_x")][0]) / (m_e * c)) + +# Convert to numpy array +x1 = np.array(x1) +x2 = np.array(x2) +beta1 = np.array(beta1) +beta2 = np.array(beta2) + +# Plot velocities, compare with theory +w = 5.0e12 +re = physical_constants["classical electron radius"][0] +beta_th = np.sqrt(beta1[0] ** 2 - 2 * w * re * np.log((x2[0] - x1[0]) / (x2 - x1))) +plt.plot(beta1, "+", label="Particle 1") +plt.plot(-beta2, "x", label="Particle 2") +plt.plot(beta_th, "*", label="Theory") +plt.legend(loc=0) +plt.xlabel("Time (a.u.)") +plt.ylabel("Normalized velocity") +plt.savefig("Comparison.png") + +# Check that the results are close to the theory +assert np.allclose(beta1[1:], beta_th[1:], atol=0.01) +assert np.allclose(-beta2[1:], beta_th[1:], atol=0.01) diff --git a/Examples/Tests/repelling_particles/analysis_default_regression.py b/Examples/Tests/repelling_particles/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/repelling_particles/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py deleted file mode 100755 index bda3d74d274..00000000000 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2021 Remi Lehe -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -""" -This script tests the interaction between 2 electron macroparticles. - -The macroparticles are moving towards each other and slow down due to the -repelling force between charges of the same sign. The particles should -be moving slow enough for relativistic effects to be negligible. In -this case the conservation of energy gives: - -beta(t)^2 = beta(0)^2 + 2 * w * re * log(d(0)/d(t)) - -where: -re is the classical electron radius -w is the weight of the individual macroparticles -d is the distance between them -beta is the velocity normalized by the speed of light -""" -import glob -import os -import re -import sys - -import matplotlib.pyplot as plt -import numpy as np -import yt -from scipy.constants import c, m_e, physical_constants - -yt.funcs.mylog.setLevel(0) - -# Check plotfile name specified in command line -last_filename = sys.argv[1] -filename_radical = re.findall(r'(.*?)\d+/*$', last_filename)[0] - -# Loop through files, and extract the position and velocity of both particles -x1 = [] -x2 = [] -beta1 = [] -beta2 = [] -for filename in sorted(glob.glob(filename_radical + '*')): - print(filename) - ds = yt.load(filename) - ad = ds.all_data() - - x1.append( float(ad[('electron1','particle_position_x')][0]) ) - x2.append( float(ad[('electron2','particle_position_x')][0]) ) - beta1.append( float(ad[('electron1','particle_momentum_x')][0])/(m_e*c) ) - beta2.append( float(ad[('electron2','particle_momentum_x')][0])/(m_e*c) ) - -# Convert to numpy array -x1 = np.array(x1) -x2 = np.array(x2) -beta1 = np.array(beta1) -beta2 = np.array(beta2) - -# Plot velocities, compare with theory -w = 5.e12 -re = physical_constants['classical electron radius'][0] -beta_th = np.sqrt( beta1[0]**2 - 2*w*re*np.log( (x2[0]-x1[0])/(x2-x1) ) ) -plt.plot( beta1, '+', label='Particle 1' ) -plt.plot( -beta2, 'x', label='Particle 2' ) -plt.plot( beta_th, '*', label='Theory' ) -plt.legend(loc=0) -plt.xlabel('Time (a.u.)') -plt.ylabel('Normalized velocity') -plt.savefig('Comparison.png') - -# Check that the results are close to the theory -assert np.allclose( beta1[1:], beta_th[1:], atol=0.01 ) -assert np.allclose( -beta2[1:], beta_th[1:], atol=0.01 ) - -# Run checksum regression test -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, last_filename) diff --git a/Examples/Tests/repelling_particles/inputs_2d b/Examples/Tests/repelling_particles/inputs_test_2d_repelling_particles similarity index 100% rename from Examples/Tests/repelling_particles/inputs_2d rename to Examples/Tests/repelling_particles/inputs_test_2d_repelling_particles diff --git a/Examples/Tests/resampling/CMakeLists.txt b/Examples/Tests/resampling/CMakeLists.txt new file mode 100644 index 00000000000..b6f8c5baecf --- /dev/null +++ b/Examples/Tests/resampling/CMakeLists.txt @@ -0,0 +1,32 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_resample_velocity_coincidence_thinning # name + 1 # dims + 2 # nprocs + inputs_test_1d_resample_velocity_coincidence_thinning # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000004" # checksum + OFF # dependency +) + +add_warpx_test( + test_1d_resample_velocity_coincidence_thinning_cartesian # name + 1 # dims + 2 # nprocs + inputs_test_1d_resample_velocity_coincidence_thinning_cartesian # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000004" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_leveling_thinning # name + 2 # dims + 2 # nprocs + inputs_test_2d_leveling_thinning # inputs + "analysis.py diags/diag1000008" # analysis + "analysis_default_regression.py --path diags/diag1000008" # checksum + OFF # dependency +) diff --git a/Examples/Tests/resampling/analysis.py b/Examples/Tests/resampling/analysis.py new file mode 100755 index 00000000000..8fff4a04a9c --- /dev/null +++ b/Examples/Tests/resampling/analysis.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Neil Zaim +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +## In this test, we check that leveling thinning works as expected on two simple cases. Each case +## corresponds to a different particle species. + +import sys + +import numpy as np +import yt +from scipy.special import erf + +fn_final = sys.argv[1] +fn0 = fn_final[:-4] + "0000" + +ds0 = yt.load(fn0) +ds = yt.load(fn_final) + +ad0 = ds0.all_data() +ad = ds.all_data() + +numcells = 16 * 16 +t_r = 1.3 # target ratio +relative_tol = 1.0e-13 # tolerance for machine precision errors + + +#### Tests for first species #### +# Particles are present in all simulation cells and all have the same weight + +ppc = 400.0 +numparts_init = numcells * ppc + +w0 = ad0["resampled_part1", "particle_weight"].to_ndarray() # weights before resampling +w = ad["resampled_part1", "particle_weight"].to_ndarray() # weights after resampling + +# Renormalize weights for easier calculations +w0 = w0 * ppc +w = w * ppc + +# Check that initial number of particles is indeed as expected +assert w0.shape[0] == numparts_init +# Check that all initial particles have the same weight +assert np.unique(w0).shape[0] == 1 +# Check that this weight is 1 (to machine precision) +assert abs(w0[0] - 1) < relative_tol + +# Check that the number of particles after resampling is as expected +numparts_final = w.shape[0] +expected_numparts_final = numparts_init / t_r**2 +error = np.abs(numparts_final - expected_numparts_final) +std_numparts_final = np.sqrt(numparts_init / t_r**2 * (1.0 - 1.0 / t_r**2)) +# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions +print( + "difference between expected and actual final number of particles (1st species): " + + str(error) +) +print("tolerance: " + str(5 * std_numparts_final)) +assert error < 5 * std_numparts_final + +# Check that the final weight is the same for all particles (to machine precision) and is as +# expected +final_weight = t_r**2 +assert np.amax(np.abs(w - final_weight)) < relative_tol * final_weight + + +#### Tests for second species #### +# Particles are only present in a single cell and have a gaussian weight distribution +# Using a single cell makes the analysis easier because leveling thinning is done separately in +# each cell + +ppc = 100000.0 +numparts_init = ppc + +w0 = ad0["resampled_part2", "particle_weight"].to_ndarray() # weights before resampling +w = ad["resampled_part2", "particle_weight"].to_ndarray() # weights after resampling + +# Renormalize and sort weights for easier calculations +w0 = np.sort(w0) * ppc +w = np.sort(w) * ppc + +## First we verify that the initial distribution is as expected + +# Check that the mean initial weight is as expected +mean_initial_weight = np.average(w0) +expected_mean_initial_weight = 2.0 * np.sqrt(2.0) +error = np.abs(mean_initial_weight - expected_mean_initial_weight) +expected_std_initial_weight = 1.0 / np.sqrt(2.0) +std_mean_initial_weight = expected_std_initial_weight / np.sqrt(numparts_init) +# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions +print( + "difference between expected and actual mean initial weight (2nd species): " + + str(error) +) +print("tolerance: " + str(5 * std_mean_initial_weight)) +assert error < 5 * std_mean_initial_weight + +# Check that the initial weight variance is as expected +variance_initial_weight = np.var(w0) +expected_variance_initial_weight = 0.5 +error = np.abs(variance_initial_weight - expected_variance_initial_weight) +std_variance_initial_weight = expected_variance_initial_weight * np.sqrt( + 2.0 / numparts_init +) +# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions +print( + "difference between expected and actual variance of initial weight (2nd species): " + + str(error) +) +print("tolerance: " + str(5 * std_variance_initial_weight)) + +## Next we verify that the resampling worked as expected + +# Check that the level weight value is as expected from the initial distribution +level_weight = w[0] +assert np.abs(level_weight - mean_initial_weight * t_r) < level_weight * relative_tol + +# Check that the number of particles at the level weight is the same as predicted from analytic +# calculations +numparts_leveled = np.argmax(w > level_weight) # This returns the first index for which +# w > level_weight, which thus corresponds to the number of particles at the level weight +expected_numparts_leveled = ( + numparts_init + / (2.0 * t_r) + * ( + 1 + + erf(expected_mean_initial_weight * (t_r - 1.0)) + - 1.0 + / (np.sqrt(np.pi) * expected_mean_initial_weight) + * np.exp(-((expected_mean_initial_weight * (t_r - 1.0)) ** 2)) + ) +) +error = np.abs(numparts_leveled - expected_numparts_leveled) +std_numparts_leveled = np.sqrt( + expected_numparts_leveled + - numparts_init + / np.sqrt(np.pi) + / (t_r * expected_mean_initial_weight) ** 2 + * ( + np.sqrt(np.pi) + / 4.0 + * (2.0 * expected_mean_initial_weight**2 + 1.0) + * (1.0 - erf(expected_mean_initial_weight * (t_r - 1.0))) + - 0.5 + * np.exp( + -((expected_mean_initial_weight * (t_r - 1.0)) ** 2) + * (expected_mean_initial_weight * (t_r + 1.0)) + ) + ) +) +# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions +print( + "difference between expected and actual number of leveled particles (2nd species): " + + str(error) +) +print("tolerance: " + str(5 * std_numparts_leveled)) + +numparts_unaffected = w.shape[0] - numparts_leveled +numparts_unaffected_anticipated = w0.shape[0] - np.argmax(w0 > level_weight) +# Check that number of particles with weight higher than level weight is the same before and after +# resampling +assert numparts_unaffected == numparts_unaffected_anticipated +# Check that particles with weight higher than level weight are unaffected by resampling. +assert np.all(w[-numparts_unaffected:] == w0[-numparts_unaffected:]) diff --git a/Examples/Tests/resampling/analysis_default_regression.py b/Examples/Tests/resampling/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/resampling/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/resampling/analysis_leveling_thinning.py b/Examples/Tests/resampling/analysis_leveling_thinning.py deleted file mode 100755 index 5f3dc8ecdff..00000000000 --- a/Examples/Tests/resampling/analysis_leveling_thinning.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Neil Zaim -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -## In this test, we check that leveling thinning works as expected on two simple cases. Each case -## corresponds to a different particle species. - -import os -import sys - -import numpy as np -import yt -from scipy.special import erf - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -fn_final = sys.argv[1] -fn0 = fn_final[:-4] + '0000' - -ds0 = yt.load(fn0) -ds = yt.load(fn_final) - -ad0 = ds0.all_data() -ad = ds.all_data() - -numcells = 16*16 -t_r = 1.3 # target ratio -relative_tol = 1.e-13 # tolerance for machine precision errors - - -#### Tests for first species #### -# Particles are present in all simulation cells and all have the same weight - -ppc = 400. -numparts_init = numcells*ppc - -w0 = ad0['resampled_part1','particle_weight'].to_ndarray() # weights before resampling -w = ad['resampled_part1','particle_weight'].to_ndarray() # weights after resampling - -# Renormalize weights for easier calculations -w0 = w0*ppc -w = w*ppc - -# Check that initial number of particles is indeed as expected -assert(w0.shape[0] == numparts_init) -# Check that all initial particles have the same weight -assert(np.unique(w0).shape[0] == 1) -# Check that this weight is 1 (to machine precision) -assert(abs(w0[0] - 1) < relative_tol) - -# Check that the number of particles after resampling is as expected -numparts_final = w.shape[0] -expected_numparts_final = numparts_init/t_r**2 -error = np.abs(numparts_final - expected_numparts_final) -std_numparts_final = np.sqrt(numparts_init/t_r**2*(1.-1./t_r**2)) -# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions -print("difference between expected and actual final number of particles (1st species): " + str(error)) -print("tolerance: " + str(5*std_numparts_final)) -assert(error<5*std_numparts_final) - -# Check that the final weight is the same for all particles (to machine precision) and is as -# expected -final_weight = t_r**2 -assert(np.amax(np.abs(w-final_weight)) < relative_tol*final_weight ) - - -#### Tests for second species #### -# Particles are only present in a single cell and have a gaussian weight distribution -# Using a single cell makes the analysis easier because leveling thinning is done separately in -# each cell - -ppc = 100000. -numparts_init = ppc - -w0 = ad0['resampled_part2','particle_weight'].to_ndarray() # weights before resampling -w = ad['resampled_part2','particle_weight'].to_ndarray() # weights after resampling - -# Renormalize and sort weights for easier calculations -w0 = np.sort(w0)*ppc -w = np.sort(w)*ppc - -## First we verify that the initial distribution is as expected - -# Check that the mean initial weight is as expected -mean_initial_weight = np.average(w0) -expected_mean_initial_weight = 2.*np.sqrt(2.) -error = np.abs(mean_initial_weight - expected_mean_initial_weight) -expected_std_initial_weight = 1./np.sqrt(2.) -std_mean_initial_weight = expected_std_initial_weight/np.sqrt(numparts_init) -# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions -print("difference between expected and actual mean initial weight (2nd species): " + str(error)) -print("tolerance: " + str(5*std_mean_initial_weight)) -assert(error<5*std_mean_initial_weight) - -# Check that the initial weight variance is as expected -variance_initial_weight = np.var(w0) -expected_variance_initial_weight = 0.5 -error = np.abs(variance_initial_weight - expected_variance_initial_weight) -std_variance_initial_weight = expected_variance_initial_weight*np.sqrt(2./numparts_init) -# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions -print("difference between expected and actual variance of initial weight (2nd species): " + str(error)) -print("tolerance: " + str(5*std_variance_initial_weight)) - -## Next we verify that the resampling worked as expected - -# Check that the level weight value is as expected from the initial distribution -level_weight = w[0] -assert(np.abs(level_weight - mean_initial_weight*t_r) < level_weight*relative_tol) - -# Check that the number of particles at the level weight is the same as predicted from analytic -# calculations -numparts_leveled = np.argmax(w > level_weight) # This returns the first index for which -# w > level_weight, which thus corresponds to the number of particles at the level weight -expected_numparts_leveled = numparts_init/(2.*t_r)*(1+erf(expected_mean_initial_weight*(t_r-1.)) \ - -1./(np.sqrt(np.pi)*expected_mean_initial_weight)* \ - np.exp(-(expected_mean_initial_weight*(t_r-1.))**2)) -error = np.abs(numparts_leveled - expected_numparts_leveled) -std_numparts_leveled = np.sqrt(expected_numparts_leveled - numparts_init/np.sqrt(np.pi)/(t_r* \ - expected_mean_initial_weight)**2*(np.sqrt(np.pi)/4.* \ - (2.*expected_mean_initial_weight**2+1.)*(1.-erf(expected_mean_initial_weight* \ - (t_r-1.)))-0.5*np.exp(-(expected_mean_initial_weight*(t_r-1.))**2* \ - (expected_mean_initial_weight*(t_r+1.))))) -# 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions -print("difference between expected and actual number of leveled particles (2nd species): " + str(error)) -print("tolerance: " + str(5*std_numparts_leveled)) - -numparts_unaffected = w.shape[0] - numparts_leveled -numparts_unaffected_anticipated = w0.shape[0] - np.argmax(w0 > level_weight) -# Check that number of particles with weight higher than level weight is the same before and after -# resampling -assert(numparts_unaffected == numparts_unaffected_anticipated) -# Check that particles with weight higher than level weight are unaffected by resampling. -assert(np.all(w[-numparts_unaffected:] == w0[-numparts_unaffected:])) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn_final) diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning b/Examples/Tests/resampling/inputs_test_1d_resample_velocity_coincidence_thinning similarity index 100% rename from Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning rename to Examples/Tests/resampling/inputs_test_1d_resample_velocity_coincidence_thinning diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian b/Examples/Tests/resampling/inputs_test_1d_resample_velocity_coincidence_thinning_cartesian similarity index 100% rename from Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian rename to Examples/Tests/resampling/inputs_test_1d_resample_velocity_coincidence_thinning_cartesian diff --git a/Examples/Tests/resampling/inputs_leveling_thinning b/Examples/Tests/resampling/inputs_test_2d_leveling_thinning similarity index 100% rename from Examples/Tests/resampling/inputs_leveling_thinning rename to Examples/Tests/resampling/inputs_test_2d_leveling_thinning diff --git a/Examples/Tests/restart/CMakeLists.txt b/Examples/Tests/restart/CMakeLists.txt new file mode 100644 index 00000000000..1be7f2d5fa7 --- /dev/null +++ b/Examples/Tests/restart/CMakeLists.txt @@ -0,0 +1,101 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_id_cpu_read_picmi # name + 2 # dims + 1 # nprocs + inputs_test_2d_id_cpu_read_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_runtime_components_picmi # name + 2 # dims + 1 # nprocs + inputs_test_2d_runtime_components_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +# FIXME +add_warpx_test( + test_2d_runtime_components_picmi_restart # name + 2 # dims + 1 # nprocs + "inputs_test_2d_runtime_components_picmi.py amr.restart='../test_2d_runtime_components_picmi/diags/chk000005'" # inputs + OFF #"analysis_default_restart.py diags/diag1000010" # analysis + OFF #"analysis_default_regression.py --path diags/diag1000010" # checksum + test_2d_runtime_components_picmi # dependency +) + +add_warpx_test( + test_3d_acceleration # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_acceleration_restart # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration_restart # inputs + "analysis_default_restart.py diags/diag1000010" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + test_3d_acceleration # dependency +) + +if(WarpX_FFT) + add_warpx_test( + test_3d_acceleration_psatd # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration_psatd # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_acceleration_psatd_restart # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration_psatd_restart # inputs + "analysis_default_restart.py diags/diag1000010" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + test_3d_acceleration_psatd # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_acceleration_psatd_time_avg # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration_psatd_time_avg # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_acceleration_psatd_time_avg_restart # name + 3 # dims + 2 # nprocs + inputs_test_3d_acceleration_psatd_time_avg_restart # inputs + "analysis_default_restart.py diags/diag1000010" # analysis + "analysis_default_regression.py --path diags/diag1000010" # checksum + test_3d_acceleration_psatd_time_avg # dependency + ) +endif() diff --git a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py deleted file mode 100755 index d400924a378..00000000000 --- a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python3 -# -# This is a script that makes sure particle ids and cpus can be read correctly -# - -import sys - -import numpy as np - -from pywarpx import callbacks, particle_containers, picmi - -########################## -# physics parameters -########################## - -dt = 7.5e-10 - -########################## -# numerics parameters -########################## - -max_steps = 10 - -nx = 64 -ny = 64 - -xmin = 0 -xmax = 0.03 -ymin = 0 -ymax = 0.03 - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - moving_window_velocity = None, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6, - warpx_self_fields_verbosity=0 -) - -########################## -# physics components -########################## - -electrons = picmi.Species( - particle_type='electron', name='electrons' -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - write_dir = '.', - warpx_file_prefix = f'Python_restart_runtime_components_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 10, - data_list = ['phi'], - write_dir = '.', - warpx_file_prefix = f'Python_restart_runtime_components_plt' -) - -checkpoint = picmi.Checkpoint( - name = 'chkpoint', - period = 5, - write_dir = '.', - warpx_file_min_digits = 5, - warpx_file_prefix = f'Python_restart_runtime_components_chk' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - verbose = 1 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[0, 0], grid=grid - ) -) - -for arg in sys.argv: - if arg.startswith("amr.restart"): - restart_file_name = arg.split("=")[1] - sim.amr_restart = restart_file_name - sys.argv.remove(arg) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) -sim.add_diagnostic(checkpoint) -sim.initialize_inputs() -sim.initialize_warpx() - -########################## -# python particle data access -########################## - -# set numpy random seed so that the particle properties generated -# below will be reproducible from run to run -np.random.seed(30025025) - -# wrap the electrons particle container -electron_wrapper = particle_containers.ParticleContainerWrapper('electrons') -electron_wrapper.add_real_comp('newPid') - -def add_particles(): - - nps = 10 - x = np.linspace(0.005, 0.025, nps) - y = np.zeros(nps) - z = np.linspace(0.005, 0.025, nps) - ux = np.random.normal(loc=0, scale=1e3, size=nps) - uy = np.random.normal(loc=0, scale=1e3, size=nps) - uz = np.random.normal(loc=0, scale=1e3, size=nps) - w = np.ones(nps) * 2.0 - newPid = 5.0 - - electron_wrapper.add_particles( - x=x, y=y, z=z, ux=ux, uy=uy, uz=uz, - w=w, newPid=newPid - ) - -callbacks.installbeforestep(add_particles) - -########################## -# simulation run -########################## - -step_number = sim.extension.warpx.getistep(lev=0) -sim.step(max_steps) - -############################################### -# check that the ids and cpus are read properly -############################################### - -assert(np.sum(np.concatenate(electron_wrapper.get_particle_id())) == 5050) -assert(np.sum(np.concatenate(electron_wrapper.get_particle_cpu())) == 0) diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py deleted file mode 100755 index 32c9f4e5808..00000000000 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -# -# This is a script that adds particle components at runtime, -# then performs checkpoint / restart and compares the result -# to the original simulation. - -import sys - -import numpy as np - -from pywarpx import callbacks, particle_containers, picmi - -########################## -# physics parameters -########################## - -dt = 7.5e-10 - -########################## -# numerics parameters -########################## - -max_steps = 10 - -nx = 64 -ny = 64 - -xmin = 0 -xmax = 0.03 -ymin = 0 -ymax = 0.03 - -########################## -# numerics components -########################## - -grid = picmi.Cartesian2DGrid( - number_of_cells = [nx, ny], - lower_bound = [xmin, ymin], - upper_bound = [xmax, ymax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], - upper_boundary_conditions_particles = ['absorbing', 'periodic'], - moving_window_velocity = None, - warpx_max_grid_size = 32 -) - -solver = picmi.ElectrostaticSolver( - grid=grid, method='Multigrid', required_precision=1e-6, - warpx_self_fields_verbosity=0 -) - -########################## -# physics components -########################## - -electrons = picmi.Species( - particle_type='electron', name='electrons' -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = 10, - write_dir = '.', - warpx_file_prefix = f'Python_restart_runtime_components_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = 10, - data_list = ['phi'], - write_dir = '.', - warpx_file_prefix = f'Python_restart_runtime_components_plt' -) - -checkpoint = picmi.Checkpoint( - name = 'chkpoint', - period = 5, - write_dir = '.', - warpx_file_min_digits = 5, - warpx_file_prefix = f'Python_restart_runtime_components_chk' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - time_step_size = dt, - max_steps = max_steps, - verbose = 1 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[0, 0], grid=grid - ) -) - -for arg in sys.argv: - if arg.startswith("amr.restart"): - restart_file_name = arg.split("=")[1] - sim.amr_restart = restart_file_name - sys.argv.remove(arg) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) -sim.add_diagnostic(checkpoint) -sim.initialize_inputs() -sim.initialize_warpx() - -########################## -# python particle data access -########################## - -# set numpy random seed so that the particle properties generated -# below will be reproducible from run to run -np.random.seed(30025025) - -electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") -electron_wrapper.add_real_comp("newPid") - -def add_particles(): - - nps = 10 - x = np.linspace(0.005, 0.025, nps) - y = np.zeros(nps) - z = np.linspace(0.005, 0.025, nps) - ux = np.random.normal(loc=0, scale=1e3, size=nps) - uy = np.random.normal(loc=0, scale=1e3, size=nps) - uz = np.random.normal(loc=0, scale=1e3, size=nps) - w = np.ones(nps) * 2.0 - newPid = 5.0 - - electron_wrapper.add_particles( - x=x, y=y, z=z, ux=ux, uy=uy, uz=uz, - w=w, newPid=newPid - ) - -callbacks.installbeforestep(add_particles) - -########################## -# simulation run -########################## - -step_number = sim.extension.warpx.getistep(lev=0) -sim.step(max_steps - 1 - step_number) - -########################## -# check that the new PIDs are properly set -########################## - -assert electron_wrapper.nps == 90 -assert electron_wrapper.particle_container.get_comp_index("w") == 2 -assert electron_wrapper.particle_container.get_comp_index("newPid") == 6 - -new_pid_vals = electron_wrapper.get_particle_real_arrays("newPid", 0) -for vals in new_pid_vals: - assert np.allclose(vals, 5) - -########################## -# take the final sim step -########################## - -sim.step(1) diff --git a/Examples/Tests/restart/analysis_default_regression.py b/Examples/Tests/restart/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/restart/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/restart/analysis_default_restart.py b/Examples/Tests/restart/analysis_default_restart.py new file mode 120000 index 00000000000..0459986eebc --- /dev/null +++ b/Examples/Tests/restart/analysis_default_restart.py @@ -0,0 +1 @@ +../../analysis_default_restart.py \ No newline at end of file diff --git a/Examples/Tests/restart/analysis_restart.py b/Examples/Tests/restart/analysis_restart.py deleted file mode 100755 index 1a5b1374672..00000000000 --- a/Examples/Tests/restart/analysis_restart.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] - -# Check restart data v. original data -sys.path.insert(0, '../../../../warpx/Examples/') -from analysis_default_restart import check_restart - -check_restart(filename) - -# Check-sum analysis -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/restart/inputs b/Examples/Tests/restart/inputs deleted file mode 100644 index f6aef120466..00000000000 --- a/Examples/Tests/restart/inputs +++ /dev/null @@ -1,164 +0,0 @@ -################################# -####### GENERAL PARAMETERS ###### -################################# -#amr.restart = diags/chk00005/ -max_step = 10 -amr.n_cell = 32 32 256 -amr.max_grid_size = 64 -amr.blocking_factor = 32 -amr.max_level = 0 -geometry.dims = 3 -geometry.prob_lo = -0.00015 -0.00015 -0.00012 -geometry.prob_hi = 0.00015 0.00015 1.e-06 - -################################# -####### Boundary condition ###### -################################# -boundary.field_lo = periodic periodic pml -boundary.field_hi = periodic periodic pml - -################################# -############ NUMERICS ########### -################################# -algo.maxwell_solver = ckc -warpx.verbose = 1 -warpx.do_dive_cleaning = 0 -warpx.use_filter = 1 -warpx.cfl = .99 -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1. # in units of the speed of light -my_constants.dens = 1e+23 - -# Order of particle shape factors -algo.particle_shape = 3 - -################################# -######### BOOSTED FRAME ######### -################################# -warpx.gamma_boost = 10.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = driver plasma_e plasma_p beam driverback -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = driver beam - -driver.charge = -q_e -driver.mass = 1.e10 -driver.injection_style = "gaussian_beam" -driver.x_rms = 2.e-6 -driver.y_rms = 2.e-6 -driver.z_rms = 4.e-6 -driver.x_m = 0. -driver.y_m = 0. -driver.z_m = -20.e-6 -driver.npart = 1000 -driver.q_tot = -1.e-9 -driver.momentum_distribution_type = "gaussian" -driver.ux_m = 0.0 -driver.uy_m = 0.0 -driver.uz_m = 200000. -driver.ux_th = 2. -driver.uy_th = 2. -driver.uz_th = 20000. -driver.zinject_plane = 0. -driver.rigid_advance = true - -driverback.charge = q_e -driverback.mass = 1.e10 -driverback.injection_style = "gaussian_beam" -driverback.x_rms = 2.e-6 -driverback.y_rms = 2.e-6 -driverback.z_rms = 4.e-6 -driverback.x_m = 0. -driverback.y_m = 0. -driverback.z_m = -20.e-6 -driverback.npart = 1000 -driverback.q_tot = 1.e-9 -driverback.momentum_distribution_type = "gaussian" -driverback.ux_m = 0.0 -driverback.uy_m = 0.0 -driverback.uz_m = 200000. -driverback.ux_th = 2. -driverback.uy_th = 2. -driverback.uz_th = 20000. -driverback.do_backward_propagation = true - -plasma_e.charge = -q_e -plasma_e.mass = m_e -plasma_e.injection_style = "NUniformPerCell" -plasma_e.zmin = -100.e-6 -plasma_e.zmax = 0.2 -plasma_e.xmin = -100.e-6 -plasma_e.xmax = 100.e-6 -plasma_e.ymin = -100.e-6 -plasma_e.ymax = 100.e-6 -plasma_e.profile = "predefined" -plasma_e.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -plasma_e.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens -plasma_e.num_particles_per_cell_each_dim = 1 1 1 -plasma_e.momentum_distribution_type = "at_rest" -plasma_e.do_continuous_injection = 1 - -plasma_p.charge = q_e -plasma_p.mass = m_p -plasma_p.injection_style = "NUniformPerCell" -plasma_p.zmin = -100.e-6 -plasma_p.zmax = 0.2 -plasma_p.profile = "predefined" -plasma_p.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -plasma_p.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens -plasma_p.xmin = -100.e-6 -plasma_p.xmax = 100.e-6 -plasma_p.ymin = -100.e-6 -plasma_p.ymax = 100.e-6 -plasma_p.num_particles_per_cell_each_dim = 1 1 1 -plasma_p.momentum_distribution_type = "at_rest" -plasma_p.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = .5e-6 -beam.y_rms = .5e-6 -beam.z_rms = 1.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -100.e-6 -beam.npart = 1000 -beam.q_tot = -5.e-10 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0.0 -beam.uy_m = 0.0 -beam.uz_m = 2000. -beam.ux_th = 2. -beam.uy_th = 2. -beam.uz_th = 200. -beam.zinject_plane = .8e-3 -beam.rigid_advance = true - -lasers.names = laser1 -laser1.profile = Gaussian -laser1.position = 0. 0. 0.e-6 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 1. 0. 0. # The main polarization vector -laser1.e_max = 16.e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 40.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 30.e-15 # The duration of the laser (in seconds) -laser1.profile_t_peak = 60.e-15 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 0.e-6 # Focal distance from the antenna (in meters) -laser1.wavelength = 5.e-6 # The wavelength of the laser (in meters) - -# Diagnostics -diagnostics.diags_names = diag1 chk -diag1.intervals = 5 -diag1.diag_type = Full -diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho -chk.intervals = 5 -chk.diag_type = Full -chk.format = checkpoint diff --git a/Examples/Tests/restart/inputs_base_3d b/Examples/Tests/restart/inputs_base_3d new file mode 100644 index 00000000000..6724bae64b3 --- /dev/null +++ b/Examples/Tests/restart/inputs_base_3d @@ -0,0 +1,163 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 10 +amr.n_cell = 32 32 256 +amr.max_grid_size = 64 +amr.blocking_factor = 32 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.00015 -0.00015 -0.00012 +geometry.prob_hi = 0.00015 0.00015 1.e-06 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic pml +boundary.field_hi = periodic periodic pml + +################################# +############ NUMERICS ########### +################################# +algo.maxwell_solver = ckc +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 1 +warpx.cfl = .99 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1. # in units of the speed of light +my_constants.dens = 1e+23 + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +######### BOOSTED FRAME ######### +################################# +warpx.gamma_boost = 10.0 +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +particles.species_names = driver plasma_e plasma_p beam driverback +particles.use_fdtd_nci_corr = 1 +particles.rigid_injected_species = driver beam + +driver.charge = -q_e +driver.mass = 1.e10 +driver.injection_style = "gaussian_beam" +driver.x_rms = 2.e-6 +driver.y_rms = 2.e-6 +driver.z_rms = 4.e-6 +driver.x_m = 0. +driver.y_m = 0. +driver.z_m = -20.e-6 +driver.npart = 1000 +driver.q_tot = -1.e-9 +driver.momentum_distribution_type = "gaussian" +driver.ux_m = 0.0 +driver.uy_m = 0.0 +driver.uz_m = 200000. +driver.ux_th = 2. +driver.uy_th = 2. +driver.uz_th = 20000. +driver.zinject_plane = 0. +driver.rigid_advance = true + +driverback.charge = q_e +driverback.mass = 1.e10 +driverback.injection_style = "gaussian_beam" +driverback.x_rms = 2.e-6 +driverback.y_rms = 2.e-6 +driverback.z_rms = 4.e-6 +driverback.x_m = 0. +driverback.y_m = 0. +driverback.z_m = -20.e-6 +driverback.npart = 1000 +driverback.q_tot = 1.e-9 +driverback.momentum_distribution_type = "gaussian" +driverback.ux_m = 0.0 +driverback.uy_m = 0.0 +driverback.uz_m = 200000. +driverback.ux_th = 2. +driverback.uy_th = 2. +driverback.uz_th = 20000. +driverback.do_backward_propagation = true + +plasma_e.charge = -q_e +plasma_e.mass = m_e +plasma_e.injection_style = "NUniformPerCell" +plasma_e.zmin = -100.e-6 +plasma_e.zmax = 0.2 +plasma_e.xmin = -100.e-6 +plasma_e.xmax = 100.e-6 +plasma_e.ymin = -100.e-6 +plasma_e.ymax = 100.e-6 +plasma_e.profile = "predefined" +plasma_e.predefined_profile_name = "parabolic_channel" +# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 +plasma_e.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens +plasma_e.num_particles_per_cell_each_dim = 1 1 1 +plasma_e.momentum_distribution_type = "at_rest" +plasma_e.do_continuous_injection = 1 + +plasma_p.charge = q_e +plasma_p.mass = m_p +plasma_p.injection_style = "NUniformPerCell" +plasma_p.zmin = -100.e-6 +plasma_p.zmax = 0.2 +plasma_p.profile = "predefined" +plasma_p.predefined_profile_name = "parabolic_channel" +# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 +plasma_p.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens +plasma_p.xmin = -100.e-6 +plasma_p.xmax = 100.e-6 +plasma_p.ymin = -100.e-6 +plasma_p.ymax = 100.e-6 +plasma_p.num_particles_per_cell_each_dim = 1 1 1 +plasma_p.momentum_distribution_type = "at_rest" +plasma_p.do_continuous_injection = 1 + +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.x_rms = .5e-6 +beam.y_rms = .5e-6 +beam.z_rms = 1.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = -100.e-6 +beam.npart = 1000 +beam.q_tot = -5.e-10 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 2000. +beam.ux_th = 2. +beam.uy_th = 2. +beam.uz_th = 200. +beam.zinject_plane = .8e-3 +beam.rigid_advance = true + +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. 0.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 1. 0. 0. # The main polarization vector +laser1.e_max = 16.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 40.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 30.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 60.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 0.e-6 # Focal distance from the antenna (in meters) +laser1.wavelength = 5.e-6 # The wavelength of the laser (in meters) + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 5 +diag1.diag_type = Full +diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho +chk.intervals = 5 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/restart/inputs_test_2d_id_cpu_read_picmi.py b/Examples/Tests/restart/inputs_test_2d_id_cpu_read_picmi.py new file mode 100755 index 00000000000..be6e621653f --- /dev/null +++ b/Examples/Tests/restart/inputs_test_2d_id_cpu_read_picmi.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# This is a script that makes sure particle ids and cpus can be read correctly +# + +import sys + +import numpy as np + +from pywarpx import callbacks, particle_containers, picmi + +########################## +# physics parameters +########################## + +dt = 7.5e-10 + +########################## +# numerics parameters +########################## + +max_steps = 10 + +nx = 64 +ny = 64 + +xmin = 0 +xmax = 0.03 +ymin = 0 +ymax = 0.03 + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + moving_window_velocity=None, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-6, + warpx_self_fields_verbosity=0, +) + +########################## +# physics components +########################## + +electrons = picmi.Species(particle_type="electron", name="electrons") + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=["phi"], +) + +checkpoint = picmi.Checkpoint( + name="chkpoint", + period=5, + write_dir=".", + warpx_file_min_digits=5, + warpx_file_prefix="Python_restart_runtime_components_chk", +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver=solver, time_step_size=dt, max_steps=max_steps, verbose=1) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[0, 0], grid=grid) +) + +for arg in sys.argv: + if arg.startswith("amr.restart"): + restart_file_name = arg.split("=")[1] + sim.amr_restart = restart_file_name + sys.argv.remove(arg) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(checkpoint) +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# python particle data access +########################## + +# set numpy random seed so that the particle properties generated +# below will be reproducible from run to run +np.random.seed(30025025) + +# wrap the electrons particle container +electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") +electron_wrapper.add_real_comp("newPid") + + +def add_particles(): + nps = 10 + x = np.linspace(0.005, 0.025, nps) + y = np.zeros(nps) + z = np.linspace(0.005, 0.025, nps) + ux = np.random.normal(loc=0, scale=1e3, size=nps) + uy = np.random.normal(loc=0, scale=1e3, size=nps) + uz = np.random.normal(loc=0, scale=1e3, size=nps) + w = np.ones(nps) * 2.0 + newPid = 5.0 + + electron_wrapper.add_particles( + x=x, y=y, z=z, ux=ux, uy=uy, uz=uz, w=w, newPid=newPid + ) + + +callbacks.installbeforestep(add_particles) + +########################## +# simulation run +########################## + +step_number = sim.extension.warpx.getistep(lev=0) +sim.step(max_steps) + +############################################### +# check that the ids and cpus are read properly +############################################### + +assert np.sum(np.concatenate(electron_wrapper.get_particle_id())) == 5050 +assert np.sum(np.concatenate(electron_wrapper.get_particle_cpu())) == 0 diff --git a/Examples/Tests/restart/inputs_test_2d_runtime_components_picmi.py b/Examples/Tests/restart/inputs_test_2d_runtime_components_picmi.py new file mode 100755 index 00000000000..746dff27a42 --- /dev/null +++ b/Examples/Tests/restart/inputs_test_2d_runtime_components_picmi.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# +# This is a script that adds particle components at runtime, +# then performs checkpoint / restart and compares the result +# to the original simulation. + +import sys + +import numpy as np + +from pywarpx import callbacks, particle_containers, picmi + +########################## +# physics parameters +########################## + +dt = 7.5e-10 + +########################## +# numerics parameters +########################## + +max_steps = 10 + +nx = 64 +ny = 64 + +xmin = 0 +xmax = 0.03 +ymin = 0 +ymax = 0.03 + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, ny], + lower_bound=[xmin, ymin], + upper_bound=[xmax, ymax], + lower_boundary_conditions=["dirichlet", "periodic"], + upper_boundary_conditions=["dirichlet", "periodic"], + lower_boundary_conditions_particles=["absorbing", "periodic"], + upper_boundary_conditions_particles=["absorbing", "periodic"], + moving_window_velocity=None, + warpx_max_grid_size=32, +) + +solver = picmi.ElectrostaticSolver( + grid=grid, + method="Multigrid", + required_precision=1e-6, + warpx_self_fields_verbosity=0, +) + +########################## +# physics components +########################## + +electrons = picmi.Species(particle_type="electron", name="electrons") + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=10, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=10, + data_list=["phi"], +) + +checkpoint = picmi.Checkpoint(name="chk", period=5) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver=solver, time_step_size=dt, max_steps=max_steps, verbose=1) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[0, 0], grid=grid) +) + +for arg in sys.argv: + if arg.startswith("amr.restart"): + restart_file_name = arg.split("=")[1] + sim.amr_restart = restart_file_name + sys.argv.remove(arg) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(checkpoint) +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# python particle data access +########################## + +# set numpy random seed so that the particle properties generated +# below will be reproducible from run to run +np.random.seed(30025025) + +electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") +if not sim.amr_restart: + electron_wrapper.add_real_comp("newPid") + + +def add_particles(): + nps = 10 + x = np.linspace(0.005, 0.025, nps) + y = np.zeros(nps) + z = np.linspace(0.005, 0.025, nps) + ux = np.random.normal(loc=0, scale=1e3, size=nps) + uy = np.random.normal(loc=0, scale=1e3, size=nps) + uz = np.random.normal(loc=0, scale=1e3, size=nps) + w = np.ones(nps) * 2.0 + newPid = 5.0 + + electron_wrapper.add_particles( + x=x, y=y, z=z, ux=ux, uy=uy, uz=uz, w=w, newPid=newPid + ) + + +callbacks.installbeforestep(add_particles) + +########################## +# simulation run +########################## + +step_number = sim.extension.warpx.getistep(lev=0) +sim.step(max_steps - 1 - step_number) + +########################## +# check that the new PIDs are properly set +########################## + +assert electron_wrapper.nps == 90 +assert electron_wrapper.particle_container.get_real_comp_index("w") == 2 +assert electron_wrapper.particle_container.get_real_comp_index("newPid") == 6 + +new_pid_vals = electron_wrapper.get_particle_real_arrays("newPid", 0) +for vals in new_pid_vals: + assert np.allclose(vals, 5) + +########################## +# take the final sim step +########################## + +sim.step(1) diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration b/Examples/Tests/restart/inputs_test_3d_acceleration new file mode 100644 index 00000000000..7665a846eef --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration @@ -0,0 +1,2 @@ +# base input parameters +FILE = inputs_base_3d diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration_psatd b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd new file mode 100644 index 00000000000..1f4e258b964 --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.maxwell_solver = psatd +boundary.field_hi = periodic periodic damped +boundary.field_lo = periodic periodic damped +particles.use_fdtd_nci_corr = 0 +psatd.current_correction = 0 +psatd.use_default_v_galilean = 1 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_restart b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_restart new file mode 100644 index 00000000000..ac2a354dcb9 --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_3d_acceleration_psatd + +# test input parameters +amr.restart = "../test_3d_acceleration_psatd/diags/chk000005" diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg new file mode 100644 index 00000000000..d9625a7f058 --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg @@ -0,0 +1,12 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +algo.maxwell_solver = psatd +boundary.field_hi = periodic periodic damped +boundary.field_lo = periodic periodic damped +particles.use_fdtd_nci_corr = 0 +psatd.current_correction = 0 +psatd.do_time_averaging = 1 +psatd.use_default_v_galilean = 1 +warpx.abort_on_warning_threshold = medium diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg_restart b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg_restart new file mode 100644 index 00000000000..44956c2259b --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration_psatd_time_avg_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_3d_acceleration_psatd_time_avg + +# test input parameters +amr.restart = "../test_3d_acceleration_psatd_time_avg/diags/chk000005" diff --git a/Examples/Tests/restart/inputs_test_3d_acceleration_restart b/Examples/Tests/restart/inputs_test_3d_acceleration_restart new file mode 100644 index 00000000000..320224f6c16 --- /dev/null +++ b/Examples/Tests/restart/inputs_test_3d_acceleration_restart @@ -0,0 +1,5 @@ +# base input parameters +FILE = inputs_test_3d_acceleration + +# test input parameters +amr.restart = "../test_3d_acceleration/diags/chk000005" diff --git a/Examples/Tests/restart_eb/CMakeLists.txt b/Examples/Tests/restart_eb/CMakeLists.txt new file mode 100644 index 00000000000..0c685340c4c --- /dev/null +++ b/Examples/Tests/restart_eb/CMakeLists.txt @@ -0,0 +1,27 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_eb_picmi # name + 3 # dims + 1 # nprocs + inputs_test_3d_eb_picmi.py # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000060" # checksum + OFF # dependency + ) +endif() + +# FIXME +#if(WarpX_EB) +# add_warpx_test( +# test_3d_eb_picmi_restart # name +# 3 # dims +# 1 # nprocs +# "inputs_test_3d_eb_picmi.py amr.restart='../test_3d_eb_picmi/diags/chk000030'" # inputs +# "analysis_default_restart.py diags/diag1000060" # analysis +# "analysis_default_regression.py --path diags/diag1000060" # checksum +# test_3d_eb_picmi # dependency +# ) +#endif() diff --git a/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py b/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py deleted file mode 100755 index a4727053334..00000000000 --- a/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -# -# Mirroring Tests/ParticleBoundaryScrape setup, this tests that restarting -# with an EB works properly. - -import sys - -from pywarpx import picmi - -########################## -# numerics parameters -########################## - -# --- Number of time steps -max_steps = 60 -diagnostic_intervals = 30 - -# --- Grid -nx = 64 -ny = 64 -nz = 128 - -cfl = 0.99 - -xmin = -125e-6 -ymin = -125e-6 -zmin = -149e-6 -xmax = 125e-6 -ymax = 125e-6 -zmax = 1e-6 - -########################## -# physics components -########################## - -uniform_plasma_elec = picmi.UniformDistribution( - density = 1e23, # number of electrons per m^3 - lower_bound = [-1e-5, -1e-5, -149e-6], - upper_bound = [1e-5, 1e-5, -129e-6], - directed_velocity = [0., 0., 2000.*picmi.constants.c] # uth the std of the (unitless) momentum -) - -electrons = picmi.Species( - particle_type='electron', name='electrons', - initial_distribution=uniform_plasma_elec, - warpx_save_particles_at_xhi=1, warpx_save_particles_at_eb=1 -) - -########################## -# numerics components -########################## - -grid = picmi.Cartesian3DGrid( - number_of_cells = [nx, ny, nz], - lower_bound = [xmin, ymin, zmin], - upper_bound = [xmax, ymax, zmax], - lower_boundary_conditions=['none', 'none', 'none'], - upper_boundary_conditions=['none', 'none', 'none'], - lower_boundary_conditions_particles=['open', 'open', 'open'], - upper_boundary_conditions_particles=['open', 'open', 'open'], - warpx_max_grid_size = 32 -) - -solver = picmi.ElectromagneticSolver( - grid=grid, cfl=cfl -) - -embedded_boundary = picmi.EmbeddedBoundary( - implicit_function="-max(max(max(x-12.5e-6,-12.5e-6-x),max(y-12.5e-6,-12.5e-6-y)),max(z-(-6.15e-5),-8.65e-5-z))" -) - -########################## -# diagnostics -########################## - -particle_diag = picmi.ParticleDiagnostic( - name = 'diag1', - period = diagnostic_intervals, - write_dir = '.', - warpx_file_prefix = 'Python_restart_eb_plt' -) -field_diag = picmi.FieldDiagnostic( - name = 'diag1', - grid = grid, - period = diagnostic_intervals, - data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz'], - write_dir = '.', - warpx_file_prefix = 'Python_restart_eb_plt' -) - -checkpoint = picmi.Checkpoint( - name = 'chkpoint', - period = diagnostic_intervals, - write_dir = '.', - warpx_file_min_digits = 5, - warpx_file_prefix = f'Python_restart_eb_chk' -) - -########################## -# simulation setup -########################## - -sim = picmi.Simulation( - solver = solver, - max_steps = max_steps, - warpx_embedded_boundary=embedded_boundary, - verbose=True, - warpx_load_balance_intervals=40, - warpx_load_balance_efficiency_ratio_threshold=0.9 -) - -sim.add_species( - electrons, - layout = picmi.GriddedLayout( - n_macroparticle_per_cell=[1, 1, 1], grid=grid - ) -) - -for arg in sys.argv: - if arg.startswith("amr.restart"): - restart_file_name = arg.split("=")[1] - sim.amr_restart = restart_file_name - sys.argv.remove(arg) - -sim.add_diagnostic(particle_diag) -sim.add_diagnostic(field_diag) -sim.add_diagnostic(checkpoint) -sim.initialize_inputs() -sim.initialize_warpx() - -########################## -# simulation run -########################## - -step_number = sim.extension.warpx.getistep(lev=0) -sim.step(max_steps - step_number) diff --git a/Examples/Tests/restart_eb/analysis_default_regression.py b/Examples/Tests/restart_eb/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/restart_eb/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/restart_eb/analysis_default_restart.py b/Examples/Tests/restart_eb/analysis_default_restart.py new file mode 120000 index 00000000000..0459986eebc --- /dev/null +++ b/Examples/Tests/restart_eb/analysis_default_restart.py @@ -0,0 +1 @@ +../../analysis_default_restart.py \ No newline at end of file diff --git a/Examples/Tests/restart_eb/inputs_test_3d_eb_picmi.py b/Examples/Tests/restart_eb/inputs_test_3d_eb_picmi.py new file mode 100755 index 00000000000..0f701ba999b --- /dev/null +++ b/Examples/Tests/restart_eb/inputs_test_3d_eb_picmi.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# +# Mirroring Tests/ParticleBoundaryScrape setup, this tests that restarting +# with an EB works properly. + +import sys + +from pywarpx import picmi + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 60 +diagnostic_intervals = 30 + +# --- Grid +nx = 64 +ny = 64 +nz = 128 + +cfl = 0.99 + +xmin = -125e-6 +ymin = -125e-6 +zmin = -149e-6 +xmax = 125e-6 +ymax = 125e-6 +zmax = 1e-6 + +########################## +# physics components +########################## + +uniform_plasma_elec = picmi.UniformDistribution( + density=1e23, # number of electrons per m^3 + lower_bound=[-1e-5, -1e-5, -149e-6], + upper_bound=[1e-5, 1e-5, -129e-6], + directed_velocity=[ + 0.0, + 0.0, + 2000.0 * picmi.constants.c, + ], # uth the std of the (unitless) momentum +) + +electrons = picmi.Species( + particle_type="electron", + name="electrons", + initial_distribution=uniform_plasma_elec, + warpx_save_particles_at_xhi=1, + warpx_save_particles_at_eb=1, +) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=["none", "none", "none"], + upper_boundary_conditions=["none", "none", "none"], + lower_boundary_conditions_particles=["open", "open", "open"], + upper_boundary_conditions_particles=["open", "open", "open"], + warpx_max_grid_size=32, +) + +solver = picmi.ElectromagneticSolver(grid=grid, cfl=cfl) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-max(max(max(x-12.5e-6,-12.5e-6-x),max(y-12.5e-6,-12.5e-6-y)),max(z-(-6.15e-5),-8.65e-5-z))" +) + +########################## +# diagnostics +########################## + +particle_diag = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_intervals, +) +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_intervals, + data_list=["Ex", "Ey", "Ez", "Bx", "By", "Bz"], +) + +checkpoint = picmi.Checkpoint( + name="chk", + period=diagnostic_intervals, +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + verbose=True, + warpx_load_balance_intervals=40, + warpx_load_balance_efficiency_ratio_threshold=0.9, +) + +sim.add_species( + electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[1, 1, 1], grid=grid) +) + +for arg in sys.argv: + if arg.startswith("amr.restart"): + restart_file_name = arg.split("=")[1] + sim.amr_restart = restart_file_name + sys.argv.remove(arg) + +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(checkpoint) +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# simulation run +########################## + +step_number = sim.extension.warpx.getistep(lev=0) +sim.step(max_steps - step_number) diff --git a/Examples/Tests/rigid_injection/CMakeLists.txt b/Examples/Tests/rigid_injection/CMakeLists.txt new file mode 100644 index 00000000000..ca0a84a87ef --- /dev/null +++ b/Examples/Tests/rigid_injection/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_rigid_injection_btd # name + 2 # dims + 2 # nprocs + inputs_test_2d_rigid_injection_btd # inputs + "analysis_rigid_injection_btd.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_rigid_injection_lab # name + 2 # dims + 2 # nprocs + inputs_test_2d_rigid_injection_lab # inputs + "analysis_rigid_injection_lab.py diags/diag1000289" # analysis + "analysis_default_regression.py --path diags/diag1000289" # checksum + OFF # dependency +) diff --git a/Examples/Tests/rigid_injection/analysis_default_regression.py b/Examples/Tests/rigid_injection/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/rigid_injection/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py deleted file mode 100755 index fd51298816b..00000000000 --- a/Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet, Revathi Jambunathan -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -''' -Analysis script of a WarpX simulation of rigid injection in a boosted frame. - -A Gaussian electron beam starts from -5 microns, propagates rigidly up to -20 microns after which it expands due to emittance only (the focal position is -20 microns). The beam width is measured after ~50 microns, and compared with -the theory (with a 1% relative error allowed). - -The simulation runs in a boosted frame, and the analysis is done in the lab -frame, i.e., on the back-transformed diagnostics. -''' - -import os -import sys - -import numpy as np -import openpmd_api as io -import yt - -yt.funcs.mylog.setLevel(0) - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] - -# Tolerances to check consistency between plotfile BTD and openPMD BTD -rtol = 1e-16 -atol = 1e-16 - -# Read data from new back-transformed diagnostics (plotfile) -ds_plotfile = yt.load(filename) -x_plotfile = ds_plotfile.all_data()['beam', 'particle_position_x'].v -z_plotfile = ds_plotfile.all_data()['beam', 'particle_position_y'].v -ux_plotfile = ds_plotfile.all_data()['beam', 'particle_momentum_x'].v -uy_plotfile = ds_plotfile.all_data()['beam', 'particle_momentum_y'].v -uz_plotfile = ds_plotfile.all_data()['beam', 'particle_momentum_z'].v - -# Read data from new back-transformed diagnostics (openPMD) -series = io.Series("./diags/diag2/openpmd_%T.h5", io.Access.read_only) -ds_openpmd = series.iterations[1] -x_openpmd = ds_openpmd.particles['beam']['position']['x'][:] -z_openpmd = ds_openpmd.particles['beam']['position']['z'][:] -ux_openpmd = ds_openpmd.particles['beam']['momentum']['x'][:] -uy_openpmd = ds_openpmd.particles['beam']['momentum']['y'][:] -uz_openpmd = ds_openpmd.particles['beam']['momentum']['z'][:] -series.flush() - -# Sort and compare arrays to check consistency between plotfile BTD and openPMD BTD -assert(np.allclose(np.sort(x_plotfile), np.sort(x_openpmd), rtol=rtol, atol=atol)) -assert(np.allclose(np.sort(z_plotfile), np.sort(z_openpmd), rtol=rtol, atol=atol)) -assert(np.allclose(np.sort(ux_plotfile), np.sort(ux_openpmd), rtol=rtol, atol=atol)) -assert(np.allclose(np.sort(uy_plotfile), np.sort(uy_openpmd), rtol=rtol, atol=atol)) -assert(np.allclose(np.sort(uz_plotfile), np.sort(uz_openpmd), rtol=rtol, atol=atol)) - -# Initial parameters -z0 = 20.e-6 -x0 = 1.e-6 -theta0 = np.arcsin(0.1) - -# Theoretical beam width after propagation with rigid injection -z = np.mean(z_plotfile) -x = np.std(x_plotfile) -print(f'Beam position = {z}') -print(f'Beam width = {x}') - -xth = np.sqrt(x0**2 + (z-z0)**2 * theta0**2) -err = np.abs((x-xth) / xth) -tol = 1e-2 -print(f'error = {err}') -print(f'tolerance = {tol}') -assert(err < tol) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py deleted file mode 100755 index ee88e32252d..00000000000 --- a/Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -''' -Analysis script of a WarpX simulation of rigid injection. - -A Gaussian electron beam starts from -5 microns, propagates rigidly up to -20 microns after which it expands due to emittance only (the focal position is -20 microns). The beam width is measured after ~50 microns, and compared with -the theory (with a 5% error allowed). - -As a help to the user, the script also compares beam width to the theory in -case rigid injection is OFF (i.e., the beam starts expanding from -5 microns), -in which case a warning is raised. - -Additionally, this script tests that runtime attributes are correctly initialized -with the gaussian_beam injection style. -''' - -import os -import sys - -import numpy as np -import yt - -yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] - -# WarpX headers include more data when rigid injection is used, -# which gives an error with the last yt release. -# To avoid this issue, the three last lines of WarpXHeader are removed if -# needed. -def remove_rigid_lines(plotfile, nlines_if_rigid): - header_name = plotfile + '/WarpXHeader' - f = open(header_name, 'r') - file_lines = f.readlines() - nlines = len(file_lines) - f.close() - if nlines == nlines_if_rigid: - f = open(header_name, 'w') - f.writelines(file_lines[:-3]) - f.close() - -# Remove rigid injection header lines -remove_rigid_lines(filename, 18) -# Read beam parameters -ds = yt.load( filename ) -ad = ds.all_data() -# Beam longitudinal position -z = np.mean(ad['beam', 'particle_position_y'].v) -# Beam width -w = np.std(ad['beam', 'particle_position_x'].v) - -# initial parameters -z0 = 20.e-6 -z0_no_rigid = -5.e-6 -w0 = 1.e-6 -theta0 = np.arcsin(0.1) - -# Theoretical beam width after propagation if rigid OFF -# Inform the user if rigid injection simply off (just to be kind) -wth_no_rigid = np.sqrt( w0**2 + (z-z0_no_rigid)**2*theta0**2 ) -error_no_rigid = np.abs((w-wth_no_rigid)/wth_no_rigid) -if ( error_no_rigid < 0.05): - print("error no rigid: " + str(error_no_rigid)) - print("Looks like the beam defocuses as if rigid injection were OFF") - -# Theoretical beam width after propagation if rigid ON -wth = np.sqrt( w0**2 + (z-z0)**2*theta0**2 ) -error_rel = np.abs((w-wth)/wth) -tolerance_rel = 0.05 - -# Print error and assert small error -print("Beam position: " + str(z)) -print("Beam width : " + str(w)) - -print("error_rel : " + str(error_rel)) -print("tolerance_rel: " + str(tolerance_rel)) - -assert( error_rel < tolerance_rel ) - - -### Check that user runtime attributes are correctly initialized -filename_start = filename[:-5] + '00000' -ds_start = yt.load( filename_start ) -ad_start = ds_start.all_data() -x = ad_start['beam', 'particle_position_x'] -z = ad_start['beam', 'particle_position_y'] -orig_z = ad_start['beam', 'particle_orig_z'] -center = ad_start['beam', 'particle_center'] -assert(np.array_equal(z, orig_z)) -assert(np.array_equal(1*(np.abs(x) < 5.e-7), center)) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_btd.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_btd.py new file mode 100755 index 00000000000..d87a680a819 --- /dev/null +++ b/Examples/Tests/rigid_injection/analysis_rigid_injection_btd.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet, Revathi Jambunathan +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +""" +Analysis script of a WarpX simulation of rigid injection in a boosted frame. + +A Gaussian electron beam starts from -5 microns, propagates rigidly up to +20 microns after which it expands due to emittance only (the focal position is +20 microns). The beam width is measured after ~50 microns, and compared with +the theory (with a 1% relative error allowed). + +The simulation runs in a boosted frame, and the analysis is done in the lab +frame, i.e., on the back-transformed diagnostics. +""" + +import sys + +import numpy as np +import openpmd_api as io +import yt + +yt.funcs.mylog.setLevel(0) + +filename = sys.argv[1] + +# Tolerances to check consistency between plotfile BTD and openPMD BTD +rtol = 1e-16 +atol = 1e-16 + +# Read data from new back-transformed diagnostics (plotfile) +ds_plotfile = yt.load(filename) +x_plotfile = ds_plotfile.all_data()["beam", "particle_position_x"].v +z_plotfile = ds_plotfile.all_data()["beam", "particle_position_y"].v +ux_plotfile = ds_plotfile.all_data()["beam", "particle_momentum_x"].v +uy_plotfile = ds_plotfile.all_data()["beam", "particle_momentum_y"].v +uz_plotfile = ds_plotfile.all_data()["beam", "particle_momentum_z"].v + +# Read data from new back-transformed diagnostics (openPMD) +series = io.Series("./diags/diag2/openpmd_%T.h5", io.Access.read_only) +ds_openpmd = series.iterations[1] +x_openpmd = ds_openpmd.particles["beam"]["position"]["x"][:] +z_openpmd = ds_openpmd.particles["beam"]["position"]["z"][:] +ux_openpmd = ds_openpmd.particles["beam"]["momentum"]["x"][:] +uy_openpmd = ds_openpmd.particles["beam"]["momentum"]["y"][:] +uz_openpmd = ds_openpmd.particles["beam"]["momentum"]["z"][:] +series.flush() + +# Sort and compare arrays to check consistency between plotfile BTD and openPMD BTD +assert np.allclose(np.sort(x_plotfile), np.sort(x_openpmd), rtol=rtol, atol=atol) +assert np.allclose(np.sort(z_plotfile), np.sort(z_openpmd), rtol=rtol, atol=atol) +assert np.allclose(np.sort(ux_plotfile), np.sort(ux_openpmd), rtol=rtol, atol=atol) +assert np.allclose(np.sort(uy_plotfile), np.sort(uy_openpmd), rtol=rtol, atol=atol) +assert np.allclose(np.sort(uz_plotfile), np.sort(uz_openpmd), rtol=rtol, atol=atol) + +# Initial parameters +z0 = 20.0e-6 +x0 = 1.0e-6 +theta0 = np.arcsin(0.1) + +# Theoretical beam width after propagation with rigid injection +z = np.mean(z_plotfile) +x = np.std(x_plotfile) +print(f"Beam position = {z}") +print(f"Beam width = {x}") + +xth = np.sqrt(x0**2 + (z - z0) ** 2 * theta0**2) +err = np.abs((x - xth) / xth) +tol = 1e-2 +print(f"error = {err}") +print(f"tolerance = {tol}") +assert err < tol diff --git a/Examples/Tests/rigid_injection/analysis_rigid_injection_lab.py b/Examples/Tests/rigid_injection/analysis_rigid_injection_lab.py new file mode 100755 index 00000000000..69fbe4cc537 --- /dev/null +++ b/Examples/Tests/rigid_injection/analysis_rigid_injection_lab.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +""" +Analysis script of a WarpX simulation of rigid injection. + +A Gaussian electron beam starts from -5 microns, propagates rigidly up to +20 microns after which it expands due to emittance only (the focal position is +20 microns). The beam width is measured after ~50 microns, and compared with +the theory (with a 5% error allowed). + +As a help to the user, the script also compares beam width to the theory in +case rigid injection is OFF (i.e., the beam starts expanding from -5 microns), +in which case a warning is raised. + +Additionally, this script tests that runtime attributes are correctly initialized +with the gaussian_beam injection style. +""" + +import sys + +import numpy as np +import yt + +yt.funcs.mylog.setLevel(0) + +filename = sys.argv[1] + + +# WarpX headers include more data when rigid injection is used, +# which gives an error with the last yt release. +# To avoid this issue, the three last lines of WarpXHeader are removed if +# needed. +def remove_rigid_lines(plotfile, nlines_if_rigid): + header_name = plotfile + "/WarpXHeader" + f = open(header_name, "r") + file_lines = f.readlines() + nlines = len(file_lines) + f.close() + if nlines == nlines_if_rigid: + f = open(header_name, "w") + f.writelines(file_lines[:-3]) + f.close() + + +# Remove rigid injection header lines +remove_rigid_lines(filename, 18) +# Read beam parameters +ds = yt.load(filename) +ad = ds.all_data() +# Beam longitudinal position +z = np.mean(ad["beam", "particle_position_y"].v) +# Beam width +w = np.std(ad["beam", "particle_position_x"].v) + +# initial parameters +z0 = 20.0e-6 +z0_no_rigid = -5.0e-6 +w0 = 1.0e-6 +theta0 = np.arcsin(0.1) + +# Theoretical beam width after propagation if rigid OFF +# Inform the user if rigid injection simply off (just to be kind) +wth_no_rigid = np.sqrt(w0**2 + (z - z0_no_rigid) ** 2 * theta0**2) +error_no_rigid = np.abs((w - wth_no_rigid) / wth_no_rigid) +if error_no_rigid < 0.05: + print("error no rigid: " + str(error_no_rigid)) + print("Looks like the beam defocuses as if rigid injection were OFF") + +# Theoretical beam width after propagation if rigid ON +wth = np.sqrt(w0**2 + (z - z0) ** 2 * theta0**2) +error_rel = np.abs((w - wth) / wth) +tolerance_rel = 0.05 + +# Print error and assert small error +print("Beam position: " + str(z)) +print("Beam width : " + str(w)) + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert error_rel < tolerance_rel + + +### Check that user runtime attributes are correctly initialized +filename_start = filename[:-5] + "00000" +ds_start = yt.load(filename_start) +ad_start = ds_start.all_data() +x = ad_start["beam", "particle_position_x"] +z = ad_start["beam", "particle_position_y"] +orig_z = ad_start["beam", "particle_orig_z"] +center = ad_start["beam", "particle_center"] +assert np.array_equal(z, orig_z) +assert np.array_equal(1 * (np.abs(x) < 5.0e-7), center) diff --git a/Examples/Tests/rigid_injection/inputs_2d_BoostedFrame b/Examples/Tests/rigid_injection/inputs_test_2d_rigid_injection_btd similarity index 100% rename from Examples/Tests/rigid_injection/inputs_2d_BoostedFrame rename to Examples/Tests/rigid_injection/inputs_test_2d_rigid_injection_btd diff --git a/Examples/Tests/rigid_injection/inputs_2d_LabFrame b/Examples/Tests/rigid_injection/inputs_test_2d_rigid_injection_lab similarity index 100% rename from Examples/Tests/rigid_injection/inputs_2d_LabFrame rename to Examples/Tests/rigid_injection/inputs_test_2d_rigid_injection_lab diff --git a/Examples/Tests/scraping/CMakeLists.txt b/Examples/Tests/scraping/CMakeLists.txt new file mode 100644 index 00000000000..71897e85b88 --- /dev/null +++ b/Examples/Tests/scraping/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_rz_scraping # name + RZ # dims + 2 # nprocs + inputs_test_rz_scraping # inputs + "analysis_rz.py diags/diag1000037" # analysis + "analysis_default_regression.py --path diags/diag1000037" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_scraping_filter # name + RZ # dims + 2 # nprocs + inputs_test_rz_scraping_filter # inputs + "analysis_rz_filter.py diags/diag1000037" # analysis + "analysis_default_regression.py --path diags/diag1000037" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/scraping/analysis_default_regression.py b/Examples/Tests/scraping/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/scraping/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 11d0194707f..c5b60350cb8 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -20,54 +20,61 @@ # tolerance: 0 # Possible running time: < 1 s -import os import sys import numpy as np import yt from openpmd_viewer import OpenPMDTimeSeries -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - tolerance = 0 fn = sys.argv[1] -ds = yt.load( fn ) +ds = yt.load(fn) ad = ds.all_data() -x = ad['electron', 'particle_position_x'].v +x = ad["electron", "particle_position_x"].v -error = len(x)-512 -print('error = ', error) -print('tolerance = ', tolerance) -assert(error==tolerance) +error = len(x) - 512 +print("error = ", error) +print("tolerance = ", tolerance) +assert error == tolerance # Check that all the removed particles are properly recorded # by making sure that, at each iteration, the sum of the number of # remaining particles and scraped particles is equal to the # original number of particles -ts_full = OpenPMDTimeSeries('./diags/diag2/') -ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') +ts_full = OpenPMDTimeSeries("./diags/diag2/") +ts_scraping = OpenPMDTimeSeries("./diags/diag3/particles_at_eb") + -def n_remaining_particles( iteration ): - w, = ts_full.get_particle(['w'], iteration=iteration) +def n_remaining_particles(iteration): + (w,) = ts_full.get_particle(["w"], iteration=iteration) return len(w) -def n_scraped_particles( iteration ): - step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + + +def n_scraped_particles(iteration): + step_scraped = ts_scraping.get_particle( + ["stepScraped"], iteration=ts_scraping.iterations[0] + ) return (step_scraped <= iteration).sum() -n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) -n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) + + +n_remaining = np.array( + [n_remaining_particles(iteration) for iteration in ts_full.iterations] +) +n_scraped = np.array( + [n_scraped_particles(iteration) for iteration in ts_full.iterations] +) n_total = n_remaining[0] -assert np.all( n_scraped+n_remaining == n_total) +assert np.all(n_scraped + n_remaining == n_total) # Check that the particle IDs match between the initial iteration # (all particles in the simulation domain) and the finall iteration (particles are either scraped or still in simulation box) -id_initial, = ts_full.get_particle(['id'], iteration=0) -id_final_scrape, = ts_scraping.get_particle(['id'], iteration=ts_scraping.iterations[0]) -id_final_box, = ts_full.get_particle(['id'], iteration=ts_full.iterations[-1]) -id_final = np.concatenate( (id_final_scrape, id_final_box)) -assert np.all( np.sort(id_initial) == np.sort(id_final) ) # Sort because particles may not be in the same order - -# Checksum test -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +(id_initial,) = ts_full.get_particle(["id"], iteration=0) +(id_final_scrape,) = ts_scraping.get_particle( + ["id"], iteration=ts_scraping.iterations[0] +) +(id_final_box,) = ts_full.get_particle(["id"], iteration=ts_full.iterations[-1]) +id_final = np.concatenate((id_final_scrape, id_final_box)) +assert np.all( + np.sort(id_initial) == np.sort(id_final) +) # Sort because particles may not be in the same order diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py index a4e0dafddbc..498f25e8422 100755 --- a/Examples/Tests/scraping/analysis_rz_filter.py +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -29,35 +29,50 @@ tolerance = 0 fn = sys.argv[1] -ds = yt.load( fn ) +ds = yt.load(fn) ad = ds.all_data() -x = ad['electron', 'particle_position_x'].v +x = ad["electron", "particle_position_x"].v -error = len(x)-512 -print('error = ', error) -print('tolerance = ', tolerance) -assert(error==tolerance) +error = len(x) - 512 +print("error = ", error) +print("tolerance = ", tolerance) +assert error == tolerance # Check that all the removed particles are properly recorded # by making sure that, at each iteration, the sum of the number of # remaining particles and scraped particles is equal to half the # original number of particles # also check that no particles with z <= 0 have been scraped -ts_full = OpenPMDTimeSeries('./diags/diag2/') -ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') +ts_full = OpenPMDTimeSeries("./diags/diag2/") +ts_scraping = OpenPMDTimeSeries("./diags/diag3/particles_at_eb") -def n_remaining_particles( iteration ): - w, = ts_full.get_particle(['w'], iteration=iteration) + +def n_remaining_particles(iteration): + (w,) = ts_full.get_particle(["w"], iteration=iteration) return len(w) -def n_scraped_particles( iteration ): - step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + + +def n_scraped_particles(iteration): + step_scraped = ts_scraping.get_particle( + ["stepScraped"], iteration=ts_scraping.iterations[0] + ) return (step_scraped <= iteration).sum() -def n_scraped_z_leq_zero( iteration ): - z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) + + +def n_scraped_z_leq_zero(iteration): + (z_pos,) = ts_scraping.get_particle(["z"], iteration=ts_scraping.iterations[0]) return (z_pos <= 0).sum() -n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) -n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) -n_z_leq_zero = np.array([ n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations ]) + + +n_remaining = np.array( + [n_remaining_particles(iteration) for iteration in ts_full.iterations] +) +n_scraped = np.array( + [n_scraped_particles(iteration) for iteration in ts_full.iterations] +) +n_z_leq_zero = np.array( + [n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations] +) n_total = n_remaining[0] -assert np.all( 2*n_scraped+n_remaining == n_total) -assert np.all( n_z_leq_zero == 0) +assert np.all(2 * n_scraped + n_remaining == n_total) +assert np.all(n_z_leq_zero == 0) diff --git a/Examples/Tests/scraping/inputs_rz b/Examples/Tests/scraping/inputs_rz deleted file mode 100644 index 0dab9ebedd2..00000000000 --- a/Examples/Tests/scraping/inputs_rz +++ /dev/null @@ -1,57 +0,0 @@ -amr.max_level = 1 -warpx.fine_tag_lo = 0.0 -0.5 -warpx.fine_tag_hi = 0.25 0.0 - -max_step = 37 - -amr.n_cell = 64 64 -amr.blocking_factor = 8 -amr.max_grid_size = 128 - -geometry.dims = RZ -geometry.prob_lo = 0.0 -0.5 -geometry.prob_hi = 0.5 0.5 - -boundary.field_lo = none periodic -boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 - -warpx.const_dt = 1.216119097e-11 -warpx.eb_implicit_function = "-(x**2-0.1**2)" - -# Do not evolve the E and B fields -algo.maxwell_solver = none -algo.field_gathering = momentum-conserving -algo.particle_shape = 1 - -particles.species_names = electron -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "NUniformPerCell" -electron.num_particles_per_cell_each_dim = 2 4 2 -electron.profile = parse_density_function -electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" -electron.momentum_distribution_type = parse_momentum_function -electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" -electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" -electron.momentum_function_uz(x,y,z) = "0" -electron.save_particles_at_eb = 1 - -diagnostics.diags_names = diag1 diag2 diag3 - -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Er - -diag2.intervals = 1 -diag2.diag_type = Full -diag2.fields_to_plot = Er -diag2.format = openpmd - -diag3.diag_type = BoundaryScraping -diag3.format = openpmd diff --git a/Examples/Tests/scraping/inputs_rz_filter b/Examples/Tests/scraping/inputs_rz_filter deleted file mode 100644 index 0d67fb96f6c..00000000000 --- a/Examples/Tests/scraping/inputs_rz_filter +++ /dev/null @@ -1,58 +0,0 @@ -amr.max_level = 1 -warpx.fine_tag_lo = 0.0 -0.5 -warpx.fine_tag_hi = 0.25 0.0 - -max_step = 37 - -amr.n_cell = 64 64 -amr.blocking_factor = 8 -amr.max_grid_size = 128 - -geometry.dims = RZ -geometry.prob_lo = 0.0 -0.5 -geometry.prob_hi = 0.5 0.5 - -boundary.field_lo = none periodic -boundary.field_hi = pec periodic -boundary.potential_lo_x = 0 -boundary.potential_hi_x = 0 -boundary.potential_lo_y = 0 -boundary.potential_hi_y = 0 -boundary.potential_lo_z = 0 -boundary.potential_hi_z = 0 - -warpx.const_dt = 1.216119097e-11 -warpx.eb_implicit_function = "-(x**2-0.1**2)" - -# Do not evolve the E and B fields -algo.maxwell_solver = none -algo.field_gathering = momentum-conserving -algo.particle_shape = 1 - -particles.species_names = electron -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "NUniformPerCell" -electron.num_particles_per_cell_each_dim = 2 4 2 -electron.profile = parse_density_function -electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" -electron.momentum_distribution_type = parse_momentum_function -electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" -electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" -electron.momentum_function_uz(x,y,z) = "0" -electron.save_particles_at_eb = 1 - -diagnostics.diags_names = diag1 diag2 diag3 - -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Er - -diag2.intervals = 1 -diag2.diag_type = Full -diag2.fields_to_plot = Er -diag2.format = openpmd - -diag3.diag_type = BoundaryScraping -diag3.format = openpmd -diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Examples/Tests/scraping/inputs_test_rz_scraping b/Examples/Tests/scraping/inputs_test_rz_scraping new file mode 100644 index 00000000000..b332de2229a --- /dev/null +++ b/Examples/Tests/scraping/inputs_test_rz_scraping @@ -0,0 +1,58 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.abort_on_warning_threshold = medium +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd diff --git a/Examples/Tests/scraping/inputs_test_rz_scraping_filter b/Examples/Tests/scraping/inputs_test_rz_scraping_filter new file mode 100644 index 00000000000..3a3ab78a226 --- /dev/null +++ b/Examples/Tests/scraping/inputs_test_rz_scraping_filter @@ -0,0 +1,59 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.abort_on_warning_threshold = medium +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd +diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Examples/Tests/secondary_ion_emission/CMakeLists.txt b/Examples/Tests/secondary_ion_emission/CMakeLists.txt new file mode 100644 index 00000000000..e6e38138a08 --- /dev/null +++ b/Examples/Tests/secondary_ion_emission/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_rz_secondary_ion_emission_picmi # name + RZ # dims + 1 # nprocs + inputs_test_rz_secondary_ion_emission_picmi.py # inputs + "analysis.py diags/diag1/" # analysis + "analysis_default_regression.py --path diags/diag1/" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/secondary_ion_emission/analysis.py b/Examples/Tests/secondary_ion_emission/analysis.py new file mode 100644 index 00000000000..8c2ed5b4af6 --- /dev/null +++ b/Examples/Tests/secondary_ion_emission/analysis.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +""" +This script checks that electron secondary emission (implemented by a callback function) works as intended. + +In this test, four ions hit a spherical embedded boundary, and produce secondary +electrons with a probability of `0.4`. We thus expect ~2 electrons to be produced. +This script tests the number of electrons emitted and checks that their position is +close to the embedded boundary. +""" + +import sys + +import numpy as np +import yt +from openpmd_viewer import OpenPMDTimeSeries + +yt.funcs.mylog.setLevel(0) + +# Open plotfile specified in command line +filename = sys.argv[1] +ts = OpenPMDTimeSeries(filename) + +it = ts.iterations +x, y, z = ts.get_particle(["x", "y", "z"], species="electrons", iteration=it[-1]) + +x_analytic = [-0.091696, 0.011599] +y_analytic = [-0.002282, -0.0111624] +z_analytic = [-0.200242, -0.201728] + +N_sec_e = np.size(z) # number of the secondary electrons + +assert N_sec_e == 2, ( + "Test did not pass: for this set up we expect 2 secondary electrons emitted" +) + +tolerance = 1e-3 + +for i in range(0, N_sec_e): + print("\n") + print(f"Electron # {i}:") + print("NUMERICAL coordinates of the emitted electrons:") + print(f"x={x[i]:5.5f}, y={y[i]:5.5f}, z={z[i]:5.5f}") + print("\n") + print("ANALYTICAL coordinates of the point of contact:") + print(f"x={x_analytic[i]:5.5f}, y={y_analytic[i]:5.5f}, z={z_analytic[i]:5.5f}") + + rel_err_x = np.abs((x[i] - x_analytic[i]) / x_analytic[i]) + rel_err_y = np.abs((y[i] - y_analytic[i]) / y_analytic[i]) + rel_err_z = np.abs((z[i] - z_analytic[i]) / z_analytic[i]) + + print("\n") + print(f"Relative percentage error for x = {rel_err_x * 100:5.4f} %") + print(f"Relative percentage error for y = {rel_err_y * 100:5.4f} %") + print(f"Relative percentage error for z = {rel_err_z * 100:5.4f} %") + + assert ( + (rel_err_x < tolerance) and (rel_err_y < tolerance) and (rel_err_z < tolerance) + ), "Test particle_boundary_interaction did not pass" diff --git a/Examples/Tests/secondary_ion_emission/analysis_default_regression.py b/Examples/Tests/secondary_ion_emission/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/secondary_ion_emission/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/secondary_ion_emission/inputs_test_rz_secondary_ion_emission_picmi.py b/Examples/Tests/secondary_ion_emission/inputs_test_rz_secondary_ion_emission_picmi.py new file mode 100644 index 00000000000..5b6248da33c --- /dev/null +++ b/Examples/Tests/secondary_ion_emission/inputs_test_rz_secondary_ion_emission_picmi.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# This is the script that tests secondary ion emission when ions hit an embedded boundary +# with a specified secondary emission yield of delta_H = 0.4. Specifically, a callback +# function at each time step ensures that the correct number of secondary electrons is +# emitted when ions impact the embedded boundary, following the given secondary emission +# model defined in sigma_nescap function. This distribution depends on the ion's energy and +# suggests that for an ion incident with 1 keV energy, an average of 0.4 secondary +# electrons will be emitted. +# Simulation is initialized with four ions with i_dist distribution and spherical +# embedded boundary given by implicit function. +import numpy as np +from scipy.constants import e, elementary_charge, m_e, proton_mass + +from pywarpx import callbacks, particle_containers, picmi + +########################## +# numerics parameters +########################## + +dt = 0.000000075 + +# --- Nb time steps +Te = 0.0259 # in eV +dist_th = np.sqrt(Te * elementary_charge / m_e) + +max_steps = 3 +diagnostic_interval = 1 + +# --- grid +nr = 64 +nz = 64 + +rmin = 0.0 +rmax = 2 +zmin = -2 +zmax = 2 +delta_H = 0.4 +E_HMax = 250 + +np.random.seed(10025015) +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + n_azimuthal_modes=1, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=["none", "dirichlet"], + upper_boundary_conditions=["dirichlet", "dirichlet"], + lower_boundary_conditions_particles=["none", "reflecting"], + upper_boundary_conditions_particles=["absorbing", "reflecting"], +) + +solver = picmi.ElectrostaticSolver( + grid=grid, method="Multigrid", warpx_absolute_tolerance=1e-7 +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-(x**2+y**2+z**2-radius**2)", radius=0.2 +) + +########################## +# physics components +########################## +i_dist = picmi.ParticleListDistribution( + x=[ + 0.025, + 0.0, + -0.1, + -0.14, + ], + y=[0.0, 0.0, 0.0, 0], + z=[-0.26, -0.29, -0.25, -0.23], + ux=[0.18e6, 0.1e6, 0.15e6, 0.21e6], + uy=[0.0, 0.0, 0.0, 0.0], + uz=[8.00e5, 7.20e5, 6.40e5, 5.60e5], + weight=[1, 1, 1, 1], +) + +electrons = picmi.Species( + particle_type="electron", # Specify the particle type + name="electrons", # Name of the species +) + +ions = picmi.Species( + name="ions", + particle_type="proton", + charge=e, + initial_distribution=i_dist, + warpx_save_particles_at_eb=1, +) + +########################## +# diagnostics +########################## + +field_diag = picmi.FieldDiagnostic( + name="diag1", + grid=grid, + period=diagnostic_interval, + data_list=["Er", "Ez", "phi", "rho"], + warpx_format="openpmd", +) + +part_diag = picmi.ParticleDiagnostic( + name="diag1", + period=diagnostic_interval, + species=[ions, electrons], + warpx_format="openpmd", +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver=solver, + time_step_size=dt, + max_steps=max_steps, + warpx_embedded_boundary=embedded_boundary, + warpx_amrex_the_arena_is_managed=1, +) + +sim.add_species( + electrons, + layout=picmi.GriddedLayout(n_macroparticle_per_cell=[0, 0, 0], grid=grid), +) + +sim.add_species( + ions, + layout=picmi.GriddedLayout(n_macroparticle_per_cell=[10, 1, 1], grid=grid), +) + +sim.add_diagnostic(part_diag) +sim.add_diagnostic(field_diag) + +sim.initialize_inputs() +sim.initialize_warpx() + +########################## +# python particle data access +########################## + + +def concat(list_of_arrays): + if len(list_of_arrays) == 0: + # Return a 1d array of size 0 + return np.empty(0) + else: + return np.concatenate(list_of_arrays) + + +def sigma_nascap(energy_kEv, delta_H, E_HMax): + """ + Compute sigma_nascap for each element in the energy array using a loop. + + Parameters: + - energy: ndarray or list, energy values in KeV + - delta_H: float, parameter for the formula + - E_HMax: float, parameter for the formula in KeV + + Returns: + - numpy array, computed probability sigma_nascap + """ + sigma_nascap = np.array([]) + # Loop through each energy value + for energy in energy_kEv: + if energy > 0.0: + sigma = ( + delta_H + * (E_HMax + 1.0) + / (E_HMax * 1.0 + energy) + * np.sqrt(energy / 1.0) + ) + else: + sigma = 0.0 + sigma_nascap = np.append(sigma_nascap, sigma) + return sigma_nascap + + +def secondary_emission(): + buffer = particle_containers.ParticleBoundaryBufferWrapper() # boundary buffer + # STEP 1: extract the different parameters of the boundary buffer (normal, time, position) + lev = 0 # level 0 (no mesh refinement here) + n = buffer.get_particle_boundary_buffer_size("ions", "eb") + elect_pc = particle_containers.ParticleContainerWrapper("electrons") + + if n != 0: + r = concat(buffer.get_particle_boundary_buffer("ions", "eb", "x", lev)) + theta = concat(buffer.get_particle_boundary_buffer("ions", "eb", "theta", lev)) + z = concat(buffer.get_particle_boundary_buffer("ions", "eb", "z", lev)) + x = r * np.cos(theta) # from RZ coordinates to 3D coordinates + y = r * np.sin(theta) + ux = concat(buffer.get_particle_boundary_buffer("ions", "eb", "ux", lev)) + uy = concat(buffer.get_particle_boundary_buffer("ions", "eb", "uy", lev)) + uz = concat(buffer.get_particle_boundary_buffer("ions", "eb", "uz", lev)) + w = concat(buffer.get_particle_boundary_buffer("ions", "eb", "w", lev)) + nx = concat(buffer.get_particle_boundary_buffer("ions", "eb", "nx", lev)) + ny = concat(buffer.get_particle_boundary_buffer("ions", "eb", "ny", lev)) + nz = concat(buffer.get_particle_boundary_buffer("ions", "eb", "nz", lev)) + delta_t = concat( + buffer.get_particle_boundary_buffer("ions", "eb", "deltaTimeScraped", lev) + ) + energy_ions = 0.5 * proton_mass * w * (ux**2 + uy**2 + uz**2) + energy_ions_in_kEv = energy_ions / (e * 1000) + sigma_nascap_ions = sigma_nascap(energy_ions_in_kEv, delta_H, E_HMax) + # Loop over all ions in the EB buffer + for i in range(0, n): + sigma = sigma_nascap_ions[i] + # Ne_sec is number of the secondary electrons to be emitted + Ne_sec = int(sigma + np.random.uniform()) + for _ in range(Ne_sec): + xe = np.array([]) + ye = np.array([]) + ze = np.array([]) + we = np.array([]) + delta_te = np.array([]) + uxe = np.array([]) + uye = np.array([]) + uze = np.array([]) + + # Random thermal momenta distribution + ux_th = np.random.normal(0, dist_th) + uy_th = np.random.normal(0, dist_th) + uz_th = np.random.normal(0, dist_th) + + un_th = nx[i] * ux_th + ny[i] * uy_th + nz[i] * uz_th + + if un_th < 0: + ux_th_reflect = ( + -2 * un_th * nx[i] + ux_th + ) # for a "mirror reflection" u(sym)=-2(u.n)n+u + uy_th_reflect = -2 * un_th * ny[i] + uy_th + uz_th_reflect = -2 * un_th * nz[i] + uz_th + + uxe = np.append(uxe, ux_th_reflect) + uye = np.append(uye, uy_th_reflect) + uze = np.append(uze, uz_th_reflect) + else: + uxe = np.append(uxe, ux_th) + uye = np.append(uye, uy_th) + uze = np.append(uze, uz_th) + + xe = np.append(xe, x[i]) + ye = np.append(ye, y[i]) + ze = np.append(ze, z[i]) + we = np.append(we, w[i]) + delta_te = np.append(delta_te, delta_t[i]) + + elect_pc.add_particles( + x=xe + (dt - delta_te) * uxe, + y=ye + (dt - delta_te) * uye, + z=ze + (dt - delta_te) * uze, + ux=uxe, + uy=uye, + uz=uze, + w=we, + ) + buffer.clear_buffer() # reinitialise the boundary buffer + + +# using the new particle container modified at the last step +callbacks.installafterstep(secondary_emission) +########################## +# simulation run +########################## +sim.step(max_steps) # the whole process is done "max_steps" times diff --git a/Examples/Tests/silver_mueller/CMakeLists.txt b/Examples/Tests/silver_mueller/CMakeLists.txt new file mode 100644 index 00000000000..6cdeeffac6f --- /dev/null +++ b/Examples/Tests/silver_mueller/CMakeLists.txt @@ -0,0 +1,42 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_silver_mueller # name + 1 # dims + 2 # nprocs + inputs_test_1d_silver_mueller # inputs + "analysis.py diags/diag1000500" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_silver_mueller_x # name + 2 # dims + 2 # nprocs + inputs_test_2d_silver_mueller_x # inputs + "analysis.py diags/diag1000500" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency +) + +add_warpx_test( + test_2d_silver_mueller_z # name + 2 # dims + 2 # nprocs + inputs_test_2d_silver_mueller_z # inputs + "analysis.py diags/diag1000500" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency +) + +add_warpx_test( + test_rz_silver_mueller_z # name + RZ # dims + 2 # nprocs + inputs_test_rz_silver_mueller_z # inputs + "analysis.py diags/diag1000500" # analysis + "analysis_default_regression.py --path diags/diag1000500" # checksum + OFF # dependency +) diff --git a/Examples/Tests/silver_mueller/analysis.py b/Examples/Tests/silver_mueller/analysis.py new file mode 100755 index 00000000000..678c5e4186a --- /dev/null +++ b/Examples/Tests/silver_mueller/analysis.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright 2021 +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +""" +This tests verifies the Silver-Mueller boundary condition. +A laser pulse is emitted and propagates towards the boundaries ; the +test check that the reflected field at the boundary is negligible. +""" + +import re +import sys + +import numpy as np +import yt + +yt.funcs.mylog.setLevel(0) + +filename = sys.argv[1] + +ds = yt.load(filename) +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +warpx_used_inputs = open("./warpx_used_inputs", "r").read() +geom_RZ = re.search("geometry.dims = RZ", warpx_used_inputs) +if geom_RZ: + Er = all_data_level_0["boxlib", "Er"].v.squeeze() + Et = all_data_level_0["boxlib", "Et"].v.squeeze() + Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() +else: + Ex = all_data_level_0["boxlib", "Ex"].v.squeeze() + Ey = all_data_level_0["boxlib", "Ey"].v.squeeze() + Ez = all_data_level_0["boxlib", "Ez"].v.squeeze() +# The peak of the initial laser pulse is on the order of 6 V/m +# Check that the amplitude after reflection is less than 0.01 V/m +max_reflection_amplitude = 0.01 + +if geom_RZ: + assert np.all(abs(Er) < max_reflection_amplitude) + assert np.all(abs(Et) < max_reflection_amplitude) + assert np.all(abs(Ez) < max_reflection_amplitude) +else: + assert np.all(abs(Ex) < max_reflection_amplitude) + assert np.all(abs(Ey) < max_reflection_amplitude) + assert np.all(abs(Ez) < max_reflection_amplitude) diff --git a/Examples/Tests/silver_mueller/analysis_default_regression.py b/Examples/Tests/silver_mueller/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/silver_mueller/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/silver_mueller/analysis_silver_mueller.py b/Examples/Tests/silver_mueller/analysis_silver_mueller.py deleted file mode 100755 index bfab40aa991..00000000000 --- a/Examples/Tests/silver_mueller/analysis_silver_mueller.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL -""" -This tests verifies the Silver-Mueller boundary condition. -A laser pulse is emitted and propagates towards the boundaries ; the -test check that the reflected field at the boundary is negligible. -""" - -import os -import re -import sys - -import numpy as np - -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -filename = sys.argv[1] - -ds = yt.load( filename ) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -warpx_used_inputs = open('./warpx_used_inputs', 'r').read() -geom_RZ = re.search('geometry.dims = RZ', warpx_used_inputs) -if geom_RZ: - Er = all_data_level_0['boxlib', 'Er'].v.squeeze() - Et = all_data_level_0['boxlib', 'Et'].v.squeeze() - Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() -else: - Ex = all_data_level_0['boxlib', 'Ex'].v.squeeze() - Ey = all_data_level_0['boxlib', 'Ey'].v.squeeze() - Ez = all_data_level_0['boxlib', 'Ez'].v.squeeze() -# The peak of the initial laser pulse is on the order of 6 V/m -# Check that the amplitude after reflection is less than 0.01 V/m -max_reflection_amplitude = 0.01 - -if geom_RZ: - assert np.all( abs(Er) < max_reflection_amplitude ) - assert np.all( abs(Et) < max_reflection_amplitude ) - assert np.all( abs(Ez) < max_reflection_amplitude ) -else: - assert np.all( abs(Ex) < max_reflection_amplitude ) - assert np.all( abs(Ey) < max_reflection_amplitude ) - assert np.all( abs(Ez) < max_reflection_amplitude ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/silver_mueller/inputs_1d b/Examples/Tests/silver_mueller/inputs_test_1d_silver_mueller similarity index 100% rename from Examples/Tests/silver_mueller/inputs_1d rename to Examples/Tests/silver_mueller/inputs_test_1d_silver_mueller diff --git a/Examples/Tests/silver_mueller/inputs_2d_x b/Examples/Tests/silver_mueller/inputs_test_2d_silver_mueller_x similarity index 100% rename from Examples/Tests/silver_mueller/inputs_2d_x rename to Examples/Tests/silver_mueller/inputs_test_2d_silver_mueller_x diff --git a/Examples/Tests/silver_mueller/inputs_2d_z b/Examples/Tests/silver_mueller/inputs_test_2d_silver_mueller_z similarity index 100% rename from Examples/Tests/silver_mueller/inputs_2d_z rename to Examples/Tests/silver_mueller/inputs_test_2d_silver_mueller_z diff --git a/Examples/Tests/silver_mueller/inputs_rz_z b/Examples/Tests/silver_mueller/inputs_test_rz_silver_mueller_z similarity index 100% rename from Examples/Tests/silver_mueller/inputs_rz_z rename to Examples/Tests/silver_mueller/inputs_test_rz_silver_mueller_z diff --git a/Examples/Tests/single_particle/CMakeLists.txt b/Examples/Tests/single_particle/CMakeLists.txt new file mode 100644 index 00000000000..fb823b39431 --- /dev/null +++ b/Examples/Tests/single_particle/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_bilinear_filter # name + 2 # dims + 2 # nprocs + inputs_test_2d_bilinear_filter # inputs + "analysis.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001" # checksum + OFF # dependency +) diff --git a/Examples/Tests/single_particle/analysis.py b/Examples/Tests/single_particle/analysis.py new file mode 100755 index 00000000000..efd3f36cfdf --- /dev/null +++ b/Examples/Tests/single_particle/analysis.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Maxence Thevenet +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import sys + +import numpy as np +import yt +from scipy import signal + +yt.funcs.mylog.setLevel(0) + +# Build Jx without filter. This can be obtained by running this test without +# a filter, e.g., execute +# > OMP_NUM_THREADS=2 mpirun -np 2 ~/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex \ +# inputs warpx.use_filter=0 +# instead of +# > OMP_NUM_THREADS=2 mpirun -np 2 ~/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex \ +# inputs warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 +# and then print the values in the array F_filtered below. +my_F_nofilter = np.zeros([16, 16]) +my_F_nofilter[8, 8] = -1.601068237523421e-11 +my_F_nofilter[8, 7] = -1.601068237523421e-11 + +# Build 2D filter +filter0 = np.array([0.25, 0.5, 0.25]) +my_order = [1, 5] +my_filterx = filter0 +my_filtery = filter0 +while my_order[0] > 1: + my_filterx = np.convolve(my_filterx, filter0) + my_order[0] -= 1 +while my_order[1] > 1: + my_filtery = np.convolve(my_filtery, filter0) + my_order[1] -= 1 +my_filter = my_filterx[:, None] * my_filtery + +# Apply filter. my_F_filtered is the theoretical value for filtered field +my_F_filtered = signal.convolve2d( + my_F_nofilter, my_filter, boundary="symm", mode="same" +) + +# Get simulation result for F_filtered +filename = sys.argv[1] +ds = yt.load(filename) +sl = yt.SlicePlot(ds, 2, "jx", aspect=1) +all_data_level_0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +F_filtered = all_data_level_0["boxlib", "jx"].v.squeeze() + +# Compare theory and PIC for filtered value +error_rel = np.sum(np.abs(F_filtered - my_F_filtered)) / np.sum(np.abs(my_F_filtered)) +tolerance_rel = 1.0e-14 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert error_rel < tolerance_rel diff --git a/Examples/Tests/single_particle/analysis_bilinear_filter.py b/Examples/Tests/single_particle/analysis_bilinear_filter.py deleted file mode 100755 index db7250dc3bb..00000000000 --- a/Examples/Tests/single_particle/analysis_bilinear_filter.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -import os -import sys - -import numpy as np -from scipy import signal - -import yt ; yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# Build Jx without filter. This can be obtained by running this test without -# a filter, e.g., execute -# > OMP_NUM_THREADS=2 mpirun -np 2 ~/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex \ -# inputs warpx.use_filter=0 -# instead of -# > OMP_NUM_THREADS=2 mpirun -np 2 ~/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex \ -# inputs warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 -# and then print the values in the array F_filtered below. -my_F_nofilter = np.zeros([16,16]) -my_F_nofilter[8,8] = -1.601068237523421e-11 -my_F_nofilter[8,7] = -1.601068237523421e-11 - -# Build 2D filter -filter0 = np.array([.25,.5,.25]) -my_order = [1,5] -my_filterx = filter0 -my_filtery = filter0 -while my_order[0]>1: - my_filterx = np.convolve(my_filterx,filter0) - my_order[0] -= 1 -while my_order[1]>1: - my_filtery = np.convolve(my_filtery,filter0) - my_order[1] -= 1 -my_filter = my_filterx[:,None]*my_filtery - -# Apply filter. my_F_filtered is the theoretical value for filtered field -my_F_filtered = signal.convolve2d(my_F_nofilter, my_filter, boundary='symm', mode='same') - -# Get simulation result for F_filtered -filename = sys.argv[1] -ds = yt.load( filename ) -sl = yt.SlicePlot(ds, 2, 'jx', aspect=1) -all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) -F_filtered = all_data_level_0['boxlib', 'jx'].v.squeeze() - -# Compare theory and PIC for filtered value -error_rel = np.sum( np.abs(F_filtered - my_F_filtered) ) / np.sum( np.abs(my_F_filtered) ) -tolerance_rel = 1.e-14 - -print("error_rel : " + str(error_rel)) -print("tolerance_rel: " + str(tolerance_rel)) - -assert( error_rel < tolerance_rel ) - -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/single_particle/analysis_default_regression.py b/Examples/Tests/single_particle/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/single_particle/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/single_particle/inputs_2d b/Examples/Tests/single_particle/inputs_2d deleted file mode 100644 index 71ca2101c22..00000000000 --- a/Examples/Tests/single_particle/inputs_2d +++ /dev/null @@ -1,32 +0,0 @@ -max_step = 1 -amr.n_cell = 16 16 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 8 -geometry.dims = 2 -geometry.prob_lo = -8 -12 -geometry.prob_hi = 8 12 - -# Boundary condition -boundary.field_lo = pec pec -boundary.field_hi = pec pec - -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -warpx.cfl = 1.0 - -# Order of particle shape factors -algo.particle_shape = 1 - -particles.species_names = electron -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 0.0 -electron.single_particle_u = 1.e20 0.0 0.0 # gamma*beta -electron.single_particle_weight = 1.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full diff --git a/Examples/Tests/single_particle/inputs_test_2d_bilinear_filter b/Examples/Tests/single_particle/inputs_test_2d_bilinear_filter new file mode 100644 index 00000000000..6f8eb6fdcea --- /dev/null +++ b/Examples/Tests/single_particle/inputs_test_2d_bilinear_filter @@ -0,0 +1,34 @@ +max_step = 1 +amr.n_cell = 16 16 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 8 +geometry.dims = 2 +geometry.prob_lo = -8 -12 +geometry.prob_hi = 8 12 + +# Boundary condition +boundary.field_lo = pec pec +boundary.field_hi = pec pec + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.cfl = 1.0 +warpx.use_filter = 1 +warpx.filter_npass_each_dir = 1 5 + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 0.0 +electron.single_particle_u = 1.e20 0.0 0.0 # gamma*beta +electron.single_particle_weight = 1.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full diff --git a/Examples/Tests/space_charge_initialization/CMakeLists.txt b/Examples/Tests/space_charge_initialization/CMakeLists.txt new file mode 100644 index 00000000000..00f9a5462fd --- /dev/null +++ b/Examples/Tests/space_charge_initialization/CMakeLists.txt @@ -0,0 +1,22 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_space_charge_initialization # name + 2 # dims + 2 # nprocs + inputs_test_2d_space_charge_initialization # inputs + "analysis.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles" # checksum + OFF # dependency +) + +add_warpx_test( + test_3d_space_charge_initialization # name + 3 # dims + 2 # nprocs + inputs_test_3d_space_charge_initialization # inputs + "analysis.py diags/diag1000001" # analysis + "analysis_default_regression.py --path diags/diag1000001 --skip-particles" # checksum + OFF # dependency +) diff --git a/Examples/Tests/space_charge_initialization/analysis.py b/Examples/Tests/space_charge_initialization/analysis.py index 48e19ea0b75..b8e5e689a87 100755 --- a/Examples/Tests/space_charge_initialization/analysis.py +++ b/Examples/Tests/space_charge_initialization/analysis.py @@ -11,12 +11,12 @@ verifying that the space-charge field of a Gaussian beam corresponds to the expected theoretical field. """ -import os + import sys import matplotlib -matplotlib.use('Agg') +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import scipy.constants as scc @@ -24,23 +24,24 @@ from scipy.special import gammainc yt.funcs.mylog.setLevel(0) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI # Parameters from the Simulation -Qtot = -1.e-20 -r0 = 2.e-6 +Qtot = -1.0e-20 +r0 = 2.0e-6 # Open data file filename = sys.argv[1] -ds = yt.load( filename ) +ds = yt.load(filename) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. -if 'force_periodicity' in dir(ds): ds.force_periodicity() +if "force_periodicity" in dir(ds): + ds.force_periodicity() # Extract data -ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +ad0 = ds.covering_grid( + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) Ex_array = ad0[("mesh", "Ex")].to_ndarray().squeeze() if ds.dimensionality == 2: # Rename the z dimension as y, so as to make this script work for 2d and 3d @@ -50,65 +51,72 @@ Ez_array = ad0[("mesh", "Ez")].to_ndarray() # Extract grid coordinates -Nx, Ny, Nz = ds.domain_dimensions +Nx, Ny, Nz = ds.domain_dimensions xmin, ymin, zmin = ds.domain_left_edge.v Lx, Ly, Lz = ds.domain_width.v -x = xmin + Lx/Nx*(0.5+np.arange(Nx)) -y = ymin + Ly/Ny*(0.5+np.arange(Ny)) -z = zmin + Lz/Nz*(0.5+np.arange(Nz)) +x = xmin + Lx / Nx * (0.5 + np.arange(Nx)) +y = ymin + Ly / Ny * (0.5 + np.arange(Ny)) +z = zmin + Lz / Nz * (0.5 + np.arange(Nz)) # Compute theoretical field if ds.dimensionality == 2: - x_2d, y_2d = np.meshgrid(x, y, indexing='ij') + x_2d, y_2d = np.meshgrid(x, y, indexing="ij") r2 = x_2d**2 + y_2d**2 - factor = (Qtot/r0)/(2*np.pi*scc.epsilon_0*r2) * (1-np.exp(-r2/(2*r0**2))) + factor = ( + (Qtot / r0) / (2 * np.pi * scc.epsilon_0 * r2) * (1 - np.exp(-r2 / (2 * r0**2))) + ) Ex_th = x_2d * factor Ey_th = y_2d * factor elif ds.dimensionality == 3: - x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') + x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing="ij") r2 = x_2d**2 + y_2d**2 + z_2d**2 - factor = Qtot/(4*np.pi*scc.epsilon_0*r2**1.5) * gammainc(3./2, r2/(2.*r0**2)) - Ex_th = factor*x_2d - Ey_th = factor*y_2d - Ez_th = factor*z_2d + factor = ( + Qtot + / (4 * np.pi * scc.epsilon_0 * r2**1.5) + * gammainc(3.0 / 2, r2 / (2.0 * r0**2)) + ) + Ex_th = factor * x_2d + Ey_th = factor * y_2d + Ez_th = factor * z_2d + # Plot theory and data def make_2d(arr): if arr.ndim == 3: - return arr[:,:,Nz//2] + return arr[:, :, Nz // 2] else: return arr -plt.figure(figsize=(10,10)) + + +plt.figure(figsize=(10, 10)) plt.subplot(221) -plt.title('Ex: Theory') +plt.title("Ex: Theory") plt.imshow(make_2d(Ex_th)) plt.colorbar() plt.subplot(222) -plt.title('Ex: Simulation') +plt.title("Ex: Simulation") plt.imshow(make_2d(Ex_array)) plt.colorbar() plt.subplot(223) -plt.title('Ey: Theory') +plt.title("Ey: Theory") plt.imshow(make_2d(Ey_th)) plt.colorbar() plt.subplot(224) -plt.title('Ey: Simulation') +plt.title("Ey: Simulation") plt.imshow(make_2d(Ey_array)) plt.colorbar() -plt.savefig('Comparison.png') +plt.savefig("Comparison.png") + # Automatically check the results def check(E, E_th, label): - print( 'Relative error in %s: %.3f'%( - label, abs(E-E_th).max()/E_th.max())) + print("Relative error in %s: %.3f" % (label, abs(E - E_th).max() / E_th.max())) tolerance_rel = 0.165 print("tolerance_rel: " + str(tolerance_rel)) - assert np.allclose( E, E_th, atol=tolerance_rel*E_th.max() ) + assert np.allclose(E, E_th, atol=tolerance_rel * E_th.max()) -check( Ex_array, Ex_th, 'Ex' ) -check( Ey_array, Ey_th, 'Ey' ) -if ds.dimensionality == 3: - check( Ez_array, Ez_th, 'Ez' ) -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, do_particles=0) +check(Ex_array, Ex_th, "Ex") +check(Ey_array, Ey_th, "Ey") +if ds.dimensionality == 3: + check(Ez_array, Ez_th, "Ez") diff --git a/Examples/Tests/space_charge_initialization/analysis_default_regression.py b/Examples/Tests/space_charge_initialization/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/space_charge_initialization/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/space_charge_initialization/inputs_3d b/Examples/Tests/space_charge_initialization/inputs_3d deleted file mode 100644 index c8058fac519..00000000000 --- a/Examples/Tests/space_charge_initialization/inputs_3d +++ /dev/null @@ -1,36 +0,0 @@ -max_step = 1 -amr.n_cell = 128 128 128 -amr.max_grid_size = 32 -amr.max_level = 0 - -geometry.dims = 3 -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 -geometry.prob_hi = 50.e-6 50.e-6 50.e-6 - -warpx.cfl = 1.e-3 - -# Order of particle shape factors -algo.particle_shape = 1 - -particles.species_names = beam -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.initialize_self_fields = 1 -beam.x_rms = 2.e-6 -beam.y_rms = 2.e-6 -beam.z_rms = 2.e-6 -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = 0.e-6 -beam.npart = 20000 -beam.q_tot = -1.e-20 -beam.momentum_distribution_type = "at_rest" - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez jx jy jz diff --git a/Examples/Tests/space_charge_initialization/inputs_test_2d_space_charge_initialization b/Examples/Tests/space_charge_initialization/inputs_test_2d_space_charge_initialization new file mode 100644 index 00000000000..4445217225c --- /dev/null +++ b/Examples/Tests/space_charge_initialization/inputs_test_2d_space_charge_initialization @@ -0,0 +1,37 @@ +max_step = 1 +amr.n_cell = 128 128 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.dims = 3 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +geometry.prob_lo = -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 +geometry.dims = 2 + +warpx.cfl = 1.e-3 + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = beam +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.initialize_self_fields = 1 +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "at_rest" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez jx jy jz diff --git a/Examples/Tests/space_charge_initialization/inputs_test_3d_space_charge_initialization b/Examples/Tests/space_charge_initialization/inputs_test_3d_space_charge_initialization new file mode 100644 index 00000000000..d7a9f42fa70 --- /dev/null +++ b/Examples/Tests/space_charge_initialization/inputs_test_3d_space_charge_initialization @@ -0,0 +1,37 @@ +max_step = 1 +amr.n_cell = 128 128 128 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.dims = 3 +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 50.e-6 +geometry.dims = 3 + +warpx.cfl = 1.e-3 + +# Order of particle shape factors +algo.particle_shape = 1 + +particles.species_names = beam +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.initialize_self_fields = 1 +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "at_rest" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez jx jy jz diff --git a/Examples/Tests/subcycling/CMakeLists.txt b/Examples/Tests/subcycling/CMakeLists.txt new file mode 100644 index 00000000000..503c9f24f34 --- /dev/null +++ b/Examples/Tests/subcycling/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_2d_subcycling_mr # name + 2 # dims + 2 # nprocs + inputs_test_2d_subcycling_mr # inputs + OFF # analysis + "analysis_default_regression.py --path diags/diag1000250" # checksum + OFF # dependency +) diff --git a/Examples/Tests/subcycling/analysis_default_regression.py b/Examples/Tests/subcycling/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/subcycling/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/subcycling/inputs_2d b/Examples/Tests/subcycling/inputs_test_2d_subcycling_mr similarity index 100% rename from Examples/Tests/subcycling/inputs_2d rename to Examples/Tests/subcycling/inputs_test_2d_subcycling_mr diff --git a/Examples/Tests/vay_deposition/CMakeLists.txt b/Examples/Tests/vay_deposition/CMakeLists.txt new file mode 100644 index 00000000000..86108530b1d --- /dev/null +++ b/Examples/Tests/vay_deposition/CMakeLists.txt @@ -0,0 +1,26 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_FFT) + add_warpx_test( + test_2d_vay_deposition # name + 2 # dims + 2 # nprocs + inputs_test_2d_vay_deposition # inputs + "analysis.py diags/diag1000050" # analysis + "analysis_default_regression.py --path diags/diag1000050" # checksum + OFF # dependency + ) +endif() + +if(WarpX_FFT) + add_warpx_test( + test_3d_vay_deposition # name + 3 # dims + 2 # nprocs + inputs_test_3d_vay_deposition # inputs + "analysis.py diags/diag1000025" # analysis + "analysis_default_regression.py --path diags/diag1000025" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/vay_deposition/analysis.py b/Examples/Tests/vay_deposition/analysis.py index cfd089f2112..def231538de 100755 --- a/Examples/Tests/vay_deposition/analysis.py +++ b/Examples/Tests/vay_deposition/analysis.py @@ -6,7 +6,6 @@ # # License: BSD-3-Clause-LBNL -import os import sys import numpy as np @@ -15,27 +14,19 @@ yt.funcs.mylog.setLevel(50) -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - # Plotfile data set fn = sys.argv[1] ds = yt.load(fn) # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) data = ds.covering_grid( - level=0, - left_edge=ds.domain_left_edge, - dims=ds.domain_dimensions) -rho = data[('boxlib','rho')].to_ndarray() -divE = data[('boxlib','divE')].to_ndarray() -error_rel = np.amax(np.abs(divE-rho/epsilon_0))/np.amax(np.abs(rho/epsilon_0)) + level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions +) +rho = data[("boxlib", "rho")].to_ndarray() +divE = data[("boxlib", "divE")].to_ndarray() +error_rel = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax(np.abs(rho / epsilon_0)) tolerance = 1e-3 print("Error on charge conservation:") print("error_rel = {}".format(error_rel)) print("tolerance = {}".format(tolerance)) -assert( error_rel < tolerance ) - -# Checksum analysis -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn) +assert error_rel < tolerance diff --git a/Examples/Tests/vay_deposition/analysis_default_regression.py b/Examples/Tests/vay_deposition/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/vay_deposition/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/vay_deposition/inputs_2d b/Examples/Tests/vay_deposition/inputs_test_2d_vay_deposition similarity index 100% rename from Examples/Tests/vay_deposition/inputs_2d rename to Examples/Tests/vay_deposition/inputs_test_2d_vay_deposition diff --git a/Examples/Tests/vay_deposition/inputs_3d b/Examples/Tests/vay_deposition/inputs_test_3d_vay_deposition similarity index 100% rename from Examples/Tests/vay_deposition/inputs_3d rename to Examples/Tests/vay_deposition/inputs_test_3d_vay_deposition diff --git a/Examples/analysis_default_openpmd_regression.py b/Examples/analysis_default_openpmd_regression.py deleted file mode 100755 index 3aadc49ac51..00000000000 --- a/Examples/analysis_default_openpmd_regression.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import sys - -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI - -# this will be the name of the plot file -fn = sys.argv[1] - -# Get name of the test -test_name = os.path.split(os.getcwd())[1] - -# Run checksum regression test -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd', rtol=2.e-6) -else: - checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd') diff --git a/Examples/analysis_default_regression.py b/Examples/analysis_default_regression.py index 453f650be01..e143e396f0c 100755 --- a/Examples/analysis_default_regression.py +++ b/Examples/analysis_default_regression.py @@ -1,20 +1,88 @@ #!/usr/bin/env python3 +import argparse import os -import re import sys -sys.path.insert(1, '../../../../warpx/Regression/Checksum/') -import checksumAPI +import yt +from openpmd_viewer import OpenPMDTimeSeries -# this will be the name of the plot file -fn = sys.argv[1] +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +from checksumAPI import evaluate_checksum -# Get name of the test -test_name = os.path.split(os.getcwd())[1] -# Run checksum regression test -if re.search( 'single_precision', fn ): - checksumAPI.evaluate_checksum(test_name, fn, rtol=2.e-6) -else: - checksumAPI.evaluate_checksum(test_name, fn) +def main(args): + # parse test name from test directory + test_name = os.path.split(os.getcwd())[1] + if "_restart" in test_name: + rtol_restart = 1e-12 + print( + f"Warning: Setting relative tolerance {rtol_restart} for restart checksum analysis" + ) + # use original test's checksums + test_name = test_name.replace("_restart", "") + # reset relative tolerance + args.rtol = rtol_restart + # TODO check environment and reset tolerance (portable, machine precision) + # compare checksums + evaluate_checksum( + test_name=test_name, + output_file=args.path, + output_format=args.format, + rtol=args.rtol, + do_fields=args.do_fields, + do_particles=args.do_particles, + ) + + +if __name__ == "__main__": + # define parser + parser = argparse.ArgumentParser() + # add arguments: output path + parser.add_argument( + "--path", + help="path to output file(s)", + type=str, + ) + # add arguments: relative tolerance + parser.add_argument( + "--rtol", + help="relative tolerance to compare checksums", + type=float, + required=False, + default=1e-9, + ) + # add arguments: skip fields + parser.add_argument( + "--skip-fields", + help="skip fields when comparing checksums", + action="store_true", + dest="skip_fields", + ) + # add arguments: skip particles + parser.add_argument( + "--skip-particles", + help="skip particles when comparing checksums", + action="store_true", + dest="skip_particles", + ) + # parse arguments + args = parser.parse_args() + # set args.format automatically + try: + yt.load(args.path) + except Exception: + try: + OpenPMDTimeSeries(args.path) + except Exception: + print("Could not open the file as a plotfile or an openPMD time series") + else: + args.format = "openpmd" + else: + args.format = "plotfile" + # set args.do_fields (not parsed, based on args.skip_fields) + args.do_fields = False if args.skip_fields else True + # set args.do_particles (not parsed, based on args.skip_particles) + args.do_particles = False if args.skip_particles else True + # execute main function + main(args) diff --git a/Examples/analysis_default_restart.py b/Examples/analysis_default_restart.py index 612851678fd..ad6bc22e60e 100755 --- a/Examples/analysis_default_restart.py +++ b/Examples/analysis_default_restart.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 +import os +import sys + import numpy as np import yt -def check_restart(filename, tolerance = 1e-12): +def check_restart(filename, tolerance=1e-12): """ Compare output data generated from initial run with output data generated after restart. @@ -21,33 +24,46 @@ def check_restart(filename, tolerance = 1e-12): # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds_restart): ds_restart.force_periodicity() + if "force_periodicity" in dir(ds_restart): + ds_restart.force_periodicity() - ad_restart = ds_restart.covering_grid(level = 0, - left_edge = ds_restart.domain_left_edge, dims = ds_restart.domain_dimensions) + ad_restart = ds_restart.covering_grid( + level=0, + left_edge=ds_restart.domain_left_edge, + dims=ds_restart.domain_dimensions, + ) # Load output data generated from initial run - benchmark = 'orig_' + filename + benchmark = os.path.join(os.getcwd().replace("_restart", ""), filename) ds_benchmark = yt.load(benchmark) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds_benchmark): ds_benchmark.force_periodicity() + if "force_periodicity" in dir(ds_benchmark): + ds_benchmark.force_periodicity() - ad_benchmark = ds_benchmark.covering_grid(level = 0, - left_edge = ds_benchmark.domain_left_edge, dims = ds_benchmark.domain_dimensions) + ad_benchmark = ds_benchmark.covering_grid( + level=0, + left_edge=ds_benchmark.domain_left_edge, + dims=ds_benchmark.domain_dimensions, + ) # Loop over all fields (all particle species, all particle attributes, all grid fields) # and compare output data generated from initial run with output data generated after restart - print('\ntolerance = {:g}'.format(tolerance)) + print(f"\ntolerance = {tolerance}") print() for field in ds_benchmark.field_list: dr = ad_restart[field].squeeze().v db = ad_benchmark[field].squeeze().v error = np.amax(np.abs(dr - db)) - if (np.amax(np.abs(db)) != 0.): + if np.amax(np.abs(db)) != 0.0: error /= np.amax(np.abs(db)) - print('field: {}; error = {:g}'.format(field, error)) - assert(error < tolerance) + print(f"field: {field}; error = {error}") + assert error < tolerance print() + + +# compare restart results against original results +output_file = sys.argv[1] +check_restart(output_file) diff --git a/Examples/test_cleanup.cmake b/Examples/test_cleanup.cmake new file mode 100644 index 00000000000..b15e31e1f5d --- /dev/null +++ b/Examples/test_cleanup.cmake @@ -0,0 +1,7 @@ +# delete all test files except backtrace +file(GLOB test_files ${CMAKE_ARGV3}/*) +foreach(file ${test_files}) + if(NOT ${file} MATCHES "Backtrace*") + execute_process(COMMAND ${CMAKE_COMMAND} -E rm -r ${file}) + endif() +endforeach() diff --git a/GNUmakefile b/GNUmakefile index a332c9cc04e..3bf69303c1a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -43,5 +43,8 @@ USE_RZ = FALSE USE_EB = FALSE +USE_LINEAR_SOLVERS_EM = TRUE +USE_LINEAR_SOLVERS_INCFLO = TRUE + WARPX_HOME := . include $(WARPX_HOME)/Source/Make.WarpX diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst index b5253b80f9f..f0efb350213 100644 --- a/GOVERNANCE.rst +++ b/GOVERNANCE.rst @@ -16,7 +16,7 @@ Current Roster - Remi Lehe - Axel Huebl -See: `GitHub team `__ +See: `GitHub team `__ Role ^^^^ @@ -54,7 +54,9 @@ Technical Committee Current Roster ^^^^^^^^^^^^^^ +- Justin Ray Angus - Luca Fedeli +- Arianna Formenti - Roelof Groenewald - David Grote - Axel Huebl @@ -66,7 +68,7 @@ Current Roster - Weiqun Zhang - Edoardo Zoni -See: `GitHub team `__ +See: `GitHub team `__ Role ^^^^ diff --git a/LICENSE.txt b/LICENSE.txt index 2965985ebb1..ba0df767288 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -14,10 +14,11 @@ this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -(3) Neither the name of the University of California, Lawrence Berkeley -National Laboratory, U.S. Dept. of Energy nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. +(3) Neither the name of the University of California, +Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, +Lawrence Livermore National Laboratory, U.S. Dept. of Energy nor the names of +its contributors may be used to endorse or promote products derived from this +software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" diff --git a/Python/pywarpx/Algo.py b/Python/pywarpx/Algo.py index f8049207078..2f637627ac5 100644 --- a/Python/pywarpx/Algo.py +++ b/Python/pywarpx/Algo.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -algo = Bucket('algo') +algo = Bucket("algo") diff --git a/Python/pywarpx/Amr.py b/Python/pywarpx/Amr.py index f9164f4d419..618db17d6d1 100644 --- a/Python/pywarpx/Amr.py +++ b/Python/pywarpx/Amr.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -amr = Bucket('amr') +amr = Bucket("amr") diff --git a/Python/pywarpx/Amrex.py b/Python/pywarpx/Amrex.py index 4dab8225a84..34b50637003 100644 --- a/Python/pywarpx/Amrex.py +++ b/Python/pywarpx/Amrex.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -amrex = Bucket('amrex') +amrex = Bucket("amrex") diff --git a/Python/pywarpx/Boundary.py b/Python/pywarpx/Boundary.py index b7ebd17af70..b84dbfe2193 100644 --- a/Python/pywarpx/Boundary.py +++ b/Python/pywarpx/Boundary.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -boundary = Bucket('boundary') +boundary = Bucket("boundary") diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 9dbe4def88e..fa595039726 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -13,9 +13,10 @@ class Bucket(object): The purpose of this class is to be a named bucket for holding attributes. This attributes will be concatenated into a string and passed into argv during initialization. """ + def __init__(self, instancename, **defaults): - self._localsetattr('instancename', instancename) - self._localsetattr('argvattrs', {}) + self._localsetattr("instancename", instancename) + self._localsetattr("argvattrs", {}) for name, value in defaults.items(): self.add_new_attr(name, value) @@ -26,7 +27,7 @@ def add_new_attr(self, name, value): """Names starting with "_" are made instance attributes. Otherwise the attribute is added to the args list. """ - if name.startswith('_'): + if name.startswith("_"): self._localsetattr(name, value) else: self.argvattrs[name] = value @@ -36,7 +37,7 @@ def add_new_group_attr(self, group, name, value): group is not an empty string, otherwise as only "name". """ if group: - self.argvattrs[f'{group}.{name}'] = value + self.argvattrs[f"{group}.{name}"] = value else: self.argvattrs[name] = value @@ -51,7 +52,9 @@ def __getattr__(self, name): def check_consistency(self, vname, value, errmsg): if vname in self.argvattrs: - assert (self.argvattrs[vname] is None) or (self.argvattrs[vname] == value), Exception(errmsg) + assert (self.argvattrs[vname] is None) or ( + self.argvattrs[vname] == value + ), Exception(errmsg) def attrlist(self): "Concatenate the attributes into a string" @@ -60,7 +63,7 @@ def attrlist(self): if value is None: continue if isinstance(value, str): - if value.find('=') > -1: + if value.find("=") > -1: # --- Expressions with temporary variables need to be inside quotes rhs = f'"{value}"' else: @@ -71,11 +74,11 @@ def attrlist(self): continue # --- For lists, tuples, and arrays make a space delimited string of the values. # --- The lambda is needed in case this is a list of strings. - rhs = ' '.join(map(lambda s : f'{s}', value)) + rhs = " ".join(map(lambda s: f"{s}", value)) elif isinstance(value, bool): rhs = 1 if value else 0 else: rhs = value - attrstring = f'{self.instancename}.{attr} = {rhs}' + attrstring = f"{self.instancename}.{attr} = {rhs}" result += [attrstring] return result diff --git a/Python/pywarpx/Collisions.py b/Python/pywarpx/Collisions.py index 9ad7f9e14a9..5269f530f4c 100644 --- a/Python/pywarpx/Collisions.py +++ b/Python/pywarpx/Collisions.py @@ -6,9 +6,10 @@ from .Bucket import Bucket -collisions = Bucket('collisions') +collisions = Bucket("collisions") collisions_list = [] + def newcollision(name): result = Bucket(name) collisions_list.append(result) diff --git a/Python/pywarpx/Constants.py b/Python/pywarpx/Constants.py index e899fa255ae..bd17c2f2dfc 100644 --- a/Python/pywarpx/Constants.py +++ b/Python/pywarpx/Constants.py @@ -13,18 +13,21 @@ class Constants(Bucket): """ The purpose of this class is to be hold user defined constants """ + def __init__(self): - Bucket.__init__(self, 'my_constants') + Bucket.__init__(self, "my_constants") def __setattr__(self, name, value): # Make sure that any constants redefined have a consistent value if name in self.argvattrs: - assert self.argvattrs[name] == value, Exception('Inconsistent values given for user defined constants') + assert self.argvattrs[name] == value, Exception( + "Inconsistent values given for user defined constants" + ) Bucket.__setattr__(self, name, value) def add_keywords(self, kwdict): mangle_dict = {} - for k,v in kwdict.items(): + for k, v in kwdict.items(): # WarpX has a single global dictionary of expression variables, my_constants, # so each variable must be unique. # Check if keyword has already been defined. If so and it has a different @@ -33,7 +36,7 @@ def add_keywords(self, kwdict): k_mangled = k while k_mangled in self.argvattrs and self.argvattrs[k_mangled] != v: mangle_number += 1 - k_mangled = f'{k}{mangle_number}' + k_mangled = f"{k}{mangle_number}" if mangle_number > 0: # The mangle_dict contains only mangled names mangle_dict[k] = k_mangled @@ -45,8 +48,8 @@ def mangle_expression(self, expression, mangle_dict): return None # For each key in mangle_dict, modify the expression replacing # the key with its value, the mangled version of key - for k,v in mangle_dict.items(): - expression = re.sub(r'\b%s\b'%k, v, expression) + for k, v in mangle_dict.items(): + expression = re.sub(r"\b%s\b" % k, v, expression) return expression diff --git a/Python/pywarpx/Diagnostics.py b/Python/pywarpx/Diagnostics.py index 19860e9b7ee..731d1d31d01 100644 --- a/Python/pywarpx/Diagnostics.py +++ b/Python/pywarpx/Diagnostics.py @@ -6,22 +6,25 @@ from .Bucket import Bucket -diagnostics = Bucket('diagnostics', _diagnostics_dict={}) -reduced_diagnostics = Bucket('warpx', _diagnostics_dict={}) +diagnostics = Bucket("diagnostics", _diagnostics_dict={}) +reduced_diagnostics = Bucket("warpx", _diagnostics_dict={}) + class Diagnostic(Bucket): """ This is the same as a Bucket, but checks that any attributes are always given the same value. """ + def add_new_attr_with_check(self, name, value): - if name.startswith('_'): + if name.startswith("_"): self._localsetattr(name, value) else: if name in self.argvattrs: - assert value == self.argvattrs[name], \ - Exception(f'Diagnostic attributes not consistent for ' - f'"{self.instancename}": ' - f'"{value}" != "{self.argvattrs[name]}"') + assert value == self.argvattrs[name], Exception( + f"Diagnostic attributes not consistent for " + f'"{self.instancename}": ' + f'"{value}" != "{self.argvattrs[name]}"' + ) self.argvattrs[name] = value def __setattr__(self, name, value): @@ -33,5 +36,5 @@ def set_or_replace_attr(self, name, value): (since __setattr__ cannot be used for replacing as it would raise an Exception) """ - assert not name.startswith('_') + assert not name.startswith("_") self.argvattrs[name] = value diff --git a/Python/pywarpx/EB2.py b/Python/pywarpx/EB2.py index 4b74aafb04f..949d362bfcc 100644 --- a/Python/pywarpx/EB2.py +++ b/Python/pywarpx/EB2.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -eb2 = Bucket('eb2') +eb2 = Bucket("eb2") diff --git a/Python/pywarpx/Geometry.py b/Python/pywarpx/Geometry.py index 2eddb9b8f05..a870a2c5a57 100644 --- a/Python/pywarpx/Geometry.py +++ b/Python/pywarpx/Geometry.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -geometry = Bucket('geometry') +geometry = Bucket("geometry") diff --git a/Python/pywarpx/HybridPICModel.py b/Python/pywarpx/HybridPICModel.py index e21bba4a240..f94f44ce931 100644 --- a/Python/pywarpx/HybridPICModel.py +++ b/Python/pywarpx/HybridPICModel.py @@ -8,4 +8,5 @@ from .Bucket import Bucket -hybridpicmodel = Bucket('hybrid_pic_model') +hybridpicmodel = Bucket("hybrid_pic_model") +external_vector_potential = Bucket("external_vector_potential") diff --git a/Python/pywarpx/Interpolation.py b/Python/pywarpx/Interpolation.py index d25539de77b..b84def573eb 100644 --- a/Python/pywarpx/Interpolation.py +++ b/Python/pywarpx/Interpolation.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -interpolation = Bucket('interpolation') +interpolation = Bucket("interpolation") diff --git a/Python/pywarpx/Lasers.py b/Python/pywarpx/Lasers.py index 60dfaca31e9..3836700215d 100644 --- a/Python/pywarpx/Lasers.py +++ b/Python/pywarpx/Lasers.py @@ -6,9 +6,10 @@ from .Bucket import Bucket -lasers = Bucket('lasers', names=[]) +lasers = Bucket("lasers", names=[]) lasers_list = [] + def newlaser(name): result = Bucket(name) lasers_list.append(result) diff --git a/Python/pywarpx/LoadThirdParty.py b/Python/pywarpx/LoadThirdParty.py index ea62d558eeb..5ec84247604 100644 --- a/Python/pywarpx/LoadThirdParty.py +++ b/Python/pywarpx/LoadThirdParty.py @@ -17,19 +17,23 @@ def load_cupy(): if amr.Config.have_gpu: try: import cupy as cp + xp = cp # Note: found and will use cupy except ImportError: status = "Warning: GPU found but cupy not available! Trying managed memory in numpy..." import numpy as np + xp = np if amr.Config.gpu_backend == "SYCL": status = "Warning: SYCL GPU backend not yet implemented for Python" import numpy as np + xp = np else: import numpy as np + xp = np # Note: found and will use numpy return xp, status diff --git a/Python/pywarpx/PSATD.py b/Python/pywarpx/PSATD.py index 0cd3038336a..a5072a81fc8 100644 --- a/Python/pywarpx/PSATD.py +++ b/Python/pywarpx/PSATD.py @@ -6,4 +6,4 @@ from .Bucket import Bucket -psatd = Bucket('psatd') +psatd = Bucket("psatd") diff --git a/Python/pywarpx/Particles.py b/Python/pywarpx/Particles.py index e05341e8203..9c87bb7083e 100644 --- a/Python/pywarpx/Particles.py +++ b/Python/pywarpx/Particles.py @@ -6,10 +6,11 @@ from .Bucket import Bucket -particles = Bucket('particles', species_names=[], rigid_injected_species=[]) +particles = Bucket("particles", species_names=[], rigid_injected_species=[]) particles_list = [] particle_dict = {} + def newspecies(name): result = Bucket(name) particles_list.append(result) diff --git a/Python/pywarpx/ProjectionDivBCleaner.py b/Python/pywarpx/ProjectionDivBCleaner.py new file mode 100644 index 00000000000..f5b0aff7bcf --- /dev/null +++ b/Python/pywarpx/ProjectionDivBCleaner.py @@ -0,0 +1,11 @@ +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: S. Eric Clark (Helion Energy) +# +# License: BSD-3-Clause-LBNL + +from .Bucket import Bucket + +projectiondivbcleaner = Bucket("projection_divb_cleaner") diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 6752b00f371..9b0446bcc79 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -20,10 +20,11 @@ from .Diagnostics import diagnostics, reduced_diagnostics from .EB2 import eb2 from .Geometry import geometry -from .HybridPICModel import hybridpicmodel +from .HybridPICModel import external_vector_potential, hybridpicmodel from .Interpolation import interpolation from .Lasers import lasers, lasers_list from .Particles import particles, particles_list +from .ProjectionDivBCleaner import projectiondivbcleaner from .PSATD import psatd @@ -37,7 +38,7 @@ def create_argv_list(self, **kw): for k, v in kw.items(): if v is not None: - argv.append(f'{k} = {v}') + argv.append(f"{k} = {v}") argv += warpx.attrlist() argv += my_constants.attrlist() @@ -45,9 +46,11 @@ def create_argv_list(self, **kw): argv += amrex.attrlist() argv += geometry.attrlist() argv += hybridpicmodel.attrlist() + argv += external_vector_potential.attrlist() argv += boundary.attrlist() argv += algo.attrlist() argv += interpolation.attrlist() + argv += projectiondivbcleaner.attrlist() argv += psatd.attrlist() argv += eb2.attrlist() @@ -62,7 +65,9 @@ def create_argv_list(self, **kw): particles_list.append(getattr(Particles, pstring)) particles_list_names.append(pstring) else: - raise Exception('Species %s listed in species_names not defined'%pstring) + raise Exception( + "Species %s listed in species_names not defined" % pstring + ) argv += particles.attrlist() for particle in particles_list: @@ -84,7 +89,9 @@ def create_argv_list(self, **kw): for species_diagnostic in diagnostic._species_dict.values(): argv += species_diagnostic.attrlist() - reduced_diagnostics.reduced_diags_names = reduced_diagnostics._diagnostics_dict.keys() + reduced_diagnostics.reduced_diags_names = ( + reduced_diagnostics._diagnostics_dict.keys() + ) argv += reduced_diagnostics.attrlist() for diagnostic in reduced_diagnostics._diagnostics_dict.values(): argv += diagnostic.attrlist() @@ -120,25 +127,25 @@ def getProbLo(self, direction): def getProbHi(self, direction): return libwarpx.libwarpx_so.warpx_getProbHi(direction) - def write_inputs(self, filename='inputs', **kw): + def write_inputs(self, filename="inputs", **kw): argv = self.create_argv_list(**kw) # Sort the argv list to make it more human readable argv.sort() - with open(filename, 'w') as ff: - - prefix_old = '' + with open(filename, "w") as ff: + prefix_old = "" for arg in argv: # This prints the name of the input group (prefix) as a header # before each group to make the input file more human readable - prefix_new = re.split(' |\.', arg)[0] + prefix_new = re.split(r" |\.", arg)[0] if prefix_new != prefix_old: - if prefix_old != '': - ff.write('\n') - ff.write(f'# {prefix_new}\n') + if prefix_old != "": + ff.write("\n") + ff.write(f"# {prefix_new}\n") prefix_old = prefix_new - ff.write(f'{arg}\n') + ff.write(f"{arg}\n") + -warpx = WarpX('warpx', _bucket_dict = {}) +warpx = WarpX("warpx", _bucket_dict={}) diff --git a/Python/pywarpx/__init__.py b/Python/pywarpx/__init__.py index 037598f4ed4..b8e025342dd 100644 --- a/Python/pywarpx/__init__.py +++ b/Python/pywarpx/__init__.py @@ -1,8 +1,8 @@ -# Copyright 2016-2023 The WarpX Community +# Copyright 2016-2024 The WarpX Community # # This file is part of WarpX. # -# Authors: Andrew Myers, David Grote, Lorenzo Giacomel, Axel Huebl +# Authors: Andrew Myers, David Grote, Lorenzo Giacomel, Axel Huebl, S. Eric Clark # License: BSD-3-Clause-LBNL import os @@ -23,35 +23,38 @@ if os.path.exists(p_abs): os.add_dll_directory(p_abs) -from ._libwarpx import libwarpx -from .Algo import algo -from .Amr import amr -from .Amrex import amrex -from .Boundary import boundary -from .Collisions import collisions -from .Constants import my_constants -from .Diagnostics import diagnostics, reduced_diagnostics -from .EB2 import eb2 -from .Geometry import geometry -from .HybridPICModel import hybridpicmodel -from .Interpolation import interpolation -from .Lasers import lasers -from .LoadThirdParty import load_cupy -from .Particles import newspecies, particles -from .PSATD import psatd -from .WarpX import warpx +from ._libwarpx import libwarpx # noqa +from .Algo import algo # noqa +from .Amr import amr # noqa +from .Amrex import amrex # noqa +from .Boundary import boundary # noqa +from .Collisions import collisions # noqa +from .Constants import my_constants # noqa +from .Diagnostics import diagnostics, reduced_diagnostics # noqa +from .EB2 import eb2 # noqa +from .Geometry import geometry # noqa +from .HybridPICModel import hybridpicmodel, external_vector_potential # noqa +from .Interpolation import interpolation # noqa +from .Lasers import lasers # noqa +from .LoadThirdParty import load_cupy # noqa +from .Particles import newspecies, particles # noqa +from .ProjectionDivBCleaner import projectiondivbcleaner # noqa +from .PSATD import psatd # noqa +from .WarpX import warpx # noqa # This is a circular import and must happen after the import of libwarpx -from . import picmi # isort:skip +from . import picmi # noqa # isort:skip + # intentionally query the value - only set once sim dimension is known def __getattr__(name): # https://stackoverflow.com/a/57263518/2719194 - if name == '__version__': + if name == "__version__": return libwarpx.__version__ raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + # TODO -#__doc__ = cxx.__doc__ -#__license__ = cxx.__license__ -#__author__ = cxx.__author__ +# __doc__ = cxx.__doc__ +# __license__ = cxx.__license__ +# __author__ = cxx.__author__ diff --git a/Python/pywarpx/_libwarpx.py b/Python/pywarpx/_libwarpx.py index 1eb410e65b8..98c10f7a8c4 100755 --- a/Python/pywarpx/_libwarpx.py +++ b/Python/pywarpx/_libwarpx.py @@ -14,13 +14,12 @@ import atexit import os - -import numpy as np +import sys from .Geometry import geometry -class LibWarpX(): +class LibWarpX: """This class manages the WarpX classes, part of the Python module from the compiled C++ code. It will only load the library when it is referenced, and this can only be done after the geometry is defined so that the version of the library that is needed can be determined. @@ -36,32 +35,36 @@ def __init__(self): self.__version__ = None def __getattr__(self, attribute): - if attribute == 'libwarpx_so': + if attribute == "libwarpx_so": # If the 'libwarpx_so' is referenced, load it. # Once loaded, it gets added to the dictionary so this code won't be called again. self.load_library() return self.__dict__[attribute] + elif attribute == "warpx": + # A `warpx` attribute has not yet been assigned, so `initialize_warpx` has not been called. + raise AttributeError( + "Trying to access libwarpx.warpx before initialize_warpx has been called!" + ) else: # For any other attribute, call the built-in routine - this should always - # return an AttributeException. + # return an AttributeError. return self.__getattribute__(attribute) def _get_package_root(self): - ''' + """ Get the path to the installation location (where libwarpx.so would be installed). - ''' + """ cur = os.path.abspath(__file__) while True: name = os.path.basename(cur) - if name == 'pywarpx': + if name == "pywarpx": return cur elif not name: - return '' + return "" cur = os.path.dirname(cur) def load_library(self): - - if 'libwarpx_so' in self.__dict__: + if "libwarpx_so" in self.__dict__: raise RuntimeError( "Invalid attempt to load the pybind11 bindings library multiple times. " "Note that multiple AMReX/WarpX geometries cannot be loaded yet into the same Python process. " @@ -74,57 +77,69 @@ def load_library(self): _prob_lo = geometry.prob_lo _dims = geometry.dims except AttributeError: - raise Exception('The shared object could not be loaded. The geometry must be setup before the WarpX pybind11 module can be accessesd. The geometry determines which version of the shared object to load.') + raise Exception( + "The shared object could not be loaded. The geometry must be setup before the WarpX pybind11 module can be accessesd. The geometry determines which version of the shared object to load." + ) - if _dims == 'RZ': - self.geometry_dim = 'rz' - elif (_dims == '1' or _dims == '2' or _dims == '3'): - self.geometry_dim = '%dd'%len(_prob_lo) + if _dims == "RZ": + self.geometry_dim = "rz" + elif _dims == "1" or _dims == "2" or _dims == "3": + self.geometry_dim = "%dd" % len(_prob_lo) else: - raise Exception('Undefined geometry %d'%_dims) + raise Exception("Undefined geometry %d" % _dims) try: if self.geometry_dim == "1d": import amrex.space1d as amr + self.amr = amr from . import warpx_pybind_1d as cxx_1d + self.libwarpx_so = cxx_1d self.dim = 1 elif self.geometry_dim == "2d": import amrex.space2d as amr + self.amr = amr from . import warpx_pybind_2d as cxx_2d + self.libwarpx_so = cxx_2d self.dim = 2 elif self.geometry_dim == "rz": import amrex.space2d as amr + self.amr = amr from . import warpx_pybind_rz as cxx_rz + self.libwarpx_so = cxx_rz self.dim = 2 elif self.geometry_dim == "3d": import amrex.space3d as amr + self.amr = amr from . import warpx_pybind_3d as cxx_3d + self.libwarpx_so = cxx_3d self.dim = 3 self.Config = self.libwarpx_so.Config except ImportError: - raise Exception(f"Dimensionality '{self.geometry_dim}' was not compiled in this Python install. Please recompile with -DWarpX_DIMS={_dims}") + raise Exception( + f"Dimensionality '{self.geometry_dim}' was not compiled in this Python install. Please recompile with -DWarpX_DIMS={_dims}" + ) self.__version__ = self.libwarpx_so.__version__ def amrex_init(self, argv, mpi_comm=None): - if mpi_comm is None: # or MPI is None: + if mpi_comm is None: # or MPI is None: self.libwarpx_so.amrex_init(argv) else: - raise Exception('mpi_comm argument not yet supported') + raise Exception("mpi_comm argument not yet supported") def initialize(self, argv=None, mpi_comm=None): - ''' + """ Initialize WarpX and AMReX. Must be called before doing anything else. - ''' + """ if argv is None: argv = sys.argv self.amrex_init(argv, mpi_comm) @@ -133,22 +148,22 @@ def initialize(self, argv=None, mpi_comm=None): self.libwarpx_so.execute_python_callback("afterinit") self.libwarpx_so.execute_python_callback("particleloader") - #self.libwarpx_so.warpx_init() - self.initialized = True def finalize(self, finalize_mpi=1): - ''' + """ Call finalize for WarpX and AMReX. Registered to run at program exit. - ''' + """ # TODO: simplify, part of pyAMReX already if self.initialized: del self.warpx # The call to warpx_finalize causes a crash - don't know why - #self.libwarpx_so.warpx_finalize() + # self.libwarpx_so.warpx_finalize() self.libwarpx_so.amrex_finalize() from pywarpx import callbacks + callbacks.clear_all() + libwarpx = LibWarpX() diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 1f32fee6620..d12293e699b 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -25,6 +25,7 @@ Functions can be called at the following times: +* ``loadExternalFields``: during ``WarpX::LoadExternalFields`` to write ``B/Efield_fp_external`` values * ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) * ``afterinit``: immediately after the init is complete * ``beforeEsolve``: before the solve for E fields @@ -100,19 +101,20 @@ class CallbackFunctions(object): installed a method in one of the call back lists. """ - def __init__(self,name=None,lcallonce=0,singlefunconly=False): + def __init__(self, name=None, lcallonce=0, singlefunconly=False): self.funcs = [] - self.time = 0. + self.time = 0.0 self.timers = {} self.name = name self.lcallonce = lcallonce self.singlefunconly = singlefunconly - def __call__(self,*args,**kw): + def __call__(self, *args, **kw): """Call all of the functions in the list""" - tt = self.callfuncsinlist(*args,**kw) + tt = self.callfuncsinlist(*args, **kw) self.time = self.time + tt - if self.lcallonce: self.funcs = [] + if self.lcallonce: + self.funcs = [] def clearlist(self): """Unregister/clear out all registered C callbacks""" @@ -131,7 +133,7 @@ def hasfuncsinstalled(self): """Checks if there are any functions installed""" return len(self.funcs) > 0 - def _getmethodobject(self,func): + def _getmethodobject(self, func): """For call backs that are methods, returns the method's instance""" return func[0] @@ -139,14 +141,15 @@ def callbackfunclist(self): """Generator returning callable functions from the list""" funclistcopy = copy.copy(self.funcs) for f in funclistcopy: - if isinstance(f,list): + if isinstance(f, list): object = self._getmethodobject(f) if object is None: self.funcs.remove(f) continue - result = getattr(object,f[1]) - elif isinstance(f,str): + result = getattr(object, f[1]) + elif isinstance(f, str): import __main__ + if f in __main__.__dict__: result = __main__.__dict__[f] # --- If the function with the name is found, then replace the @@ -159,19 +162,19 @@ def callbackfunclist(self): if not callable(result): print("\n\nWarning: a call back was found that is not callable.") if self.name is not None: - print("For %s"%self.name) + print("For %s" % self.name) print("Only callable objects can be installed.") print("It is possible that the callable's name has been overwritten") print("by something not callable. This can happen during restart") print("if a function name had later been used as a variable name.") print(self.name) - if isinstance(f,str): - print("The name of the call back is %s"%f) + if isinstance(f, str): + print(f"The name of the call back is {f}") print("\n\n") continue yield result - def installfuncinlist(self,f): + def installfuncinlist(self, f): """Check if the specified function is installed""" if self.singlefunconly and self.hasfuncsinstalled(): raise RuntimeError( @@ -182,12 +185,12 @@ def installfuncinlist(self,f): # If this is the first function installed, set the callback in the C++ # to call this class instance. libwarpx.libwarpx_so.add_python_callback(self.name, self) - if isinstance(f,types.MethodType): + if isinstance(f, types.MethodType): # --- If the function is a method of a class instance, then save a full # --- reference to that instance and the method name. finstance = f.__self__ fname = f.__name__ - self.funcs.append([finstance,fname]) + self.funcs.append([finstance, fname]) elif callable(f): # --- If a function had already been installed by name, then skip the install. # --- This is problematic, since no warning message is given, but it is unlikely @@ -202,7 +205,7 @@ def installfuncinlist(self,f): else: self.funcs.append(f) - def uninstallfuncinlist(self,f): + def uninstallfuncinlist(self, f): """Uninstall the specified function""" # --- An element by element search is needed # --- f can be a function or method object, or a name (string). @@ -212,71 +215,77 @@ def uninstallfuncinlist(self,f): if f == func: self.funcs.remove(f) break - elif isinstance(func,list) and isinstance(f,types.MethodType): + elif isinstance(func, list) and isinstance(f, types.MethodType): object = self._getmethodobject(func) if f.__self__ is object and f.__name__ == func[1]: self.funcs.remove(func) break - elif isinstance(func,str): + elif isinstance(func, str): if f.__name__ == func: self.funcs.remove(func) break - elif isinstance(f,str): - if isinstance(func,str): funcname = func - elif isinstance(func,list): funcname = None - else: funcname = func.__name__ + elif isinstance(f, str): + if isinstance(func, str): + funcname = func + elif isinstance(func, list): + funcname = None + else: + funcname = func.__name__ if f == funcname: self.funcs.remove(func) break # check that a function was removed if len(self.funcs) == len(funclistcopy): - raise Exception(f'Warning: no function, {f}, had been installed') + raise Exception(f"Warning: no function, {f}, had been installed") # if there are no functions left, remove the C callback if not self.hasfuncsinstalled(): self.clearlist() - def isinstalledfuncinlist(self,f): + def isinstalledfuncinlist(self, f): """Checks if the specified function is installed""" # --- An element by element search is needed funclistcopy = copy.copy(self.funcs) for func in funclistcopy: if f == func: return 1 - elif isinstance(func,list) and isinstance(f,types.MethodType): + elif isinstance(func, list) and isinstance(f, types.MethodType): object = self._getmethodobject(func) if f.__self__ is object and f.__name__ == func[1]: return 1 - elif isinstance(func,str): + elif isinstance(func, str): if f.__name__ == func: return 1 return 0 - def callfuncsinlist(self,*args,**kw): + def callfuncsinlist(self, *args, **kw): """Call the functions in the list""" bb = time.time() for f in self.callbackfunclist(): - #barrier() + # barrier() t1 = time.time() - f(*args,**kw) - #barrier() + f(*args, **kw) + # barrier() t2 = time.time() # --- For the timers, use the function (or method) name as the key. - self.timers[f.__name__] = self.timers.get(f.__name__,0.) + (t2 - t1) + self.timers[f.__name__] = self.timers.get(f.__name__, 0.0) + (t2 - t1) aa = time.time() return aa - bb -#============================================================================= + +# ============================================================================= callback_instances = { + "loadExternalFields": {}, "beforeInitEsolve": {}, "afterInitEsolve": {}, + "afterInitatRestart": {}, "afterinit": {}, "beforecollisions": {}, "aftercollisions": {}, "beforeEsolve": {}, - "poissonsolver": {'singlefunconly': True}, # external Poisson solver + "poissonsolver": {"singlefunconly": True}, # external Poisson solver "afterEsolve": {}, "afterBpush": {}, "afterEpush": {}, @@ -291,13 +300,14 @@ def callfuncsinlist(self,*args,**kw): "oncheckpointsignal": {}, "onbreaksignal": {}, "particleinjection": {}, - "appliedfields": {} + "appliedfields": {}, } # --- Now create the actual instances. for key, val in callback_instances.items(): callback_instances[key] = CallbackFunctions(name=key, **val) + def installcallback(name, f): """Installs a function to be called at that specified time. @@ -305,186 +315,283 @@ def installcallback(name, f): """ callback_instances[name].installfuncinlist(f) + def uninstallcallback(name, f): """Uninstalls the function (so it won't be called anymore). Removes the function from the list of functions called by this callback.""" callback_instances[name].uninstallfuncinlist(f) + def isinstalled(name, f): """Checks if a function is installed for this callback.""" return callback_instances[name].isinstalledfuncinlist(f) + def clear_all(): for key, val in callback_instances.items(): val.clearlist() -#============================================================================= -def printcallbacktimers(tmin=1.,lminmax=False,ff=None): +# ============================================================================ + + +def printcallbacktimers(tmin=1.0, lminmax=False, ff=None): """Prints timings of installed functions. - tmin=1.: only functions with time greater than tmin will be printed - lminmax=False: If True, prints the min and max times over all processors - ff=None: If given, timings will be written to the file object instead of stdout """ - if ff is None: ff = sys.stdout + if ff is None: + ff = sys.stdout for c in callback_instances.values(): - for fname, time in c.timers.items(): - #vlist = numpy.array(gather(time)) - vlist = numpy.array([time]) - #if me > 0: continue + for fname, this_time in c.timers.items(): + # vlist = numpy.array(gather(this_time)) + vlist = numpy.array([this_time]) + # if me > 0: continue vsum = numpy.sum(vlist) - if vsum <= tmin: continue - vrms = numpy.sqrt(max(0.,numpy.sum(vlist**2)/len(vlist) - (numpy.sum(vlist)/len(vlist))**2)) - npes = 1. # Only works for one processor - ff.write('%20s %s %10.4f %10.4f %10.4f'%(c.name,fname,vsum,vsum/npes,vrms)) + if vsum <= tmin: + continue + vrms = numpy.sqrt( + max( + 0.0, + numpy.sum(vlist**2) / len(vlist) + - (numpy.sum(vlist) / len(vlist)) ** 2, + ) + ) + npes = 1.0 # Only works for one processor + ff.write( + "%20s %s %10.4f %10.4f %10.4f" + % (c.name, fname, vsum, vsum / npes, vrms) + ) if lminmax: vmin = numpy.min(vlist) vmax = numpy.max(vlist) - ff.write(' %10.4f %10.4f'%(vmin,vmax)) + ff.write(" %10.4f %10.4f" % (vmin, vmax)) it = libwarpx.libwarpx_so.warpx_getistep(0) if it > 0: - ff.write(' %10.4f'%(vsum/npes/(it))) - ff.write('\n') + ff.write(" %10.4f" % (vsum / npes / (it))) + ff.write("\n") + + +# ============================================================================ + + +# ---------------------------------------------------------------------------- +def callfromloadExternalFields(f): + installcallback("loadExternalFields", f) + return f + + +def installloadExternalFields(f): + installcallback("loadExternalFields", f) -#============================================================================= # ---------------------------------------------------------------------------- def callfrombeforeInitEsolve(f): - installcallback('beforeInitEsolve', f) + installcallback("beforeInitEsolve", f) return f + + def installbeforeInitEsolve(f): - installcallback('beforeInitEsolve', f) + installcallback("beforeInitEsolve", f) + # ---------------------------------------------------------------------------- def callfromafterInitEsolve(f): - installcallback('afterInitEsolve', f) + installcallback("afterInitEsolve", f) return f + + def installafterInitEsolve(f): - installcallback('afterInitEsolve', f) + installcallback("afterInitEsolve", f) + + +# ---------------------------------------------------------------------------- +def callfromafterInitatRestart(f): + installcallback("afterInitatRestart", f) + return f + + +def installafterInitatRestart(f): + installcallback("afterInitatRestart", f) + # ---------------------------------------------------------------------------- def callfromafterinit(f): - installcallback('afterinit', f) + installcallback("afterinit", f) return f + + def installafterinit(f): - installcallback('afterinit', f) + installcallback("afterinit", f) + # ---------------------------------------------------------------------------- def callfrombeforecollisions(f): - installcallback('beforecollisions', f) + installcallback("beforecollisions", f) return f + + def installbeforecollisions(f): - installcallback('beforecollisions', f) + installcallback("beforecollisions", f) + # ---------------------------------------------------------------------------- def callfromaftercollisions(f): - installcallback('aftercollisions', f) + installcallback("aftercollisions", f) return f + + def installaftercollisions(f): - installcallback('aftercollisions', f) + installcallback("aftercollisions", f) + # ---------------------------------------------------------------------------- def callfrombeforeEsolve(f): - installcallback('beforeEsolve', f) + installcallback("beforeEsolve", f) return f + + def installbeforeEsolve(f): - installcallback('beforeEsolve', f) + installcallback("beforeEsolve", f) + # ---------------------------------------------------------------------------- def callfrompoissonsolver(f): - installcallback('poissonsolver', f) + installcallback("poissonsolver", f) return f + + def installpoissonsolver(f): - installcallback('poissonsolver', f) + installcallback("poissonsolver", f) + # ---------------------------------------------------------------------------- def callfromafterEsolve(f): - installcallback('afterEsolve', f) + installcallback("afterEsolve", f) return f + + def installafterEsolve(f): - installcallback('afterEsolve', f) + installcallback("afterEsolve", f) + # ---------------------------------------------------------------------------- def callfromafterBpush(f): - installcallback('afterBpush', f) + installcallback("afterBpush", f) return f + + def installafterBpush(f): - installcallback('afterBpush', f) + installcallback("afterBpush", f) + # ---------------------------------------------------------------------------- def callfromafterEpush(f): - installcallback('afterEpush', f) + installcallback("afterEpush", f) return f + + def installafterEpush(f): - installcallback('afterEpush', f) + installcallback("afterEpush", f) + # ---------------------------------------------------------------------------- def callfrombeforedeposition(f): - installcallback('beforedeposition', f) + installcallback("beforedeposition", f) return f + + def installbeforedeposition(f): - installcallback('beforedeposition', f) + installcallback("beforedeposition", f) + # ---------------------------------------------------------------------------- def callfromafterdeposition(f): - installcallback('afterdeposition', f) + installcallback("afterdeposition", f) return f + + def installafterdeposition(f): - installcallback('afterdeposition', f) + installcallback("afterdeposition", f) + # ---------------------------------------------------------------------------- def callfromparticlescraper(f): - installcallback('particlescraper', f) + installcallback("particlescraper", f) return f + + def installparticlescraper(f): - installcallback('particlescraper', f) + installcallback("particlescraper", f) + # ---------------------------------------------------------------------------- def callfromparticleloader(f): - installcallback('particleloader', f) + installcallback("particleloader", f) return f + + def installparticleloader(f): - installcallback('particleloader', f) + installcallback("particleloader", f) + # ---------------------------------------------------------------------------- def callfrombeforestep(f): - installcallback('beforestep', f) + installcallback("beforestep", f) return f + + def installbeforestep(f): - installcallback('beforestep', f) + installcallback("beforestep", f) + # ---------------------------------------------------------------------------- def callfromafterstep(f): - installcallback('afterstep', f) + installcallback("afterstep", f) return f + + def installafterstep(f): - installcallback('afterstep', f) + installcallback("afterstep", f) + # ---------------------------------------------------------------------------- def callfromafterdiagnostics(f): - installcallback('afterdiagnostics', f) + installcallback("afterdiagnostics", f) return f + + def installafterdiagnostics(f): - installcallback('afterdiagnostics', f) + installcallback("afterdiagnostics", f) + # ---------------------------------------------------------------------------- def oncheckpointsignal(f): - installcallback('oncheckpointsignal', f) + installcallback("oncheckpointsignal", f) return f + + def installoncheckpointsignal(f): - installcallback('oncheckpointsignal', f) + installcallback("oncheckpointsignal", f) + # ---------------------------------------------------------------------------- def onbreaksignal(f): - installcallback('onbreaksignal', f) + installcallback("onbreaksignal", f) return f + + def installonbreaksignal(f): - installcallback('onbreaksignal', f) + installcallback("onbreaksignal", f) + # ---------------------------------------------------------------------------- def callfromparticleinjection(f): - installcallback('particleinjection', f) + installcallback("particleinjection", f) return f + + def installparticleinjection(f): - installcallback('particleinjection', f) + installcallback("particleinjection", f) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 0f680595ef4..a81999103d9 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -33,7 +33,7 @@ ExFPPMLWrapper, EyFPPMLWrapper, EzFPPMLWrapper BxFPPMLWrapper, ByFPPMLWrapper, BzFPPMLWrapper JxFPPMLWrapper, JyFPPMLWrapper, JzFPPMLWrapper -JxFPAmpereWrapper, JyFPAmpereWrapper, JzFPAmpereWrapper +JxFPPlasmaWrapper, JyFPPlasmaWrapper, JzFPPlasmaWrapper FFPPMLWrapper, GFPPMLWrapper ExCPPMLWrapper, EyCPPMLWrapper, EzCPPMLWrapper @@ -41,6 +41,7 @@ JxCPPMLWrapper, JyCPPMLWrapper, JzCPPMLWrapper FCPPMLWrapper, GCPPMLWrapper """ + import numpy as np try: @@ -51,6 +52,7 @@ try: from mpi4py import MPI as mpi + comm_world = mpi.COMM_WORLD npes = comm_world.Get_size() except ImportError: @@ -75,6 +77,9 @@ class _MultiFABWrapper(object): everytime it is called if this argument is given instead of directly providing the Multifab. + idir: int, optional + For MultiFab that is an element of a vector, the direction number, 0, 1, or 2. + level: int The refinement level @@ -83,9 +88,11 @@ class _MultiFABWrapper(object): Note that when True, the first n-ghost negative indices will refer to the lower ghost cells. """ - def __init__(self, mf=None, mf_name=None, level=0, include_ghosts=False): + + def __init__(self, mf=None, mf_name=None, idir=None, level=0, include_ghosts=False): self._mf = mf self.mf_name = mf_name + self.idir = idir self.level = level self.include_ghosts = include_ghosts @@ -94,7 +101,9 @@ def __init__(self, mf=None, mf_name=None, level=0, include_ghosts=False): # The overlaps list is one along the axes where the grid boundaries overlap the neighboring grid, # which is the case with node centering. ix_type = self.mf.box_array().ix_type() - self.overlaps = self._get_indices([int(ix_type.node_centered(i)) for i in range(self.dim)], 0) + self.overlaps = self._get_indices( + [int(ix_type.node_centered(i)) for i in range(self.dim)], 0 + ) def __len__(self): "Returns the number of blocks" @@ -111,18 +120,20 @@ def mf(self): else: # Always fetch this anew in case the C++ MultiFab is recreated warpx = libwarpx.libwarpx_so.get_instance() - # All MultiFab names have the level suffix - return warpx.multifab(f'{self.mf_name}[level={self.level}]') + if self.idir is not None: + direction = libwarpx.libwarpx_so.Direction(self.idir) + return warpx.multifab(self.mf_name, direction, self.level) + else: + return warpx.multifab(self.mf_name, self.level) @property def shape(self): - """Returns the shape of the global array - """ + """Returns the shape of the global array""" min_box = self.mf.box_array().minimal_box() shape = list(min_box.size - min_box.small_end) if self.include_ghosts: nghosts = self.mf.n_grow_vect - shape = [shape[i] + 2*nghosts[i] for i in range(self.dim)] + shape = [shape[i] + 2 * nghosts[i] for i in range(self.dim)] shape.append(self.mf.nComp) return tuple(shape) @@ -139,20 +150,16 @@ def mesh(self, direction): """ try: - if libwarpx.geometry_dim == '3d': - idir = ['x', 'y', 'z'].index(direction) - celldir = idir - elif libwarpx.geometry_dim == '2d': - idir = ['x', 'z'].index(direction) - celldir = 2*idir - elif libwarpx.geometry_dim == 'rz': - idir = ['r', 'z'].index(direction) - celldir = 2*idir - elif libwarpx.geometry_dim == '1d': - idir = ['z'].index(direction) - celldir = idir + if libwarpx.geometry_dim == "3d": + idir = ["x", "y", "z"].index(direction) + elif libwarpx.geometry_dim == "2d": + idir = ["x", "z"].index(direction) + elif libwarpx.geometry_dim == "rz": + idir = ["r", "z"].index(direction) + elif libwarpx.geometry_dim == "1d": + idir = ["z"].index(direction) except ValueError: - raise Exception('Inappropriate direction given') + raise Exception("Inappropriate direction given") min_box = self.mf.box_array().minimal_box() ilo = min_box.small_end[idir] @@ -172,13 +179,13 @@ def mesh(self, direction): ix_type = self.mf.box_array().ix_type() if ix_type.node_centered(idir): # node centered - shift = 0. + shift = 0.0 else: # cell centered - shift = 0.5*dd + shift = 0.5 * dd lo = warpx.Geom(self.level).ProbLo(idir) - return lo + np.arange(ilo,ihi+1)*dd + shift + return lo + np.arange(ilo, ihi + 1) * dd + shift def _get_indices(self, index, missing): """Expand the index list to length three. @@ -214,8 +221,7 @@ def _get_min_indices(self): return imin def _get_max_indices(self): - """Returns the maximum indices, expanded to length 3. - """ + """Returns the maximum indices, expanded to length 3.""" min_box = self.mf.box_array().minimal_box() if self.include_ghosts: min_box.grow(self.mf.n_grow_vect) @@ -277,8 +283,12 @@ def _find_start_stop(self, ii, imin, imax, d): ii = self._fix_index(ii, imax, d) iistart = ii iistop = ii + 1 - assert imin <= iistart <= imax, Exception(f'Dimension {d+1} lower index is out of bounds') - assert imin <= iistop <= imax, Exception(f'Dimension {d+1} upper index is out of bounds') + assert imin <= iistart <= imax, Exception( + f"Dimension {d + 1} lower index is out of bounds" + ) + assert imin <= iistop <= imax, Exception( + f"Dimension {d + 1} upper index is out of bounds" + ) return iistart, iistop def _get_field(self, mfi): @@ -302,7 +312,9 @@ def _get_field(self, mfi): device_arr = device_arr4.to_numpy(copy=False) if not self.include_ghosts: nghosts = self._get_n_ghosts() - device_arr = device_arr[tuple([slice(ng, -ng) for ng in nghosts[:self.dim]])] + device_arr = device_arr[ + tuple([slice(ng, -ng) for ng in nghosts[: self.dim]]) + ] return device_arr def _get_intersect_slice(self, mfi, starts, stops, icstart, icstop): @@ -353,7 +365,6 @@ def _get_intersect_slice(self, mfi, starts, stops, icstart, icstop): i2 = np.minimum(stops, ihi_p1) if np.all(i1 < i2): - block_slices = [] global_slices = [] for i in range(3): @@ -384,19 +395,19 @@ def __getitem__(self, index): # Note that the index can have negative values (which wrap around) and has 1 added to the upper # limit using python style slicing if index == Ellipsis: - index = self.dim*[slice(None)] + index = self.dim * [slice(None)] elif isinstance(index, slice): # If only one slice passed in, it was not wrapped in a list index = [index] - if len(index) < self.dim+1: + if len(index) < self.dim + 1: # Add extra dims to index, including for the component. # These are the dims left out and assumed to extend over the full size of the dim index = list(index) - while len(index) < self.dim+1: + while len(index) < self.dim + 1: index.append(slice(None)) - elif len(index) > self.dim+1: - raise Exception('Too many indices given') + elif len(index) > self.dim + 1: + raise Exception("Too many indices given") # Expand the indices to length 3 ii = self._get_indices(index, None) @@ -407,9 +418,9 @@ def __getitem__(self, index): ixmax, iymax, izmax = self._get_max_indices() # Setup the size of the array to be returned - ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) - iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) - izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) + ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax + 1, 0) + iystart, iystop = self._find_start_stop(ii[1], iymin, iymax + 1, 1) + izstart, izstop = self._find_start_stop(ii[2], izmin, izmax + 1, 2) icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) # Gather the data to be included in a list to be sent to other processes @@ -417,7 +428,9 @@ def __getitem__(self, index): stops = [ixstop, iystop, izstop] datalist = [] for mfi in self.mf: - block_slices, global_slices = self._get_intersect_slice(mfi, starts, stops, icstart, icstop) + block_slices, global_slices = self._get_intersect_slice( + mfi, starts, stops, icstart, icstop + ) if global_slices is not None: # Note that the array will always have 4 dimensions. device_arr = self._get_field(mfi) @@ -434,10 +447,12 @@ def __getitem__(self, index): all_datalist = comm_world.allgather(datalist) # Create the array to be returned - result_shape = (max(0, ixstop - ixstart), - max(0, iystop - iystart), - max(0, izstop - izstart), - max(0, icstop - icstart)) + result_shape = ( + max(0, ixstop - ixstart), + max(0, iystop - iystart), + max(0, izstop - izstart), + max(0, icstop - icstart), + ) # Now, copy the data into the result array result_global = None @@ -474,19 +489,19 @@ def __setitem__(self, index, value): # Note that the index can have negative values (which wrap around) and has 1 added to the upper # limit using python style slicing if index == Ellipsis: - index = tuple(self.dim*[slice(None)]) + index = tuple(self.dim * [slice(None)]) elif isinstance(index, slice): # If only one slice passed in, it was not wrapped in a list index = [index] - if len(index) < self.dim+1: + if len(index) < self.dim + 1: # Add extra dims to index, including for the component. # These are the dims left out and assumed to extend over the full size of the dim. index = list(index) - while len(index) < self.dim+1: + while len(index) < self.dim + 1: index.append(slice(None)) - elif len(index) > self.dim+1: - raise Exception('Too many indices given') + elif len(index) > self.dim + 1: + raise Exception("Too many indices given") # Expand the indices to length 3 ii = self._get_indices(index, None) @@ -497,9 +512,9 @@ def __setitem__(self, index, value): ixmax, iymax, izmax = self._get_max_indices() # Setup the size of the global array to be set - ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) - iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) - izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) + ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax + 1, 0) + iystart, iystop = self._find_start_stop(ii[1], iymin, iymax + 1, 1) + izstart, izstop = self._find_start_stop(ii[2], izmin, izmax + 1, 2) icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) if isinstance(value, np.ndarray): @@ -511,10 +526,14 @@ def __setitem__(self, index, value): global_shape = list(value3d.shape) # The shape of 1 is added for the extra dimensions and when index is an integer # (in which case the dimension was not in the input array). - if not isinstance(ii[0], slice): global_shape[0:0] = [1] - if not isinstance(ii[1], slice): global_shape[1:1] = [1] - if not isinstance(ii[2], slice): global_shape[2:2] = [1] - if not isinstance(ic , slice) or len(global_shape) < 4: global_shape[3:3] = [1] + if not isinstance(ii[0], slice): + global_shape[0:0] = [1] + if not isinstance(ii[1], slice): + global_shape[1:1] = [1] + if not isinstance(ii[2], slice): + global_shape[2:2] = [1] + if not isinstance(ic, slice) or len(global_shape) < 4: + global_shape[3:3] = [1] value3d.shape = global_shape if libwarpx.libwarpx_so.Config.have_gpu: @@ -526,7 +545,9 @@ def __setitem__(self, index, value): starts = [ixstart, iystart, izstart] stops = [ixstop, iystop, izstop] for mfi in self.mf: - block_slices, global_slices = self._get_intersect_slice(mfi, starts, stops, icstart, icstop) + block_slices, global_slices = self._get_intersect_slice( + mfi, starts, stops, icstart, icstop + ) if global_slices is not None: mf_arr = self._get_field(mfi) if isinstance(value, np.ndarray): @@ -557,206 +578,554 @@ def norm0(self, *args): return self.mf.norm0(*args) +def CustomNamedxWrapper(mf_name, level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name=mf_name, idir=0, level=level, include_ghosts=include_ghosts + ) + + +def CustomNamedyWrapper(mf_name, level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name=mf_name, idir=1, level=level, include_ghosts=include_ghosts + ) + + +def CustomNamedzWrapper(mf_name, level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name=mf_name, idir=2, level=level, include_ghosts=include_ghosts + ) + + def ExWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_aux", idir=0, level=level, include_ghosts=include_ghosts + ) + def EyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_aux", idir=1, level=level, include_ghosts=include_ghosts + ) + def EzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_aux", idir=2, level=level, include_ghosts=include_ghosts + ) + def BxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_aux", idir=0, level=level, include_ghosts=include_ghosts + ) + def ByWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_aux", idir=1, level=level, include_ghosts=include_ghosts + ) + def BzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_aux", idir=2, level=level, include_ghosts=include_ghosts + ) + def JxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def JyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def JzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def ExFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def EyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def EzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def BxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def ByFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def BzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + + +def ExFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Efield_fp_external", idir=0, level=level, include_ghosts=include_ghosts + ) + + +def EyFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Efield_fp_external", idir=1, level=level, include_ghosts=include_ghosts + ) + + +def EzFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Efield_fp_external", idir=2, level=level, include_ghosts=include_ghosts + ) + + +def BxFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Bfield_fp_external", idir=0, level=level, include_ghosts=include_ghosts + ) + + +def ByFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Bfield_fp_external", idir=1, level=level, include_ghosts=include_ghosts + ) + + +def BzFPExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="Bfield_fp_external", idir=2, level=level, include_ghosts=include_ghosts + ) + + +def AxHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_A_fp_external", + idir=0, + level=level, + include_ghosts=include_ghosts, + ) + + +def AyHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_A_fp_external", + idir=1, + level=level, + include_ghosts=include_ghosts, + ) + + +def AzHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_A_fp_external", + idir=2, + level=level, + include_ghosts=include_ghosts, + ) + + +def ExHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_E_fp_external", + idir=0, + level=level, + include_ghosts=include_ghosts, + ) + + +def EyHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_E_fp_external", + idir=1, + level=level, + include_ghosts=include_ghosts, + ) + + +def EzHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_E_fp_external", + idir=2, + level=level, + include_ghosts=include_ghosts, + ) + + +def BxHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_B_fp_external", + idir=0, + level=level, + include_ghosts=include_ghosts, + ) + + +def ByHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_B_fp_external", + idir=1, + level=level, + include_ghosts=include_ghosts, + ) + + +def BzHybridExternalWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_B_fp_external", + idir=2, + level=level, + include_ghosts=include_ghosts, + ) + def JxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def JyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def JzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def RhoFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='rho_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="rho_fp", level=level, include_ghosts=include_ghosts + ) + def PhiFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='phi_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="phi_fp", level=level, include_ghosts=include_ghosts + ) + def FFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='F_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name="F_fp", level=level, include_ghosts=include_ghosts) + def GFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='G_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name="G_fp", level=level, include_ghosts=include_ghosts) + def AxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="vector_potential_fp_nodal", + idir=0, + level=level, + include_ghosts=include_ghosts, + ) + def AyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="vector_potential_fp_nodal", + idir=1, + level=level, + include_ghosts=include_ghosts, + ) + def AzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="vector_potential_fp_nodal", + idir=2, + level=level, + include_ghosts=include_ghosts, + ) + def ExCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def EyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def EzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Efield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Efield_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def BxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def ByCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def BzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='Bfield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="Bfield_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def JxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def JyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def JzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="current_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def RhoCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='rho_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="rho_cp", level=level, include_ghosts=include_ghosts + ) + def FCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='F_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name="F_cp", level=level, include_ghosts=include_ghosts) + def GCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='G_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name="G_cp", level=level, include_ghosts=include_ghosts) + def EdgeLengthsxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_edge_lengths[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="edge_lengths", idir=0, level=level, include_ghosts=include_ghosts + ) + def EdgeLengthsyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_edge_lengths[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="edge_lengths", idir=1, level=level, include_ghosts=include_ghosts + ) + def EdgeLengthszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_edge_lengths[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="edge_lengths", idir=2, level=level, include_ghosts=include_ghosts + ) + def FaceAreasxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_face_areas[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="face_areas", idir=0, level=level, include_ghosts=include_ghosts + ) + def FaceAreasyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_face_areas[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="face_areas", idir=1, level=level, include_ghosts=include_ghosts + ) + def FaceAreaszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='m_face_areas[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="face_areas", idir=2, level=level, include_ghosts=include_ghosts + ) + + +def JxFPPlasmaWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_current_fp_plasma", + idir=0, + level=level, + include_ghosts=include_ghosts, + ) -def JxFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp_ampere[x]', level=level, include_ghosts=include_ghosts) -def JyFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp_ampere[y]', level=level, include_ghosts=include_ghosts) +def JyFPPlasmaWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_current_fp_plasma", + idir=1, + level=level, + include_ghosts=include_ghosts, + ) + + +def JzFPPlasmaWrapper(level=0, include_ghosts=False): + return _MultiFABWrapper( + mf_name="hybrid_current_fp_plasma", + idir=2, + level=level, + include_ghosts=include_ghosts, + ) -def JzFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='current_fp_ampere[z]', level=level, include_ghosts=include_ghosts) def ExFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def EyFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def EzFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def BxFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def ByFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def BzFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def JxFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_fp", idir=0, level=level, include_ghosts=include_ghosts + ) + def JyFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_fp", idir=1, level=level, include_ghosts=include_ghosts + ) + def JzFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_fp", idir=2, level=level, include_ghosts=include_ghosts + ) + def FFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_F_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_F_fp", level=level, include_ghosts=include_ghosts + ) + def GFPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_G_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_G_fp", level=level, include_ghosts=include_ghosts + ) + def ExCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def EyCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def EzCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_E_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_E_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def BxCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def ByCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def BzCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_B_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_B_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def JxCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_cp", idir=0, level=level, include_ghosts=include_ghosts + ) + def JyCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_cp", idir=1, level=level, include_ghosts=include_ghosts + ) + def JzCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_j_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_j_cp", idir=2, level=level, include_ghosts=include_ghosts + ) + def FCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_F_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_F_cp", level=level, include_ghosts=include_ghosts + ) + def GCPPMLWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name='pml_G_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper( + mf_name="pml_G_cp", level=level, include_ghosts=include_ghosts + ) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 70471e24d1a..a66fd131aed 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -24,15 +24,35 @@ class ParticleContainerWrapper(object): def __init__(self, species_name): self.name = species_name - - # grab the desired particle container - mypc = libwarpx.warpx.multi_particle_container() - self.particle_container = mypc.get_particle_container_from_name(self.name) - - - def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, - uz=None, w=None, unique_particles=True, **kwargs): - ''' + self._particle_container = None + + @property + def particle_container(self): + if self._particle_container is None: + try: + mypc = libwarpx.warpx.multi_particle_container() + self._particle_container = mypc.get_particle_container_from_name( + self.name + ) + except AttributeError as e: + msg = "You must initialize WarpX before accessing a ParticleContainerWrapper's particle_container." + raise AttributeError(msg) from e + + return self._particle_container + + def add_particles( + self, + x=None, + y=None, + z=None, + ux=None, + uy=None, + uz=None, + w=None, + unique_particles=True, + **kwargs, + ): + """ A function for adding particles to the WarpX simulation. Parameters @@ -58,7 +78,7 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, kwargs : dict Containing an entry for all the extra particle attribute arrays. If an attribute is not given it will be set to 0. - ''' + """ # --- Get length of arrays, set to one for scalars lenx = np.size(x) @@ -87,32 +107,48 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, maxlen = max(maxlen, lenw) # --- Make sure that the lengths of the input parameters are consistent - assert x is None or lenx==maxlen or lenx==1, "Length of x doesn't match len of others" - assert y is None or leny==maxlen or leny==1, "Length of y doesn't match len of others" - assert z is None or lenz==maxlen or lenz==1, "Length of z doesn't match len of others" - assert ux is None or lenux==maxlen or lenux==1, "Length of ux doesn't match len of others" - assert uy is None or lenuy==maxlen or lenuy==1, "Length of uy doesn't match len of others" - assert uz is None or lenuz==maxlen or lenuz==1, "Length of uz doesn't match len of others" - assert w is None or lenw==maxlen or lenw==1, "Length of w doesn't match len of others" + assert x is None or lenx == maxlen or lenx == 1, ( + "Length of x doesn't match len of others" + ) + assert y is None or leny == maxlen or leny == 1, ( + "Length of y doesn't match len of others" + ) + assert z is None or lenz == maxlen or lenz == 1, ( + "Length of z doesn't match len of others" + ) + assert ux is None or lenux == maxlen or lenux == 1, ( + "Length of ux doesn't match len of others" + ) + assert uy is None or lenuy == maxlen or lenuy == 1, ( + "Length of uy doesn't match len of others" + ) + assert uz is None or lenuz == maxlen or lenuz == 1, ( + "Length of uz doesn't match len of others" + ) + assert w is None or lenw == maxlen or lenw == 1, ( + "Length of w doesn't match len of others" + ) for key, val in kwargs.items(): - assert np.size(val)==1 or len(val)==maxlen, f"Length of {key} doesn't match len of others" + assert np.size(val) == 1 or len(val) == maxlen, ( + f"Length of {key} doesn't match len of others" + ) # --- Broadcast scalars into appropriate length arrays # --- If the parameter was not supplied, use the default value if lenx == 1: - x = np.full(maxlen, (x or 0.)) + x = np.full(maxlen, (x or 0.0)) if leny == 1: - y = np.full(maxlen, (y or 0.)) + y = np.full(maxlen, (y or 0.0)) if lenz == 1: - z = np.full(maxlen, (z or 0.)) + z = np.full(maxlen, (z or 0.0)) if lenux == 1: - ux = np.full(maxlen, (ux or 0.)) + ux = np.full(maxlen, (ux or 0.0)) if lenuy == 1: - uy = np.full(maxlen, (uy or 0.)) + uy = np.full(maxlen, (uy or 0.0)) if lenuz == 1: - uz = np.full(maxlen, (uz or 0.)) + uz = np.full(maxlen, (uz or 0.0)) if lenw == 1: - w = np.full(maxlen, (w or 0.)) + w = np.full(maxlen, (w or 0.0)) for key, val in kwargs.items(): if np.size(val) == 1: kwargs[key] = np.full(maxlen, val) @@ -122,22 +158,24 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, built_in_attrs = libwarpx.dim # --- The three velocities built_in_attrs += 3 - if libwarpx.geometry_dim == 'rz': + if libwarpx.geometry_dim == "rz": # --- With RZ, there is also theta built_in_attrs += 1 # --- The number of extra attributes (including the weight) nattr = self.particle_container.num_real_comps - built_in_attrs attr = np.zeros((maxlen, nattr)) - attr[:,0] = w + attr[:, 0] = w # --- Note that the velocities are handled separately and not included in attr # --- (even though they are stored as attributes in the C++) for key, vals in kwargs.items(): - attr[:,self.particle_container.get_comp_index(key) - built_in_attrs] = vals + attr[ + :, self.particle_container.get_real_comp_index(key) - built_in_attrs + ] = vals nattr_int = 0 - attr_int = np.empty([0], dtype=np.int32) + attr_int = np.empty([0], dtype=np.int32) # TODO: expose ParticleReal through pyAMReX # and cast arrays to the correct types, before calling add_n_particles @@ -149,13 +187,23 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, # uz = uz.astype(self._numpy_particlereal_dtype, copy=False) self.particle_container.add_n_particles( - 0, x.size, x, y, z, ux, uy, uz, - nattr, attr, nattr_int, attr_int, unique_particles + 0, + x.size, + x, + y, + z, + ux, + uy, + uz, + nattr, + attr, + nattr_int, + attr_int, + unique_particles, ) - def get_particle_count(self, local=False): - ''' + """ Get the number of particles of this species in the simulation. Parameters @@ -170,13 +218,13 @@ def get_particle_count(self, local=False): int An integer count of the number of particles - ''' + """ return self.particle_container.total_number_of_particles(True, local) - nps = property(get_particle_count) + nps = property(get_particle_count) def add_real_comp(self, pid_name, comm=True): - ''' + """ Add a real component to the particle data array. Parameters @@ -187,12 +235,11 @@ def add_real_comp(self, pid_name, comm=True): comm : bool Should the component be communicated - ''' + """ self.particle_container.add_real_comp(pid_name, comm) - def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): - ''' + """ This returns a list of numpy or cupy arrays containing the particle real array data on each tile for this process. @@ -218,8 +265,8 @@ def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): List of arrays The requested particle array data - ''' - comp_idx = self.particle_container.get_comp_index(comp_name) + """ + comp_idx = self.particle_container.get_real_comp_index(comp_name) data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): @@ -236,9 +283,8 @@ def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): return data_array - def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): - ''' + """ This returns a list of numpy or cupy arrays containing the particle int array data on each tile for this process. @@ -264,8 +310,8 @@ def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): List of arrays The requested particle array data - ''' - comp_idx = self.particle_container.get_icomp_index(comp_name) + """ + comp_idx = self.particle_container.get_int_comp_index(comp_name) data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): @@ -282,9 +328,8 @@ def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): return data_array - def get_particle_idcpu_arrays(self, level, copy_to_host=False): - ''' + """ This returns a list of numpy or cupy arrays containing the particle idcpu data on each tile for this process. @@ -306,7 +351,7 @@ def get_particle_idcpu_arrays(self, level, copy_to_host=False): List of arrays The requested particle array data - ''' + """ data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() @@ -322,9 +367,8 @@ def get_particle_idcpu_arrays(self, level, copy_to_host=False): return data_array - def get_particle_idcpu(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'idcpu' numbers on each tile. @@ -343,13 +387,13 @@ def get_particle_idcpu(self, level=0, copy_to_host=False): List of arrays The requested particle idcpu - ''' + """ return self.get_particle_idcpu_arrays(level, copy_to_host=copy_to_host) - idcpu = property(get_particle_idcpu) + idcpu = property(get_particle_idcpu) def get_particle_id(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'id' numbers on each tile. @@ -368,13 +412,12 @@ def get_particle_id(self, level=0, copy_to_host=False): List of arrays The requested particle ids - ''' + """ idcpu = self.get_particle_idcpu(level, copy_to_host) return [libwarpx.amr.unpack_ids(tile) for tile in idcpu] - def get_particle_cpu(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'cpu' numbers on each tile. @@ -393,13 +436,12 @@ def get_particle_cpu(self, level=0, copy_to_host=False): List of arrays The requested particle cpus - ''' + """ idcpu = self.get_particle_idcpu(level, copy_to_host) return [libwarpx.amr.unpack_cpus(tile) for tile in idcpu] - def get_particle_x(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'x' positions on each tile. @@ -418,13 +460,13 @@ def get_particle_x(self, level=0, copy_to_host=False): List of arrays The requested particle x position - ''' - return self.get_particle_real_arrays('x', level, copy_to_host=copy_to_host) - xp = property(get_particle_x) + """ + return self.get_particle_real_arrays("x", level, copy_to_host=copy_to_host) + xp = property(get_particle_x) def get_particle_y(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'y' positions on each tile. @@ -443,13 +485,13 @@ def get_particle_y(self, level=0, copy_to_host=False): List of arrays The requested particle y position - ''' - return self.get_particle_real_arrays('y', level, copy_to_host=copy_to_host) - yp = property(get_particle_y) + """ + return self.get_particle_real_arrays("y", level, copy_to_host=copy_to_host) + yp = property(get_particle_y) def get_particle_r(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'r' positions on each tile. @@ -468,22 +510,24 @@ def get_particle_r(self, level=0, copy_to_host=False): List of arrays The requested particle r position - ''' + """ xp, cupy_status = load_cupy() - if libwarpx.geometry_dim == 'rz': + if libwarpx.geometry_dim == "rz": return self.get_particle_x(level, copy_to_host) - elif libwarpx.geometry_dim == '3d': + elif libwarpx.geometry_dim == "3d": x = self.get_particle_x(level, copy_to_host) y = self.get_particle_y(level, copy_to_host) return xp.sqrt(x**2 + y**2) - elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': - raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') - rp = property(get_particle_r) + elif libwarpx.geometry_dim == "2d" or libwarpx.geometry_dim == "1d": + raise Exception( + "get_particle_r: There is no r coordinate with 1D or 2D Cartesian" + ) + rp = property(get_particle_r) def get_particle_theta(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle theta on each tile. @@ -502,22 +546,24 @@ def get_particle_theta(self, level=0, copy_to_host=False): List of arrays The requested particle theta position - ''' + """ xp, cupy_status = load_cupy() - if libwarpx.geometry_dim == 'rz': - return self.get_particle_real_arrays('theta', level, copy_to_host) - elif libwarpx.geometry_dim == '3d': + if libwarpx.geometry_dim == "rz": + return self.get_particle_real_arrays("theta", level, copy_to_host) + elif libwarpx.geometry_dim == "3d": x = self.get_particle_x(level, copy_to_host) y = self.get_particle_y(level, copy_to_host) return xp.arctan2(y, x) - elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': - raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') - thetap = property(get_particle_theta) + elif libwarpx.geometry_dim == "2d" or libwarpx.geometry_dim == "1d": + raise Exception( + "get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian" + ) + thetap = property(get_particle_theta) def get_particle_z(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle 'z' positions on each tile. @@ -536,13 +582,13 @@ def get_particle_z(self, level=0, copy_to_host=False): List of arrays The requested particle z position - ''' - return self.get_particle_real_arrays('z', level, copy_to_host=copy_to_host) - zp = property(get_particle_z) + """ + return self.get_particle_real_arrays("z", level, copy_to_host=copy_to_host) + zp = property(get_particle_z) def get_particle_weight(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle weight on each tile. @@ -561,13 +607,13 @@ def get_particle_weight(self, level=0, copy_to_host=False): List of arrays The requested particle weight - ''' - return self.get_particle_real_arrays('w', level, copy_to_host=copy_to_host) - wp = property(get_particle_weight) + """ + return self.get_particle_real_arrays("w", level, copy_to_host=copy_to_host) + wp = property(get_particle_weight) def get_particle_ux(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle x momentum on each tile. @@ -586,13 +632,13 @@ def get_particle_ux(self, level=0, copy_to_host=False): List of arrays The requested particle x momentum - ''' - return self.get_particle_real_arrays('ux', level, copy_to_host=copy_to_host) - uxp = property(get_particle_ux) + """ + return self.get_particle_real_arrays("ux", level, copy_to_host=copy_to_host) + uxp = property(get_particle_ux) def get_particle_uy(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle y momentum on each tile. @@ -611,13 +657,13 @@ def get_particle_uy(self, level=0, copy_to_host=False): List of arrays The requested particle y momentum - ''' - return self.get_particle_real_arrays('uy', level, copy_to_host=copy_to_host) - uyp = property(get_particle_uy) + """ + return self.get_particle_real_arrays("uy", level, copy_to_host=copy_to_host) + uyp = property(get_particle_uy) def get_particle_uz(self, level=0, copy_to_host=False): - ''' + """ Return a list of numpy or cupy arrays containing the particle z momentum on each tile. @@ -636,14 +682,14 @@ def get_particle_uz(self, level=0, copy_to_host=False): List of arrays The requested particle z momentum - ''' + """ - return self.get_particle_real_arrays('uz', level, copy_to_host=copy_to_host) - uzp = property(get_particle_uz) + return self.get_particle_real_arrays("uz", level, copy_to_host=copy_to_host) + uzp = property(get_particle_uz) def get_species_charge_sum(self, local=False): - ''' + """ Returns the total charge in the simulation due to the given species. Parameters @@ -651,39 +697,38 @@ def get_species_charge_sum(self, local=False): local : bool If True return total charge per processor - ''' + """ return self.particle_container.sum_particle_charge(local) - def getex(self): - raise NotImplementedError('Particle E fields not supported') - ex = property(getex) + raise NotImplementedError("Particle E fields not supported") + ex = property(getex) def getey(self): - raise NotImplementedError('Particle E fields not supported') - ey = property(getey) + raise NotImplementedError("Particle E fields not supported") + ey = property(getey) def getez(self): - raise NotImplementedError('Particle E fields not supported') - ez = property(getez) + raise NotImplementedError("Particle E fields not supported") + ez = property(getez) def getbx(self): - raise NotImplementedError('Particle B fields not supported') - bx = property(getbx) + raise NotImplementedError("Particle B fields not supported") + bx = property(getbx) def getby(self): - raise NotImplementedError('Particle B fields not supported') - by = property(getby) + raise NotImplementedError("Particle B fields not supported") + by = property(getby) def getbz(self): - raise NotImplementedError('Particle B fields not supported') - bz = property(getbz) + raise NotImplementedError("Particle B fields not supported") + bz = property(getbz) def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): """ @@ -701,7 +746,7 @@ def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): sync_rho : bool If True, perform MPI exchange and properly set boundary cells for rho_fp. """ - rho_fp = libwarpx.warpx.multifab(f'rho_fp[level={level}]') + rho_fp = libwarpx.warpx.multifab("rho_fp", level) if rho_fp is None: raise RuntimeError("Multifab `rho_fp` is not allocated.") @@ -712,7 +757,7 @@ def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): # deposit the charge density from the desired species self.particle_container.deposit_charge(rho_fp, level) - if libwarpx.geometry_dim == 'rz': + if libwarpx.geometry_dim == "rz": libwarpx.warpx.apply_inverse_volume_scaling_to_charge_density(rho_fp, level) if sync_rho: @@ -726,11 +771,21 @@ class ParticleBoundaryBufferWrapper(object): """ def __init__(self): - self.particle_buffer = libwarpx.warpx.get_particle_boundary_buffer() + self._particle_buffer = None + @property + def particle_buffer(self): + if self._particle_buffer is None: + try: + self._particle_buffer = libwarpx.warpx.get_particle_boundary_buffer() + except AttributeError as e: + msg = "You must initialize WarpX before accessing a ParticleBoundaryBufferWrapper's particle_buffer." + raise AttributeError(msg) from e + + return self._particle_buffer def get_particle_boundary_buffer_size(self, species_name, boundary, local=False): - ''' + """ This returns the number of particles that have been scraped so far in the simulation from the specified boundary and of the specified species. @@ -747,15 +802,13 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) local : bool Whether to only return the number of particles in the current processor's buffer - ''' + """ return self.particle_buffer.get_num_particles_in_container( - species_name, self._get_boundary_number(boundary), - local=local + species_name, self._get_boundary_number(boundary), local=local ) - def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): - ''' + """ This returns a list of numpy or cupy arrays containing the particle array data for a species that has been scraped by a specific simulation boundary. @@ -783,41 +836,43 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) level : int Which AMR level to retrieve scraped particle data from. - ''' + """ xp, cupy_status = load_cupy() part_container = self.particle_buffer.get_particle_container( species_name, self._get_boundary_number(boundary) ) data_array = [] - #loop over the real attributes - if comp_name in part_container.real_comp_names: - comp_idx = part_container.real_comp_names[comp_name] - for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): + # loop over the real attributes + if comp_name in part_container.real_soa_names: + comp_idx = part_container.get_real_comp_index(comp_name) + for ii, pti in enumerate( + libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level) + ): soa = pti.soa() data_array.append(xp.array(soa.get_real_data(comp_idx), copy=False)) - #loop over the integer attributes - elif comp_name in part_container.int_comp_names: - comp_idx = part_container.int_comp_names[comp_name] - for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): + # loop over the integer attributes + elif comp_name in part_container.int_soa_names: + comp_idx = part_container.get_int_comp_index(comp_name) + for ii, pti in enumerate( + libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level) + ): soa = pti.soa() data_array.append(xp.array(soa.get_int_data(comp_idx), copy=False)) else: - raise RuntimeError('Name %s not found' %comp_name) + raise RuntimeError("Name %s not found" % comp_name) return data_array - def clear_buffer(self): - ''' + """ Clear the buffer that holds the particles lost at the boundaries. - ''' + """ self.particle_buffer.clear_particles() - def _get_boundary_number(self, boundary): - ''' + """ Utility function to find the boundary number given a boundary name. @@ -832,34 +887,36 @@ def _get_boundary_number(self, boundary): ------- int Integer index in the boundary scraper buffer for the given boundary. - ''' - if libwarpx.geometry_dim == '3d': - dimensions = {'x' : 0, 'y' : 1, 'z' : 2} - elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == 'rz': - dimensions = {'x' : 0, 'z' : 1} - elif libwarpx.geometry_dim == '1d': - dimensions = {'z' : 0} + """ + if libwarpx.geometry_dim == "3d": + dimensions = {"x": 0, "y": 1, "z": 2} + elif libwarpx.geometry_dim == "2d" or libwarpx.geometry_dim == "rz": + dimensions = {"x": 0, "z": 1} + elif libwarpx.geometry_dim == "1d": + dimensions = {"z": 0} else: raise RuntimeError(f"Unknown simulation geometry: {libwarpx.geometry_dim}") - if boundary != 'eb': + if boundary != "eb": boundary_parts = boundary.split("_") dim_num = dimensions[boundary_parts[0]] - if boundary_parts[1] == 'lo': + if boundary_parts[1] == "lo": side = 0 - elif boundary_parts[1] == 'hi': + elif boundary_parts[1] == "hi": side = 1 else: - raise RuntimeError(f'Unknown boundary specified: {boundary}') + raise RuntimeError(f"Unknown boundary specified: {boundary}") boundary_num = 2 * dim_num + side else: - if libwarpx.geometry_dim == '3d': + if libwarpx.geometry_dim == "3d": boundary_num = 6 - elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == 'rz': + elif libwarpx.geometry_dim == "2d" or libwarpx.geometry_dim == "rz": boundary_num = 4 - elif libwarpx.geometry_dim == '1d': + elif libwarpx.geometry_dim == "1d": boundary_num = 2 else: - raise RuntimeError(f"Unknown simulation geometry: {libwarpx.geometry_dim}") + raise RuntimeError( + f"Unknown simulation geometry: {libwarpx.geometry_dim}" + ) return boundary_num diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 8be6d9c6212..f660570ca7c 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -6,8 +6,8 @@ # # License: BSD-3-Clause-LBNL -"""Classes following the PICMI standard -""" +"""Classes following the PICMI standard""" + import os import re from dataclasses import dataclass @@ -17,21 +17,28 @@ import picmistandard import pywarpx +import pywarpx.callbacks -codename = 'warpx' +codename = "warpx" picmistandard.register_codename(codename) # dictionary to map field boundary conditions from picmistandard to WarpX BC_map = { - 'open':'pml', 'dirichlet':'pec', 'periodic':'periodic', 'damped':'damped', - 'absorbing_silver_mueller':'absorbing_silver_mueller', - 'neumann':'neumann', 'none':'none', None:'none' + "open": "pml", + "dirichlet": "pec", + "periodic": "periodic", + "damped": "damped", + "absorbing_silver_mueller": "absorbing_silver_mueller", + "neumann": "neumann", + "none": "none", + None: "none", } + class constants: # --- Put the constants in their own namespace # --- Values from WarpXConst.H - c = 299792458. + c = 299792458.0 ep0 = 8.8541878128e-12 mu0 = 1.25663706212e-06 q_e = 1.602176634e-19 @@ -40,6 +47,7 @@ class constants: hbar = 1.054571817e-34 kb = 1.380649e-23 + picmistandard.register_constants(constants) @@ -180,37 +188,47 @@ class Species(picmistandard.PICMI_Species): Dictionary of extra real particle attributes initialized from an expression that is a function of the variables (x, y, z, ux, uy, uz, t). """ - def init(self, kw): - if self.particle_type == 'electron': - if self.charge is None: self.charge = '-q_e' - if self.mass is None: self.mass = 'm_e' - elif self.particle_type == 'positron': - if self.charge is None: self.charge = 'q_e' - if self.mass is None: self.mass = 'm_e' - elif self.particle_type == 'proton': - if self.charge is None: self.charge = 'q_e' - if self.mass is None: self.mass = 'm_p' - elif self.particle_type == 'anti-proton': - if self.charge is None: self.charge = '-q_e' - if self.mass is None: self.mass = 'm_p' + def init(self, kw): + if self.particle_type == "electron": + if self.charge is None: + self.charge = "-q_e" + if self.mass is None: + self.mass = "m_e" + elif self.particle_type == "positron": + if self.charge is None: + self.charge = "q_e" + if self.mass is None: + self.mass = "m_e" + elif self.particle_type == "proton": + if self.charge is None: + self.charge = "q_e" + if self.mass is None: + self.mass = "m_p" + elif self.particle_type == "anti-proton": + if self.charge is None: + self.charge = "-q_e" + if self.mass is None: + self.mass = "m_p" else: if self.charge is None and self.charge_state is not None: - if self.charge_state == +1.: - self.charge = 'q_e' - elif self.charge_state == -1.: - self.charge = '-q_e' + if self.charge_state == +1.0: + self.charge = "q_e" + elif self.charge_state == -1.0: + self.charge = "-q_e" else: - self.charge = self.charge_state*constants.q_e + self.charge = self.charge_state * constants.q_e if self.particle_type is not None: # Match a string of the format '#nXx', with the '#n' optional isotope number. - m = re.match(r'(?P#[\d+])*(?P[A-Za-z]+)', self.particle_type) + m = re.match(r"(?P#[\d+])*(?P[A-Za-z]+)", self.particle_type) if m is not None: - element = periodictable.elements.symbol(m['sym']) - if m['iso'] is not None: - element = element[m['iso'][1:]] + element = periodictable.elements.symbol(m["sym"]) + if m["iso"] is not None: + element = element[m["iso"][1:]] if self.charge_state is not None: - assert self.charge_state <= element.number, Exception('%s charge state not valid'%self.particle_type) + assert self.charge_state <= element.number, Exception( + "%s charge state not valid" % self.particle_type + ) try: element = element.ion[self.charge_state] except ValueError: @@ -219,107 +237,139 @@ def init(self, kw): pass self.element = element if self.mass is None: - self.mass = element.mass*periodictable.constants.atomic_mass_constant + self.mass = ( + element.mass * periodictable.constants.atomic_mass_constant + ) else: raise Exception('The species "particle_type" is not known') - self.boost_adjust_transverse_positions = kw.pop('warpx_boost_adjust_transverse_positions', None) + self.boost_adjust_transverse_positions = kw.pop( + "warpx_boost_adjust_transverse_positions", None + ) # For the relativistic electrostatic solver - self.self_fields_required_precision = kw.pop('warpx_self_fields_required_precision', None) - self.self_fields_absolute_tolerance = kw.pop('warpx_self_fields_absolute_tolerance', None) - self.self_fields_max_iters = kw.pop('warpx_self_fields_max_iters', None) - self.self_fields_verbosity = kw.pop('warpx_self_fields_verbosity', None) - self.save_previous_position = kw.pop('warpx_save_previous_position', None) - self.do_not_deposit = kw.pop('warpx_do_not_deposit', None) - self.do_not_push = kw.pop('warpx_do_not_push', None) - self.do_not_gather = kw.pop('warpx_do_not_gather', None) - self.random_theta = kw.pop('warpx_random_theta', None) + self.self_fields_required_precision = kw.pop( + "warpx_self_fields_required_precision", None + ) + self.self_fields_absolute_tolerance = kw.pop( + "warpx_self_fields_absolute_tolerance", None + ) + self.self_fields_max_iters = kw.pop("warpx_self_fields_max_iters", None) + self.self_fields_verbosity = kw.pop("warpx_self_fields_verbosity", None) + self.save_previous_position = kw.pop("warpx_save_previous_position", None) + self.do_not_deposit = kw.pop("warpx_do_not_deposit", None) + self.do_not_push = kw.pop("warpx_do_not_push", None) + self.do_not_gather = kw.pop("warpx_do_not_gather", None) + self.random_theta = kw.pop("warpx_random_theta", None) # For particle reflection - self.reflection_model_xlo = kw.pop('warpx_reflection_model_xlo', None) - self.reflection_model_xhi = kw.pop('warpx_reflection_model_xhi', None) - self.reflection_model_ylo = kw.pop('warpx_reflection_model_ylo', None) - self.reflection_model_yhi = kw.pop('warpx_reflection_model_yhi', None) - self.reflection_model_zlo = kw.pop('warpx_reflection_model_zlo', None) - self.reflection_model_zhi = kw.pop('warpx_reflection_model_zhi', None) + self.reflection_model_xlo = kw.pop("warpx_reflection_model_xlo", None) + self.reflection_model_xhi = kw.pop("warpx_reflection_model_xhi", None) + self.reflection_model_ylo = kw.pop("warpx_reflection_model_ylo", None) + self.reflection_model_yhi = kw.pop("warpx_reflection_model_yhi", None) + self.reflection_model_zlo = kw.pop("warpx_reflection_model_zlo", None) + self.reflection_model_zhi = kw.pop("warpx_reflection_model_zhi", None) # self.reflection_model_eb = kw.pop('warpx_reflection_model_eb', None) # For the scraper buffer - self.save_particles_at_xlo = kw.pop('warpx_save_particles_at_xlo', None) - self.save_particles_at_xhi = kw.pop('warpx_save_particles_at_xhi', None) - self.save_particles_at_ylo = kw.pop('warpx_save_particles_at_ylo', None) - self.save_particles_at_yhi = kw.pop('warpx_save_particles_at_yhi', None) - self.save_particles_at_zlo = kw.pop('warpx_save_particles_at_zlo', None) - self.save_particles_at_zhi = kw.pop('warpx_save_particles_at_zhi', None) - self.save_particles_at_eb = kw.pop('warpx_save_particles_at_eb', None) + self.save_particles_at_xlo = kw.pop("warpx_save_particles_at_xlo", None) + self.save_particles_at_xhi = kw.pop("warpx_save_particles_at_xhi", None) + self.save_particles_at_ylo = kw.pop("warpx_save_particles_at_ylo", None) + self.save_particles_at_yhi = kw.pop("warpx_save_particles_at_yhi", None) + self.save_particles_at_zlo = kw.pop("warpx_save_particles_at_zlo", None) + self.save_particles_at_zhi = kw.pop("warpx_save_particles_at_zhi", None) + self.save_particles_at_eb = kw.pop("warpx_save_particles_at_eb", None) # Resampling settings - self.do_resampling = kw.pop('warpx_do_resampling', None) - self.resampling_algorithm = kw.pop('warpx_resampling_algorithm', None) - self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) - self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) - self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) - self.resampling_algorithm_target_weight = kw.pop('warpx_resampling_algorithm_target_weight', None) - self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) - self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) - self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) - self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) - self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) - if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: - self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 + self.do_resampling = kw.pop("warpx_do_resampling", None) + self.resampling_algorithm = kw.pop("warpx_resampling_algorithm", None) + self.resampling_min_ppc = kw.pop("warpx_resampling_min_ppc", None) + self.resampling_trigger_intervals = kw.pop( + "warpx_resampling_trigger_intervals", None + ) + self.resampling_triggering_max_avg_ppc = kw.pop( + "warpx_resampling_trigger_max_avg_ppc", None + ) + self.resampling_algorithm_target_weight = kw.pop( + "warpx_resampling_algorithm_target_weight", None + ) + self.resampling_algorithm_velocity_grid_type = kw.pop( + "warpx_resampling_algorithm_velocity_grid_type", None + ) + self.resampling_algorithm_delta_ur = kw.pop( + "warpx_resampling_algorithm_delta_ur", None + ) + self.resampling_algorithm_n_theta = kw.pop( + "warpx_resampling_algorithm_n_theta", None + ) + self.resampling_algorithm_n_phi = kw.pop( + "warpx_resampling_algorithm_n_phi", None + ) + self.resampling_algorithm_delta_u = kw.pop( + "warpx_resampling_algorithm_delta_u", None + ) + if ( + self.resampling_algorithm_delta_u is not None + and np.size(self.resampling_algorithm_delta_u) == 1 + ): + self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u] * 3 # extra particle attributes - self.extra_int_attributes = kw.pop('warpx_add_int_attributes', None) - self.extra_real_attributes = kw.pop('warpx_add_real_attributes', None) - - def species_initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + self.extra_int_attributes = kw.pop("warpx_add_int_attributes", None) + self.extra_real_attributes = kw.pop("warpx_add_real_attributes", None) + + def species_initialize_inputs( + self, + layout, + initialize_self_fields=False, + injection_plane_position=None, + injection_plane_normal_vector=None, + ): self.species_number = len(pywarpx.particles.species_names) if self.name is None: - self.name = 'species{}'.format(self.species_number) + self.name = "species{}".format(self.species_number) pywarpx.particles.species_names.append(self.name) if initialize_self_fields is None: initialize_self_fields = False - self.species = pywarpx.Bucket.Bucket(self.name, - mass = self.mass, - charge = self.charge, - injection_style = None, - initialize_self_fields = int(initialize_self_fields), - boost_adjust_transverse_positions = self.boost_adjust_transverse_positions, - self_fields_required_precision = self.self_fields_required_precision, - self_fields_absolute_tolerance = self.self_fields_absolute_tolerance, - self_fields_max_iters = self.self_fields_max_iters, - self_fields_verbosity = self.self_fields_verbosity, - save_particles_at_xlo = self.save_particles_at_xlo, - save_particles_at_xhi = self.save_particles_at_xhi, - save_particles_at_ylo = self.save_particles_at_ylo, - save_particles_at_yhi = self.save_particles_at_yhi, - save_particles_at_zlo = self.save_particles_at_zlo, - save_particles_at_zhi = self.save_particles_at_zhi, - save_particles_at_eb = self.save_particles_at_eb, - save_previous_position = self.save_previous_position, - do_not_deposit = self.do_not_deposit, - do_not_push = self.do_not_push, - do_not_gather = self.do_not_gather, - random_theta = self.random_theta, - do_resampling=self.do_resampling, - resampling_algorithm=self.resampling_algorithm, - resampling_min_ppc=self.resampling_min_ppc, - resampling_trigger_intervals=self.resampling_trigger_intervals, - resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, - resampling_algorithm_target_weight=self.resampling_algorithm_target_weight, - resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, - resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, - resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, - resampling_algorithm_n_phi=self.resampling_algorithm_n_phi, - resampling_algorithm_delta_u=self.resampling_algorithm_delta_u) + self.species = pywarpx.Bucket.Bucket( + self.name, + mass=self.mass, + charge=self.charge, + injection_style=None, + initialize_self_fields=int(initialize_self_fields), + boost_adjust_transverse_positions=self.boost_adjust_transverse_positions, + self_fields_required_precision=self.self_fields_required_precision, + self_fields_absolute_tolerance=self.self_fields_absolute_tolerance, + self_fields_max_iters=self.self_fields_max_iters, + self_fields_verbosity=self.self_fields_verbosity, + save_particles_at_xlo=self.save_particles_at_xlo, + save_particles_at_xhi=self.save_particles_at_xhi, + save_particles_at_ylo=self.save_particles_at_ylo, + save_particles_at_yhi=self.save_particles_at_yhi, + save_particles_at_zlo=self.save_particles_at_zlo, + save_particles_at_zhi=self.save_particles_at_zhi, + save_particles_at_eb=self.save_particles_at_eb, + save_previous_position=self.save_previous_position, + do_not_deposit=self.do_not_deposit, + do_not_push=self.do_not_push, + do_not_gather=self.do_not_gather, + random_theta=self.random_theta, + do_resampling=self.do_resampling, + resampling_algorithm=self.resampling_algorithm, + resampling_min_ppc=self.resampling_min_ppc, + resampling_trigger_intervals=self.resampling_trigger_intervals, + resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_target_weight=self.resampling_algorithm_target_weight, + resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, + resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, + resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi, + resampling_algorithm_delta_u=self.resampling_algorithm_delta_u, + ) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) @@ -334,11 +384,15 @@ def species_initialize_inputs(self, layout, if self.extra_int_attributes is not None: self.species.addIntegerAttributes = self.extra_int_attributes.keys() for attr, function in self.extra_int_attributes.items(): - self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + self.species.add_new_attr( + "attribute." + attr + "(x,y,z,ux,uy,uz,t)", function + ) if self.extra_real_attributes is not None: self.species.addRealAttributes = self.extra_real_attributes.keys() for attr, function in self.extra_real_attributes.items(): - self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + self.species.add_new_attr( + "attribute." + attr + "(x,y,z,ux,uy,uz,t)", function + ) pywarpx.Particles.particles_list.append(self.species) @@ -346,101 +400,155 @@ def species_initialize_inputs(self, layout, distributions_is_list = np.iterable(self.initial_distribution) layout_is_list = np.iterable(layout) if not distributions_is_list and not layout_is_list: - self.initial_distribution.distribution_initialize_inputs(self.species_number, layout, self.species, - self.density_scale, '') + self.initial_distribution.distribution_initialize_inputs( + self.species_number, layout, self.species, self.density_scale, "" + ) elif distributions_is_list and (layout_is_list or layout is None): - assert layout is None or (len(self.initial_distribution) == len(layout)),\ - Exception('The initial distribution and layout lists must have the same lenth') - source_names = [f'dist{i}' for i in range(len(self.initial_distribution))] + assert layout is None or ( + len(self.initial_distribution) == len(layout) + ), Exception( + "The initial distribution and layout lists must have the same lenth" + ) + source_names = [ + f"dist{i}" for i in range(len(self.initial_distribution)) + ] self.species.injection_sources = source_names for i, dist in enumerate(self.initial_distribution): layout_i = layout[i] if layout is not None else None - dist.distribution_initialize_inputs(self.species_number, layout_i, self.species, - self.density_scale, source_names[i]) + dist.distribution_initialize_inputs( + self.species_number, + layout_i, + self.species, + self.density_scale, + source_names[i], + ) else: - raise Exception('The initial distribution and layout must both be scalars or both be lists') + raise Exception( + "The initial distribution and layout must both be scalars or both be lists" + ) if injection_plane_position is not None: if injection_plane_normal_vector is not None: - assert injection_plane_normal_vector[0] == 0. and injection_plane_normal_vector[1] == 0.,\ - Exception('Rigid injection can only be done along z') + assert ( + injection_plane_normal_vector[0] == 0.0 + and injection_plane_normal_vector[1] == 0.0 + ), Exception("Rigid injection can only be done along z") pywarpx.particles.rigid_injected_species.append(self.name) self.species.rigid_advance = 1 self.species.zinject_plane = injection_plane_position picmistandard.PICMI_MultiSpecies.Species_class = Species + + class MultiSpecies(picmistandard.PICMI_MultiSpecies): - def species_initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + def species_initialize_inputs( + self, + layout, + initialize_self_fields=False, + injection_plane_position=None, + injection_plane_normal_vector=None, + ): for species in self.species_instances_list: - species.species_initialize_inputs(layout, - initialize_self_fields, - injection_plane_position, - injection_plane_normal_vector) + species.species_initialize_inputs( + layout, + initialize_self_fields, + injection_plane_position, + injection_plane_normal_vector, + ) class GaussianBunchDistribution(picmistandard.PICMI_GaussianBunchDistribution): def init(self, kw): - self.do_symmetrize = kw.pop('warpx_do_symmetrize', None) - self.symmetrization_order = kw.pop('warpx_symmetrization_order', None) - - def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): - species.add_new_group_attr(source_name, 'injection_style', "gaussian_beam") - species.add_new_group_attr(source_name, 'x_m', self.centroid_position[0]) - species.add_new_group_attr(source_name, 'y_m', self.centroid_position[1]) - species.add_new_group_attr(source_name, 'z_m', self.centroid_position[2]) - species.add_new_group_attr(source_name, 'x_rms', self.rms_bunch_size[0]) - species.add_new_group_attr(source_name, 'y_rms', self.rms_bunch_size[1]) - species.add_new_group_attr(source_name, 'z_rms', self.rms_bunch_size[2]) + self.do_symmetrize = kw.pop("warpx_do_symmetrize", None) + self.symmetrization_order = kw.pop("warpx_symmetrization_order", None) + + def distribution_initialize_inputs( + self, species_number, layout, species, density_scale, source_name + ): + species.add_new_group_attr(source_name, "injection_style", "gaussian_beam") + species.add_new_group_attr(source_name, "x_m", self.centroid_position[0]) + species.add_new_group_attr(source_name, "y_m", self.centroid_position[1]) + species.add_new_group_attr(source_name, "z_m", self.centroid_position[2]) + species.add_new_group_attr(source_name, "x_rms", self.rms_bunch_size[0]) + species.add_new_group_attr(source_name, "y_rms", self.rms_bunch_size[1]) + species.add_new_group_attr(source_name, "z_rms", self.rms_bunch_size[2]) # --- Only PseudoRandomLayout is supported - species.add_new_group_attr(source_name, 'npart', layout.n_macroparticles) + species.add_new_group_attr(source_name, "npart", layout.n_macroparticles) # --- Calculate the total charge. Note that charge might be a string instead of a number. charge = species.charge - if charge == 'q_e' or charge == '+q_e': + if charge == "q_e" or charge == "+q_e": charge = constants.q_e - elif charge == '-q_e': + elif charge == "-q_e": charge = -constants.q_e - species.add_new_group_attr(source_name, 'q_tot', self.n_physical_particles*charge) + species.add_new_group_attr( + source_name, "q_tot", self.n_physical_particles * charge + ) if density_scale is not None: - species.add_new_group_attr(source_name, 'q_tot', density_scale) + species.add_new_group_attr(source_name, "q_tot", density_scale) # --- The PICMI standard doesn't yet have a way of specifying these values. # --- They should default to the size of the domain. They are not typically # --- necessary though since any particles outside the domain are rejected. - #species.xmin - #species.xmax - #species.ymin - #species.ymax - #species.zmin - #species.zmax + # species.xmin + # species.xmax + # species.ymin + # species.ymax + # species.zmin + # species.zmax # --- Note that WarpX takes gamma*beta as input - if np.any(np.not_equal(self.velocity_divergence, 0.)): - species.add_new_group_attr(source_name, 'momentum_distribution_type', "radial_expansion") - species.add_new_group_attr(source_name, 'u_over_r', self.velocity_divergence[0]/constants.c) - #species.add_new_group_attr(source_name, 'u_over_y', self.velocity_divergence[1]/constants.c) - #species.add_new_group_attr(source_name, 'u_over_z', self.velocity_divergence[2]/constants.c) - elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") - species.add_new_group_attr(source_name, 'ux_m', self.centroid_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy_m', self.centroid_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz_m', self.centroid_velocity[2]/constants.c) - species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) + if np.any(np.not_equal(self.velocity_divergence, 0.0)): + species.add_new_group_attr( + source_name, "momentum_distribution_type", "radial_expansion" + ) + species.add_new_group_attr( + source_name, "u_over_r", self.velocity_divergence[0] / constants.c + ) + # species.add_new_group_attr(source_name, 'u_over_y', self.velocity_divergence[1]/constants.c) + # species.add_new_group_attr(source_name, 'u_over_z', self.velocity_divergence[2]/constants.c) + elif np.any(np.not_equal(self.rms_velocity, 0.0)): + species.add_new_group_attr( + source_name, "momentum_distribution_type", "gaussian" + ) + species.add_new_group_attr( + source_name, "ux_m", self.centroid_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy_m", self.centroid_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz_m", self.centroid_velocity[2] / constants.c + ) + species.add_new_group_attr( + source_name, "ux_th", self.rms_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy_th", self.rms_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz_th", self.rms_velocity[2] / constants.c + ) else: - species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") - species.add_new_group_attr(source_name, 'ux', self.centroid_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy', self.centroid_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz', self.centroid_velocity[2]/constants.c) + species.add_new_group_attr( + source_name, "momentum_distribution_type", "constant" + ) + species.add_new_group_attr( + source_name, "ux", self.centroid_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy", self.centroid_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz", self.centroid_velocity[2] / constants.c + ) - species.add_new_group_attr(source_name, 'do_symmetrize', self.do_symmetrize) - species.add_new_group_attr(source_name, 'symmetrization_order', self.symmetrization_order) + species.add_new_group_attr(source_name, "do_symmetrize", self.do_symmetrize) + species.add_new_group_attr( + source_name, "symmetrization_order", self.symmetrization_order + ) class DensityDistributionBase(object): @@ -448,7 +556,7 @@ class DensityDistributionBase(object): captures universal initialization logic.""" def set_mangle_dict(self): - if not hasattr(self, 'mangle_dict'): + if not hasattr(self, "mangle_dict"): self.mangle_dict = None if hasattr(self, "user_defined_kw") and self.mangle_dict is None: @@ -459,102 +567,266 @@ def set_mangle_dict(self): def set_species_attributes(self, species, layout, source_name): if isinstance(layout, GriddedLayout): # --- Note that the grid attribute of GriddedLayout is ignored - species.add_new_group_attr(source_name, 'injection_style', "nuniformpercell") - species.add_new_group_attr(source_name, 'num_particles_per_cell_each_dim', layout.n_macroparticle_per_cell) + species.add_new_group_attr( + source_name, "injection_style", "nuniformpercell" + ) + species.add_new_group_attr( + source_name, + "num_particles_per_cell_each_dim", + layout.n_macroparticle_per_cell, + ) elif isinstance(layout, PseudoRandomLayout): - assert (layout.n_macroparticles_per_cell is not None), Exception('WarpX only supports n_macroparticles_per_cell for the PseudoRandomLayout with this distribution') - species.add_new_group_attr(source_name, 'injection_style', "nrandompercell") - species.add_new_group_attr(source_name, 'num_particles_per_cell', layout.n_macroparticles_per_cell) + assert layout.n_macroparticles_per_cell is not None, Exception( + "WarpX only supports n_macroparticles_per_cell for the PseudoRandomLayout with this distribution" + ) + species.add_new_group_attr(source_name, "injection_style", "nrandompercell") + species.add_new_group_attr( + source_name, "num_particles_per_cell", layout.n_macroparticles_per_cell + ) else: - raise Exception('WarpX does not support the specified layout for this distribution') + raise Exception( + "WarpX does not support the specified layout for this distribution" + ) - species.add_new_group_attr(source_name, 'xmin', self.lower_bound[0]) - species.add_new_group_attr(source_name, 'xmax', self.upper_bound[0]) - species.add_new_group_attr(source_name, 'ymin', self.lower_bound[1]) - species.add_new_group_attr(source_name, 'ymax', self.upper_bound[1]) - species.add_new_group_attr(source_name, 'zmin', self.lower_bound[2]) - species.add_new_group_attr(source_name, 'zmax', self.upper_bound[2]) + species.add_new_group_attr(source_name, "xmin", self.lower_bound[0]) + species.add_new_group_attr(source_name, "xmax", self.upper_bound[0]) + species.add_new_group_attr(source_name, "ymin", self.lower_bound[1]) + species.add_new_group_attr(source_name, "ymax", self.upper_bound[1]) + species.add_new_group_attr(source_name, "zmin", self.lower_bound[2]) + species.add_new_group_attr(source_name, "zmax", self.upper_bound[2]) if self.fill_in: - species.add_new_group_attr(source_name, 'do_continuous_injection', 1) + species.add_new_group_attr(source_name, "do_continuous_injection", 1) # --- Note that WarpX takes gamma*beta as input - if (hasattr(self, "momentum_spread_expressions") - and np.any(np.not_equal(self.momentum_spread_expressions, None)) + if hasattr(self, "momentum_spread_expressions") and np.any( + np.not_equal(self.momentum_spread_expressions, None) ): - species.momentum_distribution_type = 'gaussian_parse_momentum_function' - self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '_m', self.directed_velocity) - self.setup_parse_momentum_functions(species, source_name, self.momentum_spread_expressions, '_th', [0.,0.,0.]) - elif (hasattr(self, "momentum_expressions") - and np.any(np.not_equal(self.momentum_expressions, None)) + species.momentum_distribution_type = "gaussian_parse_momentum_function" + self.setup_parse_momentum_functions( + species, + source_name, + self.momentum_expressions, + "_m", + self.directed_velocity, + ) + self.setup_parse_momentum_functions( + species, + source_name, + self.momentum_spread_expressions, + "_th", + [0.0, 0.0, 0.0], + ) + elif hasattr(self, "momentum_expressions") and np.any( + np.not_equal(self.momentum_expressions, None) ): - species.add_new_group_attr(source_name, 'momentum_distribution_type', 'parse_momentum_function') - self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '', self.directed_velocity) - elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") - species.add_new_group_attr(source_name, 'ux_m', self.directed_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy_m', self.directed_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz_m', self.directed_velocity[2]/constants.c) - species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) + species.add_new_group_attr( + source_name, "momentum_distribution_type", "parse_momentum_function" + ) + self.setup_parse_momentum_functions( + species, + source_name, + self.momentum_expressions, + "", + self.directed_velocity, + ) + elif np.any(np.not_equal(self.rms_velocity, 0.0)): + species.add_new_group_attr( + source_name, "momentum_distribution_type", "gaussian" + ) + species.add_new_group_attr( + source_name, "ux_m", self.directed_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy_m", self.directed_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz_m", self.directed_velocity[2] / constants.c + ) + species.add_new_group_attr( + source_name, "ux_th", self.rms_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy_th", self.rms_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz_th", self.rms_velocity[2] / constants.c + ) else: - species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") - species.add_new_group_attr(source_name, 'ux', self.directed_velocity[0]/constants.c) - species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) - species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) + species.add_new_group_attr( + source_name, "momentum_distribution_type", "constant" + ) + species.add_new_group_attr( + source_name, "ux", self.directed_velocity[0] / constants.c + ) + species.add_new_group_attr( + source_name, "uy", self.directed_velocity[1] / constants.c + ) + species.add_new_group_attr( + source_name, "uz", self.directed_velocity[2] / constants.c + ) - def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): - for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): + if hasattr(self, "density_min"): + species.add_new_group_attr(source_name, "density_min", self.density_min) + if hasattr(self, "density_max"): + species.add_new_group_attr(source_name, "density_max", self.density_max) + + def setup_parse_momentum_functions( + self, species, source_name, expressions, suffix, defaults + ): + for sdir, idir in zip(["x", "y", "z"], [0, 1, 2]): if expressions[idir] is not None: - expression = pywarpx.my_constants.mangle_expression(expressions[idir], self.mangle_dict) + expression = pywarpx.my_constants.mangle_expression( + expressions[idir], self.mangle_dict + ) else: - expression = f'{defaults[idir]}' - species.add_new_group_attr(source_name, f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') + expression = f"{defaults[idir]}" + species.add_new_group_attr( + source_name, + f"momentum_function_u{sdir}{suffix}(x,y,z)", + f"({expression})/{constants.c}", + ) + + +class UniformDistribution( + picmistandard.PICMI_UniformDistribution, DensityDistributionBase +): + def distribution_initialize_inputs( + self, species_number, layout, species, density_scale, source_name + ): + self.set_mangle_dict() + self.set_species_attributes(species, layout, source_name) + # --- Only constant density is supported by this class + species.add_new_group_attr(source_name, "profile", "constant") + species.add_new_group_attr(source_name, "density", self.density) + if density_scale is not None: + species.add_new_group_attr(source_name, "density", density_scale) -class UniformFluxDistribution(picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase): - def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): +class FluxDistributionBase(object): + """This is a base class for both uniform and analytic flux distributions.""" + + def init(self, kw): + self.inject_from_embedded_boundary = kw.pop( + "warpx_inject_from_embedded_boundary", False + ) + + def initialize_flux_profile_func(self, species, density_scale, source_name): + """Initialize the flux profile and flux function.""" + pass + + def distribution_initialize_inputs( + self, species_number, layout, species, density_scale, source_name + ): self.fill_in = False self.set_mangle_dict() self.set_species_attributes(species, layout, source_name) - species.add_new_group_attr(source_name, 'flux_profile', "constant") - species.add_new_group_attr(source_name, 'flux', self.flux) - if density_scale is not None: - species.add_new_group_attr(source_name, 'flux', density_scale) - species.add_new_group_attr(source_name, 'flux_normal_axis', self.flux_normal_axis) - species.add_new_group_attr(source_name, 'surface_flux_pos', self.surface_flux_position) - species.add_new_group_attr(source_name, 'flux_direction', self.flux_direction) - species.add_new_group_attr(source_name, 'flux_tmin', self.flux_tmin) - species.add_new_group_attr(source_name, 'flux_tmax', self.flux_tmax) + self.initialize_flux_profile_func(species, density_scale, source_name) + + if not self.inject_from_embedded_boundary: + species.add_new_group_attr( + source_name, "flux_normal_axis", self.flux_normal_axis + ) + species.add_new_group_attr( + source_name, "surface_flux_pos", self.surface_flux_position + ) + species.add_new_group_attr( + source_name, "flux_direction", self.flux_direction + ) + else: + species.add_new_group_attr( + source_name, "inject_from_embedded_boundary", True + ) + + species.add_new_group_attr(source_name, "flux_tmin", self.flux_tmin) + species.add_new_group_attr(source_name, "flux_tmax", self.flux_tmax) # --- Use specific attributes for flux injection - species.add_new_group_attr(source_name, 'injection_style', "nfluxpercell") - assert (isinstance(layout, PseudoRandomLayout)), Exception('UniformFluxDistribution only supports the PseudoRandomLayout in WarpX') + species.add_new_group_attr(source_name, "injection_style", "nfluxpercell") + assert isinstance(layout, PseudoRandomLayout), Exception( + "UniformFluxDistribution only supports the PseudoRandomLayout in WarpX" + ) if self.gaussian_flux_momentum_distribution: - species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussianflux") + species.add_new_group_attr( + source_name, "momentum_distribution_type", "gaussianflux" + ) -class UniformDistribution(picmistandard.PICMI_UniformDistribution, DensityDistributionBase): - def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): +class AnalyticFluxDistribution( + picmistandard.PICMI_AnalyticFluxDistribution, + FluxDistributionBase, + DensityDistributionBase, +): + """ + Parameters + ---------- - self.set_mangle_dict() - self.set_species_attributes(species, layout, source_name) + warpx_inject_from_embedded_boundary: bool + When true, the flux is injected from the embedded boundaries instead + of a plane. + """ - # --- Only constant density is supported by this class - species.add_new_group_attr(source_name, 'profile', "constant") - species.add_new_group_attr(source_name, 'density', self.density) + def init(self, kw): + FluxDistributionBase.init(self, kw) + + def initialize_flux_profile_func(self, species, density_scale, source_name): + species.add_new_group_attr(source_name, "flux_profile", "parse_flux_function") + if density_scale is not None: + species.add_new_group_attr(source_name, "flux", density_scale) + expression = pywarpx.my_constants.mangle_expression(self.flux, self.mangle_dict) + if density_scale is None: + species.add_new_group_attr( + source_name, "flux_function(x,y,z,t)", expression + ) + else: + species.add_new_group_attr( + source_name, + "flux_function(x,y,z,t)", + "{}*({})".format(density_scale, expression), + ) + + +class UniformFluxDistribution( + picmistandard.PICMI_UniformFluxDistribution, + FluxDistributionBase, + DensityDistributionBase, +): + """ + Parameters + ---------- + + warpx_inject_from_embedded_boundary: bool + When true, the flux is injected from the embedded boundaries instead + of a plane. + """ + + def init(self, kw): + FluxDistributionBase.init(self, kw) + + def initialize_flux_profile_func(self, species, density_scale, source_name): + species.add_new_group_attr(source_name, "flux_profile", "constant") + species.add_new_group_attr(source_name, "flux", self.flux) if density_scale is not None: - species.add_new_group_attr(source_name, 'density', density_scale) + species.add_new_group_attr(source_name, "flux", density_scale) -class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDistributionBase): +class AnalyticDistribution( + picmistandard.PICMI_AnalyticDistribution, DensityDistributionBase +): """ Parameters ---------- + warpx_density_min: float + Minimum plasma density. No particle is injected where the density is + below this value. + + warpx_density_max: float + Maximum plasma density. The density at each point is the minimum between + the value given in the profile, and density_max. + warpx_momentum_spread_expressions: list of string Analytic expressions describing the gamma*velocity spread for each axis [m/s]. Expressions should be in terms of the position, written as 'x', 'y', and 'z'. @@ -562,41 +834,68 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist For any axis not supplied (set to None), zero will be used. """ - def init(self, kw): - self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) - def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def init(self, kw): + self.density_min = kw.pop("warpx_density_min", None) + self.density_max = kw.pop("warpx_density_max", None) + self.momentum_spread_expressions = kw.pop( + "warpx_momentum_spread_expressions", [None, None, None] + ) + def distribution_initialize_inputs( + self, species_number, layout, species, density_scale, source_name + ): self.set_mangle_dict() self.set_species_attributes(species, layout, source_name) - species.add_new_group_attr(source_name, 'profile', "parse_density_function") - expression = pywarpx.my_constants.mangle_expression(self.density_expression, self.mangle_dict) + species.add_new_group_attr(source_name, "profile", "parse_density_function") + expression = pywarpx.my_constants.mangle_expression( + self.density_expression, self.mangle_dict + ) if density_scale is None: - species.add_new_group_attr(source_name, 'density_function(x,y,z)', expression) + species.add_new_group_attr( + source_name, "density_function(x,y,z)", expression + ) else: - species.add_new_group_attr(source_name, 'density_function(x,y,z)', "{}*({})".format(density_scale, expression)) + species.add_new_group_attr( + source_name, + "density_function(x,y,z)", + "{}*({})".format(density_scale, expression), + ) class ParticleListDistribution(picmistandard.PICMI_ParticleListDistribution): def init(self, kw): pass - def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): - - species.add_new_group_attr(source_name, 'injection_style', "multipleparticles") - species.add_new_group_attr(source_name, 'multiple_particles_pos_x', self.x) - species.add_new_group_attr(source_name, 'multiple_particles_pos_y', self.y) - species.add_new_group_attr(source_name, 'multiple_particles_pos_z', self.z) - species.add_new_group_attr(source_name, 'multiple_particles_ux', np.array(self.ux)/constants.c) - species.add_new_group_attr(source_name, 'multiple_particles_uy', np.array(self.uy)/constants.c) - species.add_new_group_attr(source_name, 'multiple_particles_uz', np.array(self.uz)/constants.c) - species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight) + def distribution_initialize_inputs( + self, species_number, layout, species, density_scale, source_name + ): + species.add_new_group_attr(source_name, "injection_style", "multipleparticles") + species.add_new_group_attr(source_name, "multiple_particles_pos_x", self.x) + species.add_new_group_attr(source_name, "multiple_particles_pos_y", self.y) + species.add_new_group_attr(source_name, "multiple_particles_pos_z", self.z) + species.add_new_group_attr( + source_name, "multiple_particles_ux", np.array(self.ux) / constants.c + ) + species.add_new_group_attr( + source_name, "multiple_particles_uy", np.array(self.uy) / constants.c + ) + species.add_new_group_attr( + source_name, "multiple_particles_uz", np.array(self.uz) / constants.c + ) + species.add_new_group_attr( + source_name, "multiple_particles_weight", self.weight + ) if density_scale is not None: - species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight*density_scale) + species.add_new_group_attr( + source_name, "multiple_particles_weight", self.weight * density_scale + ) -class ParticleDistributionPlanarInjector(picmistandard.PICMI_ParticleDistributionPlanarInjector): +class ParticleDistributionPlanarInjector( + picmistandard.PICMI_ParticleDistributionPlanarInjector +): pass @@ -607,11 +906,12 @@ class GriddedLayout(picmistandard.PICMI_GriddedLayout): class PseudoRandomLayout(picmistandard.PICMI_PseudoRandomLayout): def init(self, kw): if self.seed is not None: - print('Warning: WarpX does not support specifying the random number seed in PseudoRandomLayout') + print( + "Warning: WarpX does not support specifying the random number seed in PseudoRandomLayout" + ) class BinomialSmoother(picmistandard.PICMI_BinomialSmoother): - def smoother_initialize_inputs(self, solver): pywarpx.warpx.use_filter = 1 pywarpx.warpx.use_filter_compensation = bool(np.all(self.compensation)) @@ -623,7 +923,7 @@ def smoother_initialize_inputs(self, solver): len(self.n_pass) except TypeError: # If not, make it a vector - self.n_pass = solver.grid.number_of_dimensions*[self.n_pass] + self.n_pass = solver.grid.number_of_dimensions * [self.n_pass] pywarpx.warpx.filter_npass_each_dir = self.n_pass @@ -681,34 +981,35 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): specify the thermal speed for each species in the form {``: u_th}. Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ + def init(self, kw): - self.max_grid_size = kw.pop('warpx_max_grid_size', 32) - self.max_grid_size_x = kw.pop('warpx_max_grid_size_x', None) - self.max_grid_size_y = kw.pop('warpx_max_grid_size_y', None) - self.blocking_factor = kw.pop('warpx_blocking_factor', None) - self.blocking_factor_x = kw.pop('warpx_blocking_factor_x', None) - self.blocking_factor_y = kw.pop('warpx_blocking_factor_y', None) - - self.potential_xmin = kw.pop('warpx_potential_lo_r', None) - self.potential_xmax = kw.pop('warpx_potential_hi_r', None) + self.max_grid_size = kw.pop("warpx_max_grid_size", 32) + self.max_grid_size_x = kw.pop("warpx_max_grid_size_x", None) + self.max_grid_size_y = kw.pop("warpx_max_grid_size_y", None) + self.blocking_factor = kw.pop("warpx_blocking_factor", None) + self.blocking_factor_x = kw.pop("warpx_blocking_factor_x", None) + self.blocking_factor_y = kw.pop("warpx_blocking_factor_y", None) + + self.potential_xmin = kw.pop("warpx_potential_lo_r", None) + self.potential_xmax = kw.pop("warpx_potential_hi_r", None) self.potential_ymin = None self.potential_ymax = None - self.potential_zmin = kw.pop('warpx_potential_lo_z', None) - self.potential_zmax = kw.pop('warpx_potential_hi_z', None) - self.reflect_all_velocities = kw.pop('warpx_reflect_all_velocities', None) + self.potential_zmin = kw.pop("warpx_potential_lo_z", None) + self.potential_zmax = kw.pop("warpx_potential_hi_z", None) + self.reflect_all_velocities = kw.pop("warpx_reflect_all_velocities", None) - self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) - self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + self.start_moving_window_step = kw.pop("warpx_start_moving_window_step", None) + self.end_moving_window_step = kw.pop("warpx_end_moving_window_step", None) # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) - pywarpx.geometry.dims = 'RZ' + pywarpx.geometry.dims = "RZ" pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound # if a thermal boundary is used for particles, get the thermal speeds - self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + self.thermal_boundary_u_th = kw.pop("warpx_boundary_u_th", None) def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -722,37 +1023,56 @@ def grid_initialize_inputs(self): pywarpx.amr.blocking_factor_x = self.blocking_factor_x pywarpx.amr.blocking_factor_y = self.blocking_factor_y - assert self.lower_bound[0] >= 0., Exception('Lower radial boundary must be >= 0.') - assert self.lower_boundary_conditions[0] != 'periodic' and self.upper_boundary_conditions[0] != 'periodic', Exception('Radial boundaries can not be periodic') + assert self.lower_bound[0] >= 0.0, Exception( + "Lower radial boundary must be >= 0." + ) + assert ( + self.lower_boundary_conditions[0] != "periodic" + and self.upper_boundary_conditions[0] != "periodic" + ), Exception("Radial boundaries can not be periodic") pywarpx.warpx.n_rz_azimuthal_modes = self.n_azimuthal_modes # Boundary conditions - pywarpx.boundary.field_lo = [BC_map[bc] for bc in self.lower_boundary_conditions] - pywarpx.boundary.field_hi = [BC_map[bc] for bc in self.upper_boundary_conditions] + pywarpx.boundary.field_lo = [ + BC_map[bc] for bc in self.lower_boundary_conditions + ] + pywarpx.boundary.field_hi = [ + BC_map[bc] for bc in self.upper_boundary_conditions + ] pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles pywarpx.boundary.reflect_all_velocities = self.reflect_all_velocities if self.thermal_boundary_u_th is not None: for name, val in self.thermal_boundary_u_th.items(): - pywarpx.boundary.__setattr__(f'{name}.u_th', val) + pywarpx.boundary.__setattr__(f"{name}.u_th", val) - if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): + if self.moving_window_velocity is not None and np.any( + np.not_equal(self.moving_window_velocity, 0.0) + ): pywarpx.warpx.do_moving_window = 1 - if self.moving_window_velocity[0] != 0.: - pywarpx.warpx.moving_window_dir = 'r' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light - if self.moving_window_velocity[1] != 0.: - pywarpx.warpx.moving_window_dir = 'z' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + if self.moving_window_velocity[0] != 0.0: + pywarpx.warpx.moving_window_dir = "r" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[0] / constants.c + ) # in units of the speed of light + if self.moving_window_velocity[1] != 0.0: + pywarpx.warpx.moving_window_dir = "z" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[1] / constants.c + ) # in units of the speed of light pywarpx.warpx.start_moving_window_step = self.start_moving_window_step pywarpx.warpx.end_moving_window_step = self.end_moving_window_step if self.refined_regions: - assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') - assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') + assert len(self.refined_regions) == 1, Exception( + "WarpX only supports one refined region." + ) + assert self.refined_regions[0][0] == 1, Exception( + "The one refined region can only be level 1" + ) pywarpx.amr.max_level = 1 pywarpx.warpx.fine_tag_lo = self.refined_regions[0][1] pywarpx.warpx.fine_tag_hi = self.refined_regions[0][2] @@ -797,31 +1117,32 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): specify the thermal speed for each species in the form {``: u_th}. Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ + def init(self, kw): - self.max_grid_size = kw.pop('warpx_max_grid_size', 32) - self.max_grid_size_x = kw.pop('warpx_max_grid_size_x', None) - self.blocking_factor = kw.pop('warpx_blocking_factor', None) - self.blocking_factor_x = kw.pop('warpx_blocking_factor_x', None) + self.max_grid_size = kw.pop("warpx_max_grid_size", 32) + self.max_grid_size_x = kw.pop("warpx_max_grid_size_x", None) + self.blocking_factor = kw.pop("warpx_blocking_factor", None) + self.blocking_factor_x = kw.pop("warpx_blocking_factor_x", None) self.potential_xmin = None self.potential_xmax = None self.potential_ymin = None self.potential_ymax = None - self.potential_zmin = kw.pop('warpx_potential_lo_z', None) - self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.potential_zmin = kw.pop("warpx_potential_lo_z", None) + self.potential_zmax = kw.pop("warpx_potential_hi_z", None) - self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) - self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + self.start_moving_window_step = kw.pop("warpx_start_moving_window_step", None) + self.end_moving_window_step = kw.pop("warpx_end_moving_window_step", None) # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) - pywarpx.geometry.dims = '1' + pywarpx.geometry.dims = "1" pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound # if a thermal boundary is used for particles, get the thermal speeds - self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + self.thermal_boundary_u_th = kw.pop("warpx_boundary_u_th", None) def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -834,27 +1155,39 @@ def grid_initialize_inputs(self): pywarpx.amr.blocking_factor_x = self.blocking_factor_x # Boundary conditions - pywarpx.boundary.field_lo = [BC_map[bc] for bc in self.lower_boundary_conditions] - pywarpx.boundary.field_hi = [BC_map[bc] for bc in self.upper_boundary_conditions] + pywarpx.boundary.field_lo = [ + BC_map[bc] for bc in self.lower_boundary_conditions + ] + pywarpx.boundary.field_hi = [ + BC_map[bc] for bc in self.upper_boundary_conditions + ] pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles if self.thermal_boundary_u_th is not None: for name, val in self.thermal_boundary_u_th.items(): - pywarpx.boundary.__setattr__(f'{name}.u_th', val) + pywarpx.boundary.__setattr__(f"{name}.u_th", val) - if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): + if self.moving_window_velocity is not None and np.any( + np.not_equal(self.moving_window_velocity, 0.0) + ): pywarpx.warpx.do_moving_window = 1 - if self.moving_window_velocity[0] != 0.: - pywarpx.warpx.moving_window_dir = 'z' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light + if self.moving_window_velocity[0] != 0.0: + pywarpx.warpx.moving_window_dir = "z" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[0] / constants.c + ) # in units of the speed of light pywarpx.warpx.start_moving_window_step = self.start_moving_window_step pywarpx.warpx.end_moving_window_step = self.end_moving_window_step if self.refined_regions: - assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') - assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') + assert len(self.refined_regions) == 1, Exception( + "WarpX only supports one refined region." + ) + assert self.refined_regions[0][0] == 1, Exception( + "The one refined region can only be level 1" + ) pywarpx.amr.max_level = 1 pywarpx.warpx.fine_tag_lo = self.refined_regions[0][1] pywarpx.warpx.fine_tag_hi = self.refined_regions[0][2] @@ -862,6 +1195,7 @@ def grid_initialize_inputs(self): else: pywarpx.amr.max_level = 0 + class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): """ See `Input Parameters `__ for more information. @@ -910,33 +1244,34 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): specify the thermal speed for each species in the form {``: u_th}. Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ + def init(self, kw): - self.max_grid_size = kw.pop('warpx_max_grid_size', 32) - self.max_grid_size_x = kw.pop('warpx_max_grid_size_x', None) - self.max_grid_size_y = kw.pop('warpx_max_grid_size_y', None) - self.blocking_factor = kw.pop('warpx_blocking_factor', None) - self.blocking_factor_x = kw.pop('warpx_blocking_factor_x', None) - self.blocking_factor_y = kw.pop('warpx_blocking_factor_y', None) - - self.potential_xmin = kw.pop('warpx_potential_lo_x', None) - self.potential_xmax = kw.pop('warpx_potential_hi_x', None) + self.max_grid_size = kw.pop("warpx_max_grid_size", 32) + self.max_grid_size_x = kw.pop("warpx_max_grid_size_x", None) + self.max_grid_size_y = kw.pop("warpx_max_grid_size_y", None) + self.blocking_factor = kw.pop("warpx_blocking_factor", None) + self.blocking_factor_x = kw.pop("warpx_blocking_factor_x", None) + self.blocking_factor_y = kw.pop("warpx_blocking_factor_y", None) + + self.potential_xmin = kw.pop("warpx_potential_lo_x", None) + self.potential_xmax = kw.pop("warpx_potential_hi_x", None) self.potential_ymin = None self.potential_ymax = None - self.potential_zmin = kw.pop('warpx_potential_lo_z', None) - self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.potential_zmin = kw.pop("warpx_potential_lo_z", None) + self.potential_zmax = kw.pop("warpx_potential_hi_z", None) - self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) - self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + self.start_moving_window_step = kw.pop("warpx_start_moving_window_step", None) + self.end_moving_window_step = kw.pop("warpx_end_moving_window_step", None) # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) - pywarpx.geometry.dims = '2' + pywarpx.geometry.dims = "2" pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound # if a thermal boundary is used for particles, get the thermal speeds - self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + self.thermal_boundary_u_th = kw.pop("warpx_boundary_u_th", None) def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -951,30 +1286,44 @@ def grid_initialize_inputs(self): pywarpx.amr.blocking_factor_y = self.blocking_factor_y # Boundary conditions - pywarpx.boundary.field_lo = [BC_map[bc] for bc in self.lower_boundary_conditions] - pywarpx.boundary.field_hi = [BC_map[bc] for bc in self.upper_boundary_conditions] + pywarpx.boundary.field_lo = [ + BC_map[bc] for bc in self.lower_boundary_conditions + ] + pywarpx.boundary.field_hi = [ + BC_map[bc] for bc in self.upper_boundary_conditions + ] pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles if self.thermal_boundary_u_th is not None: for name, val in self.thermal_boundary_u_th.items(): - pywarpx.boundary.__setattr__(f'{name}.u_th', val) + pywarpx.boundary.__setattr__(f"{name}.u_th", val) - if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): + if self.moving_window_velocity is not None and np.any( + np.not_equal(self.moving_window_velocity, 0.0) + ): pywarpx.warpx.do_moving_window = 1 - if self.moving_window_velocity[0] != 0.: - pywarpx.warpx.moving_window_dir = 'x' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light - if self.moving_window_velocity[1] != 0.: - pywarpx.warpx.moving_window_dir = 'z' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + if self.moving_window_velocity[0] != 0.0: + pywarpx.warpx.moving_window_dir = "x" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[0] / constants.c + ) # in units of the speed of light + if self.moving_window_velocity[1] != 0.0: + pywarpx.warpx.moving_window_dir = "z" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[1] / constants.c + ) # in units of the speed of light pywarpx.warpx.start_moving_window_step = self.start_moving_window_step pywarpx.warpx.end_moving_window_step = self.end_moving_window_step if self.refined_regions: - assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') - assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') + assert len(self.refined_regions) == 1, Exception( + "WarpX only supports one refined region." + ) + assert self.refined_regions[0][0] == 1, Exception( + "The one refined region can only be level 1" + ) pywarpx.amr.max_level = 1 pywarpx.warpx.fine_tag_lo = self.refined_regions[0][1] pywarpx.warpx.fine_tag_hi = self.refined_regions[0][2] @@ -1043,35 +1392,36 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): specify the thermal speed for each species in the form {``: u_th}. Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ + def init(self, kw): - self.max_grid_size = kw.pop('warpx_max_grid_size', 32) - self.max_grid_size_x = kw.pop('warpx_max_grid_size_x', None) - self.max_grid_size_y = kw.pop('warpx_max_grid_size_y', None) - self.max_grid_size_z = kw.pop('warpx_max_grid_size_z', None) - self.blocking_factor = kw.pop('warpx_blocking_factor', None) - self.blocking_factor_x = kw.pop('warpx_blocking_factor_x', None) - self.blocking_factor_y = kw.pop('warpx_blocking_factor_y', None) - self.blocking_factor_z = kw.pop('warpx_blocking_factor_z', None) - - self.potential_xmin = kw.pop('warpx_potential_lo_x', None) - self.potential_xmax = kw.pop('warpx_potential_hi_x', None) - self.potential_ymin = kw.pop('warpx_potential_lo_y', None) - self.potential_ymax = kw.pop('warpx_potential_hi_y', None) - self.potential_zmin = kw.pop('warpx_potential_lo_z', None) - self.potential_zmax = kw.pop('warpx_potential_hi_z', None) - - self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) - self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + self.max_grid_size = kw.pop("warpx_max_grid_size", 32) + self.max_grid_size_x = kw.pop("warpx_max_grid_size_x", None) + self.max_grid_size_y = kw.pop("warpx_max_grid_size_y", None) + self.max_grid_size_z = kw.pop("warpx_max_grid_size_z", None) + self.blocking_factor = kw.pop("warpx_blocking_factor", None) + self.blocking_factor_x = kw.pop("warpx_blocking_factor_x", None) + self.blocking_factor_y = kw.pop("warpx_blocking_factor_y", None) + self.blocking_factor_z = kw.pop("warpx_blocking_factor_z", None) + + self.potential_xmin = kw.pop("warpx_potential_lo_x", None) + self.potential_xmax = kw.pop("warpx_potential_hi_x", None) + self.potential_ymin = kw.pop("warpx_potential_lo_y", None) + self.potential_ymax = kw.pop("warpx_potential_hi_y", None) + self.potential_zmin = kw.pop("warpx_potential_lo_z", None) + self.potential_zmax = kw.pop("warpx_potential_hi_z", None) + + self.start_moving_window_step = kw.pop("warpx_start_moving_window_step", None) + self.end_moving_window_step = kw.pop("warpx_end_moving_window_step", None) # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) - pywarpx.geometry.dims = '3' + pywarpx.geometry.dims = "3" pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound # if a thermal boundary is used for particles, get the thermal speeds - self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + self.thermal_boundary_u_th = kw.pop("warpx_boundary_u_th", None) def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -1088,33 +1438,49 @@ def grid_initialize_inputs(self): pywarpx.amr.blocking_factor_z = self.blocking_factor_z # Boundary conditions - pywarpx.boundary.field_lo = [BC_map[bc] for bc in self.lower_boundary_conditions] - pywarpx.boundary.field_hi = [BC_map[bc] for bc in self.upper_boundary_conditions] + pywarpx.boundary.field_lo = [ + BC_map[bc] for bc in self.lower_boundary_conditions + ] + pywarpx.boundary.field_hi = [ + BC_map[bc] for bc in self.upper_boundary_conditions + ] pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles if self.thermal_boundary_u_th is not None: for name, val in self.thermal_boundary_u_th.items(): - pywarpx.boundary.__setattr__(f'{name}.u_th', val) + pywarpx.boundary.__setattr__(f"{name}.u_th", val) - if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): + if self.moving_window_velocity is not None and np.any( + np.not_equal(self.moving_window_velocity, 0.0) + ): pywarpx.warpx.do_moving_window = 1 - if self.moving_window_velocity[0] != 0.: - pywarpx.warpx.moving_window_dir = 'x' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light - if self.moving_window_velocity[1] != 0.: - pywarpx.warpx.moving_window_dir = 'y' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light - if self.moving_window_velocity[2] != 0.: - pywarpx.warpx.moving_window_dir = 'z' - pywarpx.warpx.moving_window_v = self.moving_window_velocity[2]/constants.c # in units of the speed of light + if self.moving_window_velocity[0] != 0.0: + pywarpx.warpx.moving_window_dir = "x" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[0] / constants.c + ) # in units of the speed of light + if self.moving_window_velocity[1] != 0.0: + pywarpx.warpx.moving_window_dir = "y" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[1] / constants.c + ) # in units of the speed of light + if self.moving_window_velocity[2] != 0.0: + pywarpx.warpx.moving_window_dir = "z" + pywarpx.warpx.moving_window_v = ( + self.moving_window_velocity[2] / constants.c + ) # in units of the speed of light pywarpx.warpx.start_moving_window_step = self.start_moving_window_step pywarpx.warpx.end_moving_window_step = self.end_moving_window_step if self.refined_regions: - assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') - assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') + assert len(self.refined_regions) == 1, Exception( + "WarpX only supports one refined region." + ) + assert self.refined_regions[0][0] == 1, Exception( + "The one refined region can only be level 1" + ) pywarpx.amr.max_level = 1 pywarpx.warpx.fine_tag_lo = self.refined_regions[0][1] pywarpx.warpx.fine_tag_hi = self.refined_regions[0][2] @@ -1165,30 +1531,37 @@ class ElectromagneticSolver(picmistandard.PICMI_ElectromagneticSolver): warpx_do_pml_j_damping: bool, default=False Whether to do damping of J in the PML """ - def init(self, kw): - assert self.method is None or self.method in ['Yee', 'CKC', 'PSATD', 'ECT'], Exception("Only 'Yee', 'CKC', 'PSATD', and 'ECT' are supported") - - self.pml_ncell = kw.pop('warpx_pml_ncell', None) - if self.method == 'PSATD': - self.psatd_periodic_single_box_fft = kw.pop('warpx_periodic_single_box_fft', None) - self.psatd_current_correction = kw.pop('warpx_current_correction', None) - self.psatd_update_with_rho = kw.pop('warpx_psatd_update_with_rho', None) - self.psatd_do_time_averaging = kw.pop('warpx_psatd_do_time_averaging', None) - self.psatd_J_in_time = kw.pop('warpx_psatd_J_in_time', None) - self.psatd_rho_in_time = kw.pop('warpx_psatd_rho_in_time', None) + def init(self, kw): + assert self.method is None or self.method in [ + "Yee", + "CKC", + "PSATD", + "ECT", + ], Exception("Only 'Yee', 'CKC', 'PSATD', and 'ECT' are supported") + + self.pml_ncell = kw.pop("warpx_pml_ncell", None) + + if self.method == "PSATD": + self.psatd_periodic_single_box_fft = kw.pop( + "warpx_periodic_single_box_fft", None + ) + self.psatd_current_correction = kw.pop("warpx_current_correction", None) + self.psatd_update_with_rho = kw.pop("warpx_psatd_update_with_rho", None) + self.psatd_do_time_averaging = kw.pop("warpx_psatd_do_time_averaging", None) + self.psatd_J_in_time = kw.pop("warpx_psatd_J_in_time", None) + self.psatd_rho_in_time = kw.pop("warpx_psatd_rho_in_time", None) - self.do_pml_in_domain = kw.pop('warpx_do_pml_in_domain', None) - self.pml_has_particles = kw.pop('warpx_pml_has_particles', None) - self.do_pml_j_damping = kw.pop('warpx_do_pml_j_damping', None) + self.do_pml_in_domain = kw.pop("warpx_do_pml_in_domain", None) + self.pml_has_particles = kw.pop("warpx_pml_has_particles", None) + self.do_pml_j_damping = kw.pop("warpx_do_pml_j_damping", None) def solver_initialize_inputs(self): - self.grid.grid_initialize_inputs() pywarpx.warpx.pml_ncell = self.pml_ncell - if self.method == 'PSATD': + if self.method == "PSATD": pywarpx.psatd.periodic_single_box_fft = self.psatd_periodic_single_box_fft pywarpx.psatd.current_correction = self.psatd_current_correction pywarpx.psatd.update_with_rho = self.psatd_update_with_rho @@ -1210,14 +1583,19 @@ def solver_initialize_inputs(self): if self.galilean_velocity is not None: if self.grid.number_of_dimensions == 2: - self.galilean_velocity = [self.galilean_velocity[0], 0., self.galilean_velocity[1]] - pywarpx.psatd.v_galilean = np.array(self.galilean_velocity)/constants.c + self.galilean_velocity = [ + self.galilean_velocity[0], + 0.0, + self.galilean_velocity[1], + ] + pywarpx.psatd.v_galilean = ( + np.array(self.galilean_velocity) / constants.c + ) # --- Same method names are used, though mapped to lower case. pywarpx.algo.maxwell_solver = self.method - if self.cfl is not None: - pywarpx.warpx.cfl = self.cfl + pywarpx.warpx.cfl = self.cfl if self.source_smoother is not None: self.source_smoother.smoother_initialize_inputs(self) @@ -1237,8 +1615,9 @@ class ExplicitEvolveScheme(picmistandard.base._ClassWithInit): """ Sets up the explicit evolve scheme """ + def solver_scheme_initialize_inputs(self): - pywarpx.algo.evolve_scheme = 'explicit' + pywarpx.algo.evolve_scheme = "explicit" class ThetaImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): @@ -1253,13 +1632,14 @@ class ThetaImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): theta: float, optional The "theta" parameter, determining the level of implicitness """ - def __init__(self, nonlinear_solver, theta = None): + + def __init__(self, nonlinear_solver, theta=None): self.nonlinear_solver = nonlinear_solver self.theta = theta def solver_scheme_initialize_inputs(self): - pywarpx.algo.evolve_scheme = 'theta_implicit_em' - implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + pywarpx.algo.evolve_scheme = "theta_implicit_em" + implicit_evolve = pywarpx.warpx.get_bucket("implicit_evolve") implicit_evolve.theta = self.theta self.nonlinear_solver.nonlinear_solver_initialize_inputs() @@ -1274,11 +1654,12 @@ class SemiImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): nonlinear_solver: nonlinear solver instance The nonlinear solver to use for the iterations """ + def __init__(self, nonlinear_solver): self.nonlinear_solver = nonlinear_solver def solver_scheme_initialize_inputs(self): - pywarpx.algo.evolve_scheme = 'semi_implicit_em' + pywarpx.algo.evolve_scheme = "semi_implicit_em" self.nonlinear_solver.nonlinear_solver_initialize_inputs() @@ -1304,8 +1685,15 @@ class PicardNonlinearSolver(picmistandard.base._ClassWithInit): require_convergence: bool, default True Whether convergence is required. If True and convergence is not obtained, the code will exit. """ - def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, - max_iterations=None, require_convergence=None): + + def __init__( + self, + verbose=None, + absolute_tolerance=None, + relative_tolerance=None, + max_iterations=None, + require_convergence=None, + ): self.verbose = verbose self.absolute_tolerance = absolute_tolerance self.relative_tolerance = relative_tolerance @@ -1313,10 +1701,10 @@ def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=Non self.require_convergence = require_convergence def nonlinear_solver_initialize_inputs(self): - implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') - implicit_evolve.nonlinear_solver = 'picard' + implicit_evolve = pywarpx.warpx.get_bucket("implicit_evolve") + implicit_evolve.nonlinear_solver = "picard" - picard = pywarpx.warpx.get_bucket('picard') + picard = pywarpx.warpx.get_bucket("picard") picard.verbose = self.verbose picard.absolute_tolerance = self.absolute_tolerance picard.relative_tolerance = self.relative_tolerance @@ -1355,9 +1743,18 @@ class NewtonNonlinearSolver(picmistandard.base._ClassWithInit): The tolerance of parrticle quantities for convergence """ - def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, - max_iterations=None, require_convergence=None, linear_solver=None, - max_particle_iterations=None, particle_tolerance=None): + + def __init__( + self, + verbose=None, + absolute_tolerance=None, + relative_tolerance=None, + max_iterations=None, + require_convergence=None, + linear_solver=None, + max_particle_iterations=None, + particle_tolerance=None, + ): self.verbose = verbose self.absolute_tolerance = absolute_tolerance self.relative_tolerance = relative_tolerance @@ -1368,12 +1765,12 @@ def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=Non self.particle_tolerance = particle_tolerance def nonlinear_solver_initialize_inputs(self): - implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') - implicit_evolve.nonlinear_solver = 'newton' + implicit_evolve = pywarpx.warpx.get_bucket("implicit_evolve") + implicit_evolve.nonlinear_solver = "newton" implicit_evolve.max_particle_iterations = self.max_particle_iterations implicit_evolve.particle_tolerance = self.particle_tolerance - newton = pywarpx.warpx.get_bucket('newton') + newton = pywarpx.warpx.get_bucket("newton") newton.verbose = self.verbose newton.absolute_tolerance = self.absolute_tolerance newton.relative_tolerance = self.relative_tolerance @@ -1404,8 +1801,15 @@ class GMRESLinearSolver(picmistandard.base._ClassWithInit): max_iterations: integer, default=1000 Maximum number of iterations """ - def __init__(self, verbose_int=None, restart_length=None, absolute_tolerance=None, relative_tolerance=None, - max_iterations=None): + + def __init__( + self, + verbose_int=None, + restart_length=None, + absolute_tolerance=None, + relative_tolerance=None, + max_iterations=None, + ): self.verbose_int = verbose_int self.restart_length = restart_length self.absolute_tolerance = absolute_tolerance @@ -1413,7 +1817,7 @@ def __init__(self, verbose_int=None, restart_length=None, absolute_tolerance=Non self.max_iterations = max_iterations def linear_solver_initialize_inputs(self): - gmres = pywarpx.warpx.get_bucket('gmres') + gmres = pywarpx.warpx.get_bucket("gmres") gmres.verbose_int = self.verbose_int gmres.restart_length = self.restart_length gmres.absolute_tolerance = self.absolute_tolerance @@ -1449,14 +1853,56 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): substeps: int, default=100 Number of substeps to take when updating the B-field. + holmstrom_vacuum_region: bool, default=False + Flag to determine handling of vacuum region. Setting to True will solve the simplified Generalized Ohm's Law dropping the Hall and pressure terms in the vacuum region. + This flag is useful for suppressing vacuum region fluctuations. A large resistivity value must be used when rho <= rho_floor. + Jx/y/z_external_function: str Function of space and time specifying external (non-plasma) currents. + + A_external: dict + Function of space and time specifying external (non-plasma) vector potential fields. + It is expected that a nested dicitonary will be passed + into picmi for each field that has different timings + e.g. + A_external = { + '': { + 'Ax_external_function': , + 'Ay_external_function': , + 'Az_external_function': , + 'A_time_external_function': + }, + ': {...}' + } + + or if fields are to be loaded from an OpenPMD file + A_external = { + '': { + 'load_from_file': True, + 'path': , + 'A_time_external_function': + }, + ': {...}' + } """ - def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, - plasma_hyper_resistivity=None, substeps=None, - Jx_external_function=None, Jy_external_function=None, - Jz_external_function=None, **kw): + + def __init__( + self, + grid, + Te=None, + n0=None, + gamma=None, + n_floor=None, + plasma_resistivity=None, + plasma_hyper_resistivity=None, + substeps=None, + holmstrom_vacuum_region=None, + Jx_external_function=None, + Jy_external_function=None, + Jz_external_function=None, + A_external=None, + **kw, + ): self.grid = grid self.method = "hybrid" @@ -1469,10 +1915,14 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.substeps = substeps + self.holmstrom_vacuum_region = holmstrom_vacuum_region + self.Jx_external_function = Jx_external_function self.Jy_external_function = Jy_external_function self.Jz_external_function = Jz_external_function + self.A_external = A_external + # Handle keyword arguments used in expressions self.user_defined_kw = {} for k in list(kw.keys()): @@ -1482,7 +1932,6 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.handle_init(kw) def solver_initialize_inputs(self): - # Add the user defined keywords to my_constants # The keywords are mangled if there is a conflicting variable already # defined in my_constants with the same name but different value. @@ -1497,23 +1946,73 @@ def solver_initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho,J)', - pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) + "plasma_resistivity(rho,J)", + pywarpx.my_constants.mangle_expression( + self.plasma_resistivity, self.mangle_dict + ), ) pywarpx.hybridpicmodel.plasma_hyper_resistivity = self.plasma_hyper_resistivity pywarpx.hybridpicmodel.substeps = self.substeps + pywarpx.hybridpicmodel.holmstrom_vacuum_region = self.holmstrom_vacuum_region pywarpx.hybridpicmodel.__setattr__( - 'Jx_external_grid_function(x,y,z,t)', - pywarpx.my_constants.mangle_expression(self.Jx_external_function, self.mangle_dict) + "Jx_external_grid_function(x,y,z,t)", + pywarpx.my_constants.mangle_expression( + self.Jx_external_function, self.mangle_dict + ), ) pywarpx.hybridpicmodel.__setattr__( - 'Jy_external_grid_function(x,y,z,t)', - pywarpx.my_constants.mangle_expression(self.Jy_external_function, self.mangle_dict) + "Jy_external_grid_function(x,y,z,t)", + pywarpx.my_constants.mangle_expression( + self.Jy_external_function, self.mangle_dict + ), ) pywarpx.hybridpicmodel.__setattr__( - 'Jz_external_grid_function(x,y,z,t)', - pywarpx.my_constants.mangle_expression(self.Jz_external_function, self.mangle_dict) + "Jz_external_grid_function(x,y,z,t)", + pywarpx.my_constants.mangle_expression( + self.Jz_external_function, self.mangle_dict + ), ) + if self.A_external is not None: + pywarpx.hybridpicmodel.add_external_fields = True + pywarpx.external_vector_potential.__setattr__( + "fields", + pywarpx.my_constants.mangle_expression( + list(self.A_external.keys()), self.mangle_dict + ), + ) + for field_name, field_dict in self.A_external.items(): + if field_dict.get("read_from_file", False): + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.read_from_file", field_dict["read_from_file"] + ) + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.path", field_dict["path"] + ) + else: + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.Ax_external_grid_function(x,y,z)", + pywarpx.my_constants.mangle_expression( + field_dict["Ax_external_function"], self.mangle_dict + ), + ) + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.Ay_external_grid_function(x,y,z)", + pywarpx.my_constants.mangle_expression( + field_dict["Ay_external_function"], self.mangle_dict + ), + ) + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.Az_external_grid_function(x,y,z)", + pywarpx.my_constants.mangle_expression( + field_dict["Az_external_function"], self.mangle_dict + ), + ) + pywarpx.external_vector_potential.__setattr__( + f"{field_name}.A_time_external_function(t)", + pywarpx.my_constants.mangle_expression( + field_dict["A_time_external_function"], self.mangle_dict + ), + ) class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): @@ -1530,27 +2029,64 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver + + warpx_magnetostatic: bool, default=False + Whether to use the magnetostatic solver + + warpx_effective_potential: bool, default=False + Whether to use the effective potential Poisson solver (EP-PIC) + + warpx_effective_potential_factor: float, default=4 + If the effective potential Poisson solver is used, this sets the value + of C_EP (the method is marginally stable at C_EP = 1) + + warpx_dt_update_interval: integer, optional (default = -1) + How frequently the timestep is updated. Adaptive timestepping is disabled when this is <= 0. + + warpx_cfl: float, optional + Fraction of the CFL condition for particle velocity vs grid size, used to set the timestep when `warpx_dt_update_interval > 0`. + + warpx_max_dt: float, optional + The maximum allowable timestep when `warpx_dt_update_interval > 0`. + """ + def init(self, kw): - self.relativistic = kw.pop('warpx_relativistic', False) - self.absolute_tolerance = kw.pop('warpx_absolute_tolerance', None) - self.self_fields_verbosity = kw.pop('warpx_self_fields_verbosity', None) - self.magnetostatic = kw.pop('warpx_magnetostatic', False) + self.relativistic = kw.pop("warpx_relativistic", False) + self.absolute_tolerance = kw.pop("warpx_absolute_tolerance", None) + self.self_fields_verbosity = kw.pop("warpx_self_fields_verbosity", None) + self.magnetostatic = kw.pop("warpx_magnetostatic", False) + self.effective_potential = kw.pop("warpx_effective_potential", False) + self.effective_potential_factor = kw.pop( + "warpx_effective_potential_factor", None + ) + self.cfl = kw.pop("warpx_cfl", None) + self.dt_update_interval = kw.pop("warpx_dt_update_interval", None) + self.max_dt = kw.pop("warpx_max_dt", None) def solver_initialize_inputs(self): - # Open BC means FieldBoundaryType::Open for electrostatic sims, rather than perfectly-matched layer - BC_map['open'] = 'open' + BC_map["open"] = "open" self.grid.grid_initialize_inputs() + # set adaptive timestepping parameters + pywarpx.warpx.cfl = self.cfl + pywarpx.warpx.dt_update_interval = self.dt_update_interval + pywarpx.warpx.max_dt = self.max_dt + if self.relativistic: - pywarpx.warpx.do_electrostatic = 'relativistic' + pywarpx.warpx.do_electrostatic = "relativistic" else: if self.magnetostatic: - pywarpx.warpx.do_electrostatic = 'labframe-electromagnetostatic' + pywarpx.warpx.do_electrostatic = "labframe-electromagnetostatic" + elif self.effective_potential: + pywarpx.warpx.do_electrostatic = "labframe-effective-potential" + pywarpx.warpx.effective_potential_factor = ( + self.effective_potential_factor + ) else: - pywarpx.warpx.do_electrostatic = 'labframe' + pywarpx.warpx.do_electrostatic = "labframe" pywarpx.warpx.self_fields_required_precision = self.required_precision pywarpx.warpx.self_fields_absolute_tolerance = self.absolute_tolerance pywarpx.warpx.self_fields_max_iters = self.maximum_iterations @@ -1569,16 +2105,22 @@ class GaussianLaser(picmistandard.PICMI_GaussianLaser): def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: - self.name = 'laser{}'.format(self.laser_number) + self.name = "laser{}".format(self.laser_number) self.laser = pywarpx.Lasers.newlaser(self.name) self.laser.profile = "Gaussian" - self.laser.wavelength = self.wavelength # The wavelength of the laser (in meters) + self.laser.wavelength = ( + self.wavelength + ) # The wavelength of the laser (in meters) self.laser.e_max = self.E0 # Maximum amplitude of the laser field (in V/m) - self.laser.polarization = self.polarization_direction # The main polarization vector + self.laser.polarization = ( + self.polarization_direction + ) # The main polarization vector self.laser.profile_waist = self.waist # The waist of the laser (in meters) - self.laser.profile_duration = self.duration # The duration of the laser (in seconds) + self.laser.profile_duration = ( + self.duration + ) # The duration of the laser (in seconds) self.laser.direction = self.propagation_direction self.laser.zeta = self.zeta self.laser.beta = self.beta @@ -1587,6 +2129,7 @@ def laser_initialize_inputs(self): self.laser.do_continuous_injection = self.fill_in + class AnalyticLaser(picmistandard.PICMI_AnalyticLaser): def init(self, kw): self.mangle_dict = None @@ -1594,14 +2137,18 @@ def init(self, kw): def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: - self.name = 'laser{}'.format(self.laser_number) + self.name = "laser{}".format(self.laser_number) self.laser = pywarpx.Lasers.newlaser(self.name) self.laser.profile = "parse_field_function" - self.laser.wavelength = self.wavelength # The wavelength of the laser (in meters) + self.laser.wavelength = ( + self.wavelength + ) # The wavelength of the laser (in meters) self.laser.e_max = self.Emax # Maximum amplitude of the laser field (in V/m) - self.laser.polarization = self.polarization_direction # The main polarization vector + self.laser.polarization = ( + self.polarization_direction + ) # The main polarization vector self.laser.direction = self.propagation_direction self.laser.do_continuous_injection = self.fill_in @@ -1609,99 +2156,178 @@ def laser_initialize_inputs(self): # Only do this once so that the same variables are used in this distribution # is used multiple times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - expression = pywarpx.my_constants.mangle_expression(self.field_expression, self.mangle_dict) - self.laser.__setattr__('field_function(X,Y,t)', expression) + expression = pywarpx.my_constants.mangle_expression( + self.field_expression, self.mangle_dict + ) + self.laser.__setattr__("field_function(X,Y,t)", expression) class LaserAntenna(picmistandard.PICMI_LaserAntenna): def laser_antenna_initialize_inputs(self, laser): laser.laser.position = self.position # This point is on the laser plane - if ( - self.normal_vector is not None - and not np.allclose(laser.laser.direction, self.normal_vector) + if self.normal_vector is not None and not np.allclose( + laser.laser.direction, self.normal_vector ): raise AttributeError( - 'The specified laser direction does not match the ' - 'specified antenna normal.' + "The specified laser direction does not match the " + "specified antenna normal." ) - self.normal_vector = laser.laser.direction # The plane normal direction + self.normal_vector = laser.laser.direction # The plane normal direction + # Ensure the normal vector is a unit vector + self.normal_vector /= np.linalg.norm(self.normal_vector) if isinstance(laser, GaussianLaser): - # Focal distance from the antenna (in meters) - laser.laser.profile_focal_distance = np.sqrt( - (laser.focal_position[0] - self.position[0])**2 + - (laser.focal_position[1] - self.position[1])**2 + - (laser.focal_position[2] - self.position[2])**2 + # Focal displacement from the antenna (in meters) + laser.laser.profile_focal_distance = ( + (laser.focal_position[0] - self.position[0]) * self.normal_vector[0] + + (laser.focal_position[1] - self.position[1]) * self.normal_vector[1] + + (laser.focal_position[2] - self.position[2]) * self.normal_vector[2] ) # The time at which the laser reaches its peak (in seconds) - laser.laser.profile_t_peak = np.sqrt( - (self.position[0] - laser.centroid_position[0])**2 + - (self.position[1] - laser.centroid_position[1])**2 + - (self.position[2] - laser.centroid_position[2])**2 + laser.laser.profile_t_peak = ( + (self.position[0] - laser.centroid_position[0]) * self.normal_vector[0] + + (self.position[1] - laser.centroid_position[1]) + * self.normal_vector[1] + + (self.position[2] - laser.centroid_position[2]) + * self.normal_vector[2] ) / constants.c class LoadInitialField(picmistandard.PICMI_LoadGriddedField): + def init(self, kw): + self.do_divb_cleaning_external = kw.pop("warpx_do_divb_cleaning_external", None) + self.divb_cleaner_atol = kw.pop("warpx_projection_divb_cleaner_atol", None) + self.divb_cleaner_rtol = kw.pop("warpx_projection_divb_cleaner_rtol", None) + def applied_field_initialize_inputs(self): pywarpx.warpx.read_fields_from_path = self.read_fields_from_path if self.load_E: - pywarpx.warpx.E_ext_grid_init_style = 'read_from_file' + pywarpx.warpx.E_ext_grid_init_style = "read_from_file" + if self.load_B: + pywarpx.warpx.B_ext_grid_init_style = "read_from_file" + pywarpx.warpx.do_divb_cleaning_external = self.do_divb_cleaning_external + pywarpx.projectiondivbcleaner.atol = self.divb_cleaner_atol + pywarpx.projectiondivbcleaner.rtol = self.divb_cleaner_rtol + + +class LoadInitialFieldFromPython: + """ + Field Initializer that takes a function handle to be registered as a callback. + The function is expected to write the E and/or B fields into the + fields.Bx/y/zFPExternalWrapper() multifab. The callback is installed + in the beforeInitEsolve hook. This should operate identically to loading from + a file. + + Parameters + ---------- + warpx_do_divb_cleaning_external: bool, default=True + Flag that controls whether or not to execute the Projection based B-field divergence cleaner. + + load_E: bool, default=True + E field is expected to be loaded in the registered callback. + + load_B: bool, default=True + B field is expected to be loaded in the registered callback. + """ + + def __init__(self, **kw): + self.do_divb_cleaning_external = kw.pop("warpx_do_divb_cleaning_external", None) + self.divb_cleaner_atol = kw.pop("warpx_projection_divb_cleaner_atol", None) + self.divb_cleaner_rtol = kw.pop("warpx_projection_divb_cleaner_rtol", None) + + # If using load_from_python, a function handle is expected for callback + self.load_from_python = kw.pop("load_from_python") + self.load_E = kw.pop("load_E", True) + self.load_B = kw.pop("load_B", True) + + def applied_field_initialize_inputs(self): + if self.load_E: + pywarpx.warpx.E_ext_grid_init_style = "load_from_python" if self.load_B: - pywarpx.warpx.B_ext_grid_init_style = 'read_from_file' + pywarpx.warpx.B_ext_grid_init_style = "load_from_python" + pywarpx.warpx.do_divb_cleaning_external = self.do_divb_cleaning_external + pywarpx.projectiondivbcleaner.atol = self.divb_cleaner_atol + pywarpx.projectiondivbcleaner.rtol = self.divb_cleaner_rtol + + pywarpx.callbacks.installloadExternalFields(self.load_from_python) class AnalyticInitialField(picmistandard.PICMI_AnalyticAppliedField): def init(self, kw): self.mangle_dict = None - self.maxlevel_extEMfield_init = kw.pop('warpx_maxlevel_extEMfield_init', None); + self.maxlevel_extEMfield_init = kw.pop("warpx_maxlevel_extEMfield_init", None) def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX - pywarpx.warpx.maxlevel_extEMfield_init = self.maxlevel_extEMfield_init; + pywarpx.warpx.maxlevel_extEMfield_init = self.maxlevel_extEMfield_init if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution # is used multiple times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - if (self.Ex_expression is not None or - self.Ey_expression is not None or - self.Ez_expression is not None): - pywarpx.warpx.E_ext_grid_init_style = 'parse_e_ext_grid_function' - for sdir, expression in zip(['x', 'y', 'z'], [self.Ex_expression, self.Ey_expression, self.Ez_expression]): - expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) - pywarpx.warpx.__setattr__(f'E{sdir}_external_grid_function(x,y,z)', expression) - - if (self.Bx_expression is not None or - self.By_expression is not None or - self.Bz_expression is not None): - pywarpx.warpx.B_ext_grid_init_style = 'parse_b_ext_grid_function' - for sdir, expression in zip(['x', 'y', 'z'], [self.Bx_expression, self.By_expression, self.Bz_expression]): - expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) - pywarpx.warpx.__setattr__(f'B{sdir}_external_grid_function(x,y,z)', expression) + if ( + self.Ex_expression is not None + or self.Ey_expression is not None + or self.Ez_expression is not None + ): + pywarpx.warpx.E_ext_grid_init_style = "parse_e_ext_grid_function" + for sdir, expression in zip( + ["x", "y", "z"], + [self.Ex_expression, self.Ey_expression, self.Ez_expression], + ): + expression = pywarpx.my_constants.mangle_expression( + expression, self.mangle_dict + ) + pywarpx.warpx.__setattr__( + f"E{sdir}_external_grid_function(x,y,z)", expression + ) + + if ( + self.Bx_expression is not None + or self.By_expression is not None + or self.Bz_expression is not None + ): + pywarpx.warpx.B_ext_grid_init_style = "parse_b_ext_grid_function" + for sdir, expression in zip( + ["x", "y", "z"], + [self.Bx_expression, self.By_expression, self.Bz_expression], + ): + expression = pywarpx.my_constants.mangle_expression( + expression, self.mangle_dict + ) + pywarpx.warpx.__setattr__( + f"B{sdir}_external_grid_function(x,y,z)", expression + ) + class LoadAppliedField(picmistandard.PICMI_LoadAppliedField): def applied_field_initialize_inputs(self): pywarpx.particles.read_fields_from_path = self.read_fields_from_path if self.load_E: - pywarpx.particles.E_ext_particle_init_style = 'read_from_file' + pywarpx.particles.E_ext_particle_init_style = "read_from_file" if self.load_B: - pywarpx.particles.B_ext_particle_init_style = 'read_from_file' + pywarpx.particles.B_ext_particle_init_style = "read_from_file" + class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX - if (self.Ex is not None or - self.Ey is not None or - self.Ez is not None): - pywarpx.particles.E_ext_particle_init_style = 'constant' - pywarpx.particles.E_external_particle = [self.Ex or 0., self.Ey or 0., self.Ez or 0.] + if self.Ex is not None or self.Ey is not None or self.Ez is not None: + pywarpx.particles.E_ext_particle_init_style = "constant" + pywarpx.particles.E_external_particle = [ + self.Ex or 0.0, + self.Ey or 0.0, + self.Ez or 0.0, + ] - if (self.Bx is not None or - self.By is not None or - self.Bz is not None): - pywarpx.particles.B_ext_particle_init_style = 'constant' - pywarpx.particles.B_external_particle = [self.Bx or 0., self.By or 0., self.Bz or 0.] + if self.Bx is not None or self.By is not None or self.Bz is not None: + pywarpx.particles.B_ext_particle_init_style = "constant" + pywarpx.particles.B_external_particle = [ + self.Bx or 0.0, + self.By or 0.0, + self.Bz or 0.0, + ] class AnalyticAppliedField(picmistandard.PICMI_AnalyticAppliedField): @@ -1716,21 +2342,43 @@ def applied_field_initialize_inputs(self): # is used multiple times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - if (self.Ex_expression is not None or - self.Ey_expression is not None or - self.Ez_expression is not None): - pywarpx.particles.E_ext_particle_init_style = 'parse_e_ext_particle_function' - for sdir, expression in zip(['x', 'y', 'z'], [self.Ex_expression, self.Ey_expression, self.Ez_expression]): - expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) - pywarpx.particles.__setattr__(f'E{sdir}_external_particle_function(x,y,z,t)', expression) + if ( + self.Ex_expression is not None + or self.Ey_expression is not None + or self.Ez_expression is not None + ): + pywarpx.particles.E_ext_particle_init_style = ( + "parse_e_ext_particle_function" + ) + for sdir, expression in zip( + ["x", "y", "z"], + [self.Ex_expression, self.Ey_expression, self.Ez_expression], + ): + expression = pywarpx.my_constants.mangle_expression( + expression, self.mangle_dict + ) + pywarpx.particles.__setattr__( + f"E{sdir}_external_particle_function(x,y,z,t)", expression + ) - if (self.Bx_expression is not None or - self.By_expression is not None or - self.Bz_expression is not None): - pywarpx.particles.B_ext_particle_init_style = 'parse_b_ext_particle_function' - for sdir, expression in zip(['x', 'y', 'z'], [self.Bx_expression, self.By_expression, self.Bz_expression]): - expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) - pywarpx.particles.__setattr__(f'B{sdir}_external_particle_function(x,y,z,t)', expression) + if ( + self.Bx_expression is not None + or self.By_expression is not None + or self.Bz_expression is not None + ): + pywarpx.particles.B_ext_particle_init_style = ( + "parse_b_ext_particle_function" + ) + for sdir, expression in zip( + ["x", "y", "z"], + [self.Bx_expression, self.By_expression, self.Bz_expression], + ): + expression = pywarpx.my_constants.mangle_expression( + expression, self.mangle_dict + ) + pywarpx.particles.__setattr__( + f"B{sdir}_external_particle_function(x,y,z,t)", expression + ) class Mirror(picmistandard.PICMI_Mirror): @@ -1753,13 +2401,20 @@ class FieldIonization(picmistandard.PICMI_FieldIonization): """ WarpX only has ADK ionization model implemented. """ + def interaction_initialize_inputs(self): - assert self.model == 'ADK', 'WarpX only has ADK ionization model implemented' + assert self.model == "ADK", "WarpX only has ADK ionization model implemented" self.ionized_species.species.do_field_ionization = 1 - self.ionized_species.species.physical_element = self.ionized_species.particle_type - self.ionized_species.species.ionization_product_species = self.product_species.name - self.ionized_species.species.ionization_initial_level = self.ionized_species.charge_state - self.ionized_species.species.charge = 'q_e' + self.ionized_species.species.physical_element = ( + self.ionized_species.particle_type + ) + self.ionized_species.species.ionization_product_species = ( + self.product_species.name + ) + self.ionized_species.species.ionization_initial_level = ( + self.ionized_species.charge_state + ) + self.ionized_species.species.charge = "q_e" class CoulombCollisions(picmistandard.base._ClassWithInit): @@ -1783,6 +2438,7 @@ class CoulombCollisions(picmistandard.base._ClassWithInit): ndt: integer, optional The collisions will be applied every "ndt" steps. Must be 1 or larger. """ + def __init__(self, name, species, CoulombLog=None, ndt=None, **kw): self.name = name self.species = species @@ -1793,7 +2449,7 @@ def __init__(self, name, species, CoulombLog=None, ndt=None, **kw): def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) - collision.type = 'pairwisecoulomb' + collision.type = "pairwisecoulomb" collision.species = [species.name for species in self.species] collision.CoulombLog = self.CoulombLog collision.ndt = self.ndt @@ -1834,9 +2490,18 @@ class MCCCollisions(picmistandard.base._ClassWithInit): The collisions will be applied every "ndt" steps. Must be 1 or larger. """ - def __init__(self, name, species, background_density, - background_temperature, scattering_processes, - background_mass=None, max_background_density=None, ndt=None, **kw): + def __init__( + self, + name, + species, + background_density, + background_temperature, + scattering_processes, + background_mass=None, + max_background_density=None, + ndt=None, + **kw, + ): self.name = name self.species = species self.background_density = background_density @@ -1850,14 +2515,18 @@ def __init__(self, name, species, background_density, def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) - collision.type = 'background_mcc' + collision.type = "background_mcc" collision.species = self.species.name if isinstance(self.background_density, str): - collision.__setattr__('background_density(x,y,z,t)', self.background_density) + collision.__setattr__( + "background_density(x,y,z,t)", self.background_density + ) else: collision.background_density = self.background_density if isinstance(self.background_temperature, str): - collision.__setattr__('background_temperature(x,y,z,t)', self.background_temperature) + collision.__setattr__( + "background_temperature(x,y,z,t)", self.background_temperature + ) else: collision.background_temperature = self.background_temperature collision.background_mass = self.background_mass @@ -1867,9 +2536,9 @@ def collision_initialize_inputs(self): collision.scattering_processes = self.scattering_processes.keys() for process, kw in self.scattering_processes.items(): for key, val in kw.items(): - if key == 'species': + if key == "species": val = val.name - collision.add_new_attr(process+'_'+key, val) + collision.add_new_attr(process + "_" + key, val) class DSMCCollisions(picmistandard.base._ClassWithInit): @@ -1903,16 +2572,16 @@ def __init__(self, name, species, scattering_processes, ndt=None, **kw): def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) - collision.type = 'dsmc' + collision.type = "dsmc" collision.species = [species.name for species in self.species] collision.ndt = self.ndt collision.scattering_processes = self.scattering_processes.keys() for process, kw in self.scattering_processes.items(): for key, val in kw.items(): - if key == 'species': + if key == "species": val = val.name - collision.add_new_attr(process+'_'+key, val) + collision.add_new_attr(process + "_" + key, val) class EmbeddedBoundary(picmistandard.base._ClassWithInit): @@ -1951,19 +2620,35 @@ class EmbeddedBoundary(picmistandard.base._ClassWithInit): Parameters used in the analytic expressions should be given as additional keyword arguments. """ - def __init__(self, implicit_function=None, stl_file=None, stl_scale=None, stl_center=None, stl_reverse_normal=False, - potential=None, cover_multiple_cuts=None, **kw): - assert stl_file is None or implicit_function is None, Exception('Only one between implicit_function and ' - 'stl_file can be specified') + def __init__( + self, + implicit_function=None, + stl_file=None, + stl_scale=None, + stl_center=None, + stl_reverse_normal=False, + potential=None, + cover_multiple_cuts=None, + **kw, + ): + assert stl_file is None or implicit_function is None, Exception( + "Only one between implicit_function and stl_file can be specified" + ) self.implicit_function = implicit_function self.stl_file = stl_file if stl_file is None: - assert stl_scale is None, Exception('EB can only be scaled only when using an stl file') - assert stl_center is None, Exception('EB can only be translated only when using an stl file') - assert stl_reverse_normal is False, Exception('EB can only be reversed only when using an stl file') + assert stl_scale is None, Exception( + "EB can only be scaled only when using an stl file" + ) + assert stl_center is None, Exception( + "EB can only be translated only when using an stl file" + ) + assert stl_reverse_normal is False, Exception( + "EB can only be reversed only when using an stl file" + ) self.stl_scale = stl_scale self.stl_center = stl_center @@ -1976,22 +2661,26 @@ def __init__(self, implicit_function=None, stl_file=None, stl_scale=None, stl_ce # Handle keyword arguments used in expressions self.user_defined_kw = {} for k in list(kw.keys()): - if (implicit_function is not None and re.search(r'\b%s\b'%k, implicit_function) or - (potential is not None and re.search(r'\b%s\b'%k, potential))): + if ( + implicit_function is not None + and re.search(r"\b%s\b" % k, implicit_function) + or (potential is not None and re.search(r"\b%s\b" % k, potential)) + ): self.user_defined_kw[k] = kw[k] del kw[k] self.handle_init(kw) def embedded_boundary_initialize_inputs(self, solver): - # Add the user defined keywords to my_constants # The keywords are mangled if there is a conflicting variable already # defined in my_constants with the same name but different value. self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) if self.implicit_function is not None: - expression = pywarpx.my_constants.mangle_expression(self.implicit_function, self.mangle_dict) + expression = pywarpx.my_constants.mangle_expression( + self.implicit_function, self.mangle_dict + ) pywarpx.warpx.eb_implicit_function = expression if self.stl_file is not None: @@ -2004,8 +2693,10 @@ def embedded_boundary_initialize_inputs(self, solver): pywarpx.eb2.cover_multiple_cuts = self.cover_multiple_cuts if self.potential is not None: - expression = pywarpx.my_constants.mangle_expression(self.potential, self.mangle_dict) - pywarpx.warpx.__setattr__('eb_potential(x,y,z,t)', expression) + expression = pywarpx.my_constants.mangle_expression( + self.potential, self.mangle_dict + ) + pywarpx.warpx.__setattr__("eb_potential(x,y,z,t)", expression) class PlasmaLens(picmistandard.base._ClassWithInit): @@ -2042,22 +2733,25 @@ class PlasmaLens(picmistandard.base._ClassWithInit): - By = -x*strengths_B """ - def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, **kw): + + def __init__( + self, period, starts, lengths, strengths_E=None, strengths_B=None, **kw + ): self.period = period self.starts = starts self.lengths = lengths self.strengths_E = strengths_E self.strengths_B = strengths_B - assert (self.strengths_E is not None) or (self.strengths_B is not None),\ - Exception('One of strengths_E or strengths_B must be supplied') + assert (self.strengths_E is not None) or (self.strengths_B is not None), ( + Exception("One of strengths_E or strengths_B must be supplied") + ) self.handle_init(kw) def applied_field_initialize_inputs(self): - - pywarpx.particles.E_ext_particle_init_style = 'repeated_plasma_lens' - pywarpx.particles.B_ext_particle_init_style = 'repeated_plasma_lens' + pywarpx.particles.E_ext_particle_init_style = "repeated_plasma_lens" + pywarpx.particles.B_ext_particle_init_style = "repeated_plasma_lens" pywarpx.particles.repeated_plasma_lens_period = self.period pywarpx.particles.repeated_plasma_lens_starts = self.starts pywarpx.particles.repeated_plasma_lens_lengths = self.lengths @@ -2137,6 +2831,9 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_do_dynamic_scheduling: bool, default=True Whether to do dynamic scheduling with OpenMP + warpx_roundrobin_sfc: bool, default=False + Whether to use the RRSFC strategy for making DistributionMapping + warpx_load_balance_intervals: string, default='0' The intervals for doing load balancing @@ -2228,6 +2925,21 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_used_inputs_file: string, optional The name of the text file that the used input parameters is written to, + + warpx_reduced_diags_path: string, optional + Sets the default path for reduced diagnostic output files + + warpx_reduced_diags_extension: string, optional + Sets the default extension for reduced diagnostic output files + + warpx_reduced_diags_intervals: string, optional + Sets the default intervals for reduced diagnostic output files + + warpx_reduced_diags_separator: string, optional + Sets the default separator for reduced diagnostic output files + + warpx_reduced_diags_precision: integer, optional + Sets the default precision for reduced diagnostic output files """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -2236,49 +2948,69 @@ class Simulation(picmistandard.PICMI_Simulation): extension = pywarpx.libwarpx def init(self, kw): + self.evolve_scheme = kw.pop("warpx_evolve_scheme", None) + self.current_deposition_algo = kw.pop("warpx_current_deposition_algo", None) + self.charge_deposition_algo = kw.pop("warpx_charge_deposition_algo", None) + self.field_gathering_algo = kw.pop("warpx_field_gathering_algo", None) + self.particle_pusher_algo = kw.pop("warpx_particle_pusher_algo", None) + self.use_filter = kw.pop("warpx_use_filter", None) + self.do_multi_J = kw.pop("warpx_do_multi_J", None) + self.do_multi_J_n_depositions = kw.pop("warpx_do_multi_J_n_depositions", None) + self.grid_type = kw.pop("warpx_grid_type", None) + self.do_current_centering = kw.pop("warpx_do_current_centering", None) + self.field_centering_order = kw.pop("warpx_field_centering_order", None) + self.current_centering_order = kw.pop("warpx_current_centering_order", None) + self.serialize_initial_conditions = kw.pop( + "warpx_serialize_initial_conditions", None + ) + self.random_seed = kw.pop("warpx_random_seed", None) + self.do_dynamic_scheduling = kw.pop("warpx_do_dynamic_scheduling", None) + self.roundrobin_sfc = kw.pop("warpx_roundrobin_sfc", None) + self.load_balance_intervals = kw.pop("warpx_load_balance_intervals", None) + self.load_balance_efficiency_ratio_threshold = kw.pop( + "warpx_load_balance_efficiency_ratio_threshold", None + ) + self.load_balance_with_sfc = kw.pop("warpx_load_balance_with_sfc", None) + self.load_balance_knapsack_factor = kw.pop( + "warpx_load_balance_knapsack_factor", None + ) + self.load_balance_costs_update = kw.pop("warpx_load_balance_costs_update", None) + self.costs_heuristic_particles_wt = kw.pop( + "warpx_costs_heuristic_particles_wt", None + ) + self.costs_heuristic_cells_wt = kw.pop("warpx_costs_heuristic_cells_wt", None) + self.use_fdtd_nci_corr = kw.pop("warpx_use_fdtd_nci_corr", None) + self.amr_check_input = kw.pop("warpx_amr_check_input", None) + self.amr_restart = kw.pop("warpx_amr_restart", None) + self.amrex_the_arena_is_managed = kw.pop( + "warpx_amrex_the_arena_is_managed", None + ) + self.amrex_the_arena_init_size = kw.pop("warpx_amrex_the_arena_init_size", None) + self.amrex_use_gpu_aware_mpi = kw.pop("warpx_amrex_use_gpu_aware_mpi", None) + self.zmax_plasma_to_compute_max_step = kw.pop( + "warpx_zmax_plasma_to_compute_max_step", None + ) + self.compute_max_step_from_btd = kw.pop("warpx_compute_max_step_from_btd", None) + self.sort_intervals = kw.pop("warpx_sort_intervals", None) + self.sort_particles_for_deposition = kw.pop( + "warpx_sort_particles_for_deposition", None + ) + self.sort_idx_type = kw.pop("warpx_sort_idx_type", None) + self.sort_bin_size = kw.pop("warpx_sort_bin_size", None) + self.used_inputs_file = kw.pop("warpx_used_inputs_file", None) - self.evolve_scheme = kw.pop('warpx_evolve_scheme', None) - self.current_deposition_algo = kw.pop('warpx_current_deposition_algo', None) - self.charge_deposition_algo = kw.pop('warpx_charge_deposition_algo', None) - self.field_gathering_algo = kw.pop('warpx_field_gathering_algo', None) - self.particle_pusher_algo = kw.pop('warpx_particle_pusher_algo', None) - self.use_filter = kw.pop('warpx_use_filter', None) - self.do_multi_J = kw.pop('warpx_do_multi_J', None) - self.do_multi_J_n_depositions = kw.pop('warpx_do_multi_J_n_depositions', None) - self.grid_type = kw.pop('warpx_grid_type', None) - self.do_current_centering = kw.pop('warpx_do_current_centering', None) - self.field_centering_order = kw.pop('warpx_field_centering_order', None) - self.current_centering_order = kw.pop('warpx_current_centering_order', None) - self.serialize_initial_conditions = kw.pop('warpx_serialize_initial_conditions', None) - self.random_seed = kw.pop('warpx_random_seed', None) - self.do_dynamic_scheduling = kw.pop('warpx_do_dynamic_scheduling', None) - self.load_balance_intervals = kw.pop('warpx_load_balance_intervals', None) - self.load_balance_efficiency_ratio_threshold = kw.pop('warpx_load_balance_efficiency_ratio_threshold', None) - self.load_balance_with_sfc = kw.pop('warpx_load_balance_with_sfc', None) - self.load_balance_knapsack_factor = kw.pop('warpx_load_balance_knapsack_factor', None) - self.load_balance_costs_update = kw.pop('warpx_load_balance_costs_update', None) - self.costs_heuristic_particles_wt = kw.pop('warpx_costs_heuristic_particles_wt', None) - self.costs_heuristic_cells_wt = kw.pop('warpx_costs_heuristic_cells_wt', None) - self.use_fdtd_nci_corr = kw.pop('warpx_use_fdtd_nci_corr', None) - self.amr_check_input = kw.pop('warpx_amr_check_input', None) - self.amr_restart = kw.pop('warpx_amr_restart', None) - self.amrex_the_arena_is_managed = kw.pop('warpx_amrex_the_arena_is_managed', None) - self.amrex_the_arena_init_size = kw.pop('warpx_amrex_the_arena_init_size', None) - self.amrex_use_gpu_aware_mpi = kw.pop('warpx_amrex_use_gpu_aware_mpi', None) - self.zmax_plasma_to_compute_max_step = kw.pop('warpx_zmax_plasma_to_compute_max_step', None) - self.compute_max_step_from_btd = kw.pop('warpx_compute_max_step_from_btd', None) - self.sort_intervals = kw.pop('warpx_sort_intervals', None) - self.sort_particles_for_deposition = kw.pop('warpx_sort_particles_for_deposition', None) - self.sort_idx_type = kw.pop('warpx_sort_idx_type', None) - self.sort_bin_size = kw.pop('warpx_sort_bin_size', None) - self.used_inputs_file = kw.pop('warpx_used_inputs_file', None) - - self.collisions = kw.pop('warpx_collisions', None) - self.embedded_boundary = kw.pop('warpx_embedded_boundary', None) - - self.break_signals = kw.pop('warpx_break_signals', None) - self.checkpoint_signals = kw.pop('warpx_checkpoint_signals', None) - self.numprocs = kw.pop('warpx_numprocs', None) + self.collisions = kw.pop("warpx_collisions", None) + self.embedded_boundary = kw.pop("warpx_embedded_boundary", None) + + self.break_signals = kw.pop("warpx_break_signals", None) + self.checkpoint_signals = kw.pop("warpx_checkpoint_signals", None) + self.numprocs = kw.pop("warpx_numprocs", None) + + self.reduced_diags_path = kw.pop("warpx_reduced_diags_path", None) + self.reduced_diags_extension = kw.pop("warpx_reduced_diags_extension", None) + self.reduced_diags_intervals = kw.pop("warpx_reduced_diags_intervals", None) + self.reduced_diags_separator = kw.pop("warpx_reduced_diags_separator", None) + self.reduced_diags_precision = kw.pop("warpx_reduced_diags_precision", None) self.inputs_initialized = False self.warpx_initialized = False @@ -2295,9 +3027,11 @@ def initialize_inputs(self): if self.gamma_boost is not None: pywarpx.warpx.gamma_boost = self.gamma_boost - pywarpx.warpx.boost_direction = 'z' + pywarpx.warpx.boost_direction = "z" - pywarpx.warpx.zmax_plasma_to_compute_max_step = self.zmax_plasma_to_compute_max_step + pywarpx.warpx.zmax_plasma_to_compute_max_step = ( + self.zmax_plasma_to_compute_max_step + ) pywarpx.warpx.compute_max_step_from_btd = self.compute_max_step_from_btd pywarpx.warpx.sort_intervals = self.sort_intervals @@ -2313,7 +3047,9 @@ def initialize_inputs(self): pywarpx.algo.field_gathering = self.field_gathering_algo pywarpx.algo.particle_pusher = self.particle_pusher_algo pywarpx.algo.load_balance_intervals = self.load_balance_intervals - pywarpx.algo.load_balance_efficiency_ratio_threshold = self.load_balance_efficiency_ratio_threshold + pywarpx.algo.load_balance_efficiency_ratio_threshold = ( + self.load_balance_efficiency_ratio_threshold + ) pywarpx.algo.load_balance_with_sfc = self.load_balance_with_sfc pywarpx.algo.load_balance_knapsack_factor = self.load_balance_knapsack_factor pywarpx.algo.load_balance_costs_update = self.load_balance_costs_update @@ -2331,6 +3067,8 @@ def initialize_inputs(self): pywarpx.warpx.do_dynamic_scheduling = self.do_dynamic_scheduling + pywarpx.warpx.roundrobin_sfc = self.roundrobin_sfc + pywarpx.particles.use_fdtd_nci_corr = self.use_fdtd_nci_corr pywarpx.amr.check_input = self.amr_check_input @@ -2340,16 +3078,32 @@ def initialize_inputs(self): pywarpx.warpx.numprocs = self.numprocs + reduced_diags = pywarpx.warpx.get_bucket("reduced_diags") + reduced_diags.path = self.reduced_diags_path + reduced_diags.extension = self.reduced_diags_extension + reduced_diags.intervals = self.reduced_diags_intervals + reduced_diags.separator = self.reduced_diags_separator + reduced_diags.precision = self.reduced_diags_precision + particle_shape = self.particle_shape for s in self.species: if s.particle_shape is not None: - assert particle_shape is None or particle_shape == s.particle_shape, Exception('WarpX only supports one particle shape for all species') + assert particle_shape is None or particle_shape == s.particle_shape, ( + Exception("WarpX only supports one particle shape for all species") + ) # --- If this was set for any species, use that value. particle_shape = s.particle_shape - if particle_shape is not None and (len(self.species) > 0 or len(self.lasers) > 0): + if particle_shape is not None and ( + len(self.species) > 0 or len(self.lasers) > 0 + ): if isinstance(particle_shape, str): - interpolation_order = {'NGP':0, 'linear':1, 'quadratic':2, 'cubic':3}[particle_shape] + interpolation_order = { + "NGP": 0, + "linear": 1, + "quadratic": 2, + "cubic": 3, + }[particle_shape] else: interpolation_order = particle_shape pywarpx.algo.particle_shape = interpolation_order @@ -2372,13 +3126,15 @@ def initialize_inputs(self): pywarpx.warpx.current_centering_noz = self.current_centering_order[-1] for i in range(len(self.species)): - self.species[i].species_initialize_inputs(self.layouts[i], - self.initialize_self_fields[i], - self.injection_plane_positions[i], - self.injection_plane_normal_vectors[i]) + self.species[i].species_initialize_inputs( + self.layouts[i], + self.initialize_self_fields[i], + self.injection_plane_positions[i], + self.injection_plane_normal_vectors[i], + ) for interaction in self.interactions: - assert(isinstance(interaction, FieldIonization)) + assert isinstance(interaction, FieldIonization) interaction.interaction_initialize_inputs() if self.collisions is not None: @@ -2392,7 +3148,9 @@ def initialize_inputs(self): for i in range(len(self.lasers)): self.lasers[i].laser_initialize_inputs() - self.laser_injection_methods[i].laser_antenna_initialize_inputs(self.lasers[i]) + self.laser_injection_methods[i].laser_antenna_initialize_inputs( + self.lasers[i] + ) for applied_field in self.applied_fields: applied_field.applied_field_initialize_inputs() @@ -2419,9 +3177,11 @@ def initialize_warpx(self, mpi_comm=None): self.warpx_initialized = True pywarpx.warpx.init(mpi_comm, max_step=self.max_steps, stop_time=self.max_time) - def write_input_file(self, file_name='inputs'): + def write_input_file(self, file_name="inputs"): self.initialize_inputs() - pywarpx.warpx.write_inputs(file_name, max_step=self.max_steps, stop_time=self.max_time) + pywarpx.warpx.write_inputs( + file_name, max_step=self.max_steps, stop_time=self.max_time + ) def step(self, nsteps=None, mpi_comm=None): self.initialize_inputs() @@ -2443,24 +3203,26 @@ def finalize(self): # Simulation frame diagnostics # ---------------------------- + class WarpXDiagnosticBase(object): """ Base class for all WarpX diagnostic containing functionality shared by all WarpX diagnostic installations. """ + def add_diagnostic(self): # reduced diagnostics go in a different bucket than regular diagnostics if isinstance(self, ReducedDiagnostic): bucket = pywarpx.reduced_diagnostics - name_template = 'reduced_diag' + name_template = "reduced_diag" else: bucket = pywarpx.diagnostics - name_template = 'diag' + name_template = "diag" - name = getattr(self, 'name', None) + name = getattr(self, "name", None) if name is None: - diagnostics_number = (len(bucket._diagnostics_dict) + 1) - self.name = f'{name_template}{diagnostics_number}' + diagnostics_number = len(bucket._diagnostics_dict) + 1 + self.name = f"{name_template}{diagnostics_number}" try: self.diagnostic = bucket._diagnostics_dict[self.name] @@ -2472,8 +3234,8 @@ def add_diagnostic(self): def set_write_dir(self): if self.write_dir is not None or self.file_prefix is not None: - write_dir = (self.write_dir or 'diags') - file_prefix = (self.file_prefix or self.name) + write_dir = self.write_dir or "diags" + file_prefix = self.file_prefix or self.name self.diagnostic.file_prefix = os.path.join(write_dir, file_prefix) @@ -2500,6 +3262,7 @@ class ParticleFieldDiagnostic: If not specified, all particles will be included. The function arguments are the same as the `func` above. """ + name: str func: str do_average: int = 1 @@ -2550,27 +3313,26 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): be calculated separately for each specified species. If not passed, default is all of the available particle species. """ - def init(self, kw): - self.plot_raw_fields = kw.pop('warpx_plot_raw_fields', None) - self.plot_raw_fields_guards = kw.pop('warpx_plot_raw_fields_guards', None) - self.plot_finepatch = kw.pop('warpx_plot_finepatch', None) - self.plot_crsepatch = kw.pop('warpx_plot_crsepatch', None) - self.format = kw.pop('warpx_format', 'plotfile') - self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) - self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) - self.file_prefix = kw.pop('warpx_file_prefix', None) - self.file_min_digits = kw.pop('warpx_file_min_digits', None) - self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) - self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) - self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) - self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) + def init(self, kw): + self.plot_raw_fields = kw.pop("warpx_plot_raw_fields", None) + self.plot_raw_fields_guards = kw.pop("warpx_plot_raw_fields_guards", None) + self.plot_finepatch = kw.pop("warpx_plot_finepatch", None) + self.plot_crsepatch = kw.pop("warpx_plot_crsepatch", None) + self.format = kw.pop("warpx_format", "plotfile") + self.openpmd_backend = kw.pop("warpx_openpmd_backend", None) + self.openpmd_encoding = kw.pop("warpx_openpmd_encoding", None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) + self.dump_rz_modes = kw.pop("warpx_dump_rz_modes", None) + self.dump_last_timestep = kw.pop("warpx_dump_last_timestep", None) + self.particle_fields_to_plot = kw.pop("warpx_particle_fields_to_plot", []) + self.particle_fields_species = kw.pop("warpx_particle_fields_species", None) def diagnostic_initialize_inputs(self): - self.add_diagnostic() - self.diagnostic.diag_type = 'Full' + self.diagnostic.diag_type = "Full" self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.openpmd_encoding = self.openpmd_encoding @@ -2581,38 +3343,48 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound if self.number_of_cells is not None: - self.diagnostic.coarsening_ratio = (np.array(self.grid.number_of_cells)/np.array(self.number_of_cells)).astype(int) + self.diagnostic.coarsening_ratio = ( + np.array(self.grid.number_of_cells) / np.array(self.number_of_cells) + ).astype(int) # --- Use a set to ensure that fields don't get repeated. fields_to_plot = set() - if pywarpx.geometry.dims == 'RZ': - E_fields_list = ['Er', 'Et', 'Ez'] - B_fields_list = ['Br', 'Bt', 'Bz'] - J_fields_list = ['Jr', 'Jt', 'Jz'] - J_displacement_fields_list = ['Jr_displacement', 'Jt_displacement', 'Jz_displacement'] - A_fields_list = ['Ar', 'At', 'Az'] + if pywarpx.geometry.dims == "RZ": + E_fields_list = ["Er", "Et", "Ez"] + B_fields_list = ["Br", "Bt", "Bz"] + J_fields_list = ["Jr", "Jt", "Jz"] + J_displacement_fields_list = [ + "Jr_displacement", + "Jt_displacement", + "Jz_displacement", + ] + A_fields_list = ["Ar", "At", "Az"] else: - E_fields_list = ['Ex', 'Ey', 'Ez'] - B_fields_list = ['Bx', 'By', 'Bz'] - J_fields_list = ['Jx', 'Jy', 'Jz'] - J_displacement_fields_list = ['Jx_displacement', 'Jy_displacement', 'Jz_displacement'] - A_fields_list = ['Ax', 'Ay', 'Az'] + E_fields_list = ["Ex", "Ey", "Ez"] + B_fields_list = ["Bx", "By", "Bz"] + J_fields_list = ["Jx", "Jy", "Jz"] + J_displacement_fields_list = [ + "Jx_displacement", + "Jy_displacement", + "Jz_displacement", + ] + A_fields_list = ["Ax", "Ay", "Az"] if self.data_list is not None: for dataname in self.data_list: - if dataname == 'E': + if dataname == "E": for field_name in E_fields_list: fields_to_plot.add(field_name) - elif dataname == 'B': + elif dataname == "B": for field_name in B_fields_list: fields_to_plot.add(field_name) - elif dataname == 'J': + elif dataname == "J": for field_name in J_fields_list: fields_to_plot.add(field_name.lower()) - elif dataname == 'J_displacement': + elif dataname == "J_displacement": for field_name in J_displacement_fields_list: fields_to_plot.add(field_name.lower()) - elif dataname == 'A': + elif dataname == "A": for field_name in A_fields_list: fields_to_plot.add(field_name) elif dataname in E_fields_list: @@ -2621,52 +3393,61 @@ def diagnostic_initialize_inputs(self): fields_to_plot.add(dataname) elif dataname in A_fields_list: fields_to_plot.add(dataname) - elif dataname in ['rho', 'phi', 'F', 'G', 'divE', 'divB', 'proc_number', 'part_per_cell']: + elif dataname in [ + "rho", + "phi", + "F", + "G", + "divE", + "divB", + "proc_number", + "part_per_cell", + ]: fields_to_plot.add(dataname) elif dataname in J_fields_list: fields_to_plot.add(dataname.lower()) elif dataname in J_displacement_fields_list: fields_to_plot.add(dataname.lower()) - elif dataname.startswith('rho_'): + elif dataname.startswith("rho_"): # Adds rho_species diagnostic fields_to_plot.add(dataname) - elif dataname.startswith('T_'): + elif dataname.startswith("T_"): # Adds T_species diagnostic fields_to_plot.add(dataname) - elif dataname == 'dive': - fields_to_plot.add('divE') - elif dataname == 'divb': - fields_to_plot.add('divB') - elif dataname == 'raw_fields': + elif dataname == "dive": + fields_to_plot.add("divE") + elif dataname == "divb": + fields_to_plot.add("divB") + elif dataname == "raw_fields": self.plot_raw_fields = 1 - elif dataname == 'raw_fields_guards': + elif dataname == "raw_fields_guards": self.plot_raw_fields_guards = 1 - elif dataname == 'finepatch': + elif dataname == "finepatch": self.plot_finepatch = 1 - elif dataname == 'crsepatch': + elif dataname == "crsepatch": self.plot_crsepatch = 1 - elif dataname == 'none': - fields_to_plot = set(('none',)) + elif dataname == "none": + fields_to_plot = set(("none",)) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. fields_to_plot = list(fields_to_plot) fields_to_plot.sort() - self.diagnostic.set_or_replace_attr('fields_to_plot', fields_to_plot) + self.diagnostic.set_or_replace_attr("fields_to_plot", fields_to_plot) particle_fields_to_plot_names = list() for pfd in self.particle_fields_to_plot: if pfd.name in particle_fields_to_plot_names: - raise Exception('A particle fields name can not be repeated.') + raise Exception("A particle fields name can not be repeated.") particle_fields_to_plot_names.append(pfd.name) self.diagnostic.__setattr__( - f'particle_fields.{pfd.name}(x,y,z,ux,uy,uz)', pfd.func + f"particle_fields.{pfd.name}(x,y,z,ux,uy,uz)", pfd.func ) self.diagnostic.__setattr__( - f'particle_fields.{pfd.name}.do_average', pfd.do_average + f"particle_fields.{pfd.name}.do_average", pfd.do_average ) self.diagnostic.__setattr__( - f'particle_fields.{pfd.name}.filter(x,y,z,ux,uy,uz)', pfd.filter + f"particle_fields.{pfd.name}.filter(x,y,z,ux,uy,uz)", pfd.filter ) # --- Convert to a sorted list so that the order @@ -2679,7 +3460,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.plot_raw_fields_guards = self.plot_raw_fields_guards self.diagnostic.plot_finepatch = self.plot_finepatch self.diagnostic.plot_crsepatch = self.plot_crsepatch - if 'write_species' not in self.diagnostic.argvattrs: + if "write_species" not in self.diagnostic.argvattrs: self.diagnostic.write_species = False self.set_write_dir() @@ -2687,6 +3468,57 @@ def diagnostic_initialize_inputs(self): ElectrostaticFieldDiagnostic = FieldDiagnostic +class TimeAveragedFieldDiagnostic(FieldDiagnostic): + """ + See `Input Parameters `__ for more information. + + Parameters + ---------- + warpx_time_average_mode: str + Type of time averaging diagnostic + Supported values include ``"none"``, ``"fixed_start"``, and ``"dynamic_start"`` + + * ``"none"`` for no averaging (instantaneous fields) + * ``"fixed_start"`` for a diagnostic that averages all fields between the current output step and a fixed point in time + * ``"dynamic_start"`` for a constant averaging period and output at different points in time (non-overlapping) + + warpx_average_period_steps: int, optional + Configures the number of time steps in an averaging period. + Set this only in the ``"dynamic_start"`` mode and only if ``warpx_average_period_time`` has not already been set. + Will be ignored in the ``"fixed_start"`` mode (with warning). + + warpx_average_period_time: float, optional + Configures the time (SI units) in an averaging period. + Set this only in the ``"dynamic_start"`` mode and only if ``average_period_steps`` has not already been set. + Will be ignored in the ``"fixed_start"`` mode (with warning). + + warpx_average_start_steps: int, optional + Configures the time step at which time-averaging begins. + Set this only in the ``"fixed_start"`` mode. + Will be ignored in the ``"dynamic_start"`` mode (with warning). + """ + + def init(self, kw): + super().init(kw) + self.time_average_mode = kw.pop("warpx_time_average_mode", None) + self.average_period_steps = kw.pop("warpx_average_period_steps", None) + self.average_period_time = kw.pop("warpx_average_period_time", None) + self.average_start_step = kw.pop("warpx_average_start_step", None) + + def diagnostic_initialize_inputs(self): + super().diagnostic_initialize_inputs() + + self.diagnostic.set_or_replace_attr("diag_type", "TimeAveraged") + + if "write_species" not in self.diagnostic.argvattrs: + self.diagnostic.write_species = False + + self.diagnostic.time_average_mode = self.time_average_mode + self.diagnostic.average_period_steps = self.average_period_steps + self.diagnostic.average_period_time = self.average_period_time + self.diagnostic.average_start_step = self.average_start_step + + class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up checkpointing of the simulation, allowing for later restarts @@ -2703,26 +3535,24 @@ class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): directory name. """ - def __init__(self, period = 1, write_dir = None, name = None, **kw): - + def __init__(self, period=1, write_dir=None, name=None, **kw): self.period = period self.write_dir = write_dir - self.file_prefix = kw.pop('warpx_file_prefix', None) - self.file_min_digits = kw.pop('warpx_file_min_digits', None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) self.name = name if self.name is None: - self.name = 'chkpoint' + self.name = "chkpoint" self.handle_init(kw) def diagnostic_initialize_inputs(self): - self.add_diagnostic() self.diagnostic.intervals = self.period - self.diagnostic.diag_type = 'Full' - self.diagnostic.format = 'checkpoint' + self.diagnostic.diag_type = "Full" + self.diagnostic.format = "checkpoint" self.diagnostic.file_min_digits = self.file_min_digits self.set_write_dir() @@ -2769,43 +3599,44 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic """ - def init(self, kw): - self.format = kw.pop('warpx_format', 'plotfile') - self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) - self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) - self.file_prefix = kw.pop('warpx_file_prefix', None) - self.file_min_digits = kw.pop('warpx_file_min_digits', None) - self.random_fraction = kw.pop('warpx_random_fraction', None) - self.uniform_stride = kw.pop('warpx_uniform_stride', None) - self.plot_filter_function = kw.pop('warpx_plot_filter_function', None) - self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) + def init(self, kw): + self.format = kw.pop("warpx_format", "plotfile") + self.openpmd_backend = kw.pop("warpx_openpmd_backend", None) + self.openpmd_encoding = kw.pop("warpx_openpmd_encoding", None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) + self.random_fraction = kw.pop("warpx_random_fraction", None) + self.uniform_stride = kw.pop("warpx_uniform_stride", None) + self.plot_filter_function = kw.pop("warpx_plot_filter_function", None) + self.dump_last_timestep = kw.pop("warpx_dump_last_timestep", None) self.user_defined_kw = {} if self.plot_filter_function is not None: # This allows variables to be used in the plot_filter_function, but # in order not to break other codes, the variables must begin with "warpx_" for k in list(kw.keys()): - if k.startswith('warpx_') and re.search(r'\b%s\b'%k, self.plot_filter_function): + if k.startswith("warpx_") and re.search( + r"\b%s\b" % k, self.plot_filter_function + ): self.user_defined_kw[k] = kw[k] del kw[k] self.mangle_dict = None def diagnostic_initialize_inputs(self): - self.add_diagnostic() - self.diagnostic.diag_type = 'Full' + self.diagnostic.diag_type = "Full" self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period - self.diagnostic.set_or_replace_attr('write_species', True) - if 'fields_to_plot' not in self.diagnostic.argvattrs: - self.diagnostic.fields_to_plot = 'none' + self.diagnostic.set_or_replace_attr("write_species", True) + if "fields_to_plot" not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = "none" self.set_write_dir() # --- Use a set to ensure that fields don't get repeated. @@ -2813,39 +3644,59 @@ def diagnostic_initialize_inputs(self): if self.data_list is not None: for dataname in self.data_list: - if dataname == 'position': - if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z - variables.add('x') - if pywarpx.geometry.dims == '3': - variables.add('y') - variables.add('z') - if pywarpx.geometry.dims == 'RZ': - variables.add('theta') - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: - if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + if dataname == "position": + if pywarpx.geometry.dims != "1": # because then it's WARPX_DIM_1D_Z + variables.add("x") + if pywarpx.geometry.dims == "3": + variables.add("y") + variables.add("z") + if pywarpx.geometry.dims == "RZ": + variables.add("theta") + elif dataname == "momentum": + variables.add("ux") + variables.add("uy") + variables.add("uz") + elif dataname == "weighting": + variables.add("w") + elif dataname == "fields": + variables.add("Ex") + variables.add("Ey") + variables.add("Ez") + variables.add("Bx") + variables.add("By") + variables.add("Bz") + elif dataname in [ + "x", + "y", + "z", + "theta", + "ux", + "uy", + "uz", + "Ex", + "Ey", + "Ez", + "Bx", + "By", + "Bz", + "Er", + "Et", + "Br", + "Bt", + ]: + if pywarpx.geometry.dims == "1" and ( + dataname == "x" or dataname == "y" + ): raise RuntimeError( f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" f"chosen by dim={pywarpx.geometry.dims} in pywarpx." ) - elif pywarpx.geometry.dims != '3' and dataname == 'y': + elif pywarpx.geometry.dims != "3" and dataname == "y": raise RuntimeError( f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." ) - elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + elif pywarpx.geometry.dims != "RZ" and dataname == "theta": raise RuntimeError( f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." @@ -2891,21 +3742,27 @@ def diagnostic_initialize_inputs(self): self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) for name in species_names: - diag = pywarpx.Bucket.Bucket(self.name + '.' + name, - variables = variables, - random_fraction = random_fraction.get(name, random_fraction_default), - uniform_stride = uniform_stride.get(name, uniform_stride_default)) - expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) - diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) + diag = pywarpx.Bucket.Bucket( + self.name + "." + name, + variables=variables, + random_fraction=random_fraction.get(name, random_fraction_default), + uniform_stride=uniform_stride.get(name, uniform_stride_default), + ) + expression = pywarpx.my_constants.mangle_expression( + self.plot_filter_function, self.mangle_dict + ) + diag.__setattr__("plot_filter_function(t,x,y,z,ux,uy,uz)", expression) self.diagnostic._species_dict[name] = diag + # ---------------------------- # Lab frame diagnostics # ---------------------------- -class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, - WarpXDiagnosticBase): +class LabFrameFieldDiagnostic( + picmistandard.PICMI_LabFrameFieldDiagnostic, WarpXDiagnosticBase +): """ See `Input Parameters `__ for more information. @@ -2942,24 +3799,24 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_upper_bound: vector of floats, optional Passed to .upper_bound """ + def init(self, kw): """The user is using the new BTD""" - self.format = kw.pop('warpx_format', None) - self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) - self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) - self.file_prefix = kw.pop('warpx_file_prefix', None) - self.intervals = kw.pop('warpx_intervals', None) - self.file_min_digits = kw.pop('warpx_file_min_digits', None) - self.buffer_size = kw.pop('warpx_buffer_size', None) - self.lower_bound = kw.pop('warpx_lower_bound', None) - self.upper_bound = kw.pop('warpx_upper_bound', None) + self.format = kw.pop("warpx_format", None) + self.openpmd_backend = kw.pop("warpx_openpmd_backend", None) + self.openpmd_encoding = kw.pop("warpx_openpmd_encoding", None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.intervals = kw.pop("warpx_intervals", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) + self.buffer_size = kw.pop("warpx_buffer_size", None) + self.lower_bound = kw.pop("warpx_lower_bound", None) + self.upper_bound = kw.pop("warpx_upper_bound", None) def diagnostic_initialize_inputs(self): - self.add_diagnostic() - self.diagnostic.diag_type = 'BackTransformed' + self.diagnostic.diag_type = "BackTransformed" self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.openpmd_encoding = self.openpmd_encoding @@ -2980,23 +3837,23 @@ def diagnostic_initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. fields_to_plot = set() - if pywarpx.geometry.dims == 'RZ': - E_fields_list = ['Er', 'Et', 'Ez'] - B_fields_list = ['Br', 'Bt', 'Bz'] - J_fields_list = ['Jr', 'Jt', 'Jz'] + if pywarpx.geometry.dims == "RZ": + E_fields_list = ["Er", "Et", "Ez"] + B_fields_list = ["Br", "Bt", "Bz"] + J_fields_list = ["Jr", "Jt", "Jz"] else: - E_fields_list = ['Ex', 'Ey', 'Ez'] - B_fields_list = ['Bx', 'By', 'Bz'] - J_fields_list = ['Jx', 'Jy', 'Jz'] + E_fields_list = ["Ex", "Ey", "Ez"] + B_fields_list = ["Bx", "By", "Bz"] + J_fields_list = ["Jx", "Jy", "Jz"] if self.data_list is not None: for dataname in self.data_list: - if dataname == 'E': + if dataname == "E": for field_name in E_fields_list: fields_to_plot.add(field_name) - elif dataname == 'B': + elif dataname == "B": for field_name in B_fields_list: fields_to_plot.add(field_name) - elif dataname == 'J': + elif dataname == "J": for field_name in J_fields_list: fields_to_plot.add(field_name.lower()) elif dataname in E_fields_list: @@ -3005,7 +3862,7 @@ def diagnostic_initialize_inputs(self): fields_to_plot.add(dataname) elif dataname in J_fields_list: fields_to_plot.add(dataname.lower()) - elif dataname.startswith('rho_'): + elif dataname.startswith("rho_"): # Adds rho_species diagnostic fields_to_plot.add(dataname) @@ -3013,15 +3870,16 @@ def diagnostic_initialize_inputs(self): # --- is the same on all processors. fields_to_plot = list(fields_to_plot) fields_to_plot.sort() - self.diagnostic.set_or_replace_attr('fields_to_plot', fields_to_plot) + self.diagnostic.set_or_replace_attr("fields_to_plot", fields_to_plot) - if 'write_species' not in self.diagnostic.argvattrs: + if "write_species" not in self.diagnostic.argvattrs: self.diagnostic.write_species = False self.set_write_dir() -class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, - WarpXDiagnosticBase): +class LabFrameParticleDiagnostic( + picmistandard.PICMI_LabFrameParticleDiagnostic, WarpXDiagnosticBase +): """ See `Input Parameters `__ for more information. @@ -3052,20 +3910,20 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_buffer_size: integer, optional Passed to .buffer_size """ + def init(self, kw): - self.format = kw.pop('warpx_format', None) - self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) - self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) - self.file_prefix = kw.pop('warpx_file_prefix', None) - self.intervals = kw.pop('warpx_intervals', None) - self.file_min_digits = kw.pop('warpx_file_min_digits', None) - self.buffer_size = kw.pop('warpx_buffer_size', None) + self.format = kw.pop("warpx_format", None) + self.openpmd_backend = kw.pop("warpx_openpmd_backend", None) + self.openpmd_encoding = kw.pop("warpx_openpmd_encoding", None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.intervals = kw.pop("warpx_intervals", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) + self.buffer_size = kw.pop("warpx_buffer_size", None) def diagnostic_initialize_inputs(self): - self.add_diagnostic() - self.diagnostic.diag_type = 'BackTransformed' + self.diagnostic.diag_type = "BackTransformed" self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.openpmd_encoding = self.openpmd_encoding @@ -3083,9 +3941,9 @@ def diagnostic_initialize_inputs(self): self.diagnostic.do_back_transformed_fields = False - self.diagnostic.set_or_replace_attr('write_species', True) - if 'fields_to_plot' not in self.diagnostic.argvattrs: - self.diagnostic.fields_to_plot = 'none' + self.diagnostic.set_or_replace_attr("write_species", True) + if "fields_to_plot" not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = "none" self.set_write_dir() @@ -3094,39 +3952,59 @@ def diagnostic_initialize_inputs(self): if self.data_list is not None: for dataname in self.data_list: - if dataname == 'position': - if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z - variables.add('x') - if pywarpx.geometry.dims == '3': - variables.add('y') - variables.add('z') - if pywarpx.geometry.dims == 'RZ': - variables.add('theta') - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: - if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + if dataname == "position": + if pywarpx.geometry.dims != "1": # because then it's WARPX_DIM_1D_Z + variables.add("x") + if pywarpx.geometry.dims == "3": + variables.add("y") + variables.add("z") + if pywarpx.geometry.dims == "RZ": + variables.add("theta") + elif dataname == "momentum": + variables.add("ux") + variables.add("uy") + variables.add("uz") + elif dataname == "weighting": + variables.add("w") + elif dataname == "fields": + variables.add("Ex") + variables.add("Ey") + variables.add("Ez") + variables.add("Bx") + variables.add("By") + variables.add("Bz") + elif dataname in [ + "x", + "y", + "z", + "theta", + "ux", + "uy", + "uz", + "Ex", + "Ey", + "Ez", + "Bx", + "By", + "Bz", + "Er", + "Et", + "Br", + "Bt", + ]: + if pywarpx.geometry.dims == "1" and ( + dataname == "x" or dataname == "y" + ): raise RuntimeError( f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" f"chosen by dim={pywarpx.geometry.dims} in pywarpx." ) - elif pywarpx.geometry.dims != '3' and dataname == 'y': + elif pywarpx.geometry.dims != "3" and dataname == "y": raise RuntimeError( f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." ) - elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + elif pywarpx.geometry.dims != "RZ" and dataname == "theta": raise RuntimeError( f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." @@ -3148,8 +4026,7 @@ def diagnostic_initialize_inputs(self): species_names = [self.species.name] for name in species_names: - diag = pywarpx.Bucket.Bucket(self.name + '.' + name, - variables = variables) + diag = pywarpx.Bucket.Bucket(self.name + "." + name, variables=variables) self.diagnostic._species_dict[name] = diag @@ -3245,9 +4122,16 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): For diagnostic type 'FieldProbe', the vector specifying up in the 'Plane' """ - def __init__(self, diag_type, name=None, period=1, path=None, - extension=None, separator=None, **kw): - + def __init__( + self, + diag_type, + name=None, + period=None, + path=None, + extension=None, + separator=None, + **kw, + ): self.name = name self.type = diag_type self.intervals = period @@ -3262,21 +4146,31 @@ def __init__(self, diag_type, name=None, period=1, path=None, # The simple diagnostics do not require any additional arguments self._simple_reduced_diagnostics = [ - 'ParticleEnergy', 'ParticleMomentum', 'FieldEnergy', - 'FieldMomentum', 'FieldMaximum', 'RhoMaximum', 'ParticleNumber', - 'LoadBalanceCosts', 'LoadBalanceEfficiency' + "ParticleEnergy", + "ParticleMomentum", + "FieldEnergy", + "FieldMomentum", + "FieldMaximum", + "FieldPoyntingFlux", + "RhoMaximum", + "ParticleNumber", + "LoadBalanceCosts", + "LoadBalanceEfficiency", + "Timestep", ] # The species diagnostics require a species to be provided self._species_reduced_diagnostics = [ - 'BeamRelevant', 'ParticleHistogram', 'ParticleExtrema' + "BeamRelevant", + "ParticleHistogram", + "ParticleExtrema", ] if self.type in self._simple_reduced_diagnostics: pass elif self.type in self._species_reduced_diagnostics: - species = kw.pop('species') + species = kw.pop("species") self.species = species.name - if self.type == 'ParticleHistogram': + if self.type == "ParticleHistogram": kw = self._handle_particle_histogram(**kw) elif self.type == "FieldProbe": kw = self._handle_field_probe(**kw) @@ -3286,8 +4180,7 @@ def __init__(self, diag_type, name=None, period=1, path=None, kw = self._handle_charge_on_eb(**kw) else: raise RuntimeError( - f"{self.type} reduced diagnostic is not yet supported " - "in pywarpx." + f"{self.type} reduced diagnostic is not yet supported in pywarpx." ) self.handle_init(kw) @@ -3303,15 +4196,15 @@ def _handle_field_probe(self, **kw): self.integrate = kw.pop("integrate", None) self.do_moving_window_FP = kw.pop("do_moving_window_FP", None) - if self.probe_geometry.lower() != 'point': + if self.probe_geometry.lower() != "point": self.resolution = kw.pop("resolution") - if self.probe_geometry.lower() == 'line': + if self.probe_geometry.lower() == "line": self.x1_probe = kw.pop("x1_probe", None) self.y1_probe = kw.pop("y1_probe", None) self.z1_probe = kw.pop("z1_probe") - if self.probe_geometry.lower() == 'plane': + if self.probe_geometry.lower() == "plane": self.detector_radius = kw.pop("detector_radius") self.target_normal_x = kw.pop("target_normal_x", None) @@ -3329,9 +4222,15 @@ def _handle_particle_histogram(self, **kw): self.bin_max = kw.pop("bin_max") self.bin_min = kw.pop("bin_min") self.normalization = kw.pop("normalization", None) - if self.normalization not in [None, "unity_particle_weight", "max_to_unity", "area_to_unity"]: + if self.normalization not in [ + None, + "unity_particle_weight", + "max_to_unity", + "area_to_unity", + ]: raise AttributeError( - "The ParticleHistogram normalization must be one of 'unity_particle_weight', 'max_to_unity', or 'area_to_unity'") + "The ParticleHistogram normalization must be one of 'unity_particle_weight', 'max_to_unity', or 'area_to_unity'" + ) histogram_function = kw.pop("histogram_function") filter_function = kw.pop("filter_function", None) @@ -3341,8 +4240,10 @@ def _handle_particle_histogram(self, **kw): # Check the reduced function expressions for constants for k in list(kw.keys()): - if (re.search(r'\b%s\b'%k, histogram_function) or - (filter_function is not None and re.search(r'\b%s\b'%k, filter_function))): + if re.search(r"\b%s\b" % k, histogram_function) or ( + filter_function is not None + and re.search(r"\b%s\b" % k, filter_function) + ): self.user_defined_kw[k] = kw[k] del kw[k] @@ -3352,11 +4253,13 @@ def _handle_field_reduction(self, **kw): self.reduction_type = kw.pop("reduction_type") reduced_function = kw.pop("reduced_function") - self.__setattr__("reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz)", reduced_function) + self.__setattr__( + "reduced_function(x,y,z,Ex,Ey,Ez,Bx,By,Bz,jx,jy,jz)", reduced_function + ) # Check the reduced function expression for constants for k in list(kw.keys()): - if re.search(r'\b%s\b'%k, reduced_function): + if re.search(r"\b%s\b" % k, reduced_function): self.user_defined_kw[k] = kw[k] del kw[k] @@ -3369,23 +4272,199 @@ def _handle_charge_on_eb(self, **kw): # Check the reduced function expression for constants for k in list(kw.keys()): - if re.search(r'\b%s\b'%k, weighting_function): + if re.search(r"\b%s\b" % k, weighting_function): self.user_defined_kw[k] = kw[k] del kw[k] return kw def diagnostic_initialize_inputs(self): - self.add_diagnostic() self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) for key, value in self.__dict__.items(): - if not key.startswith('_') and key not in ['name', 'diagnostic']: + if not key.startswith("_") and key not in ["name", "diagnostic"]: if key.endswith(")"): # Analytic expressions require processing to deal with constants - expression = pywarpx.my_constants.mangle_expression(value, self.mangle_dict) + expression = pywarpx.my_constants.mangle_expression( + value, self.mangle_dict + ) self.diagnostic.__setattr__(key, expression) else: self.diagnostic.__setattr__(key, value) + + +class ParticleBoundaryScrapingDiagnostic( + picmistandard.PICMI_ParticleBoundaryScrapingDiagnostic, WarpXDiagnosticBase +): + """ + See `Input Parameters `__ for more information. + + Parameters + ---------- + warpx_format: openpmd + Diagnostic file format + + warpx_openpmd_backend: {bp, h5, json}, optional + Openpmd backend file format + + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + + warpx_file_prefix: string, optional + Prefix on the diagnostic file name + + warpx_file_min_digits: integer, optional + Minimum number of digits for the time step number in the file name + + warpx_random_fraction: float or dict, optional + Random fraction of particles to include in the diagnostic. If a float + is given the same fraction will be used for all species, if a dictionary + is given the keys should be species with the value specifying the random + fraction for that species. + + warpx_uniform_stride: integer or dict, optional + Stride to down select to the particles to include in the diagnostic. + If an integer is given the same stride will be used for all species, if + a dictionary is given the keys should be species with the value + specifying the stride for that species. + + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + + warpx_plot_filter_function: string, optional + Analytic expression to down select the particles to in the diagnostic + """ + + def init(self, kw): + self.format = kw.pop("warpx_format", "openpmd") + self.openpmd_backend = kw.pop("warpx_openpmd_backend", None) + self.openpmd_encoding = kw.pop("warpx_openpmd_encoding", None) + self.file_prefix = kw.pop("warpx_file_prefix", None) + self.file_min_digits = kw.pop("warpx_file_min_digits", None) + self.random_fraction = kw.pop("warpx_random_fraction", None) + self.uniform_stride = kw.pop("warpx_uniform_stride", None) + self.plot_filter_function = kw.pop("warpx_plot_filter_function", None) + self.dump_last_timestep = kw.pop("warpx_dump_last_timestep", None) + + self.user_defined_kw = {} + if self.plot_filter_function is not None: + # This allows variables to be used in the plot_filter_function, but + # in order not to break other codes, the variables must begin with "warpx_" + for k in list(kw.keys()): + if k.startswith("warpx_") and re.search( + r"\b%s\b" % k, self.plot_filter_function + ): + self.user_defined_kw[k] = kw[k] + del kw[k] + + self.mangle_dict = None + + def diagnostic_initialize_inputs(self): + self.add_diagnostic() + + self.diagnostic.diag_type = "BoundaryScraping" + self.diagnostic.format = self.format + self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding + self.diagnostic.file_min_digits = self.file_min_digits + self.diagnostic.dump_last_timestep = self.dump_last_timestep + self.diagnostic.intervals = self.period + self.diagnostic.set_or_replace_attr("write_species", True) + if "fields_to_plot" not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = "none" + + self.set_write_dir() + + # --- Use a set to ensure that fields don't get repeated. + variables = set() + + if self.data_list is not None: + for dataname in self.data_list: + if dataname == "position": + if pywarpx.geometry.dims != "1": # because then it's WARPX_DIM_1D_Z + variables.add("x") + if pywarpx.geometry.dims == "3": + variables.add("y") + variables.add("z") + if pywarpx.geometry.dims == "RZ": + variables.add("theta") + elif dataname == "momentum": + variables.add("ux") + variables.add("uy") + variables.add("uz") + elif dataname == "weighting": + variables.add("w") + elif dataname in ["x", "y", "z", "theta", "ux", "uy", "uz"]: + if pywarpx.geometry.dims == "1" and ( + dataname == "x" or dataname == "y" + ): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != "3" and dataname == "y": + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != "RZ" and dataname == "theta": + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) + else: + # possibly add user defined attributes + variables.add(dataname) + + # --- Convert the set to a sorted list so that the order + # --- is the same on all processors. + variables = list(variables) + variables.sort() + + # species list + if self.species is None: + species_names = pywarpx.particles.species_names + elif np.iterable(self.species): + species_names = [species.name for species in self.species] + else: + species_names = [self.species.name] + + # check if random fraction is specified and whether a value is given per species + random_fraction = {} + random_fraction_default = self.random_fraction + if isinstance(self.random_fraction, dict): + random_fraction_default = 1.0 + for key, val in self.random_fraction.items(): + random_fraction[key.name] = val + + # check if uniform stride is specified and whether a value is given per species + uniform_stride = {} + uniform_stride_default = self.uniform_stride + if isinstance(self.uniform_stride, dict): + uniform_stride_default = 1 + for key, val in self.uniform_stride.items(): + uniform_stride[key.name] = val + + if self.mangle_dict is None: + # Only do this once so that the same variables are used in this distribution + # is used multiple times + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) + + for name in species_names: + diag = pywarpx.Bucket.Bucket( + self.name + "." + name, + variables=variables, + random_fraction=random_fraction.get(name, random_fraction_default), + uniform_stride=uniform_stride.get(name, uniform_stride_default), + ) + expression = pywarpx.my_constants.mangle_expression( + self.plot_filter_function, self.mangle_dict + ) + diag.__setattr__("plot_filter_function(t,x,y,z,ux,uy,uz)", expression) + self.diagnostic._species_dict[name] = diag diff --git a/Python/setup.py b/Python/setup.py index 31f35eeceac..e0ec6c98a7d 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -19,8 +19,18 @@ from setuptools import setup argparser = argparse.ArgumentParser(add_help=False) -argparser.add_argument('--with-libwarpx', type=str, default=None, help='Install libwarpx with the given value as DIM. This option is only used by the GNU makefile build system.') -argparser.add_argument('--with-lib-dir', type=str, default=None, help='Install with all libwarpx* binaries found in a directory.') +argparser.add_argument( + "--with-libwarpx", + type=str, + default=None, + help="Install libwarpx with the given value as DIM. This option is only used by the GNU makefile build system.", +) +argparser.add_argument( + "--with-lib-dir", + type=str, + default=None, + help="Install with all libwarpx* binaries found in a directory.", +) args, unknown = argparser.parse_known_args() sys.argv = [sys.argv[0]] + unknown @@ -28,38 +38,39 @@ # Allow to control options via environment vars. # Work-around for https://github.com/pypa/setuptools/issues/1712 -PYWARPX_LIB_DIR = os.environ.get('PYWARPX_LIB_DIR') +PYWARPX_LIB_DIR = os.environ.get("PYWARPX_LIB_DIR") if args.with_libwarpx: # GNUmake if args.with_libwarpx not in allowed_dims: print("WARNING: '%s' is not an allowed WarpX DIM" % args.with_libwarpx) - package_data = {'pywarpx' : ['libwarpx.%s.so' % args.with_libwarpx]} + package_data = {"pywarpx": ["libwarpx.%s.so" % args.with_libwarpx]} data_files = [] elif args.with_lib_dir or PYWARPX_LIB_DIR: # CMake and Package Managers - package_data = {'pywarpx' : []} + package_data = {"pywarpx": []} lib_dir = args.with_lib_dir if args.with_lib_dir else PYWARPX_LIB_DIR my_path = os.path.dirname(os.path.realpath(__file__)) for dim in allowed_dims: - lib_name = 'libwarpx.%s.so' % dim + lib_name = "libwarpx.%s.so" % dim lib_path = os.path.join(lib_dir, lib_name) link_name = os.path.join(my_path, "pywarpx", lib_name) if os.path.isfile(link_name): os.remove(link_name) if os.path.isfile(lib_path) and os.access(lib_path, os.R_OK): os.symlink(lib_path, link_name) - package_data['pywarpx'].append(lib_name) + package_data["pywarpx"].append(lib_name) else: package_data = {} -setup(name = 'pywarpx', - version = '24.07', - packages = ['pywarpx'], - package_dir = {'pywarpx': 'pywarpx'}, - description = """Wrapper of WarpX""", - package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'], - python_requires = '>=3.8', - zip_safe=False +setup( + name="pywarpx", + version="25.02", + packages=["pywarpx"], + package_dir={"pywarpx": "pywarpx"}, + description="""Wrapper of WarpX""", + package_data=package_data, + install_requires=["numpy", "picmistandard==0.33.0", "periodictable"], + python_requires=">=3.8", # left for CI, truly ">=3.9" + zip_safe=False, ) diff --git a/Regression/Checksum/benchmark.py b/Regression/Checksum/benchmark.py index fbbe44f98b0..549900da628 100644 --- a/Regression/Checksum/benchmark.py +++ b/Regression/Checksum/benchmark.py @@ -34,8 +34,9 @@ def __init__(self, test_name, data=None): """ self.test_name = test_name - self.json_file = os.path.join(config.benchmark_location, - self.test_name + '.json') + self.json_file = os.path.join( + config.benchmark_location, self.test_name + ".json" + ) if data is None: self.data = self.get() else: @@ -45,7 +46,7 @@ def reset(self): """ Update the benchmark (overwrites reference json file). """ - with open(self.json_file, 'w') as outfile: + with open(self.json_file, "w") as outfile: json.dump(self.data, outfile, sort_keys=True, indent=2) def get(self): diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json deleted file mode 100644 index 799692135ce..00000000000 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "lev=0": { - "Bx": 958613893612.4355, - "By": 958606286084.2804, - "Bz": 50291310.73170665, - "Ex": 2.873993867215236e+20, - "Ey": 2.8740263458334176e+20, - "Ez": 1.3469564521662371e+17, - "rho_beam1": 7.96926761425663e+16, - "rho_beam2": 7.969234189119718e+16, - "rho_ele1": 325562788515568.7, - "rho_ele2": 301373974746269.7, - "rho_pos1": 314013278169396.75, - "rho_pos2": 311298336003435.56 - }, - "beam1": { - "particle_opticalDepthQSR": 104848.69019384333, - "particle_position_x": 0.0015001641452988207, - "particle_position_y": 0.0015001296473841738, - "particle_position_z": 0.004965480036291212, - "particle_momentum_x": 6.203657034942546e-15, - "particle_momentum_y": 6.161790111190829e-15, - "particle_momentum_z": 6.806194292286189e-12, - "particle_weight": 635868088.3867786 - }, - "beam2": { - "particle_opticalDepthQSR": 104197.28107371755, - "particle_position_x": 0.0015001452558405398, - "particle_position_y": 0.0015001281739351966, - "particle_position_z": 0.0049656445643994716, - "particle_momentum_x": 6.202758467582172e-15, - "particle_momentum_y": 6.18910011814166e-15, - "particle_momentum_z": 6.7994521022372906e-12, - "particle_weight": 635874794.3085052 - }, - "ele1": { - "particle_opticalDepthQSR": 398.7154177999122, - "particle_position_x": 5.2166787076833645e-06, - "particle_position_y": 5.005755590473258e-06, - "particle_position_z": 1.856829463647771e-05, - "particle_momentum_x": 6.0736878569270085e-18, - "particle_momentum_y": 5.735020185191748e-18, - "particle_momentum_z": 2.827581034346608e-15, - "particle_weight": 2602683.4209351614 - }, - "ele2": { - "particle_opticalDepthQSR": 328.6975869797729, - "particle_position_x": 4.984003903707884e-06, - "particle_position_y": 4.695016970410262e-06, - "particle_position_z": 1.606918799511055e-05, - "particle_momentum_x": 4.524294388810778e-18, - "particle_momentum_y": 4.193609622515901e-18, - "particle_momentum_z": 2.624217472737641e-15, - "particle_weight": 2432495.8168380223 - }, - "pho1": { - "particle_opticalDepthBW": 10028.214317531058, - "particle_position_x": 0.00014200324200040716, - "particle_position_y": 0.00014310262095706036, - "particle_position_z": 0.00047470309948487784, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 61455533.15171491 - }, - "pho2": { - "particle_opticalDepthBW": 10261.48950301913, - "particle_position_x": 0.0001465092909391631, - "particle_position_y": 0.00014555115652303745, - "particle_position_z": 0.00048686081947093, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 61924991.09906147 - }, - "pos1": { - "particle_opticalDepthQSR": 380.4787933889546, - "particle_position_x": 5.59226140958729e-06, - "particle_position_y": 5.199149983019462e-06, - "particle_position_z": 1.7261766049926983e-05, - "particle_momentum_x": 5.182944941041321e-18, - "particle_momentum_y": 4.665394338329992e-18, - "particle_momentum_z": 2.565450485567441e-15, - "particle_weight": 2523696.1743423166 - }, - "pos2": { - "particle_opticalDepthQSR": 378.7526306435402, - "particle_position_x": 4.812490588954386e-06, - "particle_position_y": 4.351750384371962e-06, - "particle_position_z": 1.7621416174292307e-05, - "particle_momentum_x": 4.979887438720444e-18, - "particle_momentum_y": 4.8215630209506066e-18, - "particle_momentum_z": 2.193964301475807e-15, - "particle_weight": 2513162.277112609 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/EmbeddedBoundaryDiffraction.json b/Regression/Checksum/benchmarks_json/EmbeddedBoundaryDiffraction.json deleted file mode 100644 index 0e5fad8db8a..00000000000 --- a/Regression/Checksum/benchmarks_json/EmbeddedBoundaryDiffraction.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "lev=0": { - "Br": 6.821267675779345e-19, - "Bt": 5.564905732478707e-05, - "Bz": 2.368259586613272e-19, - "Er": 16503.98082446463, - "Et": 1.5299584682447838e-10, - "Ez": 1466.854467399728 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/FluxInjection.json b/Regression/Checksum/benchmarks_json/FluxInjection.json deleted file mode 100644 index 5a80590891c..00000000000 --- a/Regression/Checksum/benchmarks_json/FluxInjection.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "electron": { - "particle_momentum_x": 7.168456345337534e-18, - "particle_momentum_y": 7.02290351254873e-18, - "particle_momentum_z": 9.565641373942318e-42, - "particle_position_x": 6962.988311042427, - "particle_position_y": 2034.5301680154264, - "particle_theta": 6397.068924320389, - "particle_weight": 3.215011942598676e-08 - }, - "lev=0": { - "Bz": 9.526664429810971e-24 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json deleted file mode 100644 index dd59536ad37..00000000000 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "lev=0": { - "Bx": 4818955.480797943, - "By": 1752.8025638791275, - "Bz": 14516.212782554387, - "Ex": 2366115503598.9224, - "Ey": 1446112025635674.2, - "Ez": 21864189507357.867, - "jx": 1996366349775593.5, - "jy": 5.312583827155926e+16, - "jz": 2.0491352624508764e+16, - "rho": 68443961.71852128 - }, - "electrons": { - "particle_momentum_x": 2.2135945391319113e-23, - "particle_momentum_y": 2.8224559499558413e-22, - "particle_momentum_z": 5.260626010214114e-22, - "particle_position_x": 0.010800577787628052, - "particle_position_y": 0.2111506062831815, - "particle_weight": 4.121554826246186e+16 - }, - "ions": { - "particle_momentum_x": 6.248472277246885e-23, - "particle_momentum_y": 4.449097689427654e-22, - "particle_momentum_z": 5.768168724998047e-22, - "particle_position_x": 0.010800001678510512, - "particle_position_y": 0.21114947608115425, - "particle_weight": 4.121554826246186e+16 - }, - "beam": { - "particle_momentum_x": 3.5357456351701565e-19, - "particle_momentum_y": 4.363391839372122e-19, - "particle_momentum_z": 5.658606416951653e-17, - "particle_position_x": 0.008314723025211468, - "particle_position_y": 1.1704335743854242, - "particle_weight": 62415090744.60765 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json deleted file mode 100644 index 2843e49ce22..00000000000 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "lev=0": { - "Bx": 14264658.38987597, - "By": 0.0, - "Bz": 0.0, - "Ex": 0.0, - "Ey": 4276056420659746.5, - "Ez": 762168740318568.1, - "jx": 0.0, - "jy": 7.47674123799233e+16, - "jz": 4.817762115932484e+17, - "rho": 1609691680.1267354 - } -} - diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc3d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc3d.json deleted file mode 100644 index de5472105d9..00000000000 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc3d.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "electrons": { - "particle_momentum_x": 1.6966182372218133e-16, - "particle_momentum_y": 2.6850066145197374e-17, - "particle_momentum_z": 2.0052710316284176e-16, - "particle_position_x": 0.3393352015355679, - "particle_position_y": 1.1078675395554147, - "particle_position_z": 0.3419438867441836, - "particle_weight": 26433181926540.81 - }, - "hydrogen": { - "particle_momentum_x": 1.7161831722699107e-16, - "particle_momentum_y": 4.9196233343263506e-17, - "particle_momentum_z": 2.1370961936359413e-16, - "particle_position_x": 0.3375134789944616, - "particle_position_y": 1.1080021730384098, - "particle_position_z": 0.33939049172256086, - "particle_weight": 26441597005520.95 - }, - "lev=0": { - "Bx": 41555976.87146437, - "By": 175750876.1712573, - "Bz": 35156983.723599546, - "Ex": 3.872657491899755e+17, - "Ey": 3.3815796095277564e+16, - "Ez": 3.937276394651024e+17, - "jx": 3.5072653955241413e+21, - "jy": 4.011484251839508e+20, - "jz": 3.787151010057889e+21, - "rho": 7429502184315.598 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Maxwell_Hybrid_QED_solver.json b/Regression/Checksum/benchmarks_json/Maxwell_Hybrid_QED_solver.json deleted file mode 100644 index 9f7b7f64dcd..00000000000 --- a/Regression/Checksum/benchmarks_json/Maxwell_Hybrid_QED_solver.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "lev=0": { - "Bx": 3.543966469013954e-05, - "By": 0.0, - "Bz": 5.103535813972088e-12, - "Ex": 0.0, - "Ey": 6553600000.005218, - "Ez": 0.0, - "jx": 0.0, - "jy": 0.0, - "jz": 0.0 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Plasma_lens_boosted.json b/Regression/Checksum/benchmarks_json/Plasma_lens_boosted.json deleted file mode 100644 index 6d5eabb492e..00000000000 --- a/Regression/Checksum/benchmarks_json/Plasma_lens_boosted.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "lev=0": { - "Bx": 1.3073041371012706e-14, - "By": 1.3033038210840872e-14, - "Bz": 5.595105968291083e-17, - "Ex": 2.801134785671445e-06, - "Ey": 2.8088613469887243e-06, - "Ez": 3.343430731047825e-06, - "jx": 2.5155716299904363e-11, - "jy": 2.013718424043256e-11, - "jz": 6.00631499206418e-09 - }, - "electrons": { - "particle_momentum_x": 7.437088723328491e-24, - "particle_momentum_y": 5.9495056615288754e-24, - "particle_momentum_z": 5.117548636687908e-22, - "particle_position_x": 0.036489969262013186, - "particle_position_y": 0.029201200231260247, - "particle_position_z": 6.9681085285694095 - } -} diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json deleted file mode 100644 index cdee4b078b0..00000000000 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "lev=0": { - "rho": 0.0 - }, - "alpha2": { - "particle_momentum_x": 4.057984510682467e-15, - "particle_momentum_y": 4.104188139725855e-15, - "particle_momentum_z": 4.17858000090827e-15, - "particle_position_x": 408793.7905852193, - "particle_position_y": 861780.8020495367, - "particle_weight": 5.078061191951185e+18 - }, - "alpha3": { - "particle_momentum_x": 5.017656304003558e-16, - "particle_momentum_y": 4.935595075276182e-16, - "particle_momentum_z": 4.867133212376827e-16, - "particle_position_x": 52678.192400911765, - "particle_position_y": 105483.59950020742, - "particle_weight": 1.5413633830148085e+27 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 - }, - "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.015821768, - "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999998307316e+28 - }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, - "alpha1": { - "particle_momentum_x": 4.691909092431811e-15, - "particle_momentum_y": 4.6836958163275755e-15, - "particle_momentum_z": 4.657203546376977e-15, - "particle_position_x": 463465.27131109464, - "particle_position_y": 978207.6061359186, - "particle_weight": 4.977661460251435e-28 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512788807322e+28 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, - "particle_weight": 128.00000000000261 - }, - "alpha5": { - "particle_momentum_x": 2.3343485859070736e-14, - "particle_momentum_y": 2.3451128753701046e-14, - "particle_momentum_z": 2.3579462789062662e-14, - "particle_position_x": 2457556.8571638423, - "particle_position_y": 4914659.635379322, - "particle_weight": 3.839999999999998e-19 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399486212205656e+30 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, - "proton3": { - "particle_momentum_x": 1.6847290893972251e-15, - "particle_momentum_y": 1.6827074502304075e-15, - "particle_momentum_z": 1.6802489646490975e-15, - "particle_position_x": 2457270.6999197667, - "particle_position_y": 4914315.665267942, - "particle_weight": 1.279486212205663e+30 - }, - "alpha4": { - "particle_momentum_x": 2.338084461204216e-14, - "particle_momentum_y": 2.3436156778849828e-14, - "particle_momentum_z": 2.35535708386288e-14, - "particle_position_x": 2457367.4582781536, - "particle_position_y": 4915112.044373056, - "particle_weight": 384.0000000000002 - }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 - } -} diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json deleted file mode 100644 index ec7e047c537..00000000000 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "alpha3": { - "particle_momentum_x": 4.764404554793872e-16, - "particle_momentum_y": 4.655900875811434e-16, - "particle_momentum_z": 4.578927372510084e-16, - "particle_position_x": 50987.442011759704, - "particle_position_y": 48999.674675246955, - "particle_position_z": 101142.57224226737, - "particle_weight": 1.0633705344227063e+28 - }, - "alpha1": { - "particle_momentum_x": 4.665933695243743e-15, - "particle_momentum_y": 4.603805875733438e-15, - "particle_momentum_z": 4.706765986105302e-15, - "particle_position_x": 461871.79172011977, - "particle_position_y": 461162.2166206925, - "particle_position_z": 969262.7809050508, - "particle_weight": 3.2387855108185994e-27 - }, - "alpha5": { - "particle_momentum_x": 2.3388206254864998e-14, - "particle_momentum_y": 2.334372885765467e-14, - "particle_momentum_z": 2.363588638941874e-14, - "particle_position_x": 2457556.857163843, - "particle_position_y": 2457059.6353793247, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.022781024024315e+29 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.1196455431551905e+31 - }, - "alpha4": { - "particle_momentum_x": 2.33898275612641e-14, - "particle_momentum_y": 2.3423797451957437e-14, - "particle_momentum_z": 2.3516107929259732e-14, - "particle_position_x": 2457367.458278153, - "particle_position_y": 2457512.0443730573, - "particle_position_z": 4914475.7765130745, - "particle_weight": 3072.000000000002 - }, - "proton3": { - "particle_momentum_x": 1.6847282386883186e-15, - "particle_momentum_y": 1.6828065767793222e-15, - "particle_momentum_z": 1.6803456707569493e-15, - "particle_position_x": 2457343.371083716, - "particle_position_y": 2457033.3891170574, - "particle_position_z": 4914529.855222688, - "particle_weight": 1.023645543155193e+31 - }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - }, - "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999998887592e+29 - }, - "alpha2": { - "particle_momentum_x": 4.1179548991012315e-15, - "particle_momentum_y": 4.110026665992801e-15, - "particle_momentum_z": 4.169802553223462e-15, - "particle_position_x": 408575.75269073684, - "particle_position_y": 413407.5155277014, - "particle_position_z": 863983.4313441743, - "particle_weight": 3.3372246639840338e+19 - } -} diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json deleted file mode 100644 index 53c3708056f..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "lev=0": { - "rho_electrons": 0.004434311371276535, - "rho_he_ions": 0.005200489146365628 - }, - "he_ions": { - "particle_momentum_x": 2.7688639005236794e-19, - "particle_momentum_y": 2.760648864116014e-19, - "particle_momentum_z": 3.6226628630061563e-19, - "particle_position_x": 2201.614485497906, - "particle_weight": 17190996093750.002 - }, - "neutrals": { - "particle_momentum_x": 1.4040775167811838e-19, - "particle_momentum_y": 1.4009514702703257e-19, - "particle_momentum_z": 1.4093144247152345e-19, - "particle_position_x": 1119.524782452131, - "particle_weight": 6.4588e+19 - }, - "electrons": { - "particle_momentum_x": 3.5193566020597954e-20, - "particle_momentum_y": 3.5368803263788353e-20, - "particle_momentum_z": 1.2572273121326582e-19, - "particle_position_x": 2139.3709122461873, - "particle_weight": 14579566406250.002 - } -} diff --git a/Regression/Checksum/benchmarks_json/Python_magnetostatic_eb_3d.json b/Regression/Checksum/benchmarks_json/Python_magnetostatic_eb_3d.json deleted file mode 100644 index abe91ac9e9d..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_magnetostatic_eb_3d.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "beam": { - "particle_momentum_x": 1.3878812158350944e-21, - "particle_momentum_y": 1.387881215835094e-21, - "particle_momentum_z": 7.150872953138685e-16, - "particle_position_x": 11163.999973134894, - "particle_position_y": 11163.999973134896, - "particle_position_z": 131662.5003103311, - "particle_weight": 20895107655113.465 - }, - "lev=0": { - "Ax": 1.408892468360627e-05, - "Ay": 1.4088924683606269e-05, - "Az": 11.423480469161868, - "Bx": 112.23826555908032, - "By": 112.2382655590803, - "Bz": 0.00019186770330025167, - "Ex": 31418238386.183773, - "Ey": 31418238386.183773, - "Ez": 3461330433.5026026, - "jx": 1961.0003914783667, - "jy": 1961.0003914783663, - "jz": 1038931606.7573991, - "phi": 3157908107.1102533, - "rho": 3.46977258905983 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json deleted file mode 100644 index ec1b6272092..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "lev=0": {}, - "ions": { - "particle_momentum_x": 5.0438993756415296e-17, - "particle_momentum_y": 5.0444406612873916e-17, - "particle_momentum_z": 5.0519292431385393e-17, - "particle_position_x": 143164.41713467025, - "particle_position_y": 143166.51845281923, - "particle_theta": 2573261.8729711357, - "particle_weight": 8.128680645366887e+18 - } -} diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json deleted file mode 100644 index e61206d19e3..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "lev=0": { - "Bx": 0.0, - "By": 7.079679609748623e-06, - "Bz": 0.0, - "Ex": 2726044.0536666242, - "Ey": 0.0, - "Ez": 4060168.6414095857, - "jx": 177543428.8941278, - "jy": 187432087.03814715, - "jz": 594259755.4658135 - }, - "ions": { - "particle_momentum_x": 9.141594694084731e-17, - "particle_momentum_y": 9.135546407258978e-17, - "particle_momentum_z": 9.137866220861254e-17, - "particle_position_x": 1197.3344862524336, - "particle_position_y": 153269.17690371818, - "particle_weight": 8.032598963696067e+16 - } -} diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json deleted file mode 100644 index 316bcb5b30d..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "lev=0": { - "Bx": 1524.8789585554075, - "By": 639.8314135126764, - "Bz": 7.476064371022787, - "Ex": 56500024.54347144, - "Ey": 75064104.7364582, - "Ez": 309548482.989921 - }, - "ions": { - "particle_momentum_x": 7.142955224072594e-15, - "particle_momentum_y": 7.13807805985974e-15, - "particle_momentum_z": 7.141134511815448e-15, - "particle_position_x": 11170689.202853566, - "particle_position_y": 5585328.08326772, - "particle_weight": 9.036667901693183e+18 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_restart_eb.json b/Regression/Checksum/benchmarks_json/Python_restart_eb.json deleted file mode 100644 index ad0d2cee5a3..00000000000 --- a/Regression/Checksum/benchmarks_json/Python_restart_eb.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "lev=0": { - "Bx": 148673.005859208, - "By": 148673.00585920806, - "Bz": 3371.758117878558, - "Ex": 55378581103426.71, - "Ey": 55378581103426.72, - "Ez": 68412803445328.25 - } -} diff --git a/Regression/Checksum/benchmarks_json/RigidInjection_BTD.json b/Regression/Checksum/benchmarks_json/RigidInjection_BTD.json deleted file mode 100644 index 90cf134201f..00000000000 --- a/Regression/Checksum/benchmarks_json/RigidInjection_BTD.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "beam": { - "particle_momentum_x": 2.2080215038948936e-16, - "particle_momentum_y": 2.18711072170811e-16, - "particle_momentum_z": 2.730924530737497e-15, - "particle_position_x": 0.0260823588888081, - "particle_position_y": 0.5049438607316916, - "particle_weight": 62415.090744607645 - }, - "lev=0": { - "Bx": 3.721807007218884e-05, - "By": 0.004860056238272468, - "Bz": 5.5335765596325185e-06, - "Ex": 1466447.517373168, - "Ey": 11214.10223280318, - "Ez": 283216.0961218869, - "jx": 16437877.898892513, - "jy": 2492340.3149980404, - "jz": 215102423.57036877, - "rho": 0.7246235591902177 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json deleted file mode 100644 index 2c9859b037d..00000000000 --- a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "lev=0": { - "Bx": 3559.0541122456157, - "By": 1685.942868827529, - "Bz": 0.0, - "Ex": 796541204346.5195, - "Ey": 961740397927.6577, - "Ez": 3528140764527.8877, - "divE": 2.0829159085076083e+21, - "jx": 7.683674095607745e+17, - "jy": 1.4132459141738875e+18, - "jz": 1.350650739310074e+18, - "rho": 18442528652.19583 - }, - "protons": { - "particle_momentum_x": 5.231109104020749e-19, - "particle_momentum_y": 5.367985047933474e-19, - "particle_momentum_z": 5.253213505843665e-19, - "particle_position_x": 0.00010628272743703473, - "particle_weight": 5.314093261582036e+22 - }, - "electrons": { - "particle_momentum_x": 1.196357181461066e-20, - "particle_momentum_y": 1.2271903040162696e-20, - "particle_momentum_z": 1.2277743615209627e-20, - "particle_position_x": 0.0001064956905491333, - "particle_weight": 5.314093261582036e+22 - } -} diff --git a/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json b/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json deleted file mode 100644 index aaf04f8a74c..00000000000 --- a/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "electron1": { - "particle_momentum_x": 3.346088201352191e-29, - "particle_momentum_y": 3.346088199932699e-29, - "particle_momentum_z": 3.3460881978884573e-29, - "particle_position_x": 0.1545496421786394, - "particle_position_y": 0.15454964213891717, - "particle_position_z": 0.15454964208395047, - "particle_weight": 1.0 - }, - "electron2": { - "particle_momentum_x": 3.346088199424244e-29, - "particle_momentum_y": 3.346088202432085e-29, - "particle_momentum_z": 3.346088202108714e-29, - "particle_position_x": 0.15454964215048347, - "particle_position_y": 0.15454964222866666, - "particle_position_z": 0.15454964222208387, - "particle_weight": 1.0 - }, - "lev=0": { - "Ex": 7.101527209952963e-05, - "Ey": 7.10152721046017e-05, - "Ez": 7.101527211163835e-05, - "rho": 1.3125030985727997e-15 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json deleted file mode 100644 index 350848d4aee..00000000000 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "lev=0": { - "Bx": 0.0, - "By": 0.0, - "Bz": 0.0, - "Ex": 0.0, - "Ey": 0.0, - "Ez": 0.0, - "jx": 0.0, - "jy": 0.0, - "jz": 0.0 - }, - "electron": { - "particle_momentum_x": 3.5790777034053853e-19, - "particle_momentum_y": 3.5815348106229496e-19, - "particle_momentum_z": 3.577963316718249e-19, - "particle_position_x": 1.024180253191667, - "particle_position_y": 1.023919590453571, - "particle_position_z": 1.0240653505082926, - "particle_weight": 714240000000.0 - } -} diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json deleted file mode 100644 index 726573ef8b2..00000000000 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "lev=0": { - "Bx": 0.0, - "By": 0.0, - "Bz": 0.0, - "Ex": 0.0, - "Ey": 0.0, - "Ez": 0.0, - "T_electron": 351982.0169218243, - "T_ion": 349599.6939052666 - }, - "electron": { - "particle_momentum_x": 8.359982321196841e-19, - "particle_momentum_y": 8.192841151167721e-19, - "particle_momentum_z": 8.182985690701241e-19, - "particle_position_x": 21255110.08090505, - "particle_position_y": 21303488.6242626, - "particle_position_z": 21238676.122703437, - "particle_weight": 7.168263344048695e+28 - }, - "ion": { - "particle_momentum_x": 2.0034830240966893e-18, - "particle_momentum_y": 1.8323959076577197e-18, - "particle_momentum_z": 1.827953230828629e-18, - "particle_position_x": 21246214.748882487, - "particle_position_y": 21280709.710960124, - "particle_position_z": 21206153.002106402, - "particle_weight": 7.168263344048695e+28 - } -} diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json deleted file mode 100644 index f90c34bc86d..00000000000 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "lev=0": { - "Bx": 0.0, - "By": 0.0, - "Bz": 0.0, - "Ex": 0.0, - "Ey": 0.0, - "Ez": 0.0 - }, - "ion": { - "particle_momentum_x": 2.4932317055825563e-19, - "particle_momentum_y": 2.274916403334278e-19, - "particle_momentum_z": 2.2528161767665816e-19, - "particle_position_x": 2648815.601036139, - "particle_position_y": 2662836.7581390506, - "particle_weight": 1.7256099431746894e+26 - }, - "electron": { - "particle_momentum_x": 1.0489203687862582e-19, - "particle_momentum_y": 1.0209657029567292e-19, - "particle_momentum_z": 1.0248962872393911e-19, - "particle_position_x": 2657004.8285825616, - "particle_position_y": 2670174.272797987, - "particle_weight": 1.7256099431746894e+26 - } -} diff --git a/Regression/Checksum/benchmarks_json/collisionZ.json b/Regression/Checksum/benchmarks_json/collisionZ.json deleted file mode 100644 index 3be8d5ae893..00000000000 --- a/Regression/Checksum/benchmarks_json/collisionZ.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lev=0": { - "Bx": 0.0, - "By": 0.0, - "Bz": 0.0, - "Ex": 0.0, - "Ey": 0.0, - "Ez": 0.0, - "jx": 0.0, - "jy": 0.0, - "jz": 0.0 - }, - "ions": { - "particle_momentum_x": 3.425400072687143e-16, - "particle_momentum_y": 3.421937133999805e-16, - "particle_momentum_z": 5.522701882677923e-16, - "particle_position_x": 720.0011611411148, - "particle_weight": 1.0999999999999999e+24 - } -} diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json deleted file mode 100644 index 8b03899369b..00000000000 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "lev=0": { - "Bx": 1118808.3686978193, - "By": 3248970.5506422943, - "Bz": 280612.7921641442, - "Ex": 975536649649286.1, - "Ey": 402861835403418.1, - "Ez": 159049265640492.28, - "jx": 2.9996888133195436e+16, - "jy": 8.866654944519546e+16, - "jz": 3.164008885453435e+17, - "rho": 1059988299.6088305 - }, - "ions": { - "particle_momentum_x": 1.6150513873065298e-18, - "particle_momentum_y": 2.233426695677123e-18, - "particle_momentum_z": 4.279249529993671e-13, - "particle_position_x": 1.4883816864183497, - "particle_position_y": 16.452386504127254, - "particle_weight": 1.234867369440658e+18 - }, - "electrons": { - "particle_momentum_x": 7.058167362825288e-19, - "particle_momentum_y": 2.204239326446281e-18, - "particle_momentum_z": 2.530521998715408e-16, - "particle_position_x": 1.5006581263609764, - "particle_position_y": 16.454388313398017, - "particle_weight": 1.234867020725368e+18 - }, - "beam": { - "particle_momentum_x": 6.869222298759882e-19, - "particle_momentum_y": 4.374719809060106e-19, - "particle_momentum_z": 6.4523206583503136e-18, - "particle_position_x": 0.001290816359726098, - "particle_position_y": 0.3586691102823157, - "particle_weight": 3120754537230.3823 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json b/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json deleted file mode 100644 index 71e147cb1f7..00000000000 --- a/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "beam": { - "particle_Bx": 0.0, - "particle_By": 1.4229748768905527e-19, - "particle_Bz": 0.0, - "particle_Ex": 210305.84591470752, - "particle_Ey": 0.0, - "particle_Ez": 210741.1714121227, - "particle_momentum_x": 1.111019933689776e-26, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.113270980745195e-26, - "particle_position_x": 0.03183627816789909, - "particle_position_y": 0.03171922054171794, - "particle_weight": 31207.545372303823 - }, - "lev=0": { - "Bx": 0.0, - "By": 1.5468538800972258e-20, - "Bz": 0.0, - "Ex": 8533.638650013556, - "Ey": 0.0, - "Ez": 8534.98921988922, - "jx": 0.0, - "jy": 0.0, - "jz": 0.0 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json b/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json deleted file mode 100644 index db23c26f9a7..00000000000 --- a/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "beam": { - "particle_Bx": 1.6547943661629038e-20, - "particle_By": 1.7166945226626064e-20, - "particle_Bz": 1.851357799836734e-20, - "particle_Ex": 39363.91309372786, - "particle_Ey": 39331.17437154593, - "particle_Ez": 39396.18209787599, - "particle_momentum_x": 1.700384472207379e-27, - "particle_momentum_y": 1.69889110346099e-27, - "particle_momentum_z": 1.7017928140329036e-27, - "particle_position_x": 0.031880969779242374, - "particle_position_y": 0.03175704658379704, - "particle_position_z": 0.03183674192208247, - "particle_weight": 0.06241509074460764 - }, - "lev=0": { - "Bx": 2.2090009624207165e-20, - "By": 2.2307246822783936e-20, - "Bz": 2.1967888687392684e-20, - "Ex": 8888.956516621029, - "Ey": 8838.45337149105, - "Ez": 8837.421045658291, - "jx": 0.0, - "jy": 0.0, - "jz": 0.0 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json deleted file mode 100644 index d0edafb9f0e..00000000000 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "lev=0": { - "Bx": 3.769898030127477e-18, - "By": 0.006628374119786834, - "Bz": 0.006628374119786834, - "Ex": 5102618.4711524295, - "Ey": 6.323755130400527e-05, - "Ez": 6.323755130400527e-05 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json deleted file mode 100644 index a3e609bd9a9..00000000000 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_2d.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "lev=0": { - "Bx": 9.263694545408503e-05, - "By": 0.00031905198933489145, - "Bz": 7.328424783762594e-05, - "Ex": 8553.906698053046, - "Ey": 60867.04830538045, - "Ez": 8.439422682267567e-07 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json b/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json deleted file mode 100644 index c759f36ac5d..00000000000 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_cube_macroscopic.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "lev=0": { - "Bx": 3.898540288712651e-18, - "By": 0.005101824310293573, - "Bz": 0.005101824310293573, - "Ex": 4414725.184731114, - "Ey": 6.323754812712063e-05, - "Ez": 6.323754812712063e-05 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_rotated_cube.json b/Regression/Checksum/benchmarks_json/embedded_boundary_rotated_cube.json deleted file mode 100644 index b2b4aa569c1..00000000000 --- a/Regression/Checksum/benchmarks_json/embedded_boundary_rotated_cube.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "lev=0": { - "Bx": 1.280747509243305e-05, - "By": 2.473900144296397e-02, - "Bz": 2.473890786894079e-02, - "Ex": 1.025322901921306e+07, - "Ey": 1.042254197269831e+04, - "Ez": 1.040011664019071e+04 - } -} - diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json deleted file mode 100644 index dd56f8170a9..00000000000 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "lev=0": { - "Bx": 1086729.9718613266, - "By": 2886554.482275311, - "Bz": 264259.55093734514, - "Ex": 867387781289915.2, - "Ey": 392666724461952.5, - "Ez": 146897592531660.03, - "jx": 2.702866174672266e+16, - "jy": 8.615938361747776e+16, - "jz": 2.7329155817806224e+17, - "rho": 915945723.7934376 - }, - "ions": { - "particle_momentum_x": 1.4394902513923003e-18, - "particle_momentum_y": 1.5967629157922875e-18, - "particle_momentum_z": 4.287340658051679e-13, - "particle_position_x": 1.4911814217142487, - "particle_position_y": 16.521964978771, - "particle_weight": 1.2372405194129536e+18 - }, - "electrons": { - "particle_momentum_x": 6.240933687389075e-19, - "particle_momentum_y": 1.5790611427694247e-18, - "particle_momentum_z": 2.5064357834741096e-16, - "particle_position_x": 1.501413766926399, - "particle_position_y": 16.523781713952324, - "particle_weight": 1.2372401466086835e+18 - }, - "beam": { - "particle_momentum_x": 7.000932845220306e-19, - "particle_momentum_y": 4.374936866729326e-19, - "particle_momentum_z": 6.194468548032543e-18, - "particle_position_x": 0.0016030835496557787, - "particle_position_y": 0.3589262705964349, - "particle_weight": 3120754537230.3823 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_boosted.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_boosted.json deleted file mode 100644 index acec34286f7..00000000000 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_boosted.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "electron": { - "particle_momentum_x": 5.955475926588059e-26, - "particle_momentum_y": 1.4612764777454504e-35, - "particle_momentum_z": 3.4687284535374423e-23, - "particle_position_x": 0.049960237123814574, - "particle_position_y": 8.397636119991403e-15, - "particle_position_z": 0.10931687737912647, - "particle_weight": 1.0 - }, - "lev=0": { - "Bx": 3.254531465641299e-14, - "By": 3.2768092409497234e-14, - "Bz": 1.0615286316115558e-16, - "Ex": 2.30845657253269e-05, - "Ey": 2.2656898931877975e-05, - "Ez": 1.997747654112569e-05, - "jx": 1.7819477343635878e-10, - "jy": 4.2163030523377745e-20, - "jz": 1.0378839382497739e-07 - } -} diff --git a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json deleted file mode 100644 index a1ec0b4c831..00000000000 --- a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "lev=0": { - "Az": 11.358663326449284, - "Bx": 111.55929407644248, - "By": 111.55929407644244, - "Ex": 31257180402.55472, - "Ey": 31257180402.55473, - "jz": 1034841325.9848926, - "phi": 3143521213.0157924, - "rho": 3.449203918900721 - }, - "beam": { - "particle_momentum_x": 1.3604657334742729e-21, - "particle_momentum_y": 1.3604657334742772e-21, - "particle_momentum_z": 7.150873450281544e-16, - "particle_position_x": 11163.99997371537, - "particle_position_y": 11163.999973715368, - "particle_position_z": 131662.50031035842, - "particle_weight": 20895107655113.465 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json b/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json deleted file mode 100644 index f30e773e7e0..00000000000 --- a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "lev=0": { - "Bt": 24080.416715463354, - "Er": 4536303784778.672, - "Ez": 4298815071343.07, - "jr": 361182004529074.8, - "jz": 1802215706551553.5, - "rho": 4884623.957368025, - "rho_driver": 6288266.101815153, - "rho_plasma_e": 49568366.40537152, - "rho_plasma_p": 50769182.21072973 - }, - "driver": { - "particle_momentum_x": 8.723405122353729e-16, - "particle_momentum_y": 8.719285567351437e-16, - "particle_momentum_z": 5.461771334692466e-13, - "particle_position_x": 6.269566322411488, - "particle_position_y": 17.934200805964075, - "particle_theta": 1570790.0436095877, - "particle_weight": 6241484108.424456 - }, - "plasma_p": { - "particle_momentum_x": 6.650539058831456e-20, - "particle_momentum_y": 6.776260514923786e-20, - "particle_momentum_z": 5.470216831432835e-20, - "particle_position_x": 1.1365201443471713, - "particle_position_y": 0.6152066517828133, - "particle_theta": 20286.92798337582, - "particle_weight": 1002457942911.3788 - }, - "plasma_e": { - "particle_momentum_x": 6.655027717314839e-20, - "particle_momentum_y": 6.730480164464723e-20, - "particle_momentum_z": 2.8073811669581434e-20, - "particle_position_x": 1.1423427658689635, - "particle_position_y": 0.6140113094028803, - "particle_theta": 20188.939948727297, - "particle_weight": 1002457942911.3788 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json deleted file mode 100644 index e4ff1fc68a8..00000000000 --- a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lev=0": { - "Bx": 100915933.44993827, - "By": 157610622.1855512, - "Bz": 9.717358898362187e-14, - "Ex": 4.7250652706211096e+16, - "Ey": 3.0253948990559976e+16, - "Ez": 3276573.9514776524, - "rho": 10994013582437.193 - }, - "electron": { - "particle_momentum_x": 5.701277606050295e-19, - "particle_momentum_y": 3.6504516641520437e-19, - "particle_momentum_z": 1.145432768297242e-10, - "particle_position_x": 17.314086912497864, - "particle_position_y": 0.2583691267187796, - "particle_position_z": 10066.329600000008, - "particle_weight": 19969036501.910976 - } -} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/scraping.json b/Regression/Checksum/benchmarks_json/scraping.json deleted file mode 100644 index 8f85b956b26..00000000000 --- a/Regression/Checksum/benchmarks_json/scraping.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "lev=0": { - "Er": 0.0 - }, - "lev=1": { - "Er": 0.0 - } -} diff --git a/Regression/Checksum/benchmarks_json/test_1d_background_mcc_picmi.json b/Regression/Checksum/benchmarks_json/test_1d_background_mcc_picmi.json new file mode 100644 index 00000000000..029294deb66 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_background_mcc_picmi.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "rho_electrons": 0.0044328572492614605, + "rho_he_ions": 0.005198609403474849 + }, + "electrons": { + "particle_momentum_x": 3.5020450942268976e-20, + "particle_momentum_y": 3.5342700024993965e-20, + "particle_momentum_z": 1.2596017960675146e-19, + "particle_position_x": 2139.5967568101983, + "particle_weight": 14577210937500.002 + }, + "he_ions": { + "particle_momentum_x": 2.770046913680294e-19, + "particle_momentum_y": 2.755651798947783e-19, + "particle_momentum_z": 3.619494241595636e-19, + "particle_position_x": 2200.218124999781, + "particle_weight": 17184714843750.002 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_1d_collision_z.json b/Regression/Checksum/benchmarks_json/test_1d_collision_z.json new file mode 100644 index 00000000000..e517fc2b281 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_collision_z.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "ions": { + "particle_momentum_x": 3.4242515316949867e-16, + "particle_momentum_y": 3.441515880563085e-16, + "particle_momentum_z": 5.537993210573513e-16, + "particle_position_x": 720.0714079424476, + "particle_weight": 1.0999999999999997e+24 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_1d_dsmc_picmi.json b/Regression/Checksum/benchmarks_json/test_1d_dsmc_picmi.json new file mode 100644 index 00000000000..62f915cf2e8 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_dsmc_picmi.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "rho_electrons": 0.004437338851654305, + "rho_he_ions": 0.005200276265886133 + }, + "he_ions": { + "particle_momentum_x": 2.768463746716725e-19, + "particle_momentum_y": 2.7585450668167785e-19, + "particle_momentum_z": 3.6189671443598533e-19, + "particle_position_x": 2201.408357891233, + "particle_weight": 17190472656250.002 + }, + "electrons": { + "particle_momentum_x": 3.523554668287801e-20, + "particle_momentum_y": 3.515628626179393e-20, + "particle_momentum_z": 1.258711379033217e-19, + "particle_position_x": 2140.8168584833174, + "particle_weight": 14588988281250.002 + }, + "neutrals": { + "particle_momentum_x": 1.4054952479597137e-19, + "particle_momentum_y": 1.403311018061206e-19, + "particle_momentum_z": 1.411491089895956e-19, + "particle_position_x": 1119.82858839282, + "particle_weight": 6.4588e+19 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_1d_fel.json b/Regression/Checksum/benchmarks_json/test_1d_fel.json new file mode 100644 index 00000000000..ffcc97fa057 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_fel.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 473.98537926589177, + "Bz": 0.0, + "Ex": 142097845843.78326, + "Ey": 0.0, + "Ez": 0.0, + "jx": 1260205974.7220135, + "jy": 0.0, + "jz": 0.0 + }, + "electrons": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 13607.572916093213, + "particle_momentum_x": 3.2646797960476606e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.541338620507345e-16, + "particle_weight": 1349823909946836.0 + }, + "positrons": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 13607.572916093213, + "particle_momentum_x": 3.2646797960476606e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.541338620507345e-16, + "particle_weight": 1349823909946836.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json b/Regression/Checksum/benchmarks_json/test_1d_langmuir_fluid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json rename to Regression/Checksum/benchmarks_json/test_1d_langmuir_fluid.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_1d.json b/Regression/Checksum/benchmarks_json/test_1d_langmuir_multi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_langmuir_multi.json diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d.json b/Regression/Checksum/benchmarks_json/test_1d_laser_acceleration.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAcceleration_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_acceleration.json diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json b/Regression/Checksum/benchmarks_json/test_1d_laser_acceleration_fluid_boosted.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_acceleration_fluid_boosted.json diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration_1d.json b/Regression/Checksum/benchmarks_json/test_1d_laser_acceleration_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LaserAcceleration_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_acceleration_picmi.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjection_1d.json b/Regression/Checksum/benchmarks_json/test_1d_laser_injection.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjection_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_injection.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_1d.json b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_from_lasy_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_injection_from_lasy_file.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_1d_boost.json b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_from_lasy_file_boost.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_1d_boost.json rename to Regression/Checksum/benchmarks_json/test_1d_laser_injection_from_lasy_file_boost.json diff --git a/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json new file mode 100644 index 00000000000..a89d6ccc2d4 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 374596.7817425552, + "By": 374596.7817425552, + "Bz": 0.0, + "Ex": 111502789524279.0, + "Ey": 111502789524279.0, + "Ez": 0.0, + "jx": 73098054407.2772, + "jy": 73098054407.2772, + "jz": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/test_1d_ohm_solver_em_modes_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_ohm_solver_em_modes_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json b/Regression/Checksum/benchmarks_json/test_1d_ohm_solver_ion_beam_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_ohm_solver_ion_beam_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration1d.json b/Regression/Checksum/benchmarks_json/test_1d_plasma_acceleration_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration1d.json rename to Regression/Checksum/benchmarks_json/test_1d_plasma_acceleration_picmi.json diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json b/Regression/Checksum/benchmarks_json/test_1d_resample_velocity_coincidence_thinning.json similarity index 100% rename from Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json rename to Regression/Checksum/benchmarks_json/test_1d_resample_velocity_coincidence_thinning.json diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json b/Regression/Checksum/benchmarks_json/test_1d_resample_velocity_coincidence_thinning_cartesian.json similarity index 100% rename from Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json rename to Regression/Checksum/benchmarks_json/test_1d_resample_velocity_coincidence_thinning_cartesian.json diff --git a/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json b/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json new file mode 100644 index 00000000000..758db4355ec --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3625.566538877196, + "By": 1684.1769211109035, + "Bz": 0.0, + "Ex": 796541204346.5195, + "Ey": 961740397927.6577, + "Ez": 3528140764527.8877, + "divE": 2.0829159085076083e+21, + "jx": 7.683674095607745e+17, + "jy": 1.4132459141738875e+18, + "jz": 1.350650739310074e+18, + "rho": 18442528652.19583 + }, + "protons": { + "particle_momentum_x": 5.231109104020749e-19, + "particle_momentum_y": 5.367985047933474e-19, + "particle_momentum_z": 5.253213505843665e-19, + "particle_position_x": 0.00010628272743703473, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196357181461066e-20, + "particle_momentum_y": 1.2271903040162696e-20, + "particle_momentum_z": 1.2277743615209627e-20, + "particle_position_x": 0.0001064956905491333, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/silver_mueller_1d.json b/Regression/Checksum/benchmarks_json/test_1d_silver_mueller.json similarity index 100% rename from Regression/Checksum/benchmarks_json/silver_mueller_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_silver_mueller.json diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/test_1d_theta_implicit_picard.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json rename to Regression/Checksum/benchmarks_json/test_1d_theta_implicit_picard.json diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_averaged_galilean_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd.json rename to Regression/Checksum/benchmarks_json/test_2d_averaged_galilean_psatd.json diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_2d_averaged_galilean_psatd_hybrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd_hybrid.json rename to Regression/Checksum/benchmarks_json/test_2d_averaged_galilean_psatd_hybrid.json diff --git a/Regression/Checksum/benchmarks_json/background_mcc.json b/Regression/Checksum/benchmarks_json/test_2d_background_mcc.json similarity index 100% rename from Regression/Checksum/benchmarks_json/background_mcc.json rename to Regression/Checksum/benchmarks_json/test_2d_background_mcc.json diff --git a/Regression/Checksum/benchmarks_json/background_mcc_dp_psp.json b/Regression/Checksum/benchmarks_json/test_2d_background_mcc_dp_psp.json similarity index 100% rename from Regression/Checksum/benchmarks_json/background_mcc_dp_psp.json rename to Regression/Checksum/benchmarks_json/test_2d_background_mcc_dp_psp.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_background_mcc_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_background_mcc_picmi.json new file mode 100644 index 00000000000..579f46d33ab --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_background_mcc_picmi.json @@ -0,0 +1,22 @@ +{ + "electrons": { + "particle_momentum_x": 1.011638818664759e-18, + "particle_momentum_y": 2.81974298744432e-19, + "particle_momentum_z": 2.809194032519318e-19, + "particle_position_x": 17136.01865460215, + "particle_position_y": 936.3651769897449, + "particle_weight": 61113170379.63868 + }, + "he_ions": { + "particle_momentum_x": 2.883076633513297e-18, + "particle_momentum_y": 2.195704870583595e-18, + "particle_momentum_z": 2.198216553980008e-18, + "particle_position_x": 17607.42545752183, + "particle_position_y": 1100.024786059151, + "particle_weight": 71976747650.1465 + }, + "lev=0": { + "rho_electrons": 0.03558889419586454, + "rho_he_ions": 0.04176234095111594 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/bilinear_filter.json b/Regression/Checksum/benchmarks_json/test_2d_bilinear_filter.json similarity index 100% rename from Regression/Checksum/benchmarks_json/bilinear_filter.json rename to Regression/Checksum/benchmarks_json/test_2d_bilinear_filter.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_collision_xz.json b/Regression/Checksum/benchmarks_json/test_2d_collision_xz.json new file mode 100644 index 00000000000..1e01ddcd291 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_collision_xz.json @@ -0,0 +1,26 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0 + }, + "ion": { + "particle_momentum_x": 2.4848259831544605e-19, + "particle_momentum_y": 2.268622463170605e-19, + "particle_momentum_z": 2.266703976728937e-19, + "particle_position_x": 2661759.7677191226, + "particle_position_y": 2654536.6931685647, + "particle_weight": 1.7256099431746894e+26 + }, + "electron": { + "particle_momentum_x": 1.0437899826798073e-19, + "particle_momentum_y": 1.0175192222562617e-19, + "particle_momentum_z": 1.0262602372143781e-19, + "particle_position_x": 2653326.05816203, + "particle_position_y": 2659804.729475678, + "particle_weight": 1.7256099431746894e+26 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_collision_xz_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_collision_xz_picmi.json new file mode 100644 index 00000000000..5407fe62374 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_collision_xz_picmi.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "electron": { + "particle_momentum_x": 1.0340540111403247e-19, + "particle_momentum_y": 1.0154008257709287e-19, + "particle_momentum_z": 1.0325278275532687e-19, + "particle_position_x": 2667286.170170416, + "particle_position_y": 2647299.0368030528, + "particle_weight": 1.7256099431746894e+26 + }, + "ion": { + "particle_momentum_x": 2.489595454855792e-19, + "particle_momentum_y": 2.295864697447754e-19, + "particle_momentum_z": 2.2800373850644785e-19, + "particle_position_x": 2657799.867468604, + "particle_position_y": 2664764.1491634008, + "particle_weight": 1.7256099431746894e+26 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json new file mode 100644 index 00000000000..930d48ed713 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json @@ -0,0 +1,38 @@ +{ + "lev=0": { + "Bx": 1118808.3708538802, + "By": 3248949.0437452313, + "Bz": 280612.7768961371, + "Ex": 975530336896144.1, + "Ey": 402861838033488.6, + "Ez": 159049784131625.12, + "jx": 2.9997142632475216e+16, + "jy": 8.866655055001146e+16, + "jz": 3.163953981093208e+17, + "rho": 1059970922.1974506 + }, + "electrons": { + "particle_momentum_x": 7.058252826278211e-19, + "particle_momentum_y": 2.204239315713169e-18, + "particle_momentum_z": 2.530521235191952e-16, + "particle_position_x": 1.5006579649318788, + "particle_position_y": 16.454388304724286, + "particle_weight": 1.234867020725368e+18 + }, + "beam": { + "particle_momentum_x": 6.88879318082965e-19, + "particle_momentum_y": 4.37466174746362e-19, + "particle_momentum_z": 6.4299296650127095e-18, + "particle_position_x": 0.0012936414423443238, + "particle_position_y": 0.3587414953163842, + "particle_weight": 3120754537230.3823 + }, + "ions": { + "particle_momentum_x": 1.6150618501530563e-18, + "particle_momentum_y": 2.2334266731098355e-18, + "particle_momentum_z": 4.279249530957972e-13, + "particle_position_x": 1.488381686539698, + "particle_position_y": 16.4523865041322, + "particle_weight": 1.234867369440658e+18 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc.json b/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc.json new file mode 100644 index 00000000000..41567dc3bf2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc.json @@ -0,0 +1,5 @@ +{ + "lev=0": { + "phi": 10817.97280547637 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc_picmi.json new file mode 100644 index 00000000000..41567dc3bf2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_dirichlet_bc_picmi.json @@ -0,0 +1,5 @@ +{ + "lev=0": { + "phi": 10817.97280547637 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_dive_cleaning.json b/Regression/Checksum/benchmarks_json/test_2d_dive_cleaning.json new file mode 100644 index 00000000000..06216490281 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_dive_cleaning.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 1.2286794953440962e-20, + "Bz": 0.0, + "Ex": 8214.62652670017, + "Ey": 0.0, + "Ez": 8214.876065863818, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "beam": { + "particle_momentum_x": 1.580805184936455e-26, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.5759037267515492e-26, + "particle_position_x": 0.0319918174870481, + "particle_position_y": 0.03196785829873647, + "particle_weight": 31207.545372303823 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json new file mode 100644 index 00000000000..dbb5ffa39ae --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 9.263694545408503e-05, + "By": 0.00031905198933489145, + "Bz": 7.328424783762594e-05, + "Ex": 8553.90669811286, + "Ey": 60867.04830538045, + "Ez": 4.223902107031194e-06 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..de3d125c744 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 3.059581906777539e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..481aba1f36b --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 2.4521053721245334e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..82b6c6849ac --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 2.2059346534892452e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..632c48df963 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.3792992316172512e-15, + "Bz": 0.0 , + "Ex": 6.177046470842443e-07, + "Ey": 0.0, + "Ez": 7.259396011803518e-07, + "divE": 2.809306467366024e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..9112c931b55 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.3948084603369097e-15, + "Bz": 0.0, + "Ex": 6.747158562891953e-07, + "Ey": 0.0, + "Ez": 5.541309886315263e-07, + "divE": 2.091715826275267e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..8d6ddf169af --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.379299231617251e-15, + "Bz": 0.0, + "Ex": 6.177046470842443e-07, + "Ey": 0.0, + "Ez": 7.259396011803522e-07, + "divE": 2.8093064673660275e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + diff --git a/Regression/Checksum/benchmarks_json/embedded_boundary_rotated_cube_2d.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_rotated_cube.json similarity index 100% rename from Regression/Checksum/benchmarks_json/embedded_boundary_rotated_cube_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_rotated_cube.json diff --git a/Regression/Checksum/benchmarks_json/embedded_circle.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_circle.json similarity index 100% rename from Regression/Checksum/benchmarks_json/embedded_circle.json rename to Regression/Checksum/benchmarks_json/test_2d_embedded_circle.json diff --git a/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json b/Regression/Checksum/benchmarks_json/test_2d_energy_conserving_thermal_plasma.json similarity index 100% rename from Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json rename to Regression/Checksum/benchmarks_json/test_2d_energy_conserving_thermal_plasma.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_field_probe.json b/Regression/Checksum/benchmarks_json/test_2d_field_probe.json new file mode 100644 index 00000000000..8aabe6c8301 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_field_probe.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 123510.69657444415, + "Bz": 0.0, + "Ex": 31206368949280.34, + "Ey": 0.0, + "Ez": 16921005306450.537 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json new file mode 100644 index 00000000000..d4fe12f759f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json @@ -0,0 +1,11 @@ +{ + "lev=0": {}, + "electron": { + "particle_momentum_x": 1.4013860393698154e-18, + "particle_momentum_y": 1.0934049057929508e-19, + "particle_momentum_z": 1.4066623146535866e-18, + "particle_position_x": 72129.9049362857, + "particle_position_y": 72178.76505490103, + "particle_weight": 6.279375e-08 + } +} diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_2d_psatd.json rename to Regression/Checksum/benchmarks_json/test_2d_galilean_psatd.json diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_2d_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_current_correction_psb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_2d_psatd_current_correction_psb.json rename to Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_current_correction_psb.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json new file mode 100644 index 00000000000..a163d063134 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json @@ -0,0 +1,38 @@ +{ + "lev=0": { + "Bx": 1086729.9879225595, + "By": 2886531.8361456757, + "Bz": 264259.55266959703, + "Ex": 867381192933999.0, + "Ey": 392666738858258.7, + "Ez": 146898030091111.84, + "jx": 2.702892158065604e+16, + "jy": 8.615938867870698e+16, + "jz": 2.7328506574305683e+17, + "rho": 915924567.6956444 + }, + "beam": { + "particle_momentum_x": 7.006049777955171e-19, + "particle_momentum_y": 4.374916846096741e-19, + "particle_momentum_z": 6.173292885825711e-18, + "particle_position_x": 0.0016046573777589298, + "particle_position_y": 0.35899824939059793, + "particle_weight": 3120754537230.3823 + }, + "ions": { + "particle_momentum_x": 1.4395010524514718e-18, + "particle_momentum_y": 1.596762923413923e-18, + "particle_momentum_z": 4.2873406589510426e-13, + "particle_position_x": 1.4911814218338595, + "particle_position_y": 16.52196497877563, + "particle_weight": 1.2372405194129536e+18 + }, + "electrons": { + "particle_momentum_x": 6.241019323257125e-19, + "particle_momentum_y": 1.5790611706036782e-18, + "particle_momentum_z": 2.5064350576384073e-16, + "particle_position_x": 1.501413593465263, + "particle_position_y": 16.52378170448397, + "particle_weight": 1.2372401466086835e+18 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_id_cpu_read_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_id_cpu_read_picmi.json new file mode 100644 index 00000000000..a59b780ba38 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_id_cpu_read_picmi.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "phi": 0.001516261625969309 + }, + "electrons": { + "particle_momentum_x": 7.751654441658017e-26, + "particle_momentum_y": 6.938526597814195e-26, + "particle_momentum_z": 6.572520038890184e-26, + "particle_newPid": 500.0, + "particle_position_x": 1.4999588764815643, + "particle_position_y": 1.4999551809411737, + "particle_weight": 200.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/test_2d_ionization_boost.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ionization_boost.json rename to Regression/Checksum/benchmarks_json/test_2d_ionization_boost.json diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/test_2d_ionization_lab.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ionization_lab.json rename to Regression/Checksum/benchmarks_json/test_2d_ionization_lab.json diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/test_2d_ionization_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_ionization.json rename to Regression/Checksum/benchmarks_json/test_2d_ionization_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_fluid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_fluid.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json new file mode 100644 index 00000000000..899352c45ba --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 5.726296856755232, + "Bz": 0.0, + "Ex": 3751589134191.326, + "Ey": 0.0, + "Ez": 3751589134191.332, + "jx": 1.0100623329922576e+16, + "jy": 0.0, + "jz": 1.0100623329922578e+16 + }, + "electrons": { + "particle_momentum_x": 5.668407513430198e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407513430198e-20, + "particle_position_x": 0.6553599999999999, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + }, + "positrons": { + "particle_momentum_x": 5.668407513430198e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407513430198e-20, + "particle_position_x": 0.6553599999999999, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_anisotropic.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_anisotropic.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_anisotropic.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_anisotropic.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_momentum_conserving.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_momentum_conserving.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_momentum_conserving.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_momentum_conserving.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_mr_psatd.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction_nodal.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_current_correction_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction_nodal.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_current_correction_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_momentum_conserving.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_momentum_conserving.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_multiJ.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_multiJ.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ_nodal.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_multiJ_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ_nodal.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_multiJ_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_nodal.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_nodal.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json rename to Regression/Checksum/benchmarks_json/test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4.json diff --git a/Regression/Checksum/benchmarks_json/Larmor.json b/Regression/Checksum/benchmarks_json/test_2d_larmor.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Larmor.json rename to Regression/Checksum/benchmarks_json/test_2d_larmor.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json new file mode 100644 index 00000000000..b8592e87c82 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json @@ -0,0 +1,38 @@ +{ + "lev=0": { + "Bx": 4818955.485307051, + "By": 1752.8020185365554, + "Bz": 14516.212849649737, + "Ex": 2366115529014.2324, + "Ey": 1446112026998942.5, + "Ez": 21864189485739.55, + "jx": 1996366372981548.5, + "jy": 5.312583836344946e+16, + "jz": 2.049135259966133e+16, + "rho": 68443961.64027263 + }, + "beam": { + "particle_momentum_x": 3.535736052190267e-19, + "particle_momentum_y": 4.363217976210739e-19, + "particle_momentum_z": 5.658515465395611e-17, + "particle_position_x": 0.008314855161869274, + "particle_position_y": 1.170433573157185, + "particle_weight": 62415090744.60765 + }, + "electrons": { + "particle_momentum_x": 2.213594541883545e-23, + "particle_momentum_y": 2.8224559261549207e-22, + "particle_momentum_z": 5.260626007410037e-22, + "particle_position_x": 0.010800577787636243, + "particle_position_y": 0.2111506062831794, + "particle_weight": 4.121554826246186e+16 + }, + "ions": { + "particle_momentum_x": 6.24847229412907e-23, + "particle_momentum_y": 4.449097671673176e-22, + "particle_momentum_z": 5.768168722032957e-22, + "particle_position_x": 0.010800001678510512, + "particle_position_y": 0.21114947608115425, + "particle_weight": 4.121554826246186e+16 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationMR.json b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAccelerationMR.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_mr.json diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_mr_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_mr_picmi.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjection_2d.json b/Regression/Checksum/benchmarks_json/test_2d_laser_injection.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjection_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_injection.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromBINARYFile.json b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_from_binary_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromBINARYFile.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_injection_from_binary_file.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_2d.json b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_from_lasy_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_injection_from_lasy_file.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json new file mode 100644 index 00000000000..b77b951e92a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 19699314.38858362, + "By": 101297372.8536657, + "Bz": 39796093.072294116, + "Ex": 1.3881256464656438e+16, + "Ey": 1.322100107139857e+16, + "Ez": 2.6833518029118908e+16, + "jx": 3.669364941403736e+16, + "jy": 3.669364586262695e+16, + "jz": 7.338729883115621e+16 + } +} diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserIonAcc2d.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc.json diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d_no_field_diag.json b/Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc_no_field_diag.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserIonAcc2d_no_field_diag.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc_no_field_diag.json diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_ion_acc_picmi.json diff --git a/Regression/Checksum/benchmarks_json/LaserOnFine.json b/Regression/Checksum/benchmarks_json/test_2d_laser_on_fine.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserOnFine.json rename to Regression/Checksum/benchmarks_json/test_2d_laser_on_fine.json diff --git a/Regression/Checksum/benchmarks_json/leveling_thinning.json b/Regression/Checksum/benchmarks_json/test_2d_leveling_thinning.json similarity index 100% rename from Regression/Checksum/benchmarks_json/leveling_thinning.json rename to Regression/Checksum/benchmarks_json/test_2d_leveling_thinning.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_maxwell_hybrid_qed_solver.json b/Regression/Checksum/benchmarks_json/test_2d_maxwell_hybrid_qed_solver.json new file mode 100644 index 00000000000..a5c1c1fd700 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_maxwell_hybrid_qed_solver.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 3.5439667773344274e-05, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 6553599999.996383, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/nci_corrector.json b/Regression/Checksum/benchmarks_json/test_2d_nci_corrector.json similarity index 100% rename from Regression/Checksum/benchmarks_json/nci_corrector.json rename to Regression/Checksum/benchmarks_json/test_2d_nci_corrector.json diff --git a/Regression/Checksum/benchmarks_json/nci_correctorMR.json b/Regression/Checksum/benchmarks_json/test_2d_nci_corrector_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/nci_correctorMR.json rename to Regression/Checksum/benchmarks_json/test_2d_nci_corrector_mr.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_landau_damping_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_landau_damping_picmi.json new file mode 100644 index 00000000000..b8328651b2e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_landau_damping_picmi.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 2726044.0531522455, + "Ey": 0.0, + "Ez": 4060168.641739453, + "jx": 177543428.93227622, + "jy": 187432087.0391676, + "jz": 594259755.4796929 + }, + "ions": { + "particle_momentum_x": 9.14159469411586e-17, + "particle_momentum_y": 9.135546407258978e-17, + "particle_momentum_z": 9.137866220831626e-17, + "particle_position_x": 1197.3344862525519, + "particle_position_y": 153269.1769037183, + "particle_weight": 8.032598963696067e+16 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_magnetic_reconnection_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_magnetic_reconnection_picmi.json new file mode 100644 index 00000000000..9a4e9056250 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_ohm_solver_magnetic_reconnection_picmi.json @@ -0,0 +1,18 @@ +{ + "lev=0": { + "Bx": 1524.8789586064377, + "By": 639.8314135126764, + "Bz": 7.475443144751569, + "Ex": 56498215.028332025, + "Ey": 75045238.66546318, + "Ez": 309589044.22395706 + }, + "ions": { + "particle_momentum_x": 7.142955285069654e-15, + "particle_momentum_y": 7.138077993721551e-15, + "particle_momentum_z": 7.141133937530391e-15, + "particle_position_x": 11170689.202379484, + "particle_position_y": 5585328.091626547, + "particle_weight": 9.036667901693183e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/parabolic_channel_initialization_2d_single_precision.json b/Regression/Checksum/benchmarks_json/test_2d_parabolic_channel_initialization.json similarity index 100% rename from Regression/Checksum/benchmarks_json/parabolic_channel_initialization_2d_single_precision.json rename to Regression/Checksum/benchmarks_json/test_2d_parabolic_channel_initialization.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_picmi.json new file mode 100644 index 00000000000..92a1ccbe638 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_picmi.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "phi": 0.0023065875621041164 + }, + "electrons": { + "particle_momentum_x": 1.1623026977941542e-25, + "particle_momentum_y": 1.0012020618770149e-25, + "particle_momentum_z": 1.0768794697418634e-25, + "particle_newPid": 750.0, + "particle_position_x": 2.4984316660445582, + "particle_position_y": 2.498475649375752, + "particle_weight": 300.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_unique_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_unique_picmi.json new file mode 100644 index 00000000000..d1630437fe0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_particle_attr_access_unique_picmi.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "phi": 0.004613047318603685 + }, + "electrons": { + "particle_momentum_x": 2.285288009419423e-25, + "particle_momentum_y": 2.0816449298979767e-25, + "particle_momentum_z": 2.0646896248001752e-25, + "particle_newPid": 1500.0, + "particle_position_x": 4.499791671680002, + "particle_position_y": 4.499957554820931, + "particle_weight": 600.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_particle_reflection_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_particle_reflection_picmi.json new file mode 100644 index 00000000000..97d0c1f5e58 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_particle_reflection_picmi.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Ex": 4.865922376234882e-11, + "Ey": 0.0, + "Ez": 2.3293326580399806e-10 + } +} diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/test_2d_particle_thermal_boundary.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_thermal_boundary.json rename to Regression/Checksum/benchmarks_json/test_2d_particle_thermal_boundary.json diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml_2d.json b/Regression/Checksum/benchmarks_json/test_2d_particles_in_pml.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particles_in_pml_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_particles_in_pml.json diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json b/Regression/Checksum/benchmarks_json/test_2d_particles_in_pml_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json rename to Regression/Checksum/benchmarks_json/test_2d_particles_in_pml_mr.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json new file mode 100644 index 00000000000..b389856e66b --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.3605690508580849, + "Bz": 0.0, + "Ex": 37101418.32484252, + "Ey": 0.0, + "Ez": 108228802.9434361, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit.json new file mode 100644 index 00000000000..fcb3081f6ae --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.35907571934346943, + "Bz": 0.0, + "Ex": 36840284.366667606, + "Ey": 0.0, + "Ez": 107777138.0847348, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} + diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit_restart.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit_restart.json new file mode 100644 index 00000000000..fcb3081f6ae --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator_implicit_restart.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.35907571934346943, + "Bz": 0.0, + "Ex": 36840284.366667606, + "Ey": 0.0, + "Ez": 107777138.0847348, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} + diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json b/Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_boosted.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json rename to Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_boosted.json diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json b/Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json rename to Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_mr.json diff --git a/Regression/Checksum/benchmarks_json/momentum-conserving-gather.json b/Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_mr_momentum_conserving.json similarity index 100% rename from Regression/Checksum/benchmarks_json/momentum-conserving-gather.json rename to Regression/Checksum/benchmarks_json/test_2d_plasma_acceleration_mr_momentum_conserving.json diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/test_2d_plasma_mirror.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PlasmaMirror.json rename to Regression/Checksum/benchmarks_json/test_2d_plasma_mirror.json diff --git a/Regression/Checksum/benchmarks_json/pml_x_ckc.json b/Regression/Checksum/benchmarks_json/test_2d_pml_x_ckc.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_x_ckc.json rename to Regression/Checksum/benchmarks_json/test_2d_pml_x_ckc.json diff --git a/Regression/Checksum/benchmarks_json/pml_x_galilean.json b/Regression/Checksum/benchmarks_json/test_2d_pml_x_galilean.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_x_galilean.json rename to Regression/Checksum/benchmarks_json/test_2d_pml_x_galilean.json diff --git a/Regression/Checksum/benchmarks_json/pml_x_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_pml_x_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_x_psatd.json rename to Regression/Checksum/benchmarks_json/test_2d_pml_x_psatd.json diff --git a/Regression/Checksum/benchmarks_json/pml_x_yee.json b/Regression/Checksum/benchmarks_json/test_2d_pml_x_yee.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_x_yee.json rename to Regression/Checksum/benchmarks_json/test_2d_pml_x_yee.json diff --git a/Regression/Checksum/benchmarks_json/pml_x_yee_eb.json b/Regression/Checksum/benchmarks_json/test_2d_pml_x_yee_eb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_x_yee_eb.json rename to Regression/Checksum/benchmarks_json/test_2d_pml_x_yee_eb.json diff --git a/Regression/Checksum/benchmarks_json/Python_prev_positions.json b/Regression/Checksum/benchmarks_json/test_2d_prev_positions_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_prev_positions.json rename to Regression/Checksum/benchmarks_json/test_2d_prev_positions_picmi.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_proton_boron_fusion.json b/Regression/Checksum/benchmarks_json/test_2d_proton_boron_fusion.json new file mode 100644 index 00000000000..2de146e45ce --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_proton_boron_fusion.json @@ -0,0 +1,117 @@ +{ + "lev=0": { + "rho": 0.0 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, + "particle_weight": 128.00000000000261 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399499907503547e+30 + }, + "alpha5": { + "particle_momentum_x": 2.3859593638404282e-14, + "particle_momentum_y": 2.38840514351394e-14, + "particle_momentum_z": 2.3991308560597987e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 4914659.635379324, + "particle_weight": 3.839999999999998e-19 + }, + "boron2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.015821768, + "particle_position_y": 819270.9858143466, + "particle_weight": 1.2799999998307316e+28 + }, + "alpha1": { + "particle_momentum_x": 4.758865177822356e-15, + "particle_momentum_y": 4.7625215232714745e-15, + "particle_momentum_z": 4.734412976565431e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 4.977661460251435e-28 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 + }, + "proton3": { + "particle_momentum_x": 1.6842001035542442e-15, + "particle_momentum_y": 1.6822700479972656e-15, + "particle_momentum_z": 1.6797740090740159e-15, + "particle_position_x": 2456930.3827292607, + "particle_position_y": 4913715.277730561, + "particle_weight": 1.2794999075035503e+30 + }, + "alpha2": { + "particle_momentum_x": 4.140109871048478e-15, + "particle_momentum_y": 4.159837925039718e-15, + "particle_momentum_z": 4.251680389225861e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 + }, + "alpha4": { + "particle_momentum_x": 2.3844872070315362e-14, + "particle_momentum_y": 2.3880317964250153e-14, + "particle_momentum_z": 2.4010930856446718e-14, + "particle_position_x": 2457367.4582781526, + "particle_position_y": 4915112.044373058, + "particle_weight": 384.0000000000002 + }, + "proton2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 + }, + "alpha3": { + "particle_momentum_x": 4.758972533179803e-16, + "particle_momentum_y": 4.744688612812961e-16, + "particle_momentum_z": 4.703298089277572e-16, + "particle_position_x": 49191.438625552386, + "particle_position_y": 98893.17267714937, + "particle_weight": 1.500277489351144e+27 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_wrappers.json b/Regression/Checksum/benchmarks_json/test_2d_python_wrappers_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_wrappers.json rename to Regression/Checksum/benchmarks_json/test_2d_python_wrappers_picmi.json diff --git a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_2d.json b/Regression/Checksum/benchmarks_json/test_2d_qed_breit_wheeler.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_breit_wheeler_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_qed_breit_wheeler.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_qed_breit_wheeler_opmd.json b/Regression/Checksum/benchmarks_json/test_2d_qed_breit_wheeler_opmd.json new file mode 100644 index 00000000000..fdfddfcbf2d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_qed_breit_wheeler_opmd.json @@ -0,0 +1,134 @@ +{ + "lev=0": { + "Ex": 0.0 + }, + "dummy_phot": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "ele1": { + "particle_opticalDepthQSR": 94805.7653371546, + "particle_position_x": 0.02366929620831116, + "particle_position_y": 0.0, + "particle_position_z": 0.023729402832031253, + "particle_momentum_x": 2.6017428344962994e-14, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 904932.0220947266 + }, + "ele2": { + "particle_opticalDepthQSR": 11848.411715651348, + "particle_position_x": 0.00294089697265625, + "particle_position_y": 0.0, + "particle_position_z": 0.00291261572265625, + "particle_momentum_x": 0.0, + "particle_momentum_y": 8.023686137822937e-15, + "particle_momentum_z": 0.0, + "particle_weight": 112028.12194824219 + }, + "ele3": { + "particle_opticalDepthQSR": 126169.82836793675, + "particle_position_x": 0.03160814697265625, + "particle_position_y": 0.0, + "particle_position_z": 0.03157382483413338, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7279204307644868e-13, + "particle_weight": 1203489.3035888672 + }, + "ele4": { + "particle_opticalDepthQSR": 49124.07007571301, + "particle_position_x": 0.012249532599865704, + "particle_position_y": 0.0, + "particle_position_z": 0.012296328078458279, + "particle_momentum_x": 3.86289566432756e-13, + "particle_momentum_y": 3.86289566432756e-13, + "particle_momentum_z": 3.86289566432756e-13, + "particle_weight": 467967.9870605469 + }, + "p1": { + "particle_opticalDepthBW": 871455.4785435478, + "particle_position_x": 0.2384708675227609, + "particle_position_y": 0.0, + "particle_position_z": 0.23841459716796876, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 9095067.977905273 + }, + "p2": { + "particle_opticalDepthBW": 1026305.8359152814, + "particle_position_x": 0.25920310302734373, + "particle_position_y": 0.0, + "particle_position_z": 0.25923138427734393, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 9887971.878051758 + }, + "p3": { + "particle_opticalDepthBW": 819556.3040882011, + "particle_position_x": 0.23053585302734383, + "particle_position_y": 0.0, + "particle_position_z": 0.230565112772128, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 8796510.696411133 + }, + "p4": { + "particle_opticalDepthBW": 953481.8671852223, + "particle_position_x": 0.24989221486908522, + "particle_position_y": 0.0, + "particle_position_z": 0.24984575334030557, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 9532032.012939453 + }, + "pos1": { + "particle_opticalDepthQSR": 95351.53381764909, + "particle_position_x": 0.02366929620831116, + "particle_position_y": 0.0, + "particle_position_z": 0.023729402832031253, + "particle_momentum_x": 2.580947118885758e-14, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 904932.0220947266 + }, + "pos2": { + "particle_opticalDepthQSR": 11715.614695181783, + "particle_position_x": 0.00294089697265625, + "particle_position_y": 0.0, + "particle_position_z": 0.00291261572265625, + "particle_momentum_x": 0.0, + "particle_momentum_y": 8.016397398195943e-15, + "particle_momentum_z": 0.0, + "particle_weight": 112028.12194824219 + }, + "pos3": { + "particle_opticalDepthQSR": 126049.551252123, + "particle_position_x": 0.03160814697265625, + "particle_position_y": 0.0, + "particle_position_z": 0.03157382483413338, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.718369575300312e-13, + "particle_weight": 1203489.3035888672 + }, + "pos4": { + "particle_opticalDepthQSR": 48953.578114257085, + "particle_position_x": 0.012249532599865704, + "particle_position_y": 0.0, + "particle_position_z": 0.012296328078458279, + "particle_momentum_x": 3.873971285219934e-13, + "particle_momentum_y": 3.873971285219934e-13, + "particle_momentum_z": 3.873971285219934e-13, + "particle_weight": 467967.9870605469 + } +} diff --git a/Regression/Checksum/benchmarks_json/qed_quantum_sync_2d.json b/Regression/Checksum/benchmarks_json/test_2d_qed_quantum_sync.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_quantum_sync_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_qed_quantum_sync.json diff --git a/Regression/Checksum/benchmarks_json/RefinedInjection.json b/Regression/Checksum/benchmarks_json/test_2d_refined_injection.json similarity index 100% rename from Regression/Checksum/benchmarks_json/RefinedInjection.json rename to Regression/Checksum/benchmarks_json/test_2d_refined_injection.json diff --git a/Regression/Checksum/benchmarks_json/RepellingParticles.json b/Regression/Checksum/benchmarks_json/test_2d_repelling_particles.json similarity index 100% rename from Regression/Checksum/benchmarks_json/RepellingParticles.json rename to Regression/Checksum/benchmarks_json/test_2d_repelling_particles.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json b/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json new file mode 100644 index 00000000000..9e876d5c23e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 3.719030475087696e-05, + "By": 0.004843257051761486, + "Bz": 5.522765606391185e-06, + "Ex": 1461264.5033270014, + "Ey": 11205.64142004876, + "Ez": 282020.7784731542, + "jx": 16437877.898892798, + "jy": 2492340.3149980744, + "jz": 215102423.57036853, + "rho": 0.7246235591902171 + }, + "beam": { + "particle_momentum_x": 2.2080215038948934e-16, + "particle_momentum_y": 2.18711072170811e-16, + "particle_momentum_z": 2.730924530737456e-15, + "particle_position_x": 0.026082358888808558, + "particle_position_y": 0.5049438607316916, + "particle_weight": 62415.090744607645 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/RigidInjection_lab.json b/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_lab.json similarity index 100% rename from Regression/Checksum/benchmarks_json/RigidInjection_lab.json rename to Regression/Checksum/benchmarks_json/test_2d_rigid_injection_lab.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_runtime_components_picmi.json b/Regression/Checksum/benchmarks_json/test_2d_runtime_components_picmi.json new file mode 100644 index 00000000000..f1eb0047d49 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_runtime_components_picmi.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "phi": 0.001516261626005395 + }, + "electrons": { + "particle_momentum_x": 7.75165529536844e-26, + "particle_momentum_y": 6.938526597814195e-26, + "particle_momentum_z": 6.572519525636007e-26, + "particle_newPid": 500.0, + "particle_position_x": 1.4999588764814886, + "particle_position_y": 1.4999551809410656, + "particle_weight": 200.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/silver_mueller_2d_x.json b/Regression/Checksum/benchmarks_json/test_2d_silver_mueller_x.json similarity index 100% rename from Regression/Checksum/benchmarks_json/silver_mueller_2d_x.json rename to Regression/Checksum/benchmarks_json/test_2d_silver_mueller_x.json diff --git a/Regression/Checksum/benchmarks_json/silver_mueller_2d_z.json b/Regression/Checksum/benchmarks_json/test_2d_silver_mueller_z.json similarity index 100% rename from Regression/Checksum/benchmarks_json/silver_mueller_2d_z.json rename to Regression/Checksum/benchmarks_json/test_2d_silver_mueller_z.json diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json b/Regression/Checksum/benchmarks_json/test_2d_space_charge_initialization.json similarity index 100% rename from Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_space_charge_initialization.json diff --git a/Regression/Checksum/benchmarks_json/subcyclingMR.json b/Regression/Checksum/benchmarks_json/test_2d_subcycling_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/subcyclingMR.json rename to Regression/Checksum/benchmarks_json/test_2d_subcycling_mr.json diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json new file mode 100644 index 00000000000..d342c49e2fd --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 65625.24877705125, + "By": 71913.65275407257, + "Bz": 59768.79247890749, + "Ex": 56341360261928.086, + "Ey": 13926508614721.855, + "Ez": 56508162715968.17, + "divE": 5.5816922509658905e+22, + "jx": 1.8114330881270456e+19, + "jy": 2.0727708668063334e+19, + "jz": 1.7843765469944717e+19, + "rho": 494213515033.04443 + }, + "electrons": { + "particle_momentum_x": 4.888781979240524e-19, + "particle_momentum_y": 4.879904653089102e-19, + "particle_momentum_z": 4.878388335258947e-19, + "particle_position_x": 0.0042514822919144084, + "particle_position_y": 0.0042515394083575886, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0873319751377048e-17, + "particle_momentum_y": 2.0858882863041667e-17, + "particle_momentum_z": 2.0877426824914187e-17, + "particle_position_x": 0.004251275869325256, + "particle_position_y": 0.0042512738905204584, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json rename to Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_picmi.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json new file mode 100644 index 00000000000..5281804abba --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 60642.062637340816, + "By": 89855.09371265332, + "Bz": 54561.47120738846, + "Ex": 81536346169528.28, + "Ey": 13888711042388.54, + "Ez": 86853122458391.0, + "divE": 9.492653438830812e+22, + "jx": 2.5941826848709296e+19, + "jy": 2.9929071160915993e+19, + "jz": 2.692985701872205e+19, + "rho": 851978517887.51 + }, + "electrons": { + "particle_momentum_x": 4.864385990952573e-19, + "particle_momentum_y": 4.879723483907468e-19, + "particle_momentum_z": 4.865564630727981e-19, + "particle_position_x": 0.004250851253052539, + "particle_position_y": 0.0042513622554793, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0934469726422704e-17, + "particle_momentum_y": 2.0929630794865952e-17, + "particle_momentum_z": 2.093085625201003e-17, + "particle_position_x": 0.004251276208274589, + "particle_position_y": 0.004251274670600805, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/Uniform_2d.json b/Regression/Checksum/benchmarks_json/test_2d_uniform_plasma.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Uniform_2d.json rename to Regression/Checksum/benchmarks_json/test_2d_uniform_plasma.json diff --git a/Regression/Checksum/benchmarks_json/VayDeposition2D.json b/Regression/Checksum/benchmarks_json/test_2d_vay_deposition.json similarity index 100% rename from Regression/Checksum/benchmarks_json/VayDeposition2D.json rename to Regression/Checksum/benchmarks_json/test_2d_vay_deposition.json diff --git a/Regression/Checksum/benchmarks_json/restart.json b/Regression/Checksum/benchmarks_json/test_3d_acceleration.json similarity index 100% rename from Regression/Checksum/benchmarks_json/restart.json rename to Regression/Checksum/benchmarks_json/test_3d_acceleration.json diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_acceleration_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/restart_psatd.json rename to Regression/Checksum/benchmarks_json/test_3d_acceleration_psatd.json diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/test_3d_acceleration_psatd_time_avg.json similarity index 100% rename from Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json rename to Regression/Checksum/benchmarks_json/test_3d_acceleration_psatd_time_avg.json diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_averaged_galilean_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd.json rename to Regression/Checksum/benchmarks_json/test_3d_averaged_galilean_psatd.json diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_3d_averaged_galilean_psatd_hybrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd_hybrid.json rename to Regression/Checksum/benchmarks_json/test_3d_averaged_galilean_psatd_hybrid.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_beam_beam_collision.json b/Regression/Checksum/benchmarks_json/test_3d_beam_beam_collision.json new file mode 100644 index 00000000000..ad478e96e2f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_beam_beam_collision.json @@ -0,0 +1,96 @@ +{ + "lev=0": { + "Bx": 461189750202.8409, + "By": 461177227889.41614, + "Bz": 20074725.954957746, + "Ex": 1.3827102525768512e+20, + "Ey": 1.3827414851469086e+20, + "Ez": 3.5091001092648376e+16, + "rho_beam1": 3.829927705533972e+16, + "rho_beam2": 3.830015035966193e+16, + "rho_ele1": 162411820580232.94, + "rho_ele2": 150511095307150.62, + "rho_pos1": 159308102449286.12, + "rho_pos2": 156118194376152.7 + }, + "beam1": { + "particle_opticalDepthQSR": 52563.18675345914, + "particle_position_x": 0.0007501135479816878, + "particle_position_y": 0.0007501095180907413, + "particle_position_z": 0.002483074586668512, + "particle_momentum_x": 3.0935661449588394e-15, + "particle_momentum_y": 3.070048977445414e-15, + "particle_momentum_z": 3.4017266957145416e-12, + "particle_weight": 636083515.3729652 + }, + "beam2": { + "particle_opticalDepthQSR": 52275.42552501091, + "particle_position_x": 0.0007500428425956199, + "particle_position_y": 0.0007500867178448842, + "particle_position_z": 0.0024830812114940977, + "particle_momentum_x": 3.1124995623090863e-15, + "particle_momentum_y": 3.094827769550167e-15, + "particle_momentum_z": 3.4015150389915676e-12, + "particle_weight": 636114264.930704 + }, + "ele1": { + "particle_opticalDepthQSR": 156.022199846285, + "particle_position_x": 2.4401923319868757e-06, + "particle_position_y": 2.399150249907213e-06, + "particle_position_z": 8.791577071017722e-06, + "particle_momentum_x": 2.7299291171683895e-18, + "particle_momentum_y": 2.6551510668418435e-18, + "particle_momentum_z": 1.33445643731407e-15, + "particle_weight": 2656838.9769653436 + }, + "ele2": { + "particle_opticalDepthQSR": 163.79686010701988, + "particle_position_x": 2.724737203764692e-06, + "particle_position_y": 2.9829403746737846e-06, + "particle_position_z": 9.127382649103148e-06, + "particle_momentum_x": 2.1813342297510976e-18, + "particle_momentum_y": 2.7643067192718357e-18, + "particle_momentum_z": 1.259574872512064e-15, + "particle_weight": 2517356.358594387 + }, + "pho1": { + "particle_opticalDepthBW": 5007.597539698783, + "particle_position_x": 7.214053121897416e-05, + "particle_position_y": 7.223804186317301e-05, + "particle_position_z": 0.00024115699590772453, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 62860453.544321 + }, + "pho2": { + "particle_opticalDepthBW": 5113.753887045111, + "particle_position_x": 7.271625175781002e-05, + "particle_position_y": 7.311023374122331e-05, + "particle_position_z": 0.00024123464128276151, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 59821371.52413007 + }, + "pos1": { + "particle_opticalDepthQSR": 176.87865344045926, + "particle_position_x": 2.597218595285766e-06, + "particle_position_y": 2.5071002403671178e-06, + "particle_position_z": 8.190923176799435e-06, + "particle_momentum_x": 2.409544640420923e-18, + "particle_momentum_y": 2.5096320511498773e-18, + "particle_momentum_z": 1.3349884612525734e-15, + "particle_weight": 2604339.6419650833 + }, + "pos2": { + "particle_opticalDepthQSR": 229.50925371797547, + "particle_position_x": 2.6205324097963396e-06, + "particle_position_y": 2.8134541282576216e-06, + "particle_position_z": 9.865542956073817e-06, + "particle_momentum_x": 2.536744632018102e-18, + "particle_momentum_y": 3.035517414633681e-18, + "particle_momentum_z": 1.3203905663185877e-15, + "particle_weight": 2570091.732557297 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/collider_diagnostics.json b/Regression/Checksum/benchmarks_json/test_3d_collider_diagnostics.json similarity index 100% rename from Regression/Checksum/benchmarks_json/collider_diagnostics.json rename to Regression/Checksum/benchmarks_json/test_3d_collider_diagnostics.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_collision_iso.json b/Regression/Checksum/benchmarks_json/test_3d_collision_iso.json new file mode 100644 index 00000000000..c14fcbe99d8 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_collision_iso.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "electron": { + "particle_momentum_x": 3.578935809964031e-19, + "particle_momentum_y": 3.5778028343192025e-19, + "particle_momentum_z": 3.579884355240226e-19, + "particle_position_x": 1.0241442531780067, + "particle_position_y": 1.0238915904698023, + "particle_position_z": 1.024005350488445, + "particle_weight": 714240000000.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_collision_xyz.json b/Regression/Checksum/benchmarks_json/test_3d_collision_xyz.json new file mode 100644 index 00000000000..a3769ba7fa0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_collision_xyz.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "T_electron": 351188.7347045234, + "T_ion": 350097.5453711827 + }, + "electron": { + "particle_momentum_x": 8.348736016166693e-19, + "particle_momentum_y": 8.175754844970833e-19, + "particle_momentum_z": 8.209507928471627e-19, + "particle_position_x": 21258578.13072786, + "particle_position_y": 21266758.00828195, + "particle_position_z": 21207386.447255243, + "particle_weight": 7.168263344048695e+28 + }, + "ion": { + "particle_momentum_x": 2.008818164997072e-18, + "particle_momentum_y": 1.829384185025146e-18, + "particle_momentum_z": 1.8289477333243715e-18, + "particle_position_x": 21233610.538310427, + "particle_position_y": 21280892.516510233, + "particle_position_z": 21213150.945697505, + "particle_weight": 7.168263344048695e+28 + } +} diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/test_3d_deuterium_deuterium_fusion.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json rename to Regression/Checksum/benchmarks_json/test_3d_deuterium_deuterium_fusion.json diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/test_3d_deuterium_deuterium_fusion_intraspecies.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json rename to Regression/Checksum/benchmarks_json/test_3d_deuterium_deuterium_fusion_intraspecies.json diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/test_3d_deuterium_tritium_fusion.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json rename to Regression/Checksum/benchmarks_json/test_3d_deuterium_tritium_fusion.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_leptons.json b/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_leptons.json new file mode 100644 index 00000000000..5764278a0c0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_leptons.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "rho_beam1": 656097730.398061, + "rho_beam2": 656073566.2600528 + }, + "beam2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.3357156607472946e-11, + "particle_position_x": 0.07980004798883583, + "particle_position_y": 0.0015964156825818534, + "particle_position_z": 240184.81987149065, + "particle_weight": 11997240000.0 + }, + "beam1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.3357730733347833e-11, + "particle_position_x": 0.0796200211671279, + "particle_position_y": 0.001592794429510446, + "particle_position_z": 239913.37896451252, + "particle_weight": 11997780000.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_photons.json b/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_photons.json new file mode 100644 index 00000000000..86659bafd79 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_diff_lumi_diag_photons.json @@ -0,0 +1,25 @@ +{ + "lev=0": { + "rho_beam1": 0.0, + "rho_beam2": 0.0 + }, + "beam2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7511853009715152e-11, + "particle_position_x": 0.2621440000000001, + "particle_position_y": 0.005242880000000004, + "particle_position_z": 314572.8000000002, + "particle_weight": 11997744756.909575 + }, + "beam1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.75121641230803e-11, + "particle_position_x": 0.2621440000000001, + "particle_position_y": 0.005242880000000004, + "particle_position_z": 314572.8000000004, + "particle_weight": 11997744756.909573 + } +} + diff --git a/Regression/Checksum/benchmarks_json/divb_cleaning_3d.json b/Regression/Checksum/benchmarks_json/test_3d_divb_cleaning.json similarity index 100% rename from Regression/Checksum/benchmarks_json/divb_cleaning_3d.json rename to Regression/Checksum/benchmarks_json/test_3d_divb_cleaning.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_dive_cleaning.json b/Regression/Checksum/benchmarks_json/test_3d_dive_cleaning.json new file mode 100644 index 00000000000..b8206240500 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_dive_cleaning.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 1.9159134471952935e-20, + "By": 1.8827238279614072e-20, + "Bz": 1.8885687211875642e-20, + "Ex": 8648.536817097653, + "Ey": 8613.089981021956, + "Ez": 8626.889465408336, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "beam": { + "particle_momentum_x": 2.3895400750846334e-27, + "particle_momentum_y": 2.3729444184823814e-27, + "particle_momentum_z": 2.390801909783316e-27, + "particle_position_x": 0.031866021988156114, + "particle_position_y": 0.032281276107277185, + "particle_position_z": 0.03185444043880588, + "particle_weight": 0.06241509074460764 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json new file mode 100644 index 00000000000..1f9f0a77b5a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 144495.08082507108, + "By": 144495.08082507114, + "Bz": 8481.958724628861, + "Ex": 54500496182517.92, + "Ey": 54500496182517.91, + "Ez": 70231240245509.39 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_effective_potential_electrostatic_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_effective_potential_electrostatic_picmi.json new file mode 100644 index 00000000000..b61da644114 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_effective_potential_electrostatic_picmi.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Ex": 148992.08170563323, + "Ey": 148964.62980386653, + "Ez": 149085.76473996745, + "T_electrons": 279.2796849134268, + "T_ions": 32.09677271761641, + "jx": 31.077856013948246, + "jy": 31.425549493321245, + "jz": 31.424168300110658, + "phi": 2002.2518068289028, + "rho_electrons": 0.007909027251581852, + "rho_ions": 0.008266762306092332 + } +} diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphere.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json new file mode 100644 index 00000000000..561fbf86669 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json @@ -0,0 +1,17 @@ +{ + "lev=0": { + "Ex": 5.177444767224255, + "Ey": 5.177444767224254, + "Ez": 5.177444767224256, + "rho": 2.6092568008333797e-10 + }, + "electron": { + "particle_momentum_x": 1.3215019655285216e-23, + "particle_momentum_y": 1.3215019655285214e-23, + "particle_momentum_z": 1.3215019655285217e-23, + "particle_position_x": 912.2310003741203, + "particle_position_y": 912.2310003741203, + "particle_position_z": 912.2310003741202, + "particle_weight": 6212.501525878906 + } +} diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereEB.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_mixedBCs.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb_mixed_bc.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_mixedBCs.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb_mixed_bc.json diff --git a/Regression/Checksum/benchmarks_json/Python_ElectrostaticSphereEB.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_ElectrostaticSphereEB.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_eb_picmi.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_lab_frame.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_lab_frame.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_lab_frame_mr_emass_10.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_lab_frame_mr_emass_10.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_rel_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json rename to Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_rel_nodal.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json new file mode 100644 index 00000000000..9563c52adbe --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 4.166971025838921e-18, + "By": 0.006628374119786834, + "Bz": 0.006628374119786834, + "Ex": 5102618.471153786, + "Ey": 1.4283859321773714e-05, + "Ez": 1.4283859321773714e-05 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json new file mode 100644 index 00000000000..67bdbea18ca --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 4.228863291892693e-18, + "By": 0.005101824310293573, + "Bz": 0.005101824310293573, + "Ex": 4414725.184732471, + "Ey": 1.4283895626502055e-05, + "Ez": 1.4283895626502055e-05 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..d3e08d9723e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 4.928354322096152e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..d3e08d9723e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 4.928354322096152e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..23c03c7e7bc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 4.3355127342920327e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_picmi.json new file mode 100644 index 00000000000..f3483a544b5 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_picmi.json @@ -0,0 +1,5 @@ +{ + "lev=0": { + "Ex": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..109f5fb4d35 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 3.835377401535272e-15, + "By": 7.634560527952392e-15, + "Bz": 7.670097149513554e-15, + "Ex": 1.837433604148419e-06, + "Ey": 1.4507850267928362e-06, + "Ez": 1.4325637523931794e-06, + "divE": 7.330866223540695e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..9ff91af6550 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 2.3948084603369097e-15, + "By": 0.0, + "Bz": 6.747158562891953e-07, + "Ex": 0.0, + "Ey": 5.541309886315263e-07, + "Ez": 0.0, + "divE": 2.091715826275267e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..0e0cdf5eb0f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 2.9100687916345874e-15, + "By": 6.121275580503632e-15, + "Bz": 5.9043095451081354e-15, + "Ex": 1.4574231057224582e-06, + "Ey": 1.1648744803916206e-06, + "Ez": 1.16164257835401e-06, + "divE": 6.781029990295743e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json new file mode 100644 index 00000000000..118214948a5 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 1.280747509243305e-05, + "By": 2.473900144296397e-02, + "Bz": 2.473890786894079e-02, + "Ex": 1.025322901921306e+07, + "Ey": 1.042254197269831e+04, + "Ez": 1.040011664019071e+04 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/FluxInjection3D.json b/Regression/Checksum/benchmarks_json/test_3d_flux_injection.json similarity index 100% rename from Regression/Checksum/benchmarks_json/FluxInjection3D.json rename to Regression/Checksum/benchmarks_json/test_3d_flux_injection.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json new file mode 100644 index 00000000000..c1c888ff808 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json @@ -0,0 +1,12 @@ +{ + "lev=0": {}, + "electron": { + "particle_momentum_x": 1.7587772989573373e-17, + "particle_momentum_y": 1.7608560965806728e-17, + "particle_momentum_z": 1.7596701993624562e-17, + "particle_position_x": 902783.9285213391, + "particle_position_y": 902981.7980528818, + "particle_position_z": 902777.1246066706, + "particle_weight": 2.503818749999996e-07 + } +} diff --git a/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json b/Regression/Checksum/benchmarks_json/test_3d_focusing_gaussian_beam.json similarity index 100% rename from Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json rename to Regression/Checksum/benchmarks_json/test_3d_focusing_gaussian_beam.json diff --git a/Regression/Checksum/benchmarks_json/galilean_3d_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_galilean_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_3d_psatd.json rename to Regression/Checksum/benchmarks_json/test_3d_galilean_psatd.json diff --git a/Regression/Checksum/benchmarks_json/galilean_3d_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_3d_galilean_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_3d_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_3d_galilean_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/galilean_3d_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/test_3d_galilean_psatd_current_correction_psb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_3d_psatd_current_correction_psb.json rename to Regression/Checksum/benchmarks_json/test_3d_galilean_psatd_current_correction_psb.json diff --git a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json b/Regression/Checksum/benchmarks_json/test_3d_gaussian_beam_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_gaussian_beam.json rename to Regression/Checksum/benchmarks_json/test_3d_gaussian_beam_picmi.json diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json b/Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles.json similarity index 100% rename from Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json rename to Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles_boosted.json b/Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles_boosted.json new file mode 100644 index 00000000000..0a601b7b437 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles_boosted.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 3.254604354043409e-14, + "By": 3.2768679907552955e-14, + "Bz": 1.0615351421410278e-16, + "Ex": 2.3084916770539354e-05, + "Ey": 2.2657235922655432e-05, + "Ez": 1.9978004351148e-05, + "jx": 1.781971994166362e-10, + "jy": 4.2163624424546344e-20, + "jz": 1.0378980680353126e-07 + }, + "electron": { + "particle_momentum_x": 5.955475927655105e-26, + "particle_momentum_y": 1.4613271542201658e-35, + "particle_momentum_z": 3.468728453537439e-23, + "particle_position_x": 0.04996023704063194, + "particle_position_y": 8.398113230295983e-15, + "particle_position_z": 0.10931682580470406, + "particle_weight": 1.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json b/Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles_moving.json similarity index 100% rename from Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json rename to Regression/Checksum/benchmarks_json/test_3d_hard_edged_quadrupoles_moving.json diff --git a/Regression/Checksum/benchmarks_json/initial_distribution.json b/Regression/Checksum/benchmarks_json/test_3d_initial_distribution.json similarity index 100% rename from Regression/Checksum/benchmarks_json/initial_distribution.json rename to Regression/Checksum/benchmarks_json/test_3d_initial_distribution.json diff --git a/Regression/Checksum/benchmarks_json/ion_stopping.json b/Regression/Checksum/benchmarks_json/test_3d_ion_stopping.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ion_stopping.json rename to Regression/Checksum/benchmarks_json/test_3d_ion_stopping.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_fluid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_fluid.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_Langmuir.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction_nodal.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_current_correction_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction_nodal.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_current_correction_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_div_cleaning.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_div_cleaning.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_momentum_conserving.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_momentum_conserving.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_multiJ.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_multiJ.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ_nodal.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_multiJ_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ_nodal.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_multiJ_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_single_precision.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_single_precision.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_Vay_deposition.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_vay_deposition.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_Vay_deposition.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_vay_deposition.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_Vay_deposition_nodal.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_vay_deposition_nodal.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_Vay_deposition_nodal.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_psatd_vay_deposition_nodal.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json b/Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_single_precision.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json rename to Regression/Checksum/benchmarks_json/test_3d_langmuir_multi_single_precision.json diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration.json b/Regression/Checksum/benchmarks_json/test_3d_laser_acceleration.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAcceleration.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_acceleration.json diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_btd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_btd.json diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_picmi.json diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_single_precision_comms.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_acceleration_single_precision_comms.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjection.json b/Regression/Checksum/benchmarks_json/test_3d_laser_injection.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjection.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_injection.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile.json b/Regression/Checksum/benchmarks_json/test_3d_laser_injection_from_lasy_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile.json rename to Regression/Checksum/benchmarks_json/test_3d_laser_injection_from_lasy_file.json diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json b/Regression/Checksum/benchmarks_json/test_3d_load_external_field_grid_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json rename to Regression/Checksum/benchmarks_json/test_3d_load_external_field_grid_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json b/Regression/Checksum/benchmarks_json/test_3d_load_external_field_particle_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json rename to Regression/Checksum/benchmarks_json/test_3d_load_external_field_particle_picmi.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb.json b/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb.json new file mode 100644 index 00000000000..6415fc3e930 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Az": 11.358663299932457, + "Bx": 111.55929615203162, + "By": 111.55929615203165, + "Ex": 31463410849.74626, + "Ey": 31463410849.746258, + "jz": 1034841323.6861029, + "phi": 3164328318.15416, + "rho": 3.4565836983918676 + }, + "beam": { + "particle_momentum_x": 1.3829464728617761e-21, + "particle_momentum_y": 1.3829464728617792e-21, + "particle_momentum_z": 7.150871807235339e-16, + "particle_position_x": 11163.99997715059, + "particle_position_y": 11163.999977150592, + "particle_position_z": 131662.5003102683, + "particle_weight": 20895107655113.465 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb_picmi.json new file mode 100644 index 00000000000..2c99a4218c2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_magnetostatic_eb_picmi.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "Ax": 1.40889223759456e-05, + "Ay": 1.4088922375945606e-05, + "Az": 11.423480450267745, + "Bx": 112.23826705481486, + "By": 112.23826705481484, + "Bz": 0.00019199345672949735, + "Ex": 31557746267.686367, + "Ey": 31557746267.686363, + "Ez": 3339526660.3539834, + "jx": 1980.6549408566577, + "jy": 1980.6549408566577, + "jz": 1038931605.1197203, + "phi": 3171976204.251914, + "rho": 3.4840085919357926 + }, + "beam": { + "particle_momentum_x": 1.4011190163358655e-21, + "particle_momentum_y": 1.401119016335865e-21, + "particle_momentum_z": 7.15087179293042e-16, + "particle_position_x": 11163.99997543546, + "particle_position_y": 11163.999975435456, + "particle_position_z": 131662.50031026747, + "particle_weight": 20895107655113.465 + } +} diff --git a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json b/Regression/Checksum/benchmarks_json/test_3d_nodal_electrostatic_solver.json similarity index 100% rename from Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json rename to Regression/Checksum/benchmarks_json/test_3d_nodal_electrostatic_solver.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_ohm_solver_cylinder_compression_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_ohm_solver_cylinder_compression_picmi.json new file mode 100644 index 00000000000..6cde3a9450e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_ohm_solver_cylinder_compression_picmi.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.5334253070691776, + "By": 0.5318560243634998, + "Bz": 2252.108905639938, + "Ex": 10509838.331420777, + "Ey": 10512676.798857061, + "Ez": 8848.113963901804, + "rho": 384112.2912140536 + }, + "ions": { + "particle_momentum_x": 2.161294367543349e-16, + "particle_momentum_y": 2.161870747294985e-16, + "particle_momentum_z": 2.0513400435256855e-16, + "particle_position_x": 769864.202585846, + "particle_position_y": 769908.6569812088, + "particle_position_z": 620721.1900338201, + "particle_weight": 1.008292384042714e+19 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json new file mode 100644 index 00000000000..80561aaa4e1 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 100915933.44604117, + "By": 157610622.18548763, + "Bz": 9.614441087794229e-14, + "Ex": 4.725065270619209e+16, + "Ey": 3.025394898938681e+16, + "Ez": 3276573.9514776673, + "rho": 10994013582437.193 + }, + "electron": { + "particle_momentum_x": 5.7012776060557455e-19, + "particle_momentum_y": 3.650451663685222e-19, + "particle_momentum_z": 1.145432768297242e-10, + "particle_position_x": 17.314086912497864, + "particle_position_y": 0.25836912671877954, + "particle_position_z": 10066.329600000008, + "particle_weight": 19969036501.910976 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver_sliced.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver_sliced.json new file mode 100644 index 00000000000..fd4a9afbc29 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver_sliced.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Bx": 100915933.44551668, + "By": 157610622.18551716, + "Bz": 2.598515299403035e-15, + "Ex": 4.725065270620093e+16, + "Ey": 3.0253948989229424e+16, + "Ez": 2787743.3330717986, + "rho": 10994013582437.193 + }, + "electron": { + "particle_momentum_x": 5.701277606056779e-19, + "particle_momentum_y": 3.650451663675671e-19, + "particle_momentum_z": 1.145432768297242e-10, + "particle_position_x": 17.314086912497864, + "particle_position_y": 0.25836912671877954, + "particle_position_z": 10066.329600000008, + "particle_weight": 19969036501.910976 + } +} + diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json b/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json new file mode 100644 index 00000000000..3dc9d956b79 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 198610.0530604908, + "By": 198610.0530604909, + "Bz": 8482.656173586969, + "Ex": 37232105734622.53, + "Ey": 37232105734622.54, + "Ez": 85094015810307.19 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particle_boundaries_3d.json b/Regression/Checksum/benchmarks_json/test_3d_particle_boundaries.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_boundaries_3d.json rename to Regression/Checksum/benchmarks_json/test_3d_particle_boundaries.json diff --git a/Regression/Checksum/benchmarks_json/particle_fields_diags.json b/Regression/Checksum/benchmarks_json/test_3d_particle_fields_diags.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_fields_diags.json rename to Regression/Checksum/benchmarks_json/test_3d_particle_fields_diags.json diff --git a/Regression/Checksum/benchmarks_json/particle_fields_diags_single_precision.json b/Regression/Checksum/benchmarks_json/test_3d_particle_fields_diags_single_precision.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_fields_diags_single_precision.json rename to Regression/Checksum/benchmarks_json/test_3d_particle_fields_diags_single_precision.json diff --git a/Regression/Checksum/benchmarks_json/particle_pusher.json b/Regression/Checksum/benchmarks_json/test_3d_particle_pusher.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_pusher.json rename to Regression/Checksum/benchmarks_json/test_3d_particle_pusher.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json new file mode 100644 index 00000000000..9437ebed275 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 144495.08082507108, + "By": 144495.0808250711, + "Bz": 8481.95872462886, + "Ex": 54500496182517.914, + "Ey": 54500496182517.914, + "Ez": 70231240245509.4 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json new file mode 100644 index 00000000000..1f9f0a77b5a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bx": 144495.08082507108, + "By": 144495.08082507114, + "Bz": 8481.958724628861, + "Ex": 54500496182517.92, + "Ey": 54500496182517.91, + "Ez": 70231240245509.39 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml.json b/Regression/Checksum/benchmarks_json/test_3d_particles_in_pml.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particles_in_pml.json rename to Regression/Checksum/benchmarks_json/test_3d_particles_in_pml.json diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml_3d_MR.json b/Regression/Checksum/benchmarks_json/test_3d_particles_in_pml_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particles_in_pml_3d_MR.json rename to Regression/Checksum/benchmarks_json/test_3d_particles_in_pml_mr.json diff --git a/Regression/Checksum/benchmarks_json/PEC_field.json b/Regression/Checksum/benchmarks_json/test_3d_pec_field.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PEC_field.json rename to Regression/Checksum/benchmarks_json/test_3d_pec_field.json diff --git a/Regression/Checksum/benchmarks_json/PEC_field_mr.json b/Regression/Checksum/benchmarks_json/test_3d_pec_field_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PEC_field_mr.json rename to Regression/Checksum/benchmarks_json/test_3d_pec_field_mr.json diff --git a/Regression/Checksum/benchmarks_json/PEC_particle.json b/Regression/Checksum/benchmarks_json/test_3d_pec_particle.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PEC_particle.json rename to Regression/Checksum/benchmarks_json/test_3d_pec_particle.json diff --git a/Regression/Checksum/benchmarks_json/photon_pusher.json b/Regression/Checksum/benchmarks_json/test_3d_photon_pusher.json similarity index 100% rename from Regression/Checksum/benchmarks_json/photon_pusher.json rename to Regression/Checksum/benchmarks_json/test_3d_photon_pusher.json diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_boosted.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_boosted.json diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d_hybrid.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_boosted_hybrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d_hybrid.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_boosted_hybrid.json diff --git a/Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_mr_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_mr_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_acceleration_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Plasma_lens.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Plasma_lens.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_lens.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_boosted.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_boosted.json new file mode 100644 index 00000000000..e1fa54618ee --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_boosted.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Bx": 1.307357220398482e-14, + "By": 1.3033571630685163e-14, + "Bz": 5.594998319468307e-17, + "Ex": 2.8010832905044288e-06, + "Ey": 2.8088096742407935e-06, + "Ez": 3.3433681277560495e-06, + "jx": 2.5151718871714067e-11, + "jy": 2.013398608921663e-11, + "jz": 6.0063967622563335e-09 + }, + "electrons": { + "particle_momentum_x": 7.43708887164806e-24, + "particle_momentum_y": 5.949505779760011e-24, + "particle_momentum_z": 5.117548636790359e-22, + "particle_position_x": 0.03648994812700447, + "particle_position_y": 0.029201183320618985, + "particle_position_z": 6.968107021318396 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/hard_edged_plasma_lens.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_hard_edged.json similarity index 100% rename from Regression/Checksum/benchmarks_json/hard_edged_plasma_lens.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_lens_hard_edged.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_picmi.json new file mode 100644 index 00000000000..205bf8204dd --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_picmi.json @@ -0,0 +1,21 @@ +{ + "lev=0": { + "Bx": 3.742282886653039e-14, + "By": 3.733653562337366e-14, + "Bz": 3.159003724979974e-16, + "Ex": 4.413173824952238e-06, + "Ey": 4.440807110847932e-06, + "Ez": 8.994610621212147e-06, + "jx": 2.294712258669695e-10, + "jy": 1.8314117733996873e-10, + "jz": 2.1990787829485306e-08 + }, + "electrons": { + "particle_momentum_x": 7.424668333878816e-24, + "particle_momentum_y": 5.9396389377972404e-24, + "particle_momentum_z": 2.730924530737562e-22, + "particle_position_x": 0.03608389438974155, + "particle_position_y": 0.028872102262786022, + "particle_position_z": 3.894799963324232 + } +} diff --git a/Regression/Checksum/benchmarks_json/Plasma_lens_short.json b/Regression/Checksum/benchmarks_json/test_3d_plasma_lens_short.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Plasma_lens_short.json rename to Regression/Checksum/benchmarks_json/test_3d_plasma_lens_short.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_pmc_field.json b/Regression/Checksum/benchmarks_json/test_3d_pmc_field.json new file mode 100644 index 00000000000..486f8bb965d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_pmc_field.json @@ -0,0 +1,6 @@ +{ + "lev=0": { + "Bx": 4.1354151621557795, + "Ey": 8373879983.480644 + } +} diff --git a/Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json b/Regression/Checksum/benchmarks_json/test_3d_pml_psatd_dive_divb_cleaning.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json rename to Regression/Checksum/benchmarks_json/test_3d_pml_psatd_dive_divb_cleaning.json diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json b/Regression/Checksum/benchmarks_json/test_3d_point_of_contact_eb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json rename to Regression/Checksum/benchmarks_json/test_3d_point_of_contact_eb.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_callback_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_callback_picmi.json new file mode 100644 index 00000000000..bd44f9d2c44 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_callback_picmi.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Bx": 16281.68118911512, + "By": 16281.681189115117, + "Bz": 78850.29928317585 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_picmi.json new file mode 100644 index 00000000000..7bf927205ee --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_projection_divb_cleaner_picmi.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Bx": 51.23782716855388, + "By": 51.23782716855387, + "Bz": 341.0698792433061 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_proton_boron_fusion.json b/Regression/Checksum/benchmarks_json/test_3d_proton_boron_fusion.json new file mode 100644 index 00000000000..5c9d00d6f70 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_proton_boron_fusion.json @@ -0,0 +1,132 @@ +{ + "lev=0": { + "rho": 0.0 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119677303391259e+31 + }, + "alpha2": { + "particle_momentum_x": 4.172264972648105e-15, + "particle_momentum_y": 4.1838781269703316e-15, + "particle_momentum_z": 4.256769380749126e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 + }, + + "alpha1": { + "particle_momentum_x": 4.743550273664842e-15, + "particle_momentum_y": 4.703189539479841e-15, + "particle_momentum_z": 4.75829084832661e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 3.2387855108185994e-27 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 + }, + "proton3": { + "particle_momentum_x": 1.6843593058271961e-15, + "particle_momentum_y": 1.6823892851456668e-15, + "particle_momentum_z": 1.6800222030208115e-15, + "particle_position_x": 2457072.7669634065, + "particle_position_y": 2456771.196542574, + "particle_position_z": 4914021.8271464575, + "particle_weight": 1.0236773033912632e+31 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 + }, + "alpha3": { + "particle_momentum_x": 4.529369912319074e-16, + "particle_momentum_y": 4.55762819982174e-16, + "particle_momentum_z": 4.501223434274883e-16, + "particle_position_x": 48732.63289890166, + "particle_position_y": 46262.85664926968, + "particle_position_z": 95593.72900482587, + "particle_weight": 9.680898262134895e+27 + }, + "alpha5": { + "particle_momentum_x": 2.3875154301762454e-14, + "particle_momentum_y": 2.3859525730749002e-14, + "particle_momentum_z": 2.4029799910201597e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 2457059.6353793237, + "particle_position_z": 4915847.043341332, + "particle_weight": 3.0719999999999984e-18 + }, + "alpha4": { + "particle_momentum_x": 2.3842314895045605e-14, + "particle_momentum_y": 2.386245392919704e-14, + "particle_momentum_z": 2.4007606987094537e-14, + "particle_position_x": 2457367.4582781545, + "particle_position_y": 2457512.044373057, + "particle_position_z": 4914475.7765130745, + "particle_weight": 3072.000000000002 + }, + "proton2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 + }, + "boron2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.0239999998887592e+29 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + } +} diff --git a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_3d.json b/Regression/Checksum/benchmarks_json/test_3d_qed_breit_wheeler.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_breit_wheeler_3d.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_breit_wheeler.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_qed_breit_wheeler_opmd.json b/Regression/Checksum/benchmarks_json/test_3d_qed_breit_wheeler_opmd.json new file mode 100644 index 00000000000..bf98f8bd963 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_qed_breit_wheeler_opmd.json @@ -0,0 +1,134 @@ +{ + "lev=0": { + "Ex": 0.0 + }, + "dummy_phot": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "ele1": { + "particle_opticalDepthQSR": 150084.76186868473, + "particle_position_x": 0.037522814130005, + "particle_position_y": 0.0375364375, + "particle_position_z": 0.0375473125, + "particle_momentum_x": 4.111367826819435e-14, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 1.4303398132324217 + }, + "ele2": { + "particle_opticalDepthQSR": 19166.68554224755, + "particle_position_x": 0.0047233984375, + "particle_position_y": 0.004773031169456623, + "particle_position_z": 0.004769421875, + "particle_momentum_x": 0.0, + "particle_momentum_y": 1.3120815641048112e-14, + "particle_momentum_z": 0.0, + "particle_weight": 0.18199920654296872 + }, + "ele3": { + "particle_opticalDepthQSR": 197996.95086384623, + "particle_position_x": 0.049603601562499995, + "particle_position_y": 0.049503953125, + "particle_position_z": 0.049484700428793646, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.703275178685535e-13, + "particle_weight": 1.8893241882324217 + }, + "ele4": { + "particle_opticalDepthQSR": 79277.54339646622, + "particle_position_x": 0.019832326822916666, + "particle_position_y": 0.019875475260416667, + "particle_position_z": 0.019791898437499997, + "particle_momentum_x": 6.270940742446365e-13, + "particle_momentum_y": 6.270940742446365e-13, + "particle_momentum_z": 6.270940742446365e-13, + "particle_weight": 0.7561206817626952 + }, + "p1": { + "particle_opticalDepthBW": 779753.6899265796, + "particle_position_x": 0.22462570742050836, + "particle_position_y": 0.22460756249999997, + "particle_position_z": 0.22459668749999998, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 8.569660186767576 + }, + "p2": { + "particle_opticalDepthBW": 1011632.5072099702, + "particle_position_x": 0.2574206015625001, + "particle_position_y": 0.25736832040839486, + "particle_position_z": 0.2573745781250001, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 9.81800079345703 + }, + "p3": { + "particle_opticalDepthBW": 707943.7338077008, + "particle_position_x": 0.2125403984375, + "particle_position_y": 0.21264004687499996, + "particle_position_z": 0.21265919757150045, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 8.110675811767576 + }, + "p4": { + "particle_opticalDepthBW": 899116.8515237333, + "particle_position_x": 0.24232183723958334, + "particle_position_y": 0.24226657421875006, + "particle_position_z": 0.24235025781250008, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 9.243879318237303 + }, + "pos1": { + "particle_opticalDepthQSR": 150411.2911918669, + "particle_position_x": 0.037522814130005, + "particle_position_y": 0.0375364375, + "particle_position_z": 0.0375473125, + "particle_momentum_x": 4.0804162983963274e-14, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 1.4303398132324217 + }, + "pos2": { + "particle_opticalDepthQSR": 19071.695614845616, + "particle_position_x": 0.0047233984375, + "particle_position_y": 0.004773031169456623, + "particle_position_z": 0.004769421875, + "particle_momentum_x": 0.0, + "particle_momentum_y": 1.2937663481149343e-14, + "particle_momentum_z": 0.0, + "particle_weight": 0.18199920654296872 + }, + "pos3": { + "particle_opticalDepthQSR": 197682.0402897762, + "particle_position_x": 0.049603601562499995, + "particle_position_y": 0.049503953125, + "particle_position_z": 0.049484700428793646, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.7069590910941717e-13, + "particle_weight": 1.8893241882324217 + }, + "pos4": { + "particle_opticalDepthQSR": 79817.78242989839, + "particle_position_x": 0.019832326822916666, + "particle_position_y": 0.019875475260416667, + "particle_position_z": 0.019791898437499997, + "particle_momentum_x": 6.229925285542024e-13, + "particle_momentum_y": 6.229925285542024e-13, + "particle_momentum_z": 6.229925285542024e-13, + "particle_weight": 0.7561206817626952 + } +} diff --git a/Regression/Checksum/benchmarks_json/qed_quantum_sync_3d.json b/Regression/Checksum/benchmarks_json/test_3d_qed_quantum_sync.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_quantum_sync_3d.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_quantum_sync.json diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger1.json b/Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_1.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_schwinger1.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_1.json diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger2.json b/Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_2.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_schwinger2.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_2.json diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger3.json b/Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_3.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_schwinger3.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_3.json diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger4.json b/Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_4.json similarity index 100% rename from Regression/Checksum/benchmarks_json/qed_schwinger4.json rename to Regression/Checksum/benchmarks_json/test_3d_qed_schwinger_4.json diff --git a/Regression/Checksum/benchmarks_json/radiation_reaction.json b/Regression/Checksum/benchmarks_json/test_3d_radiation_reaction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/radiation_reaction.json rename to Regression/Checksum/benchmarks_json/test_3d_radiation_reaction.json diff --git a/Regression/Checksum/benchmarks_json/reduced_diags.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags.json similarity index 100% rename from Regression/Checksum/benchmarks_json/reduced_diags.json rename to Regression/Checksum/benchmarks_json/test_3d_reduced_diags.json diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_heuristic.json similarity index 100% rename from Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json rename to Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_heuristic.json diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers.json similarity index 100% rename from Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json rename to Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers.json diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers_psatd.json rename to Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers_picmi.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers_psatd.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers_psatd.json new file mode 100644 index 00000000000..a77d93b9621 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_load_balance_costs_timers_psatd.json @@ -0,0 +1,22 @@ +{ + "electrons": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 262144.0, + "particle_position_y": 262144.0, + "particle_position_z": 65536.0, + "particle_weight": 1600000000000000.0 + }, + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json b/Regression/Checksum/benchmarks_json/test_3d_reduced_diags_single_precision.json similarity index 100% rename from Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json rename to Regression/Checksum/benchmarks_json/test_3d_reduced_diags_single_precision.json diff --git a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json b/Regression/Checksum/benchmarks_json/test_3d_relativistic_space_charge_initialization.json similarity index 100% rename from Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json rename to Regression/Checksum/benchmarks_json/test_3d_relativistic_space_charge_initialization.json diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization.json b/Regression/Checksum/benchmarks_json/test_3d_space_charge_initialization.json similarity index 100% rename from Regression/Checksum/benchmarks_json/space_charge_initialization.json rename to Regression/Checksum/benchmarks_json/test_3d_space_charge_initialization.json diff --git a/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json new file mode 100644 index 00000000000..2346ffd8124 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json @@ -0,0 +1,35 @@ +{ + "lev=0": { + "rho_carbon12_4": 8.391105120785595e-13, + "rho_carbon12_6": 1.2586657681178396e-12, + "rho_hydrogen1_1": 0.0 + }, + "carbon12_4": { + "particle_position_x": 0.24746482639048117, + "particle_position_y": 0.3712831550411343, + "particle_position_z": 291.92951822527056, + "particle_momentum_x": 7.446857998192906e-19, + "particle_momentum_y": 6.58876061665569e-18, + "particle_momentum_z": 3.0678537977188415e-16, + "particle_weight": 1000.0 + }, + "carbon12_6": { + "particle_position_x": 0.3706220153511513, + "particle_position_y": 0.5770046251488395, + "particle_position_z": 291.70616446343365, + "particle_momentum_x": 1.1143091694186902e-18, + "particle_momentum_y": 1.015840779649768e-17, + "particle_momentum_z": 3.063311157322583e-16, + "particle_weight": 1000.0 + }, + "hydrogen1_1": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} + diff --git a/Regression/Checksum/benchmarks_json/uniform_plasma_restart.json b/Regression/Checksum/benchmarks_json/test_3d_uniform_plasma.json similarity index 100% rename from Regression/Checksum/benchmarks_json/uniform_plasma_restart.json rename to Regression/Checksum/benchmarks_json/test_3d_uniform_plasma.json diff --git a/Regression/Checksum/benchmarks_json/uniform_plasma_multiJ.json b/Regression/Checksum/benchmarks_json/test_3d_uniform_plasma_multiJ.json similarity index 100% rename from Regression/Checksum/benchmarks_json/uniform_plasma_multiJ.json rename to Regression/Checksum/benchmarks_json/test_3d_uniform_plasma_multiJ.json diff --git a/Regression/Checksum/benchmarks_json/VayDeposition3D.json b/Regression/Checksum/benchmarks_json/test_3d_vay_deposition.json similarity index 100% rename from Regression/Checksum/benchmarks_json/VayDeposition3D.json rename to Regression/Checksum/benchmarks_json/test_3d_vay_deposition.json diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/test_rz_btd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/BTD_rz.json rename to Regression/Checksum/benchmarks_json/test_rz_btd.json diff --git a/Regression/Checksum/benchmarks_json/collisionRZ.json b/Regression/Checksum/benchmarks_json/test_rz_collision.json similarity index 100% rename from Regression/Checksum/benchmarks_json/collisionRZ.json rename to Regression/Checksum/benchmarks_json/test_rz_collision.json diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json b/Regression/Checksum/benchmarks_json/test_rz_deuterium_tritium_fusion.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json rename to Regression/Checksum/benchmarks_json/test_rz_deuterium_tritium_fusion.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json b/Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json rename to Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ.json b/Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere_eb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ.json rename to Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere_eb.json diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json b/Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere_eb_mr.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json rename to Regression/Checksum/benchmarks_json/test_rz_electrostatic_sphere_eb_mr.json diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json new file mode 100644 index 00000000000..e4b9d9c07ff --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Br": 6.7914286131989935e-19, + "Bt": 5.4557350206853276e-05, + "Bz": 2.357229221622199e-19, + "Er": 16481.39008058988, + "Et": 1.5258937379236053e-10, + "Ez": 1508.1064116028576 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..30d7d0ba081 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 1.4599714697029335e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..30d7d0ba081 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 1.4599714697029335e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..76baf73cc3a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 1.3292471881599093e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json b/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json new file mode 100644 index 00000000000..2ba80d4fb0e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Bz": 9.524453851623612e-24 + }, + "electron": { + "particle_momentum_x": 7.146168286112378e-18, + "particle_momentum_y": 7.073108431229069e-18, + "particle_momentum_z": 9.282175511339672e-42, + "particle_position_x": 6978.157994231982, + "particle_position_y": 2044.6981840260364, + "particle_theta": 6298.956888689097, + "particle_weight": 3.2236798669537214e-08 + } +} + diff --git a/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json new file mode 100644 index 00000000000..f8043c5c3e2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json @@ -0,0 +1,12 @@ +{ + "lev=0": {}, + "electron": { + "particle_momentum_x": 1.3547613622259754e-18, + "particle_momentum_y": 1.3539614160696825e-18, + "particle_momentum_z": 2.102305484242805e-18, + "particle_position_x": 108281.74349700565, + "particle_position_y": 108222.91506078152, + "particle_theta": 118597.06004310239, + "particle_weight": 2.5087578786544294e-07 + } +} diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json b/Regression/Checksum/benchmarks_json/test_rz_galilean_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_rz_psatd.json rename to Regression/Checksum/benchmarks_json/test_rz_galilean_psatd.json diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_rz_galilean_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_rz_galilean_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/test_rz_galilean_psatd_current_correction_psb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json rename to Regression/Checksum/benchmarks_json/test_rz_galilean_psatd_current_correction_psb.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_fluid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_fluid.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_multi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_multi.json diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_picmi.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd_current_correction.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd_current_correction.json diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd_multiJ.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_multiJ.json rename to Regression/Checksum/benchmarks_json/test_rz_langmuir_multi_psatd_multiJ.json diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/test_rz_laser_acceleration.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json rename to Regression/Checksum/benchmarks_json/test_rz_laser_acceleration.json diff --git a/Regression/Checksum/benchmarks_json/test_rz_laser_acceleration_opmd.json b/Regression/Checksum/benchmarks_json/test_rz_laser_acceleration_opmd.json new file mode 100644 index 00000000000..de631f4767a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_laser_acceleration_opmd.json @@ -0,0 +1,35 @@ +{ + "lev=0": { + "Bt": 4299.677335258863, + "Bz": 34749.512290662635, + "Er": 1343319090029.9607, + "jr": 5229952989213.152, + "jt": 9.287962600874053e+17, + "jz": 3712414162446391.5, + "part_per_cell": 6288.0, + "part_per_grid": 25755648.0, + "rho": 102920475.65331206, + "rho_beam": 12377109.352622943, + "rho_electrons": 90543366.3006891 + }, + "beam": { + "particle_position_x": 3.651481908823126e-05, + "particle_position_y": 4.275668879776449e-05, + "particle_position_z": 0.0025531549045483943, + "particle_momentum_x": 3.879691286254116e-20, + "particle_momentum_y": 5.0782566944104114e-20, + "particle_momentum_z": 1.3503182565048374e-17, + "particle_weight": 6241509.074460764 + }, + "electrons": { + "particle_origX": 0.03652440297475791, + "particle_origZ": 0.06924276562500002, + "particle_position_x": 0.036524412900510936, + "particle_position_y": 0.03652445428108603, + "particle_position_z": 0.06924303765442104, + "particle_momentum_x": 5.508781425380743e-23, + "particle_momentum_y": 7.236141259605716e-21, + "particle_momentum_z": 4.4528442530356535e-22, + "particle_weight": 1118799420.1067173 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/test_rz_laser_acceleration_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json rename to Regression/Checksum/benchmarks_json/test_rz_laser_acceleration_picmi.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json b/Regression/Checksum/benchmarks_json/test_rz_laser_injection_from_RZ_lasy_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json rename to Regression/Checksum/benchmarks_json/test_rz_laser_injection_from_RZ_lasy_file.json diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/test_rz_laser_injection_from_lasy_file.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json rename to Regression/Checksum/benchmarks_json/test_rz_laser_injection_from_lasy_file.json diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json b/Regression/Checksum/benchmarks_json/test_rz_load_external_field_grid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json rename to Regression/Checksum/benchmarks_json/test_rz_load_external_field_grid.json diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json b/Regression/Checksum/benchmarks_json/test_rz_load_external_field_particles.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json rename to Regression/Checksum/benchmarks_json/test_rz_load_external_field_particles.json diff --git a/Regression/Checksum/benchmarks_json/Python_magnetostatic_eb_rz.json b/Regression/Checksum/benchmarks_json/test_rz_magnetostatic_eb_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_magnetostatic_eb_rz.json rename to Regression/Checksum/benchmarks_json/test_rz_magnetostatic_eb_picmi.json diff --git a/Regression/Checksum/benchmarks_json/test_rz_multiJ_psatd.json b/Regression/Checksum/benchmarks_json/test_rz_multiJ_psatd.json new file mode 100644 index 00000000000..d9ca66cf0c3 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_multiJ_psatd.json @@ -0,0 +1,40 @@ +{ + "lev=0": { + "Bt": 5436.145827903481, + "Er": 1084033329144.5951, + "Ez": 1031727477244.4946, + "jr": 86689049352046.02, + "jz": 384358875752423.8, + "rho": 863406.9511533659, + "rho_driver": 1142390.635353391, + "rho_plasma_e": 12207428.74629265, + "rho_plasma_p": 12452342.269855343 + }, + "driver": { + "particle_momentum_x": 8.723365602723476e-16, + "particle_momentum_y": 8.719184082046568e-16, + "particle_momentum_z": 5.461727890375063e-13, + "particle_position_x": 6.269474429349755, + "particle_position_y": 17.933429487411857, + "particle_theta": 1570777.477238974, + "particle_weight": 6241434176.35186 + }, + "plasma_e": { + "particle_momentum_x": 1.369719083664514e-20, + "particle_momentum_y": 1.5823095211640957e-20, + "particle_momentum_z": 7.189697105606772e-21, + "particle_position_x": 0.28510775565436686, + "particle_position_y": 0.1491507116912657, + "particle_theta": 5011.181913404598, + "particle_weight": 1001422925327.429 + }, + "plasma_p": { + "particle_momentum_x": 1.4836311665634252e-20, + "particle_momentum_y": 1.3653689459385548e-20, + "particle_momentum_z": 1.2505361981099512e-20, + "particle_position_x": 0.2838368160801851, + "particle_position_y": 0.14950442866368444, + "particle_theta": 4995.577541103406, + "particle_weight": 1001422925327.4291 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_cylinder_compression_picmi.json b/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_cylinder_compression_picmi.json new file mode 100644 index 00000000000..6fd2ca04fce --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_cylinder_compression_picmi.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Br": 0.01190012639573578, + "Bt": 0.011313481779415917, + "Bz": 11.684908684984164, + "Er": 154581.58512851578, + "Et": 4798.276941148807, + "Ez": 193.22344271401872, + "rho": 7968.182346905438 + }, + "ions": { + "particle_momentum_x": 3.1125151786241107e-18, + "particle_momentum_y": 3.119385993047207e-18, + "particle_momentum_z": 3.0289560038617916e-18, + "particle_position_x": 13628.662686419664, + "particle_position_y": 2285.6952310457755, + "particle_theta": 115055.48935725243, + "particle_weight": 2.525423582445981e+18 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_em_modes_picmi.json b/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_em_modes_picmi.json new file mode 100644 index 00000000000..feca88922e2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_ohm_solver_em_modes_picmi.json @@ -0,0 +1,12 @@ +{ + "lev=0": {}, + "ions": { + "particle_momentum_x": 5.043784704795177e-17, + "particle_momentum_y": 5.0444695502620235e-17, + "particle_momentum_z": 5.05193106847111e-17, + "particle_position_x": 143164.53685279266, + "particle_position_y": 143166.5185853012, + "particle_theta": 2573262.446840369, + "particle_weight": 8.128680645366886e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/particle_boundary_interaction.json b/Regression/Checksum/benchmarks_json/test_rz_particle_boundary_interaction_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/particle_boundary_interaction.json rename to Regression/Checksum/benchmarks_json/test_rz_particle_boundary_interaction_picmi.json diff --git a/Regression/Checksum/benchmarks_json/pml_psatd_rz.json b/Regression/Checksum/benchmarks_json/test_rz_pml_psatd.json similarity index 100% rename from Regression/Checksum/benchmarks_json/pml_psatd_rz.json rename to Regression/Checksum/benchmarks_json/test_rz_pml_psatd.json diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json b/Regression/Checksum/benchmarks_json/test_rz_point_of_contact_eb.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json rename to Regression/Checksum/benchmarks_json/test_rz_point_of_contact_eb.json diff --git a/Regression/Checksum/benchmarks_json/test_rz_projection_divb_cleaner.json b/Regression/Checksum/benchmarks_json/test_rz_projection_divb_cleaner.json new file mode 100644 index 00000000000..df8024c11b1 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_projection_divb_cleaner.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Br": 0.8165863784977822, + "Bt": 0.0, + "Bz": 6.203503481560605 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_scraping.json b/Regression/Checksum/benchmarks_json/test_rz_scraping.json new file mode 100644 index 00000000000..3a97a2dc651 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_scraping.json @@ -0,0 +1,17 @@ +{ + "lev=0": { + "Er": 0.0 + }, + "lev=1": { + "Er": 0.0 + }, + "electron": { + "particle_momentum_x": 8.802233511708275e-20, + "particle_momentum_y": 8.865573181381068e-20, + "particle_momentum_z": 0.0, + "particle_position_x": 52.1624916491251, + "particle_position_y": 128.0, + "particle_theta": 776.9665451756912, + "particle_weight": 4.841626861764053e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_scraping_filter.json b/Regression/Checksum/benchmarks_json/test_rz_scraping_filter.json new file mode 100644 index 00000000000..3a97a2dc651 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_scraping_filter.json @@ -0,0 +1,17 @@ +{ + "lev=0": { + "Er": 0.0 + }, + "lev=1": { + "Er": 0.0 + }, + "electron": { + "particle_momentum_x": 8.802233511708275e-20, + "particle_momentum_y": 8.865573181381068e-20, + "particle_momentum_z": 0.0, + "particle_position_x": 52.1624916491251, + "particle_position_y": 128.0, + "particle_theta": 776.9665451756912, + "particle_weight": 4.841626861764053e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_secondary_ion_emission_picmi.json b/Regression/Checksum/benchmarks_json/test_rz_secondary_ion_emission_picmi.json new file mode 100644 index 00000000000..cfc84819e97 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_secondary_ion_emission_picmi.json @@ -0,0 +1,26 @@ +{ + "electrons": { + "particle_momentum_x": 5.621885683102775e-26, + "particle_momentum_y": 1.2079178196118306e-25, + "particle_momentum_z": 1.2496342823828099e-25, + "particle_position_x": 0.10329568998704057, + "particle_position_y": 0.013444257249267193, + "particle_position_z": 0.4019696082583948, + "particle_weight": 2.0 + }, + "ions": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "Er": 1.772547702166409e-06, + "Ez": 2.2824957684716966e-06, + "phi": 4.338168233265556e-07, + "rho": 1.933391680367631e-15 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/silver_mueller_rz_z.json b/Regression/Checksum/benchmarks_json/test_rz_silver_mueller_z.json similarity index 100% rename from Regression/Checksum/benchmarks_json/silver_mueller_rz_z.json rename to Regression/Checksum/benchmarks_json/test_rz_silver_mueller_z.json diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/test_rz_spacecraft_charging_picmi.json similarity index 100% rename from Regression/Checksum/benchmarks_json/spacecraft_charging.json rename to Regression/Checksum/benchmarks_json/test_rz_spacecraft_charging_picmi.json diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index 727c8beb7f7..8c93f4ea6ea 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -1,10 +1,10 @@ """ - Copyright 2020 +Copyright 2020 - This file is part of WarpX. +This file is part of WarpX. - License: BSD-3-Clause-LBNL - """ +License: BSD-3-Clause-LBNL +""" import json import sys @@ -19,10 +19,16 @@ class Checksum: - """Class for checksum comparison of one test. - """ + """Class for checksum comparison of one test.""" - def __init__(self, test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): + def __init__( + self, + test_name, + output_file, + output_format="plotfile", + do_fields=True, + do_particles=True, + ): """ Checksum constructor. Store test_name, output file name and format, compute checksum @@ -49,8 +55,9 @@ def __init__(self, test_name, output_file, output_format='plotfile', do_fields=T self.test_name = test_name self.output_file = output_file self.output_format = output_format - self.data = self.read_output_file(do_fields=do_fields, - do_particles=do_particles) + self.data = self.read_output_file( + do_fields=do_fields, do_particles=do_particles + ) def read_output_file(self, do_fields=True, do_particles=True): """ @@ -68,38 +75,45 @@ def read_output_file(self, do_fields=True, do_particles=True): Whether to read particles from the output file. """ - if self.output_format == 'plotfile': + if self.output_format == "plotfile": ds = yt.load(self.output_file) # yt 4.0+ has rounding issues with our domain data: # RuntimeError: yt attempted to read outside the boundaries # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() - grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] + if "force_periodicity" in dir(ds): + ds.force_periodicity() + grid_fields = [item for item in ds.field_list if item[0] == "boxlib"] # "fields"/"species" we remove: # - nbody: added by yt by default, unused by us - species_list = set([item[0] for item in ds.field_list if - item[1][:9] == 'particle_' and - item[0] != 'all' and - item[0] != 'nbody']) + species_list = set( + [ + item[0] + for item in ds.field_list + if item[1][:9] == "particle_" + and item[0] != "all" + and item[0] != "nbody" + ] + ) data = {} # Compute checksum for field quantities if do_fields: - for lev in range(ds.max_level+1): + for lev in range(ds.max_level + 1): data_lev = {} - lev_grids = [grid for grid in ds.index.grids - if grid.Level == lev] + lev_grids = [grid for grid in ds.index.grids if grid.Level == lev] # Warning: For now, we assume all levels are rectangular LeftEdge = np.min( - np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) + np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0 + ) all_data_level = ds.covering_grid( - level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) + level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions + ) for field in grid_fields: Q = all_data_level[field].v.squeeze() data_lev[field[1]] = np.sum(np.abs(Q)) - data['lev=' + str(lev)] = data_lev + data["lev=" + str(lev)] = data_lev # Compute checksum for particle quantities if do_particles: @@ -109,49 +123,63 @@ def read_output_file(self, do_fields=True, do_particles=True): # - particle_cpu/id: they depend on the parallelism: MPI-ranks and # on-node acceleration scheme, thus not portable # and irrelevant for physics benchmarking - part_fields = [item[1] for item in ds.field_list - if item[0] == species and - item[1] != 'particle_cpu' and - item[1] != 'particle_id' - ] + part_fields = [ + item[1] + for item in ds.field_list + if item[0] == species + and item[1] != "particle_cpu" + and item[1] != "particle_id" + ] data_species = {} for field in part_fields: Q = ad[(species, field)].v data_species[field] = np.sum(np.abs(Q)) data[species] = data_species - elif self.output_format == 'openpmd': + elif self.output_format == "openpmd": # Load time series ts = OpenPMDTimeSeries(self.output_file) data = {} # Compute number of MR levels # TODO This calculation of nlevels assumes that the last element # of level_fields is by default on the highest MR level. - level_fields = [field for field in ts.avail_fields if 'lvl' in field] + level_fields = [field for field in ts.avail_fields if "lvl" in field] nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) # Compute checksum for field quantities if do_fields: - for lev in range(nlevels+1): + for lev in range(nlevels + 1): # Create list of fields specific to level lev grid_fields = [] if lev == 0: - grid_fields = [field for field in ts.avail_fields if 'lvl' not in field] + grid_fields = [ + field for field in ts.avail_fields if "lvl" not in field + ] else: - grid_fields = [field for field in ts.avail_fields if f'lvl{lev}' in field] + grid_fields = [ + field for field in ts.avail_fields if f"lvl{lev}" in field + ] data_lev = {} for field in grid_fields: - vector_components = ts.fields_metadata[field]['avail_components'] + vector_components = ts.fields_metadata[field][ + "avail_components" + ] if vector_components != []: for coord in vector_components: - Q, info = ts.get_field(field=field, iteration=ts.iterations[-1], coord=coord) + Q, info = ts.get_field( + field=field, + iteration=ts.iterations[-1], + coord=coord, + ) # key stores strings composed of field name and vector components # (e.g., field='B' or field='B_lvl1' + coord='y' results in key='By') - key = field.replace(f'_lvl{lev}', '') + coord + key = field.replace(f"_lvl{lev}", "") + coord data_lev[key] = np.sum(np.abs(Q)) - else: # scalar field - Q, info = ts.get_field(field=field, iteration=ts.iterations[-1]) + else: # scalar field + Q, info = ts.get_field( + field=field, iteration=ts.iterations[-1] + ) data_lev[field] = np.sum(np.abs(Q)) - data[f'lev={lev}'] = data_lev + data[f"lev={lev}"] = data_lev # Compute checksum for particle quantities if do_particles: species_list = [] @@ -159,27 +187,36 @@ def read_output_file(self, do_fields=True, do_particles=True): species_list = ts.avail_record_components.keys() for species in species_list: data_species = {} - part_fields = [item for item in ts.avail_record_components[species] - if item != 'id' and item != 'charge' and item != 'mass'] + part_fields = [ + item + for item in ts.avail_record_components[species] + if item != "id" and item != "charge" and item != "mass" + ] # Convert the field name to the name used in plotfiles for field in part_fields: - Q = ts.get_particle(var_list=[field], species=species, iteration=ts.iterations[-1]) - if field in ['x', 'y', 'z']: - field_name = 'particle_position_' + field - elif field in ['ux', 'uy', 'uz']: - field_name = 'particle_momentum_' + field[-1] - m, = ts.get_particle(['mass'], species=species, iteration=ts.iterations[-1]) - Q *= m*c - elif field in ['w']: - field_name = 'particle_weight' + Q = ts.get_particle( + var_list=[field], + species=species, + iteration=ts.iterations[-1], + ) + if field in ["x", "y", "z"]: + field_name = "particle_position_" + field + elif field in ["ux", "uy", "uz"]: + field_name = "particle_momentum_" + field[-1] + (m,) = ts.get_particle( + ["mass"], species=species, iteration=ts.iterations[-1] + ) + Q *= m * c + elif field in ["w"]: + field_name = "particle_weight" else: - field_name = 'particle_' + field + field_name = "particle_" + field data_species[field_name] = np.sum(np.abs(Q)) data[species] = data_species return data - def evaluate(self, rtol=1.e-9, atol=1.e-40): + def evaluate(self, rtol=1.0e-9, atol=1.0e-40): """ Compare output file checksum with benchmark. Read checksum from output file, read benchmark @@ -199,56 +236,65 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): ref_benchmark = Benchmark(self.test_name) # Dictionaries have same outer keys (levels, species)? - if (self.data.keys() != ref_benchmark.data.keys()): - print("ERROR: Benchmark and output file checksum " - "have different outer keys:") + if self.data.keys() != ref_benchmark.data.keys(): + print( + "ERROR: Benchmark and output file checksum have different outer keys:" + ) print("Benchmark: %s" % ref_benchmark.data.keys()) print("Test file: %s" % self.data.keys()) - print("\n----------------\nNew file for " + self.test_name + ":") + print(f"\nNew checksums file {self.test_name}.json:") print(json.dumps(self.data, indent=2)) - print("----------------") sys.exit(1) # Dictionaries have same inner keys (field and particle quantities)? for key1 in ref_benchmark.data.keys(): - if (self.data[key1].keys() != ref_benchmark.data[key1].keys()): - print("ERROR: Benchmark and output file checksum have " - "different inner keys:") + if self.data[key1].keys() != ref_benchmark.data[key1].keys(): + print( + "ERROR: Benchmark and output file checksum have " + "different inner keys:" + ) print("Common outer keys: %s" % ref_benchmark.data.keys()) - print("Benchmark inner keys in %s: %s" - % (key1, ref_benchmark.data[key1].keys())) - print("Test file inner keys in %s: %s" - % (key1, self.data[key1].keys())) - print("\n----------------\nNew file for " + self.test_name + ":") + print( + "Benchmark inner keys in %s: %s" + % (key1, ref_benchmark.data[key1].keys()) + ) + print("Test file inner keys in %s: %s" % (key1, self.data[key1].keys())) + print(f"\nNew checksums file {self.test_name}.json:") print(json.dumps(self.data, indent=2)) - print("----------------") sys.exit(1) # Dictionaries have same values? checksums_differ = False for key1 in ref_benchmark.data.keys(): for key2 in ref_benchmark.data[key1].keys(): - passed = np.isclose(self.data[key1][key2], - ref_benchmark.data[key1][key2], - rtol=rtol, atol=atol) + passed = np.isclose( + self.data[key1][key2], + ref_benchmark.data[key1][key2], + rtol=rtol, + atol=atol, + ) if not passed: - print("ERROR: Benchmark and output file checksum have " - "different value for key [%s,%s]" % (key1, key2)) - print("Benchmark: [%s,%s] %.15e" - % (key1, key2, ref_benchmark.data[key1][key2])) - print("Test file: [%s,%s] %.15e" - % (key1, key2, self.data[key1][key2])) + print( + "ERROR: Benchmark and output file checksum have " + "different value for key [%s,%s]" % (key1, key2) + ) + print( + "Benchmark: [%s,%s] %.15e" + % (key1, key2, ref_benchmark.data[key1][key2]) + ) + print( + "Test file: [%s,%s] %.15e" % (key1, key2, self.data[key1][key2]) + ) checksums_differ = True # Print absolute and relative error for each failing key x = ref_benchmark.data[key1][key2] y = self.data[key1][key2] abs_err = np.abs(x - y) print("Absolute error: {:.2e}".format(abs_err)) - if (np.abs(x) != 0.): + if np.abs(x) != 0.0: rel_err = abs_err / np.abs(x) print("Relative error: {:.2e}".format(rel_err)) if checksums_differ: - print("\n----------------\nNew file for " + self.test_name + ":") + print(f"\nNew checksums file {self.test_name}.json:") print(json.dumps(self.data, indent=2)) - print("----------------") sys.exit(1) diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index cc13ceefa28..85aac38c3d4 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -1,12 +1,12 @@ #! /usr/bin/env python3 """ - Copyright 2020 +Copyright 2020 - This file is part of WarpX. +This file is part of WarpX. - License: BSD-3-Clause-LBNL - """ +License: BSD-3-Clause-LBNL +""" import argparse import glob @@ -35,13 +35,23 @@ """ -def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e-9, atol=1.e-40, - do_fields=True, do_particles=True): +def evaluate_checksum( + test_name, + output_file, + output_format="plotfile", + rtol=1.0e-9, + atol=1.0e-40, + do_fields=True, + do_particles=True, +): """ Compare output file checksum with benchmark. Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. + If the environment variable CHECKSUM_RESET is set while this function is run, + the evaluation will be replaced with a call to reset_benchmark (see below). + Parameters ---------- test_name: string @@ -65,12 +75,35 @@ def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, output_file, output_format, - do_fields=do_fields, do_particles=do_particles) - test_checksum.evaluate(rtol=rtol, atol=atol) - - -def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): + # Reset benchmark? + reset = os.getenv("CHECKSUM_RESET", "False").lower() in [ + "true", + "1", + "t", + "y", + "yes", + "on", + ] + + if reset: + print( + f"Environment variable CHECKSUM_RESET is set, resetting benchmark for {test_name}" + ) + reset_benchmark(test_name, output_file, output_format, do_fields, do_particles) + else: + test_checksum = Checksum( + test_name, + output_file, + output_format, + do_fields=do_fields, + do_particles=do_particles, + ) + test_checksum.evaluate(rtol=rtol, atol=atol) + + +def reset_benchmark( + test_name, output_file, output_format="plotfile", do_fields=True, do_particles=True +): """ Update the benchmark (overwrites reference json file). Overwrite value of benchmark corresponding to @@ -93,13 +126,18 @@ def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields= do_particles: bool, default=True Whether to write particles checksums in the benchmark. """ - ref_checksum = Checksum(test_name, output_file, output_format, - do_fields=do_fields, do_particles=do_particles) + ref_checksum = Checksum( + test_name, + output_file, + output_format, + do_fields=do_fields, + do_particles=do_particles, + ) ref_benchmark = Benchmark(test_name, ref_checksum.data) ref_benchmark.reset() -def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): +def reset_all_benchmarks(path_to_all_output_files, output_format="plotfile"): """ Update all benchmarks (overwrites reference json files) found in path_to_all_output_files @@ -116,8 +154,9 @@ def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): """ # Get list of output files in path_to_all_output_files - output_file_list = glob.glob(path_to_all_output_files + '*_plt*[0-9]', - recursive=True) + output_file_list = glob.glob( + path_to_all_output_files + "*_plt*[0-9]", recursive=True + ) output_file_list.sort() # Loop over output files and reset the corresponding benchmark @@ -126,68 +165,122 @@ def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): reset_benchmark(test_name, output_file, output_format) -if __name__ == '__main__': - +if __name__ == "__main__": parser = argparse.ArgumentParser() # Options relevant to evaluate a checksum or reset a benchmark - parser.add_argument('--evaluate', dest='evaluate', action='store_true', - default=False, help='Evaluate a checksum.') - parser.add_argument('--reset-benchmark', dest='reset_benchmark', - default=False, - action='store_true', help='Reset a benchmark.') - parser.add_argument('--test-name', dest='test_name', type=str, default='', - required='--evaluate' in sys.argv or - '--reset-benchmark' in sys.argv, - help='Name of the test (as in WarpX-tests.ini)') - parser.add_argument('--output-file', dest='output_file', type=str, default='', - required='--evaluate' in sys.argv or - '--reset-benchmark' in sys.argv, - help='Name of WarpX output file') - parser.add_argument('--output-format', dest='output_format', type=str, default='plotfile', - required='--evaluate' in sys.argv or - '--reset-benchmark' in sys.argv, - help='Format of the output file (plotfile, openpmd)') - parser.add_argument('--skip-fields', dest='do_fields', - default=True, action='store_false', - help='If used, do not read/write field checksums') - parser.add_argument('--skip-particles', dest='do_particles', - default=True, action='store_false', - help='If used, do not read/write particle checksums') + parser.add_argument( + "--evaluate", + dest="evaluate", + action="store_true", + default=False, + help="Evaluate a checksum.", + ) + parser.add_argument( + "--reset-benchmark", + dest="reset_benchmark", + default=False, + action="store_true", + help="Reset a benchmark.", + ) + parser.add_argument( + "--test-name", + dest="test_name", + type=str, + default="", + required="--evaluate" in sys.argv or "--reset-benchmark" in sys.argv, + help="Name of the test (as in WarpX-tests.ini)", + ) + parser.add_argument( + "--output-file", + dest="output_file", + type=str, + default="", + required="--evaluate" in sys.argv or "--reset-benchmark" in sys.argv, + help="Name of WarpX output file", + ) + parser.add_argument( + "--output-format", + dest="output_format", + type=str, + default="plotfile", + required="--evaluate" in sys.argv or "--reset-benchmark" in sys.argv, + help="Format of the output file (plotfile, openpmd)", + ) + parser.add_argument( + "--skip-fields", + dest="do_fields", + default=True, + action="store_false", + help="If used, do not read/write field checksums", + ) + parser.add_argument( + "--skip-particles", + dest="do_particles", + default=True, + action="store_false", + help="If used, do not read/write particle checksums", + ) # Fields and/or particles are read from output file/written to benchmark? - parser.add_argument('--rtol', dest='rtol', - type=float, default=1.e-9, - help='relative tolerance for comparison') - parser.add_argument('--atol', dest='atol', - type=float, default=1.e-40, - help='absolute tolerance for comparison') + parser.add_argument( + "--rtol", + dest="rtol", + type=float, + default=1.0e-9, + help="relative tolerance for comparison", + ) + parser.add_argument( + "--atol", + dest="atol", + type=float, + default=1.0e-40, + help="absolute tolerance for comparison", + ) # Option to reset all benchmarks present in a folder. - parser.add_argument('--reset-all-benchmarks', dest='reset_all_benchmarks', - action='store_true', default=False, - help='Reset all benchmarks.') - parser.add_argument('--path-to-all-output-files', - dest='path_to_all_output_files', type=str, default='', - required='--reset-all-benchmarks' in sys.argv, - help='Directory containing all benchmark output files, \ + parser.add_argument( + "--reset-all-benchmarks", + dest="reset_all_benchmarks", + action="store_true", + default=False, + help="Reset all benchmarks.", + ) + parser.add_argument( + "--path-to-all-output-files", + dest="path_to_all_output_files", + type=str, + default="", + required="--reset-all-benchmarks" in sys.argv, + help="Directory containing all benchmark output files, \ typically WarpX-benchmarks generated by \ - regression_testing/regtest.py') + regression_testing/regtest.py", + ) args = parser.parse_args() if args.reset_benchmark: - reset_benchmark(args.test_name, args.output_file, args.output_format, - do_fields=args.do_fields, - do_particles=args.do_particles) + reset_benchmark( + args.test_name, + args.output_file, + args.output_format, + do_fields=args.do_fields, + do_particles=args.do_particles, + ) if args.evaluate: - evaluate_checksum(args.test_name, args.output_file, args.output_format, - rtol=args.rtol, atol=args.atol, - do_fields=args.do_fields, do_particles=args.do_particles) + evaluate_checksum( + args.test_name, + args.output_file, + args.output_format, + rtol=args.rtol, + atol=args.atol, + do_fields=args.do_fields, + do_particles=args.do_particles, + ) if args.reset_all_benchmarks: - if args.output_format == 'openpmd': - sys.exit('Option --reset-all-benchmarks does not work with openPMD format') + if args.output_format == "openpmd": + sys.exit("Option --reset-all-benchmarks does not work with openPMD format") # WARNING: this mode does not support skip-fields/particles and tolerances reset_all_benchmarks(args.path_to_all_output_files, args.output_format) diff --git a/Regression/Checksum/config.py b/Regression/Checksum/config.py index fb9dbb223c8..8d19d82beaa 100644 --- a/Regression/Checksum/config.py +++ b/Regression/Checksum/config.py @@ -1,11 +1,11 @@ """ - Copyright 2020 +Copyright 2020 - This file is part of WarpX. +This file is part of WarpX. - License: BSD-3-Clause-LBNL - """ +License: BSD-3-Clause-LBNL +""" import os -benchmark_location = os.path.split(__file__)[0] + '/benchmarks_json' +benchmark_location = os.path.split(__file__)[0] + "/benchmarks_json" diff --git a/Regression/PostProcessingUtils/__init__.py b/Regression/PostProcessingUtils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Regression/PostProcessingUtils/post_processing_utils.py b/Regression/PostProcessingUtils/post_processing_utils.py index 50026ca1076..cbd55c433d3 100644 --- a/Regression/PostProcessingUtils/post_processing_utils.py +++ b/Regression/PostProcessingUtils/post_processing_utils.py @@ -13,118 +13,157 @@ ## This is a generic function to test a particle filter. We reproduce the filter in python and ## verify that the results are the same as with the WarpX filtered diagnostic. def check_particle_filter(fn, filtered_fn, filter_expression, dim, species_name): - ds = yt.load( fn ) - ds_filtered = yt.load( filtered_fn ) - ad = ds.all_data() - ad_filtered = ds_filtered.all_data() + ds = yt.load(fn) + ds_filtered = yt.load(filtered_fn) + ad = ds.all_data() + ad_filtered = ds_filtered.all_data() ## Load arrays from the unfiltered diagnostic - ids = ad[species_name, 'particle_id'].to_ndarray() - cpus = ad[species_name, 'particle_cpu'].to_ndarray() - px = ad[species_name, 'particle_momentum_x'].to_ndarray() - pz = ad[species_name, 'particle_momentum_z'].to_ndarray() - py = ad[species_name, 'particle_momentum_y'].to_ndarray() - w = ad[species_name, 'particle_weight'].to_ndarray() - if (dim == "2d"): - x = ad[species_name, 'particle_position_x'].to_ndarray() - z = ad[species_name, 'particle_position_y'].to_ndarray() - elif (dim == "3d"): - x = ad[species_name, 'particle_position_x'].to_ndarray() - y = ad[species_name, 'particle_position_y'].to_ndarray() - z = ad[species_name, 'particle_position_z'].to_ndarray() - elif (dim == "rz"): - r = ad[species_name, 'particle_position_x'].to_ndarray() - z = ad[species_name, 'particle_position_y'].to_ndarray() - theta = ad[species_name, 'particle_theta'].to_ndarray() + ids = ad[species_name, "particle_id"].to_ndarray() + cpus = ad[species_name, "particle_cpu"].to_ndarray() + px = ad[species_name, "particle_momentum_x"].to_ndarray() + pz = ad[species_name, "particle_momentum_z"].to_ndarray() + py = ad[species_name, "particle_momentum_y"].to_ndarray() + w = ad[species_name, "particle_weight"].to_ndarray() + if dim == "2d": + x = ad[species_name, "particle_position_x"].to_ndarray() + z = ad[species_name, "particle_position_y"].to_ndarray() + elif dim == "3d": + x = ad[species_name, "particle_position_x"].to_ndarray() + y = ad[species_name, "particle_position_y"].to_ndarray() + z = ad[species_name, "particle_position_z"].to_ndarray() + elif dim == "rz": + r = ad[species_name, "particle_position_x"].to_ndarray() + z = ad[species_name, "particle_position_y"].to_ndarray() + theta = ad[species_name, "particle_theta"].to_ndarray() ## Load arrays from the filtered diagnostic - ids_filtered_warpx = ad_filtered[species_name, 'particle_id'].to_ndarray() - cpus_filtered_warpx = ad_filtered[species_name, 'particle_cpu'].to_ndarray() - px_filtered_warpx = ad_filtered[species_name, 'particle_momentum_x'].to_ndarray() - pz_filtered_warpx = ad_filtered[species_name, 'particle_momentum_z'].to_ndarray() - py_filtered_warpx = ad_filtered[species_name, 'particle_momentum_y'].to_ndarray() - w_filtered_warpx = ad_filtered[species_name, 'particle_weight'].to_ndarray() - if (dim == "2d"): - x_filtered_warpx = ad_filtered[species_name, 'particle_position_x'].to_ndarray() - z_filtered_warpx = ad_filtered[species_name, 'particle_position_y'].to_ndarray() - elif (dim == "3d"): - x_filtered_warpx = ad_filtered[species_name, 'particle_position_x'].to_ndarray() - y_filtered_warpx = ad_filtered[species_name, 'particle_position_y'].to_ndarray() - z_filtered_warpx = ad_filtered[species_name, 'particle_position_z'].to_ndarray() - elif (dim == "rz"): - r_filtered_warpx = ad_filtered[species_name, 'particle_position_x'].to_ndarray() - z_filtered_warpx = ad_filtered[species_name, 'particle_position_y'].to_ndarray() - theta_filtered_warpx = ad_filtered[species_name, 'particle_theta'].to_ndarray() + ids_filtered_warpx = ad_filtered[species_name, "particle_id"].to_ndarray() + cpus_filtered_warpx = ad_filtered[species_name, "particle_cpu"].to_ndarray() + px_filtered_warpx = ad_filtered[species_name, "particle_momentum_x"].to_ndarray() + pz_filtered_warpx = ad_filtered[species_name, "particle_momentum_z"].to_ndarray() + py_filtered_warpx = ad_filtered[species_name, "particle_momentum_y"].to_ndarray() + w_filtered_warpx = ad_filtered[species_name, "particle_weight"].to_ndarray() + if dim == "2d": + x_filtered_warpx = ad_filtered[species_name, "particle_position_x"].to_ndarray() + z_filtered_warpx = ad_filtered[species_name, "particle_position_y"].to_ndarray() + elif dim == "3d": + x_filtered_warpx = ad_filtered[species_name, "particle_position_x"].to_ndarray() + y_filtered_warpx = ad_filtered[species_name, "particle_position_y"].to_ndarray() + z_filtered_warpx = ad_filtered[species_name, "particle_position_z"].to_ndarray() + elif dim == "rz": + r_filtered_warpx = ad_filtered[species_name, "particle_position_x"].to_ndarray() + z_filtered_warpx = ad_filtered[species_name, "particle_position_y"].to_ndarray() + theta_filtered_warpx = ad_filtered[species_name, "particle_theta"].to_ndarray() ## Reproduce the filter in python: this returns the indices of the filtered particles in the ## unfiltered arrays. - ind_filtered_python, = np.where(eval(filter_expression)) + (ind_filtered_python,) = np.where(eval(filter_expression)) ## Sort the indices of the filtered arrays by particle id. - sorted_ind_filtered_python = ind_filtered_python[np.argsort(ids[ind_filtered_python])] + sorted_ind_filtered_python = ind_filtered_python[ + np.argsort(ids[ind_filtered_python]) + ] sorted_ind_filtered_warpx = np.argsort(ids_filtered_warpx) ## Check that the sorted ids are exactly the same with the warpx filter and the filter ## reproduced in python - assert(np.array_equal(ids[sorted_ind_filtered_python], - ids_filtered_warpx[sorted_ind_filtered_warpx])) - assert(np.array_equal(cpus[sorted_ind_filtered_python], - cpus_filtered_warpx[sorted_ind_filtered_warpx])) + assert np.array_equal( + ids[sorted_ind_filtered_python], ids_filtered_warpx[sorted_ind_filtered_warpx] + ) + assert np.array_equal( + cpus[sorted_ind_filtered_python], cpus_filtered_warpx[sorted_ind_filtered_warpx] + ) ## Finally, we check that the sum of the particles quantities are the same to machine precision - tolerance_checksum = 1.e-12 - check_array_sum(px[sorted_ind_filtered_python], - px_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(pz[sorted_ind_filtered_python], - pz_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(py[sorted_ind_filtered_python], - py_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(w[sorted_ind_filtered_python], - w_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(z[sorted_ind_filtered_python], - z_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - if (dim == "2d"): - check_array_sum(x[sorted_ind_filtered_python], - x_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - elif (dim == "3d"): - check_array_sum(x[sorted_ind_filtered_python], - x_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(y[sorted_ind_filtered_python], - y_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - elif (dim == "rz"): - check_array_sum(r[sorted_ind_filtered_python], - r_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) - check_array_sum(theta[sorted_ind_filtered_python], - theta_filtered_warpx[sorted_ind_filtered_warpx], tolerance_checksum) + tolerance_checksum = 1.0e-12 + check_array_sum( + px[sorted_ind_filtered_python], + px_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + pz[sorted_ind_filtered_python], + pz_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + py[sorted_ind_filtered_python], + py_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + w[sorted_ind_filtered_python], + w_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + z[sorted_ind_filtered_python], + z_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + if dim == "2d": + check_array_sum( + x[sorted_ind_filtered_python], + x_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + elif dim == "3d": + check_array_sum( + x[sorted_ind_filtered_python], + x_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + y[sorted_ind_filtered_python], + y_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + elif dim == "rz": + check_array_sum( + r[sorted_ind_filtered_python], + r_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + check_array_sum( + theta[sorted_ind_filtered_python], + theta_filtered_warpx[sorted_ind_filtered_warpx], + tolerance_checksum, + ) + ## This function checks that the absolute sums of two arrays are the same to a required precision def check_array_sum(array1, array2, tolerance_checksum): sum1 = np.sum(np.abs(array1)) sum2 = np.sum(np.abs(array2)) - assert(abs(sum2-sum1)/sum1 < tolerance_checksum) + assert abs(sum2 - sum1) / sum1 < tolerance_checksum + ## This function is specifically used to test the random filter. First, we check that the number of ## dumped particles is as expected. Next, we call the generic check_particle_filter function. def check_random_filter(fn, filtered_fn, random_fraction, dim, species_name): - ds = yt.load( fn ) - ds_filtered = yt.load( filtered_fn ) - ad = ds.all_data() - ad_filtered = ds_filtered.all_data() + ds = yt.load(fn) + ds_filtered = yt.load(filtered_fn) + ad = ds.all_data() + ad_filtered = ds_filtered.all_data() ## Check that the number of particles is as expected - numparts = ad[species_name, 'particle_id'].to_ndarray().shape[0] - numparts_filtered = ad_filtered['particle_id'].to_ndarray().shape[0] - expected_numparts_filtered = random_fraction*numparts + numparts = ad[species_name, "particle_id"].to_ndarray().shape[0] + numparts_filtered = ad_filtered["particle_id"].to_ndarray().shape[0] + expected_numparts_filtered = random_fraction * numparts # 5 sigma test that has an intrinsic probability to fail of 1 over ~2 millions std_numparts_filtered = np.sqrt(expected_numparts_filtered) - error = abs(numparts_filtered-expected_numparts_filtered) - print("Random filter: difference between expected and actual number of dumped particles: " \ - + str(error)) - print("tolerance: " + str(5*std_numparts_filtered)) - assert(error<5*std_numparts_filtered) + error = abs(numparts_filtered - expected_numparts_filtered) + print( + "Random filter: difference between expected and actual number of dumped particles: " + + str(error) + ) + print("tolerance: " + str(5 * std_numparts_filtered)) + assert error < 5 * std_numparts_filtered ## Dirty trick to find particles with the same ID + same CPU (does not work with more than 10 ## MPI ranks) - random_filter_expression = 'np.isin(ids + 0.1*cpus,' \ - 'ids_filtered_warpx + 0.1*cpus_filtered_warpx)' + random_filter_expression = ( + "np.isin(ids + 0.1*cpus,ids_filtered_warpx + 0.1*cpus_filtered_warpx)" + ) check_particle_filter(fn, filtered_fn, random_filter_expression, dim, species_name) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini deleted file mode 100644 index 68d59d850a0..00000000000 --- a/Regression/WarpX-GPU-tests.ini +++ /dev/null @@ -1,678 +0,0 @@ -# This file is used both for the nightly regression tests -# on the garunda server, and for CI tests. -# In the case of CI, some of the parameters entered -# below are overwritten, see prepare_file_ci.py -[main] -# repeat captured errors to stderr, e.g., for CI runs -verbose = 1 - -testTopDir = /home/regtester/RegTesting/rt-WarpX/ -webTopDir = /home/regtester/RegTesting/rt-WarpX/web - -sourceTree = C_Src - -# suiteName is the name prepended to all output directories -suiteName = WarpX-GPU - -COMP = g++ -add_to_c_make_command = TEST=TRUE USE_ASSERTION=TRUE WarpxBinDir= - -archive_output = 0 -purge_output = 1 - -MAKE = make -numMakeJobs = 8 - -# We build by default a few tools for output comparison. -# The build time for those can be skipped if they are not needed. -ftools = - -# Control the build of the particle_compare tool. -# Needed for test particle_tolerance option. -use_ctools = 0 - -# MPIcommand should use the placeholders: -# @host@ to indicate where to put the hostname to run on -# @nprocs@ to indicate where to put the number of processors -# @command@ to indicate where to put the command to run -# -# only tests with useMPI = 1 will run in parallel -# nprocs is problem dependent and specified in the individual problem -# sections. - -#MPIcommand = mpiexec -host @host@ -n @nprocs@ @command@ -MPIcommand = mpiexec -n @nprocs@ @command@ -MPIhost = - -reportActiveTestsOnly = 1 - -# Add "GO UP" link at the top of the web page? -goUpLink = 1 - -# string queried to change plotfiles and checkpoint files -plot_file_name = diag1.file_prefix -check_file_name = none - -# email -sendEmailWhenFail = 1 -emailTo = weiqunzhang@lbl.gov, jlvay@lbl.gov, rlehe@lbl.gov, atmyers@lbl.gov, mthevenet@lbl.gov, oshapoval@lbl.gov, ldianaamorim@lbl.gov, rjambunathan@lbl.gov, axelhuebl@lbl.gov, ezoni@lbl.gov -emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more details. - -[AMReX] -dir = /home/regtester/git/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 - -[source] -dir = /home/regtester/git/WarpX -branch = development - -[extra-PICSAR] -dir = /home/regtester/git/picsar/ -branch = 7b5449f92a4b30a095cc4a67f0a8b1fc69680e15 - -# individual problems follow - -[pml_x_yee] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py - -[pml_x_ckc] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=ckc -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py - -#[pml_x_psatd] -#buildDir = . -#inputFile = Examples/Tests/pml/inputs_2d -#runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 -#dim = 2 -#addToCompileString = USE_FFT=TRUE USE_GPU=TRUE -#restartTest = 0 -#useMPI = 1 -#numprocs = 2 -#useOMP = 0 -#numthreads = 1 -#compileTest = 0 -#doVis = 0 -#analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py -# -[RigidInjection_lab] -buildDir = . -inputFile = Examples/Tests/RigidInjection/inputs_2d_LabFrame -runtime_params = -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/RigidInjection/analysis_rigid_injection_LabFrame.py - -[RigidInjection_boost_backtransformed] -buildDir = . -inputFile = Examples/Tests/RigidInjection/inputs_2d_BoostedFrame -runtime_params = -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 -aux1File = Tools/PostProcessing/read_raw_data.py -analysisRoutine = Examples/Tests/RigidInjection/analysis_rigid_injection_BoostedFrame.py - -[nci_corrector] -buildDir = . -inputFile = Examples/Tests/nci_fdtd_stability/inputs_2d -runtime_params = amr.max_level=0 particles.use_fdtd_nci_corr=1 -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 -analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py - -# [nci_correctorMR] -# buildDir = . -# inputFile = Examples/Tests/nci_fdtd_stability/inputs_2d -# runtime_params = amr.max_level=1 particles.use_fdtd_nci_corr=1 -# dim = 2 -# addToCompileString = USE_GPU=TRUE -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# doComparison = 0 -# analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py -# -# [ionization_lab] -# buildDir = . -# inputFile = Examples/Tests/ionization/inputs_2d_rt -# runtime_params = -# dim = 2 -# addToCompileString = USE_GPU=TRUE -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# analysisRoutine = Examples/Tests/ionization/analysis_ionization.py -# -# [ionization_boost] -# buildDir = . -# inputFile = Examples/Tests/ionization/inputs_2d_bf_rt -# runtime_params = -# dim = 2 -# addToCompileString = USE_GPU=TRUE -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# analysisRoutine = Examples/Tests/ionization/analysis_ionization.py -# -[bilinear_filter] -buildDir = . -inputFile = Examples/Tests/single_particle/inputs_2d -runtime_params = warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -analysisRoutine = Examples/Tests/single_particle/analysis_bilinear_filter.py - -[Langmuir_2d] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir2d.py -analysisOutputImage = langmuir2d_analysis.png - -[Langmuir_2d_single_precision] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux -dim = 2 -addToCompileString = USE_GPU=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir2d.py -analysisOutputImage = langmuir2d_analysis.png - -[Langmuir_2d_nompi] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 0 -numprocs = 1 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir2d.py -analysisOutputImage = langmuir2d_analysis.png - -[Langmuir_x] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex jx diag1.electrons.variables=w ux -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir.py -analysisOutputImage = langmuir_x_analysis.png - -[Langmuir_y] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -runtime_params = electrons.uy=0.01 electrons.ymax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ey jy diag1.electrons.variables=w uy -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir.py -analysisOutputImage = langmuir_y_analysis.png - -[Langmuir_z] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_rt -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -runtime_params = electrons.uz=0.01 electrons.zmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ez jz diag1.electrons.variables=w uz -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir.py -analysisOutputImage = langmuir_z_analysis.png - -[Langmuir_multi] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi.py -analysisOutputImage = langmuir_multi_analysis.png - -[Langmuir_multi_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi.py -analysisOutputImage = langmuir_multi_analysis.png - -[Langmuir_multi_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt -runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi.py -analysisOutputImage = langmuir_multi_analysis.png - -[Langmuir_multi_psatd_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt -runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi.py -analysisOutputImage = langmuir_multi_analysis.png - -[Langmuir_multi_2d_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png - -[Langmuir_multi_2d_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons positrons -analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png - -# [Langmuir_multi_2d_psatd_nodal] -# buildDir = . -# inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt -# runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell -# dim = 2 -# addToCompileString = USE_FFT=TRUE USE_GPU=TRUE -# restartTest = 0 -# useMPI = 1 -# numprocs = 4 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# particleTypes = electrons positrons -# analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi_2d.py -# analysisOutputImage = langmuir_multi_2d_analysis.png -# -# [Langmuir_multi_rz] -# buildDir = . -# inputFile = Examples/Tests/langmuir/inputs_2d_multi_rz_rt -# dim = 2 -# addToCompileString = USE_RZ=TRUE USE_GPU=TRUE -# restartTest = 0 -# useMPI = 1 -# numprocs = 4 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz -# compareParticles = 0 -# particleTypes = electrons ions -# analysisRoutine = Examples/Tests/langmuir/analysis_langmuir_multi_rz.py -# analysisOutputImage = langmuir_multi_rz_analysis.png -# -# [Langmuir_rz_multimode] -# buildDir = . -# inputFile = Examples/Tests/langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py -# customRunCmd = python PICMI_inputs_langmuir_rz_multimode_analyze.py -# runtime_params = -# dim = 2 -# addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE USE_GPU=TRUE PYINSTALLOPTIONS="--user --prefix=" -# restartTest = 0 -# useMPI = 1 -# numprocs = 4 -# useOMP = 0 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# particleTypes = electrons protons -# outputFile = diags/plotfiles/plt00040 -# -[LaserInjection] -buildDir = . -inputFile = Examples/Tests/laser_injection/inputs_3d_rt -dim = 3 -runtime_params = max_step=20 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py -analysisOutputImage = laser_analysis.png - -[LaserInjection_2d] -buildDir = . -inputFile = Examples/Tests/laser_injection/inputs_2d_rt -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 -compareParticles = 0 - -#xxxxx -#[LaserAcceleration] -#buildDir = . -#inputFile = Examples/Physics_applications/laser_acceleration/inputs_3d -#runtime_params = warpx.do_dynamic_scheduling=0 amr.n_cell=32 32 256 max_step=100 electrons.zmin=0.e-6 warpx.serialize_initial_conditions=1 -#dim = 3 -#addToCompileString = USE_GPU=TRUE -#restartTest = 0 -#useMPI = 1 -#numprocs = 2 -#useOMP = 0 -#numthreads = 1 -#compileTest = 0 -#doVis = 0 -#compareParticles = 0 -#particleTypes = electrons -# -[subcyclingMR] -buildDir = . -inputFile = Examples/Tests/subcycling/inputs_2d -runtime_params = warpx.serialize_initial_conditions=1 warpx.do_dynamic_scheduling=0 -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 - -[LaserAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d -runtime_params = amr.max_level=1 max_step=200 warpx.serialize_initial_conditions=1 warpx.fine_tag_lo=-5.e-6 -35.e-6 warpx.fine_tag_hi=5.e-6 -25.e-6 -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons beam - -[PlasmaAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 warpx.serialize_initial_conditions=1 warpx.do_dynamic_scheduling=0 -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam driver plasma_e - -[Python_Langmuir] -buildDir = . -inputFile = Examples/Tests/langmuir/PICMI_inputs_langmuir_rt.py -customRunCmd = python PICMI_inputs_langmuir_rt.py -runtime_params = -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_GPU=TRUE PYINSTALLOPTIONS="--user --prefix=" -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons -outputFile = diags/diag200040 - -[uniform_plasma_restart] -buildDir = . -inputFile = Examples/Physics_applications/uniform_plasma/inputs_3d -runtime_params = chk.file_prefix=uniform_plasma_restart_chk -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 1 -restartFileNum = 6 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons - -[particles_in_pml_2d] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_2d -runtime_params = -dim = 2 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[particles_in_pml] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_3d -runtime_params = -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[photon_pusher] -buildDir = . -inputFile = Examples/Tests/photon_pusher/inputs_3d -runtime_params = -dim = 3 -addToCompileString = USE_GPU=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini deleted file mode 100644 index 96dbea5c380..00000000000 --- a/Regression/WarpX-tests.ini +++ /dev/null @@ -1,3967 +0,0 @@ -# This file is used both for the nightly regression tests -# on the battra server, and for CI tests. -# In the case of CI, some of the parameters entered -# below are overwritten, see prepare_file_ci.py -[main] -# repeat captured errors to stderr, e.g., for CI runs -verbose = 1 - -testTopDir = /home/regtester/AMReX_RegTesting/rt-WarpX/ -webTopDir = /home/regtester/AMReX_RegTesting/rt-WarpX/web - -sourceTree = C_Src - -# suiteName is the name prepended to all output directories -suiteName = WarpX - -archive_output = 0 -purge_output = 1 - -useCmake = 1 -isSuperbuild = 1 -MAKE = make -numMakeJobs = 8 - -# We build by default a few tools for output comparison. -# The build time for those can be skipped if they are not needed. -ftools = - -# Control the build of the particle_compare tool. -# Needed for test particle_tolerance option. -use_ctools = 0 - -# MPIcommand should use the placeholders: -# @host@ to indicate where to put the hostname to run on -# @nprocs@ to indicate where to put the number of processors -# @command@ to indicate where to put the command to run -# -# only tests with useMPI = 1 will run in parallel -# nprocs is problem dependent and specified in the individual problem -# sections. - -#MPIcommand = mpiexec -host @host@ -n @nprocs@ @command@ -MPIcommand = mpiexec -n @nprocs@ @command@ -MPIhost = - -reportActiveTestsOnly = 1 - -# Add "GO UP" link at the top of the web page? -goUpLink = 1 - -# string queried to change plotfiles and checkpoint files -plot_file_name = diag1.file_prefix -check_file_name = none - -# email -sendEmailWhenFail = 1 -emailTo = weiqunzhang@lbl.gov, jlvay@lbl.gov, rlehe@lbl.gov, atmyers@lbl.gov, oshapoval@lbl.gov, henri.vincenti@cea.fr, rjambunathan@lbl.gov, yinjianzhao@lbl.gov -emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more details. - -[AMReX] -dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 - -[source] -dir = /home/regtester/AMReX_RegTesting/warpx -branch = development -cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_PYTHON_IPO=OFF -DpyAMReX_IPO=OFF -# -DPYINSTALLOPTIONS="--disable-pip-version-check" - -# individual problems follow - -[averaged_galilean_2d_psatd] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d -runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[averaged_galilean_2d_psatd_hybrid] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d -runtime_params = amr.max_grid_size_x=128 amr.max_grid_size_y=64 warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[averaged_galilean_3d_psatd] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d -runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[averaged_galilean_3d_psatd_hybrid] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d -runtime_params = warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[background_mcc] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[background_mcc_dp_psp] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PRECISION=DOUBLE -DWarpX_PARTICLE_PRECISION=SINGLE -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[bilinear_filter] -buildDir = . -inputFile = Examples/Tests/single_particle/inputs_2d -runtime_params = warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/single_particle/analysis_bilinear_filter.py - -[BTD_rz] -buildDir = . -inputFile = Examples/Tests/btd_rz/inputs_rz_z_boosted_BTD -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py - -[collider_diagnostics] -buildDir = . -inputFile = Examples/Tests/collider_relevant_diags/inputs_3d_multiple_particles -runtime_params = warpx.abort_on_warning_threshold=high -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py - -[collisionZ] -buildDir = . -inputFile = Examples/Tests/collision/inputs_1d -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_1d.py - -[collisionISO] -buildDir = . -inputFile = Examples/Tests/collision/inputs_3d_isotropization -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_3d_isotropization.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[collisionRZ] -buildDir = . -inputFile = Examples/Tests/collision/inputs_rz -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=FALSE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=OFF -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[collisionXYZ] -buildDir = . -inputFile = Examples/Tests/collision/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[collisionXZ] -buildDir = . -inputFile = Examples/Tests/collision/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[comoving_2d_psatd_hybrid] -buildDir = . -inputFile = Examples/Tests/comoving/inputs_2d_hybrid -runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Deuterium_Deuterium_Fusion_3D] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py - -[Deuterium_Deuterium_Fusion_3D_intraspecies] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py - -[Deuterium_Tritium_Fusion_3D] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py - -[Deuterium_Tritium_Fusion_RZ] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_rz -runtime_params = warpx.abort_on_warning_threshold=high -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py - -[dirichletbc] -buildDir = . -inputFile = Examples/Tests/electrostatic_dirichlet_bc/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py - -[divb_cleaning_3d] -buildDir = . -inputFile = Examples/Tests/divb_cleaning/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/divb_cleaning/analysis.py - -[dive_cleaning_2d] -buildDir = . -inputFile = Examples/Tests/dive_cleaning/inputs_3d -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = geometry.dims=2 -analysisRoutine = Examples/Tests/dive_cleaning/analysis.py - -[dive_cleaning_3d] -buildDir = . -inputFile = Examples/Tests/dive_cleaning/inputs_3d -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = -analysisRoutine = Examples/Tests/dive_cleaning/analysis.py - -[ElectrostaticSphere] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py - -[ElectrostaticSphereLabFrame_MR_emass_10] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py - -[ElectrostaticSphereEB] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py - -[ElectrostaticSphereEB_mixedBCs] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_3d_mixed_BCs -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[EmbeddedBoundaryDiffraction] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_diffraction/inputs_rz -runtime_params = -dim = 2 -addToCompileString = USE_EB=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = EmbeddedBoundaryDiffraction_plt -analysisRoutine = Examples/Tests/embedded_boundary_diffraction/analysis_fields.py - -[ElectrostaticSphereEB_RZ] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_rz -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = USE_EB=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py - -[ElectrostaticSphereEB_RZ_MR] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr -runtime_params = warpx.abort_on_warning_threshold = medium amr.ref_ratio_vect = 2 2 2 -dim = 2 -addToCompileString = USE_EB=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = ElectrostaticSphereEB_RZ_MR_plt -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py - -[ElectrostaticSphereLabFrame] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w phi -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py - -[ElectrostaticSphereRZ] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere/inputs_rz -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py - -[ElectrostaticSphereRelNodal] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = medium warpx.grid_type = collocated -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py - -[embedded_boundary_cube] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py - -[embedded_boundary_cube_2d] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_cube/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py - -[embedded_boundary_cube_macroscopic] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_cube/inputs_3d -runtime_params = algo.em_solver_medium=macroscopic macroscopic.epsilon=1.5*8.8541878128e-12 macroscopic.sigma=0 macroscopic.mu=1.25663706212e-06 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py - -[embedded_boundary_python_API] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py -runtime_params = -customRunCmd = python PICMI_inputs_EB_API.py -dim = 3 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_python_api/analysis.py - -[embedded_boundary_rotated_cube] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_rotated_cube/inputs_3d -runtime_params = warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py - -[embedded_boundary_rotated_cube_2d] -buildDir = . -inputFile = Examples/Tests/embedded_boundary_rotated_cube/inputs_2d -runtime_params = warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py - -[embedded_circle] -buildDir = . -inputFile = Examples/Tests/embedded_circle/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = USE_EB=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/embedded_circle/analysis.py - -[FieldProbe] -buildDir = . -inputFile = Examples/Tests/field_probe/inputs_2d -runtime_params = -dim = 2 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/field_probe/analysis_field_probe.py - -[FluxInjection] -buildDir = . -inputFile = Examples/Tests/flux_injection/inputs_rz -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_rz.py - -[FluxInjection3D] -buildDir = . -inputFile = Examples/Tests/flux_injection/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_3d.py - -[galilean_2d_psatd] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_2d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_2d_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_2d -runtime_params = psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE amr.max_grid_size=64 amr.blocking_factor=64 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_2d_psatd_current_correction_psb] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_2d -runtime_params = psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_2d_psatd_hybrid] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_2d_hybrid -runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[galilean_3d_psatd] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_3d -runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_3d_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_3d -runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 2 psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_3d_psatd_current_correction_psb] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_3d -runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 1 psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_rz_psatd] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_rz -runtime_params = electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_rz_psatd_current_correction_psb] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_rz -runtime_params = psatd.periodic_single_box_fft=1 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[galilean_rz_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_rz -runtime_params = psatd.periodic_single_box_fft=0 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 amr.max_grid_size=32 amr.blocking_factor=32 -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py - -[hard_edged_plasma_lens] -buildDir = . -inputFile = Examples/Tests/plasma_lens/inputs_lattice_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/plasma_lens/analysis.py - -[hard_edged_quadrupoles] -buildDir = . -inputFile = Examples/Tests/AcceleratorLattice/inputs_quad_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py - -[hard_edged_quadrupoles_boosted] -buildDir = . -inputFile = Examples/Tests/AcceleratorLattice/inputs_quad_boosted_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py - -[hard_edged_quadrupoles_moving] -buildDir = . -inputFile = Examples/Tests/AcceleratorLattice/inputs_quad_moving_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py - -[initial_distribution] -buildDir = . -inputFile = Examples/Tests/initial_distribution/inputs -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/initial_distribution/analysis_distribution.py -aux1File = Tools/PostProcessing/read_raw_data.py - -[ionization_boost] -buildDir = . -inputFile = Examples/Tests/ionization/inputs_2d_bf_rt -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ionization/analysis_ionization.py - -[ionization_lab] -buildDir = . -inputFile = Examples/Tests/ionization/inputs_2d_rt -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ionization/analysis_ionization.py - -[ion_stopping] -buildDir = . -inputFile = Examples/Tests/ion_stopping/inputs_3d -runtime_params = warpx.cfl=0.7 -dim = 3 -addToCompileString = -cmakeSetupOpts = -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py - -[Langmuir_multi] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_fluid_1D] -buildDir = . -inputFile = Examples/Tests/langmuir_fluids/inputs_1d -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py - -[Langmuir_fluid_RZ] -buildDir = . -inputFile = Examples/Tests/langmuir_fluids/inputs_rz -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py - -[Langmuir_fluid_2D] -buildDir = . -inputFile = Examples/Tests/langmuir_fluids/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py - -[Langmuir_fluid_multi] -buildDir = . -inputFile = Examples/Tests/langmuir_fluids/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py - -[Langmuir_multi_1d] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_1d -runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=z w ux uy uz diag1.positrons.variables=z w ux uy uz -dim = 1 -addToCompileString = USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_1d.py - -[Langmuir_multi_2d_MR] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_MR_anisotropic] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_MR_momentum_conserving] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_MR_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_current_correction_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_momentum_conserving] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_multiJ] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_multiJ_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_Vay_deposition] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_2d_psatd_Vay_deposition_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_2d.py - -[Langmuir_multi_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_current_correction_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_div_cleaning] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 psatd.update_with_rho = 1 algo.current_deposition = direct warpx.do_dive_cleaning = 1 warpx.do_divb_cleaning = 1 diag1.intervals = 0, 38:40:1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE F warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_momentum_conserving] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_multiJ] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_multiJ_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_single_precision] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PRECISION=SINGLE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_Vay_deposition] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_psatd_Vay_deposition_nodal] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Langmuir_multi_rz] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[Langmuir_multi_rz_psatd] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[Langmuir_multi_rz_psatd_current_correction] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[Langmuir_multi_rz_psatd_multiJ] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[Langmuir_multi_single_precision] -buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = -dim = 3 -addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/langmuir/analysis_3d.py - -[Larmor] -buildDir = . -inputFile = Examples/Tests/larmor/inputs_2d_mr -runtime_params = max_step=10 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[LaserAcceleration] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = LaserAcceleration_plt -analysisRoutine = Examples/analysis_default_openpmd_regression.py - -[LaserAcceleration_1d] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d -runtime_params = -dim = 1 -addToCompileString = USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[LaserAcceleration_1d_fluid] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids -runtime_params = -dim = 1 -addToCompileString = USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py - -[LaserAcceleration_1d_fluid_boosted] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted -runtime_params = -dim = 1 -addToCompileString = USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py - -[LaserAccelerationBoost] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d_boost -runtime_params = amr.n_cell=64 512 max_step=300 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[LaserAcceleration_BTD] -buildDir = . -inputFile = Examples/Tests/boosted_diags/inputs_3d -runtime_params = -dim = 3 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/boosted_diags/analysis.py - -[LaserAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[LaserAccelerationRZ] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_rz -runtime_params = diag1.dump_rz_modes=1 warpx.abort_on_warning_threshold=high -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[LaserAccelerationRZ_opmd] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_rz -runtime_params = diag1.format=openpmd diag1.openpmd_backend=h5 max_step=20 diag1.fields_to_plot=Er Bt Bz jr jt jz rho part_per_cell part_per_grid rho_beam rho_electrons warpx.abort_on_warning_threshold=high -dim = 2 -addToCompileString = USE_RZ=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = LaserAccelerationRZ_opmd_plt -analysisRoutine = Examples/Tests/openpmd_rz/analysis_openpmd_rz.py - -[LaserAcceleration_single_precision_comms] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_3d -runtime_params = warpx.do_single_precision_comms=1 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = LaserAcceleration_single_precision_comms_plt -analysisRoutine = Examples/analysis_default_openpmd_regression.py - -[LaserInjection] -buildDir = . -inputFile = Examples/Tests/laser_injection/inputs_3d_rt -runtime_params = max_step=20 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py - -[LaserInjection_1d] -buildDir = . -inputFile = Examples/Tests/laser_injection/inputs_1d_rt -runtime_params = -dim = 1 -addToCompileString = USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/laser_injection/analysis_1d.py - -[LaserInjection_2d] -buildDir = . -inputFile = Examples/Tests/laser_injection/inputs_2d_rt -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/laser_injection/analysis_2d.py - -[LaserInjectionFromBINARYFile] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_2d_binary.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.2d_test_binary -customRunCmd = ./analysis_2d_binary.py -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromLASYFile] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_3d.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.3d_test -customRunCmd = ./analysis_3d.py -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromLASYFile_1d] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_1d.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.1d_test -customRunCmd = ./analysis_1d.py -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromLASYFile_1d_boost] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_1d_boost.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.1d_boost_test -customRunCmd = ./analysis_1d_boost.py -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromLASYFile_2d] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_2d.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.2d_test -customRunCmd = ./analysis_2d.py -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromLASYFile_RZ] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_RZ.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.RZ_test -customRunCmd = ./analysis_RZ.py -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserInjectionFromRZLASYFile] -buildDir = . -inputFile = Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py -aux1File = Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test -customRunCmd = ./analysis_from_RZ_file.py -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -selfTest = 1 -stSuccessString = Passed - -[LaserIonAcc2d] -buildDir = . -inputFile = Examples/Physics_applications/laser_ion/inputs_2d -outputFile = LaserIonAcc2d_plt -runtime_params = -dim = 2 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_openpmd_regression.py - -[LaserOnFine] -buildDir = . -inputFile = Examples/Tests/laser_on_fine/inputs_2d -runtime_params = max_step=50 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[leveling_thinning] -buildDir = . -inputFile = Examples/Tests/resampling/inputs_leveling_thinning -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py - -[LoadExternalFieldRZGrid] -buildDir = . -inputFile = Examples/Tests/LoadExternalField/inputs_rz_grid_fields -runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZGrid_chk chk.file_min_digits=5 -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 1 -restartFileNum = 150 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py - -[LoadExternalFieldRZParticles] -buildDir = . -inputFile = Examples/Tests/LoadExternalField/inputs_rz_particle_fields -runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZParticles_chk chk.file_min_digits=5 -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 1 -restartFileNum = 150 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py - -[magnetostatic_eb_3d] -buildDir = . -inputFile = Examples/Tests/magnetostatic_eb/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 2 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Maxwell_Hybrid_QED_solver] -buildDir = . -inputFile = Examples/Tests/maxwell_hybrid_qed/inputs_2d -runtime_params = warpx.cfl=0.7071067811865475 -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py - -[momentum-conserving-gather] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 algo.field_gathering=momentum-conserving -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[multi_J_rz_psatd] -buildDir = . -inputFile = Examples/Tests/multi_j/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[nci_corrector] -buildDir = . -inputFile = Examples/Tests/nci_fdtd_stability/inputs_2d -runtime_params = amr.max_level=0 particles.use_fdtd_nci_corr=1 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py - -[nci_correctorMR] -buildDir = . -inputFile = Examples/Tests/nci_fdtd_stability/inputs_2d -runtime_params = amr.max_level=1 particles.use_fdtd_nci_corr=1 amr.n_cell=64 64 warpx.fine_tag_lo=-20.e-6 -20.e-6 warpx.fine_tag_hi=20.e-6 20.e-6 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py - -[parabolic_channel_initialization_2d_single_precision] -buildDir = . -inputFile = Examples/Tests/initial_plasma_profile/inputs -dim = 2 -addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PRECISION=SINGLE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = -analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py - -[particle_absorption] -buildDir = . -inputFile = Examples/Tests/particle_boundary_process/inputs_absorption -runtime_params = -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_boundary_process/analysis_absorption.py - -[particle_boundaries_3d] -buildDir = . -inputFile = Examples/Tests/boundaries/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/boundaries/analysis.py - -[particle_fields_diags] -buildDir = . -inputFile = Examples/Tests/particle_fields_diags/inputs -aux1File = Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py -runtime_params = -dim = 3 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags.py - -[particle_fields_diags_single_precision] -buildDir = . -inputFile = Examples/Tests/particle_fields_diags/inputs -aux1File = Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py -runtime_params = -dim = 3 -addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py - -[particle_pusher] -buildDir = . -inputFile = Examples/Tests/particle_pusher/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_pusher/analysis_pusher.py - -[particle_scrape] -buildDir = . -inputFile = Examples/Tests/particle_boundary_scrape/inputs_scrape -runtime_params = -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py - -[particles_in_pml] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[particles_in_pml_2d] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[particles_in_pml_2d_MR] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_mr_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[particles_in_pml_3d_MR] -buildDir = . -inputFile = Examples/Tests/particles_in_pml/inputs_mr_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py - -[PEC_field] -buildDir = . -inputFile = Examples/Tests/pec/inputs_field_PEC_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pec/analysis_pec.py - -[PEC_field_mr] -buildDir = . -inputFile = Examples/Tests/pec/inputs_field_PEC_mr_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pec/analysis_pec_mr.py - -[PEC_particle] -buildDir = . -inputFile = Examples/Tests/pec/inputs_particle_PEC_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Performance_works_1_uniform_rest_32ppc] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_1_uniform_rest_32ppc -runtime_params = amr.max_grid_size=32 amr.n_cell=32 32 32 max_step=5 diagnostics.diags_names=diag1 diag1.intervals=0 diag1.diag_type=Full -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Performance_works_2_uniform_rest_1ppc] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_2_uniform_rest_1ppc -runtime_params = amr.max_grid_size=32 amr.n_cell=32 32 32 max_step=5 diagnostics.diags_names=diag1 diag1.intervals=0 diag1.diag_type=Full -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Performance_works_3_uniform_drift_4ppc] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_3_uniform_drift_4ppc -runtime_params = amr.max_grid_size=32 amr.n_cell=32 32 32 max_step=5 diagnostics.diags_names=diag1 diag1.intervals=0 diag1.diag_type=Full -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Performance_works_4_labdiags_2ppc] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_4_labdiags_2ppc -runtime_params = amr.n_cell=64 64 64 max_step=10 diagnostics.diags_names=diag1 diag1.intervals=0 diag1.diag_type=Full -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Performance_works_5_loadimbalance] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_5_loadimbalance -runtime_params = amr.max_grid_size=32 amr.n_cell=32 32 32 max_step=5 diagnostics.diags_names=diag1 diag1.intervals=0 diag1.diag_type=Full -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Performance_works_6_output_2ppc] -buildDir = . -inputFile = Examples/Tests/performance_tests/automated_test_6_output_2ppc -runtime_params = amr.n_cell=64 64 64 max_step=10 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[photon_pusher] -buildDir = . -inputFile = Examples/Tests/photon_pusher/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py - -[PlasmaAccelerationBoost2d] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d_boost -runtime_params = amr.n_cell=64 256 max_step=20 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[PlasmaAccelerationBoost3d] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_3d_boost -runtime_params = amr.n_cell=64 64 128 max_step=5 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[PlasmaAccelerationBoost3d_hybrid] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_3d_boost -runtime_params = amr.n_cell=64 64 128 max_step=25 warpx.grid_type=hybrid warpx.do_current_centering=0 -dim = 3 -addToCompileString = -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[PlasmaAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Plasma_lens] -buildDir = . -inputFile = Examples/Tests/plasma_lens/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/plasma_lens/analysis.py - -[Plasma_lens_boosted] -buildDir = . -inputFile = Examples/Tests/plasma_lens/inputs_boosted_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/plasma_lens/analysis.py - -[Plasma_lens_short] -buildDir = . -inputFile = Examples/Tests/plasma_lens/inputs_short_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/plasma_lens/analysis.py - -[PlasmaMirror] -buildDir = . -inputFile = Examples/Physics_applications/plasma_mirror/inputs_2d -runtime_params = amr.n_cell=256 128 max_step=20 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[pml_psatd_dive_divb_cleaning] -buildDir = . -inputFile = Examples/Tests/pml/inputs_3d -runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium ablastr.fillboundary_always_sync=1 -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[pml_psatd_rz] -buildDir = . -inputFile = Examples/Tests/pml/inputs_rz -runtime_params = warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_RZ=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_psatd_rz.py - -[pml_x_ckc] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=ckc -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py - -[pml_x_galilean] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py - -[pml_x_psatd] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 1 -restartFileNum = 150 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py - -[pml_x_yee] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_chk chk.file_min_digits=5 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 1 -restartFileNum = 150 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py - -[pml_x_yee_eb] -buildDir = . -inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_eb_chk chk.file_min_digits=5 -dim = 2 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON -restartTest = 1 -restartFileNum = 150 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py - -[Proton_Boron_Fusion_2D] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_proton_boron_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py - -[Proton_Boron_Fusion_3D] -buildDir = . -inputFile = Examples/Tests/nuclear_fusion/inputs_proton_boron_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py - -[Python_background_mcc] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_2d.py - -[Python_background_mcc_1d] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py --test --pythonsolver -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py - -[Python_background_mcc_1d_tridiag] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py --test -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py - -[Python_collisionXZ] -buildDir = . -inputFile = Examples/Tests/collision/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py -aux1File = Regression/PostProcessingUtils/post_processing_utils.py - -[Python_dirichletbc] -buildDir = . -inputFile = Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py - -[Python_dsmc_1d] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py - -[Python_ElectrostaticSphereEB] -buildDir = . -inputFile = Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py -dim = 3 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py - -[Python_gaussian_beam] -buildDir = . -inputFile = Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py -customRunCmd = python3 PICMI_inputs_gaussian_beam.py -runtime_params = -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_gaussian_beam_no_field_output] -buildDir = . -inputFile = Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py -customRunCmd = python3 PICMI_inputs_gaussian_beam.py --fields_to_plot none -runtime_params = -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Python_gaussian_beam_opmd] -buildDir = . -inputFile = Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py -customRunCmd = python3 PICMI_inputs_gaussian_beam.py --diagformat=openpmd -runtime_params = -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Python_gaussian_beam_opmd_no_field_output] -buildDir = . -inputFile = Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py -customRunCmd = python PICMI_inputs_gaussian_beam.py --diagformat=openpmd --fields_to_plot none -runtime_params = -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = - -[Python_id_cpu_read] -buildDir = . -inputFile = Examples/Tests/restart/PICMI_inputs_id_cpu_read.py -runtime_params = -customRunCmd = python3 PICMI_inputs_id_cpu_read.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 - -[Python_ionization] -buildDir = . -inputFile = Examples/Tests/ionization/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ionization/analysis_ionization.py - -[Python_Langmuir] -buildDir = . -inputFile = Examples/Tests/langmuir/PICMI_inputs_3d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_Langmuir_2d] -buildDir = . -inputFile = Examples/Tests/langmuir/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_Langmuir_rz_multimode] -buildDir = . -inputFile = Examples/Tests/langmuir/PICMI_inputs_rz.py -runtime_params = -customRunCmd = python3 PICMI_inputs_rz.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_LaserAcceleration] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_LaserAcceleration_1d] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_LaserAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_LaserAccelerationRZ] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py -runtime_params = -customRunCmd = python3 PICMI_inputs_rz.py -dim = 2 -addToCompileString = USE_RZ=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_LaserIonAcc2d] -buildDir = . -inputFile = Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py -outputFile= diags/Python_LaserIonAcc2d_plt -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_openpmd_regression.py - -[Python_LoadExternalGridField3D] -buildDir = . -inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d_grid_fields.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py - -[Python_LoadExternalParticleField3D] -buildDir = . -inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d_particle_fields.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py - -[Python_magnetostatic_eb_3d] -buildDir = . -inputFile = Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py -dim = 3 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_EB=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 2 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_magnetostatic_eb_rz] -buildDir = . -inputFile = Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py -runtime_params = -customRunCmd = python3 PICMI_inputs_rz.py -dim = 2 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_EB=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 2 -analysisRoutine = Examples/Tests/magnetostatic_eb/analysis_rz.py - -[Python_ohms_law_solver_EM_modes_1d] -buildDir = . -inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py -runtime_params = warpx.abort_on_warning_threshold = medium -customRunCmd = python3 PICMI_inputs.py --test --dim 1 --bdir z -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis.py - -[Python_ohms_law_solver_EM_modes_rz] -buildDir = . -inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py -runtime_params = warpx.abort_on_warning_threshold = medium -customRunCmd = python3 PICMI_inputs_rz.py --test -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis_rz.py - -[Python_ohms_law_solver_ion_beam_1d] -buildDir = . -inputFile = Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py -runtime_params = warpx.abort_on_warning_threshold = medium -customRunCmd = python3 PICMI_inputs.py --test --dim 1 --resonant -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ohm_solver_ion_beam_instability/analysis.py - -[Python_ohms_law_solver_landau_damping_2d] -buildDir = . -inputFile = Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py -runtime_params = warpx.abort_on_warning_threshold = medium -customRunCmd = python3 PICMI_inputs.py --test --dim 2 --temp_ratio 0.1 -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py - -[Python_ohms_law_solver_magnetic_reconnection_2d] -buildDir = . -inputFile = Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py -runtime_params = warpx.abort_on_warning_threshold = medium -customRunCmd = python3 PICMI_inputs.py --test -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py - -[Python_particle_attr_access] -buildDir = . -inputFile = Examples/Tests/particle_data_python/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_data_python/analysis.py - -[Python_particle_attr_access_unique] -buildDir = . -inputFile = Examples/Tests/particle_data_python/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py --unique -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_data_python/analysis.py - -[Python_particle_reflection] -buildDir = . -inputFile = Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py -runtime_params = -customRunCmd = python3 PICMI_inputs_reflection.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_boundary_process/analysis_reflection.py - -[Python_particle_scrape] -buildDir = . -inputFile = Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py -runtime_params = -customRunCmd = python3 PICMI_inputs_scrape.py -dim = 3 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py - -# TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again -# https://github.com/AMReX-Codes/pyamrex/issues/163 -[Python_pass_mpi_comm] -buildDir = . -inputFile = Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -# TODO: comment in again once enabled -#analysisRoutine = Examples/Tests/pass_mpi_communicator/analysis.py - -[Python_PlasmaAcceleration] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py -runtime_params = -customRunCmd = python3 PICMI_inputs_plasma_acceleration.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_PlasmaAcceleration1d] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_plasma_acceleration_1d.py -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_PlasmaAccelerationMR] -buildDir = . -inputFile = Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py -runtime_params = -customRunCmd = python3 PICMI_inputs_plasma_acceleration_mr.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_plasma_lens] -buildDir = . -inputFile = Examples/Tests/plasma_lens/PICMI_inputs_3d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/plasma_lens/analysis.py - -[Python_prev_positions] -buildDir = . -inputFile = Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_prev_pos_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Python_reduced_diags_loadbalancecosts_timers] -buildDir = . -inputFile = Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py -runtime_params = algo.load_balance_costs_update=Timers -customRunCmd = python3 PICMI_inputs_loadbalancecosts.py -dim = 3 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py - -[Python_restart_eb] -buildDir = . -inputFile = Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py -runtime_params = -customRunCmd = python3 PICMI_inputs_restart_eb.py -dim = 3 -addToCompileString = USE_EB=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 1 -restartFileNum = 30 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/restart/analysis_restart.py - -[Python_restart_runtime_components] -buildDir = . -inputFile = Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py -runtime_params = -customRunCmd = python3 PICMI_inputs_runtime_component_analyze.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 1 -restartFileNum = 5 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 - -[Python_wrappers] -buildDir = . -inputFile = Examples/Tests/python_wrappers/PICMI_inputs_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_2d.py -dim = 2 -addToCompileString = USE_FFT=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[qed_breit_wheeler_2d] -buildDir = . -inputFile = Examples/Tests/qed/breit_wheeler/inputs_2d -aux1File = Examples/Tests/qed/breit_wheeler/analysis_core.py -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py - -[qed_breit_wheeler_2d_opmd] -buildDir = . -inputFile = Examples/Tests/qed/breit_wheeler/inputs_2d -aux1File = Examples/Tests/qed/breit_wheeler/analysis_core.py -runtime_params = diag1.format = openpmd diag1.openpmd_backend = h5 warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = QED=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_QED=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = qed_breit_wheeler_2d_opmd_plt -analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py - -[qed_breit_wheeler_3d] -buildDir = . -inputFile = Examples/Tests/qed/breit_wheeler/inputs_3d -aux1File = Examples/Tests/qed/breit_wheeler/analysis_core.py -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py - -[qed_breit_wheeler_3d_opmd] -buildDir = . -inputFile = Examples/Tests/qed/breit_wheeler/inputs_3d -aux1File = Examples/Tests/qed/breit_wheeler/analysis_core.py -runtime_params = diag1.format = openpmd diag1.openpmd_backend = h5 warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = QED=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = qed_breit_wheeler_3d_opmd_plt -analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py - -[qed_quantum_sync_2d] -buildDir = . -inputFile = Examples/Tests/qed/quantum_synchrotron/inputs_2d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 2 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py - -[qed_quantum_sync_3d] -buildDir = . -inputFile = Examples/Tests/qed/quantum_synchrotron/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py - -[qed_schwinger1] -buildDir = . -inputFile = Examples/Tests/qed/schwinger/inputs_3d_schwinger -runtime_params = warpx.E_external_grid = 1.e16 0 0 warpx.B_external_grid = 16792888.570516706 5256650.141557486 18363530.799561853 -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py - -[qed_schwinger2] -buildDir = . -inputFile = Examples/Tests/qed/schwinger/inputs_3d_schwinger -runtime_params = warpx.E_external_grid = 1.e18 0 0 warpx.B_external_grid = 1679288857.0516706 525665014.1557486 1836353079.9561853 qed_schwinger.xmin = -2.5e-7 qed_schwinger.xmax = 2.49e-7 -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py - -[qed_schwinger3] -buildDir = . -inputFile = Examples/Tests/qed/schwinger/inputs_3d_schwinger -runtime_params = warpx.E_external_grid = 0 1.090934525450495e+17 0 -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py - -[qed_schwinger4] -buildDir = . -inputFile = Examples/Tests/qed/schwinger/inputs_3d_schwinger -runtime_params = warpx.E_external_grid = 0 0 2.5e+20 warpx.B_external_grid = 0 833910140000. 0 qed_schwinger.ymin = -2.5e-7 qed_schwinger.zmax = 2.49e-7 -dim = 3 -addToCompileString = QED=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_QED=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py - -[radiation_reaction] -buildDir = . -inputFile = Examples/Tests/radiation_reaction/test_const_B_analytical/inputs_3d -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py - -[reduced_diags] -buildDir = . -inputFile = Examples/Tests/reduced_diags/inputs -aux1File = Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py -runtime_params = -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags.py - -[reduced_diags_loadbalancecosts_heuristic] -buildDir = . -inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = algo.load_balance_costs_update=Heuristic -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py - -[reduced_diags_loadbalancecosts_timers] -buildDir = . -inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = algo.load_balance_costs_update=Timers -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py - -[reduced_diags_loadbalancecosts_timers_psatd] -buildDir = . -inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = algo.load_balance_costs_update=Timers -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py - -[reduced_diags_single_precision] -buildDir = . -inputFile = Examples/Tests/reduced_diags/inputs -aux1File = Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py -runtime_params = -dim = 3 -addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_single.py - -[RefinedInjection] -buildDir = . -inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d -runtime_params = warpx.refine_plasma=1 amr.ref_ratio_vect = 2 1 -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py - -[relativistic_space_charge_initialization] -buildDir = . -inputFile = Examples/Tests/relativistic_space_charge_initialization/inputs_3d -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = -analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py - -[RepellingParticles] -buildDir = . -inputFile = Examples/Tests/repelling_particles/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py - -[resample_velocity_coincidence_thinning] -buildDir = . -inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[resample_velocity_coincidence_thinning_cartesian] -buildDir = . -inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[restart] -buildDir = . -inputFile = Examples/Tests/restart/inputs -runtime_params = chk.file_prefix=restart_chk chk.file_min_digits=5 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 1 -restartFileNum = 5 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/restart/analysis_restart.py - -[restart_psatd] -buildDir = . -inputFile = Examples/Tests/restart/inputs -runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 1 -restartFileNum = 5 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/restart/analysis_restart.py - -[restart_psatd_time_avg] -buildDir = . -inputFile = Examples/Tests/restart/inputs -runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 1 -restartFileNum = 5 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/restart/analysis_restart.py - -[RigidInjection_BTD] -buildDir = . -inputFile = Examples/Tests/rigid_injection/inputs_2d_BoostedFrame -runtime_params = -dim = 2 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py - -[RigidInjection_lab] -buildDir = . -inputFile = Examples/Tests/rigid_injection/inputs_2d_LabFrame -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py - -[scraping] -buildDir = . -inputFile = Examples/Tests/scraping/inputs_rz -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/scraping/analysis_rz.py - -[scraping_filter] -buildDir = . -inputFile = Examples/Tests/scraping/inputs_rz_filter -runtime_params = warpx.abort_on_warning_threshold = medium -dim = 2 -addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py - -[silver_mueller_1d] -buildDir = . -inputFile = Examples/Tests/silver_mueller/inputs_1d -runtime_params = -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py - -[silver_mueller_2d_x] -buildDir = . -inputFile = Examples/Tests/silver_mueller/inputs_2d_x -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py - -[silver_mueller_2d_z] -buildDir = . -inputFile = Examples/Tests/silver_mueller/inputs_2d_z -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py - -[silver_mueller_rz_z] -buildDir = . -inputFile = Examples/Tests/silver_mueller/inputs_rz_z -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py - -[space_charge_initialization] -buildDir = . -inputFile = Examples/Tests/space_charge_initialization/inputs_3d -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = -analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py - -[space_charge_initialization_2d] -buildDir = . -inputFile = Examples/Tests/space_charge_initialization/inputs_3d -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -runtime_params = geometry.dims=2 -analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py - -[subcyclingMR] -buildDir = . -inputFile = Examples/Tests/subcycling/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[Uniform_2d] -buildDir = . -inputFile = Examples/Physics_applications/uniform_plasma/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/analysis_default_regression.py - -[uniform_plasma_restart] -buildDir = . -inputFile = Examples/Physics_applications/uniform_plasma/inputs_3d -runtime_params = chk.file_prefix=uniform_plasma_restart_chk chk.file_min_digits=5 -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 1 -restartFileNum = 6 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/restart/analysis_restart.py - -[uniform_plasma_multiJ] -buildDir = . -inputFile = Examples/Tests/nci_psatd_stability/inputs_3d -runtime_params = psatd.solution_type=first-order psatd.J_in_time=constant psatd.rho_in_time=constant warpx.do_dive_cleaning=1 warpx.do_divb_cleaning=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=1 diag1.fields_to_plot=Bx By Bz divE Ex Ey Ez F G jx jy jz rho warpx.abort_on_warning_threshold=medium -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_multiJ.py - -[VayDeposition2D] -buildDir = . -inputFile = Examples/Tests/vay_deposition/inputs_2d -runtime_params = -dim = 2 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/vay_deposition/analysis.py - -[VayDeposition3D] -buildDir = . -inputFile = Examples/Tests/vay_deposition/inputs_3d -runtime_params = -dim = 3 -addToCompileString = USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/vay_deposition/analysis.py - -[NodalElectrostaticSolver] -buildDir = . -inputFile = Examples/Tests/nodal_electrostatic/inputs_3d -runtime_params = warpx.abort_on_warning_threshold=high -dim = 3 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=3 -restartTest = 0 -useMPI = 1 -numprocs = 1 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py - -[BeamBeamCollision] -buildDir = . -inputFile = Examples/Physics_applications/beam-beam_collision/inputs -runtime_params = warpx.abort_on_warning_threshold=high -dim = 3 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = BeamBeamCollision_plt -analysisRoutine = Examples/analysis_default_openpmd_regression.py - -[spacecraft_charging] -buildDir = . -inputFile = Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py -runtime_params = -customRunCmd = python3 PICMI_inputs_rz.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS="RZ" -DWarpX_EB=ON -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = spacecraft_charging_plt -analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py - -[Point_of_contact_EB_3d] -buildDir = . -inputFile = Examples/Tests/point_of_contact_EB/inputs_3d -runtime_params = -dim = 3 -addToCompileString = USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -outputFile = Point_of_contact_EB_3d_plt -analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py - -[Point_of_contact_EB_rz] -buildDir = . -inputFile = Examples/Tests/point_of_contact_EB/inputs_rz -runtime_params = -dim = 2 -addToCompileString = USE_RZ=TRUE USE_EB=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -outputFile = Point_of_contact_EB_rz_plt -analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py - -[ThetaImplicitPicard_1d] -buildDir = . -inputFile = Examples/Tests/Implicit/inputs_1d -runtime_params = warpx.abort_on_warning_threshold=high -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_1d.py - -[ThetaImplicitJFNK_VandB_2d] -buildDir = . -inputFile = Examples/Tests/Implicit/inputs_vandb_jfnk_2d -runtime_params = warpx.abort_on_warning_threshold=high -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py - -[ThetaImplicitJFNK_VandB_2d_PICMI] -buildDir = . -inputFile = Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_vandb_jfnk_2d.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py - -[SemiImplicitPicard_1d] -buildDir = . -inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit -runtime_params = warpx.abort_on_warning_threshold=high -dim = 1 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=1 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_1d.py - -[EnergyConservingThermalPlasma] -buildDir = . -inputFile = Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 0 -numthreads = 1 -analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py - -[focusing_gaussian_beam] -buildDir = . -inputFile = Examples/Tests/gaussian_beam/inputs_focusing_beam -runtime_params = -dim = 3 -addToCompileString = USE_OPENPMD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py - -[particle_boundary_interaction] -buildDir = . -inputFile = Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py -runtime_params = -customRunCmd = python3 PICMI_inputs_rz.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE -cmakeSetupOpts = -DWarpX_DIMS="RZ" -DWarpX_EB=ON -DWarpX_PYTHON=ON -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -outputFile = particle_boundary_interaction_plt -analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py - -[particle_thermal_boundary] -buildDir = . -inputFile = Examples/Tests/particle_thermal_boundary/inputs_2d -runtime_params = -dim = 2 -addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py - -[openbc_poisson_solver] -buildDir = . -inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py deleted file mode 100644 index f7b610dace9..00000000000 --- a/Regression/prepare_file_ci.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2018-2019 Andrew Myers, Luca Fedeli, Maxence Thevenet -# Remi Lehe -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import os - -# This script modifies `WarpX-test.ini` (which is used for nightly builds) -# and creates the file `ci-test.ini` (which is used for continuous -# integration) -# The subtests that are selected are controlled by WARPX_TEST_DIM -# The architecture (CPU/GPU) is selected by WARPX_TEST_ARCH -import re - -# Get relevant environment variables -arch = os.environ.get('WARPX_TEST_ARCH', 'CPU') - -ci_regular_cartesian_1d = os.environ.get('WARPX_CI_REGULAR_CARTESIAN_1D') == 'TRUE' -ci_regular_cartesian_2d = os.environ.get('WARPX_CI_REGULAR_CARTESIAN_2D') == 'TRUE' -ci_regular_cartesian_3d = os.environ.get('WARPX_CI_REGULAR_CARTESIAN_3D') == 'TRUE' -ci_psatd = os.environ.get('WARPX_CI_PSATD', 'TRUE') == 'TRUE' -ci_single_precision = os.environ.get('WARPX_CI_SINGLE_PRECISION') == 'TRUE' -ci_rz_or_nompi = os.environ.get('WARPX_CI_RZ_OR_NOMPI') == 'TRUE' -ci_qed = os.environ.get('WARPX_CI_QED') == 'TRUE' -ci_eb = os.environ.get('WARPX_CI_EB') == 'TRUE' -ci_openpmd = os.environ.get('WARPX_CI_OPENPMD') == 'TRUE' -ci_ccache = os.environ.get('WARPX_CI_CCACHE') == 'TRUE' -ci_num_make_jobs = os.environ.get('WARPX_CI_NUM_MAKE_JOBS', None) - -# Find the directory in which the tests should be run -current_dir = os.getcwd() -test_dir = re.sub('warpx/Regression', '', current_dir ) - -with open('WarpX-tests.ini') as f: - text = f.read() - -# Replace default folder name -text = re.sub('/home/regtester/AMReX_RegTesting', test_dir, text) -# Remove the web directory -text = re.sub('[\w\-\/]*/web', '', text) - -# Add doComparison = 0 for each test -text = re.sub( '\[(?P.*)\]\nbuildDir = ', - '[\g]\ndoComparison = 0\nbuildDir = ', text ) - -# Change compile options when running on GPU -if arch == 'GPU': - text = re.sub( 'addToCompileString =', - 'addToCompileString = USE_GPU=TRUE USE_OMP=FALSE ', text) -print('Compiling for %s' %arch) - -# Extra dependencies -if ci_openpmd: - text = re.sub('addToCompileString =', - 'addToCompileString = USE_OPENPMD=TRUE ', text) - -# always build with PSATD support (runtime controlled if used) -if ci_psatd: - text = re.sub('addToCompileString =', - 'addToCompileString = USE_FFT=TRUE ', text) - text = re.sub('USE_FFT=FALSE', - '', text) - -# CCache -if ci_ccache: - text = re.sub('addToCompileString =', - 'addToCompileString = USE_CCACHE=TRUE ', text) - -# Add runtime options: -# > crash for unused variables -# > trap NaNs, divisions by zero, and overflows -# > abort upon any warning message by default -text = re.sub('runtime_params =', - 'runtime_params = amrex.abort_on_unused_inputs=1 '+ - 'amrex.fpe_trap_invalid=1 amrex.fpe_trap_zero=1 amrex.fpe_trap_overflow=1 '+ - 'warpx.always_warn_immediately=1 warpx.abort_on_warning_threshold=low', - text) - -# Add runtime options for CPU: -# > serialize initial conditions and no dynamic scheduling in OpenMP -if arch == 'CPU': - text = re.sub('runtime_params =', - 'runtime_params = '+ - 'warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1', - text) - -# Use less/more cores for compiling, e.g. public CI only provides 2 cores -if ci_num_make_jobs is not None: - text = re.sub( 'numMakeJobs = \d+', 'numMakeJobs = {}'.format(ci_num_make_jobs), text ) - -# Prevent emails from being sent -text = re.sub( 'sendEmailWhenFail = 1', 'sendEmailWhenFail = 0', text ) - -# Select the tests to be run -# -------------------------- - -# - Extract test blocks (they are identified by the fact that they contain "inputFile") -select_test_regex = r'(\[(.+\n)*inputFile(.+\n)*)' -test_blocks = [ match[0] for match in re.findall(select_test_regex, text) ] -# - Remove the test blocks from `text` (only the selected ones will be added back) -text = re.sub( select_test_regex, '', text ) - -def select_tests(blocks, match_string_list, do_test): - """Remove or keep tests from list in WarpX-tests.ini according to do_test variable""" - if do_test not in [True, False]: - raise ValueError("do_test must be True or False") - if (do_test == False): - for match_string in match_string_list: - print('Selecting tests without ' + match_string) - blocks = [ block for block in blocks if not match_string in block ] - else: - for match_string in match_string_list: - print('Selecting tests with ' + match_string) - blocks = [ block for block in blocks if match_string in block ] - return blocks - -if ci_regular_cartesian_1d: - test_blocks = select_tests(test_blocks, ['dim = 1'], True) - test_blocks = select_tests(test_blocks, ['USE_RZ=TRUE'], False) - test_blocks = select_tests(test_blocks, ['PRECISION=FLOAT', 'USE_SINGLE_PRECISION_PARTICLES=TRUE'], False) - test_blocks = select_tests(test_blocks, ['useMPI = 0'], False) - test_blocks = select_tests(test_blocks, ['QED=TRUE'], False) - test_blocks = select_tests(test_blocks, ['USE_EB=TRUE'], False) - -if ci_regular_cartesian_2d: - test_blocks = select_tests(test_blocks, ['dim = 2'], True) - test_blocks = select_tests(test_blocks, ['USE_RZ=TRUE'], False) - test_blocks = select_tests(test_blocks, ['PRECISION=FLOAT', 'USE_SINGLE_PRECISION_PARTICLES=TRUE'], False) - test_blocks = select_tests(test_blocks, ['useMPI = 0'], False) - test_blocks = select_tests(test_blocks, ['QED=TRUE'], False) - test_blocks = select_tests(test_blocks, ['USE_EB=TRUE'], False) - -if ci_regular_cartesian_3d: - test_blocks = select_tests(test_blocks, ['dim = 3'], True) - test_blocks = select_tests(test_blocks, ['PRECISION=FLOAT', 'USE_SINGLE_PRECISION_PARTICLES=TRUE'], False) - test_blocks = select_tests(test_blocks, ['useMPI = 0'], False) - test_blocks = select_tests(test_blocks, ['QED=TRUE'], False) - test_blocks = select_tests(test_blocks, ['USE_EB=TRUE'], False) - -if ci_single_precision: - test_blocks = select_tests(test_blocks, ['PRECISION=FLOAT', 'USE_SINGLE_PRECISION_PARTICLES=TRUE'], True) - -if ci_rz_or_nompi: - block1 = select_tests(test_blocks, ['USE_RZ=TRUE'], True) - block2 = select_tests(test_blocks, ['useMPI = 0'], True) - test_blocks = block1 + block2 - -if ci_qed: - test_blocks = select_tests(test_blocks, ['QED=TRUE'], True) - -if ci_eb: - test_blocks = select_tests(test_blocks, ['USE_RZ=TRUE'], False) - test_blocks = select_tests(test_blocks, ['USE_EB=TRUE'], True) - -# - Add the selected test blocks to the text -text = text + '\n' + '\n'.join(test_blocks) - -with open('ci-tests.ini', 'w') as f: - f.write(text) diff --git a/Regression/requirements.txt b/Regression/requirements.txt index 5bdd04ba106..509123899ba 100644 --- a/Regression/requirements.txt +++ b/Regression/requirements.txt @@ -1,10 +1,12 @@ +-r ../requirements.txt dill -lasy +lasy>=0.5.0 matplotlib mpi4py numpy openpmd-api openpmd-viewer pandas +periodictable scipy yt diff --git a/Source/AcceleratorLattice/AcceleratorLattice.H b/Source/AcceleratorLattice/AcceleratorLattice.H index 4b3eff46094..e8acc2c8743 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.H +++ b/Source/AcceleratorLattice/AcceleratorLattice.H @@ -42,10 +42,15 @@ public: * \brief Initialize the element finder instance at the given level of refinement * * @param[in] lev the level of refinement + * @param[in] gamma_boost the Lorentz factor of the boosted frame * @param[in] ba the box array at the level of refinement * @param[in] dm the distribution map at the level of refinement */ - void InitElementFinder (int lev, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm); + void InitElementFinder ( + int lev, + amrex::Real gamma_boost, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm); /** * \brief Update the element finder, needed when the simulation frame has moved relative to the lab frame diff --git a/Source/AcceleratorLattice/AcceleratorLattice.cpp b/Source/AcceleratorLattice/AcceleratorLattice.cpp index edccae9374a..b0513f767a0 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.cpp +++ b/Source/AcceleratorLattice/AcceleratorLattice.cpp @@ -76,13 +76,15 @@ AcceleratorLattice::ReadLattice (std::string const & root_name, amrex::ParticleR } void -AcceleratorLattice::InitElementFinder (int const lev, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm) +AcceleratorLattice::InitElementFinder ( + int const lev, amrex::Real const gamma_boost, + amrex::BoxArray const & ba, amrex::DistributionMapping const & dm) { if (m_lattice_defined) { m_element_finder = std::make_unique>(ba, dm); for (amrex::MFIter mfi(*m_element_finder); mfi.isValid(); ++mfi) { - (*m_element_finder)[mfi].InitElementFinder(lev, mfi, *this); + (*m_element_finder)[mfi].InitElementFinder(lev, gamma_boost, mfi, *this); } } } diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index 6773ed56a65..f7eb5c66531 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -30,10 +30,12 @@ struct LatticeElementFinder * \brief Initialize the element finder at the level and grid * * @param[in] lev the refinement level + * @param[in] gamma_boost the Lorentz factor of the boosted frame * @param[in] a_mfi specifies the grid where the finder is defined * @param[in] accelerator_lattice a reference to the accelerator lattice at the refinement level */ - void InitElementFinder (int lev, amrex::MFIter const& a_mfi, + void InitElementFinder (int lev, amrex::Real gamma_boost, + amrex::MFIter const& a_mfi, AcceleratorLattice const& accelerator_lattice); /** diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index ec784049760..64e593aee30 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -15,7 +15,8 @@ using namespace amrex::literals; void -LatticeElementFinder::InitElementFinder (int const lev, amrex::MFIter const& a_mfi, +LatticeElementFinder::InitElementFinder (int const lev, const amrex::Real gamma_boost, + amrex::MFIter const& a_mfi, AcceleratorLattice const& accelerator_lattice) { @@ -26,8 +27,8 @@ LatticeElementFinder::InitElementFinder (int const lev, amrex::MFIter const& a_m m_dz = WarpX::CellSize(lev)[2]; - m_gamma_boost = WarpX::gamma_boost; - m_uz_boost = std::sqrt(WarpX::gamma_boost*WarpX::gamma_boost - 1._prt)*PhysConst::c; + m_gamma_boost = gamma_boost; + m_uz_boost = std::sqrt(m_gamma_boost*m_gamma_boost - 1._prt)*PhysConst::c; AllocateIndices(accelerator_lattice); diff --git a/Source/BoundaryConditions/CMakeLists.txt b/Source/BoundaryConditions/CMakeLists.txt index 751e52abdd9..c560d121385 100644 --- a/Source/BoundaryConditions/CMakeLists.txt +++ b/Source/BoundaryConditions/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + PEC_Insulator.cpp PML.cpp WarpXEvolvePML.cpp WarpXFieldBoundaries.cpp diff --git a/Source/BoundaryConditions/Make.package b/Source/BoundaryConditions/Make.package index 43d18425ffc..452c9c18b7e 100644 --- a/Source/BoundaryConditions/Make.package +++ b/Source/BoundaryConditions/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += PEC_Insulator.cpp CEXE_sources += PML.cpp WarpXEvolvePML.cpp CEXE_sources += WarpXFieldBoundaries.cpp WarpX_PEC.cpp diff --git a/Source/BoundaryConditions/PEC_Insulator.H b/Source/BoundaryConditions/PEC_Insulator.H new file mode 100644 index 00000000000..5cfdf6488f0 --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator.H @@ -0,0 +1,180 @@ +#ifndef PEC_INSULATOR_H_ +#define PEC_INSULATOR_H_ + +#include "Utils/WarpXAlgorithmSelection.H" + +#include +#include +#include + +#include + +#include +#include + +class PEC_Insulator +{ +public: + + PEC_Insulator(); + + /** + * \brief Apply either the PEC or insulator boundary condition on the boundary and in the + * guard cells. + * In the PEC, the nodal fields (in a Yee mesh) are made even relative to the boundary, + * the non-nodal fields are made odd. + * In the insulator, the tangential fields are set to the value if specified, otherwise unchanged, + * and the normal fields extrapolated from the valid cells. + * + * \param[in,out] Efield + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field + */ + void ApplyPEC_InsulatortoEfield (std::array Efield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field = false); + /** + * \brief Apply either the PEC or insulator boundary condition on the boundary and in the + * guard cells. + * In the PEC, the nodal fields (in a Yee mesh) are made even relative to the boundary, + * the non-nodal fields are made odd. + * In the insulator, the tangential fields are set to the value if specified, otherwise unchanged, + * and the normal fields extrapolated from the valid cells. + * + * \param[in,out] Bfield + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + */ + void ApplyPEC_InsulatortoBfield (std::array Bfield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time); + + /** + * \brief The work routine applying the boundary condition + * + * \param[in,out] field + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field + * \param[in] E_like whether the field is E like or B like + * \param[in] set_F_x_lo whether the tangential field at the boundary was specified + * \param[in] set_F_x_hi whether the tangential field at the boundary was specified + * \param[in] a_Fy_x_lo the parser for the tangential field at the boundary + * \param[in] a_Fz_x_lo the parser for the tangential field at the boundary + * \param[in] a_Fy_x_hi the parser for the tangential field at the boundary + * \param[in] a_Fz_x_hi the parser for the tangential field at the boundary + * \param[in] set_F_y_lo whether the tangential field at the boundary was specified + * \param[in] set_F_y_hi whether the tangential field at the boundary was specified + * \param[in] a_Fx_y_lo the parser for the tangential field at the boundary + * \param[in] a_Fz_y_lo the parser for the tangential field at the boundary + * \param[in] a_Fx_y_hi the parser for the tangential field at the boundary + * \param[in] a_Fz_y_hi the parser for the tangential field at the boundary + * \param[in] set_F_z_lo whether the tangential field at the boundary was specified + * \param[in] set_F_z_hi whether the tangential field at the boundary was specified + * \param[in] a_Fx_z_lo the parser for the tangential field at the boundary + * \param[in] a_Fy_z_lo the parser for the tangential field at the boundary + * \param[in] a_Fx_z_hi the parser for the tangential field at the boundary + * \param[in] a_Fy_z_hi the parser for the tangential field at the boundary + */ + void + ApplyPEC_InsulatortoField (std::array field, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field, + bool E_like, +#if (AMREX_SPACEDIM > 1) + bool set_F_x_lo, bool set_F_x_hi, + std::unique_ptr const & a_Fy_x_lo, std::unique_ptr const & a_Fz_x_lo, + std::unique_ptr const & a_Fy_x_hi, std::unique_ptr const & a_Fz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + bool set_F_y_lo, bool set_F_y_hi, + std::unique_ptr const & a_Fx_y_lo, std::unique_ptr const & a_Fz_y_lo, + std::unique_ptr const & a_Fx_y_hi, std::unique_ptr const & a_Fz_y_hi, +#endif + bool set_F_z_lo, bool set_F_z_hi, + std::unique_ptr const & a_Fx_z_lo, std::unique_ptr const & a_Fy_z_lo, + std::unique_ptr const & a_Fx_z_hi, std::unique_ptr const & a_Fy_z_hi); + +private: + + /* \brief Reads in the parsers for the tangential fields, returning whether + * the input parameter was specified. + * \param[in] pp_insulator ParmParse instance + * \param[out] parser the parser generated from the input + * \param[in] input_name the name of the input parameter + * \param[in] coord1 the first coordinate in the plane + * \param[in] coord2 the second coordinate in the plane + */ + bool ReadTangentialFieldParser (amrex::ParmParse const & pp_insulator, + std::unique_ptr & parser, + std::string const & input_name, + std::string const & coord1, + std::string const & coord2); + + std::vector> m_insulator_area_lo; + std::vector> m_insulator_area_hi; + +#if (AMREX_SPACEDIM > 1) + bool m_set_B_x_lo = false, m_set_B_x_hi = false; + std::unique_ptr m_By_x_lo, m_Bz_x_lo; + std::unique_ptr m_By_x_hi, m_Bz_x_hi; +#endif +#if defined(WARPX_DIM_3D) + bool m_set_B_y_lo = false, m_set_B_y_hi = false; + std::unique_ptr m_Bx_y_lo, m_Bz_y_lo; + std::unique_ptr m_Bx_y_hi, m_Bz_y_hi; +#endif + bool m_set_B_z_lo = false, m_set_B_z_hi = false; + std::unique_ptr m_Bx_z_lo, m_By_z_lo; + std::unique_ptr m_Bx_z_hi, m_By_z_hi; + + +#if (AMREX_SPACEDIM > 1) + bool m_set_E_x_lo = false, m_set_E_x_hi = false; + std::unique_ptr m_Ey_x_lo, m_Ez_x_lo; + std::unique_ptr m_Ey_x_hi, m_Ez_x_hi; +#endif +#if defined(WARPX_DIM_3D) + bool m_set_E_y_lo = false, m_set_E_y_hi = false; + std::unique_ptr m_Ex_y_lo, m_Ez_y_lo; + std::unique_ptr m_Ex_y_hi, m_Ez_y_hi; +#endif + bool m_set_E_z_lo = false, m_set_E_z_hi = false; + std::unique_ptr m_Ex_z_lo, m_Ey_z_lo; + std::unique_ptr m_Ex_z_hi, m_Ey_z_hi; + + +}; +#endif // PEC_INSULATOR_H_ diff --git a/Source/BoundaryConditions/PEC_Insulator.cpp b/Source/BoundaryConditions/PEC_Insulator.cpp new file mode 100644 index 00000000000..b6b6bcf08ea --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator.cpp @@ -0,0 +1,573 @@ +#include "BoundaryConditions/PEC_Insulator.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + /** + * \brief At the specified grid location, apply either the PEC or insulator boundary condition if + * the cell is on the boundary or in the guard cells. + * + * \param[in] icomp component of the field being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of field Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] field field data to be updated if (ijk) is at the boundary + * or a guard cell + * \param[in] E_like whether the field behaves like E field or B field + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] is_insulator_lo Specifies whether lower boundaries are insulators + * \param[in] is_insulator_hi Specifies whether upper boundaries are insulators + * \param[in] field_lo the values of the field for the lower insulator boundary cell + * \param[in] field_hi the values of the field for the upper insulator boundary cell + * \param[in] set_field_lo whether to set the field for the direction on the lower boundary + * \param[in] set_field_hi whether to set the field for the direction on the upper boundary + * \param[in] fbndry_lo specified values of the field at the lower boundaries in the insulator + * \param[in] fbndry_hi specified values of the field at the upper boundaries in the insulator + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetFieldOnPEC_Insulator (int icomp, + amrex::IntVect const & dom_lo, + amrex::IntVect const & dom_hi, + amrex::IntVect const & ijk_vec, int n, + amrex::Array4 const & field, + bool const E_like, + amrex::IntVect const & is_nodal, + amrex::IntVect const & is_insulator_lo, + amrex::IntVect const & is_insulator_hi, + amrex::RealVect const & field_lo, + amrex::RealVect const & field_hi, + amrex::IntVect const & set_field_lo, + amrex::IntVect const & set_field_hi, + amrex::GpuArray const fbndry_lo, + amrex::GpuArray const fbndry_hi) + { + using namespace amrex::literals; + amrex::IntVect ijk_next = ijk_vec; + amrex::IntVect ijk_nextp1 = ijk_vec; + amrex::IntVect ijk_mirror = ijk_vec; + bool OnBoundary = false; + bool GuardCell = false; + bool isInsulatorBoundary = false; + amrex::Real sign = +1._rt; + bool is_normal_to_boundary = false; + amrex::Real field_value = 0._rt; + bool set_field = false; + // Loop over all dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = -1 (lo), iside = +1 (hi) + for (int iside = -1; iside <= +1; iside += 2) { + bool const isPEC_InsulatorBoundary = ( (iside == -1) + ? fbndry_lo[idim] == FieldBoundaryType::PECInsulator + : fbndry_hi[idim] == FieldBoundaryType::PECInsulator ); + if (isPEC_InsulatorBoundary) { + isInsulatorBoundary = ( (iside == -1) + ? is_insulator_lo[idim] == 1 + : is_insulator_hi[idim] == 1 ); + } + if (isPEC_InsulatorBoundary) { + // Calculates the number of grid points ijk_vec is beyond the + // domain boundary i.e. a value of +1 means the current cell is + // outside of the simulation domain by 1 cell. Note that the high + // side domain boundary is between cell dom_hi and dom_hi+1 for cell + // centered grids and on cell dom_hi+1 for nodal grid. This is why + // (dom_hi[idim] + is_nodal[idim]) is used. + int const ig = ((iside == -1) ? (dom_lo[idim] - ijk_vec[idim]) + : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); + +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (Fy in XZ, Ftheta in RZ), + // icomp=1 is not normal to x or z boundary + // The logic below ensures that the flags are set right for 2D + is_normal_to_boundary = (icomp == (2*idim)); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Fx and Fy are not normal to the z boundary) + // The logic below ensures that the flags are set right for 1D + is_normal_to_boundary = (icomp == 2); +#else + is_normal_to_boundary = (icomp == idim); +#endif + + if (ig == 0) { + // Check if field is on the boundary + if (is_nodal[idim] == 1) { + OnBoundary = true; + } + } else if (ig > 0) { + GuardCell = true; + + // Location of the next cells inward + ijk_next[idim] = ijk_vec[idim] - ig*iside; + ijk_nextp1[idim] = ijk_next[idim] - ig*iside; + + // Mirror location inside the domain by "ig" number of cells + ijk_mirror[idim] = ( (iside == -1) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] - ig + 1)); + + // Check for components with even symmetry. + // True for E_like and tangential, and B_like and normal + if (E_like ^ is_normal_to_boundary) { sign *= -1._rt; } + + field_value = ( (iside == -1) ? field_lo[idim] : field_hi[idim] ); + set_field = ( (iside == -1) ? set_field_lo[idim]==1 : set_field_hi[idim]==1 ); + +#if (defined WARPX_DIM_RZ) + if (idim == 0 && iside == +1) { + // Upper radial boundary + amrex::Real const rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + if (icomp == 0) { + // Add radial scale so that the divergence, drFr/dr, is 0. + // This only works for the first guard cell and with + // Fr cell centered in r. + amrex::Real const rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + // Calculate radial scale factor + sign *= rmirror/rguard; + } + if (isInsulatorBoundary) { + // Apply radial scale factor + field_value *= dom_hi[idim]/rguard; + } + } +#endif + } + } // is pec_insulator boundary + } // loop over iside + } // loop over dimensions + + if (isInsulatorBoundary) { + if (is_normal_to_boundary) { + // The value on the boundary is left unmodified + // The values in the guard cells are extrapolated + if (GuardCell) { + field(ijk_vec, n) = 2._rt*field(ijk_next, n) - field(ijk_nextp1, n); + } + } else if ((OnBoundary || GuardCell) && set_field) { + field(ijk_vec, n) = field_value; + } else if (GuardCell) { + field(ijk_vec, n) = 2._rt*field(ijk_next, n) - field(ijk_nextp1, n); + } + } else { + if (OnBoundary && (E_like ^ is_normal_to_boundary)) { + // If ijk_vec is on a boundary, set to zero if + // E_like and tangential or B_like and normal + field(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + // Fnormal and Ftangential is set opposite and equal to the value + // in the mirror location, respectively. + field(ijk_vec,n) = sign * field(ijk_mirror,n); + } + } + } +} + + +bool +PEC_Insulator::ReadTangentialFieldParser (amrex::ParmParse const & pp_insulator, + std::unique_ptr & parser, + std::string const & input_name, + std::string const & coord1, + std::string const & coord2) +{ + std::string str = "0"; + bool const specified = utils::parser::Query_parserString(pp_insulator, input_name, str); + parser = std::make_unique(utils::parser::makeParser(str, {coord1, coord2, "t"})); + return specified; +} + +PEC_Insulator::PEC_Insulator () +{ + + amrex::ParmParse const pp_insulator("insulator"); + +#if (AMREX_SPACEDIM > 1) + std::string str_area_x_lo = "0"; + std::string str_area_x_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_x_lo(y,z)", str_area_x_lo); + utils::parser::Query_parserString( pp_insulator, "area_x_hi(y,z)", str_area_x_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_x_lo, {"y", "z"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_x_hi, {"y", "z"}))); + + m_set_B_x_lo |= ReadTangentialFieldParser(pp_insulator, m_By_x_lo, "By_x_lo(y,z,t)", "y", "z"); + m_set_B_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Bz_x_lo, "Bz_x_lo(y,z,t)", "y", "z"); + m_set_B_x_hi |= ReadTangentialFieldParser(pp_insulator, m_By_x_hi, "By_x_hi(y,z,t)", "y", "z"); + m_set_B_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Bz_x_hi, "Bz_x_hi(y,z,t)", "y", "z"); + + m_set_E_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Ey_x_lo, "Ey_x_lo(y,z,t)", "y", "z"); + m_set_E_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Ez_x_lo, "Ez_x_lo(y,z,t)", "y", "z"); + m_set_E_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Ey_x_hi, "Ey_x_hi(y,z,t)", "y", "z"); + m_set_E_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Ez_x_hi, "Ez_x_hi(y,z,t)", "y", "z"); +#endif +#if defined(WARPX_DIM_3D) + std::string str_area_y_lo = "0"; + std::string str_area_y_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_y_lo(x,z)", str_area_y_lo); + utils::parser::Query_parserString( pp_insulator, "area_y_hi(x,z)", str_area_y_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_y_lo, {"x", "z"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_y_hi, {"x", "z"}))); + + m_set_B_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Bx_y_lo, "Bx_y_lo(x,z,t)", "x", "z"); + m_set_B_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Bz_y_lo, "Bz_y_lo(x,z,t)", "x", "z"); + m_set_B_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Bx_y_hi, "Bx_y_hi(x,z,t)", "x", "z"); + m_set_B_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Bz_y_hi, "Bz_y_hi(x,z,t)", "x", "z"); + + m_set_E_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Ex_y_lo, "Ex_y_lo(x,z,t)", "x", "z"); + m_set_E_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Ez_y_lo, "Ez_y_lo(x,z,t)", "x", "z"); + m_set_E_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Ex_y_hi, "Ex_y_hi(x,z,t)", "x", "z"); + m_set_E_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Ez_y_hi, "Ez_y_hi(x,z,t)", "x", "z"); +#endif + + std::string str_area_z_lo = "0"; + std::string str_area_z_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_z_lo(x,y)", str_area_z_lo); + utils::parser::Query_parserString( pp_insulator, "area_z_hi(x,y)", str_area_z_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_z_lo, {"x", "y"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_z_hi, {"x", "y"}))); + + m_set_B_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Bx_z_lo, "Bx_z_lo(x,y,t)", "x", "y"); + m_set_B_z_lo |= ReadTangentialFieldParser(pp_insulator, m_By_z_lo, "By_z_lo(x,y,t)", "x", "y"); + m_set_B_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Bx_z_hi, "Bx_z_hi(x,y,t)", "x", "y"); + m_set_B_z_hi |= ReadTangentialFieldParser(pp_insulator, m_By_z_hi, "By_z_hi(x,y,t)", "x", "y"); + + m_set_E_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Ex_z_lo, "Ex_z_lo(x,y,t)", "x", "y"); + m_set_E_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Ey_z_lo, "Ey_z_lo(x,y,t)", "x", "y"); + m_set_E_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Ex_z_hi, "Ex_z_hi(x,y,t)", "x", "y"); + m_set_E_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Ey_z_hi, "Ey_z_hi(x,y,t)", "x", "y"); + +} + +void +PEC_Insulator::ApplyPEC_InsulatortoEfield ( + std::array Efield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field) +{ + bool const E_like = true; + ApplyPEC_InsulatortoField(Efield, field_boundary_lo, field_boundary_hi, ng_fieldgather, geom, + lev, patch_type, ref_ratios, time, split_pml_field, + E_like, +#if (AMREX_SPACEDIM > 1) + m_set_E_x_lo, m_set_E_x_hi, + m_Ey_x_lo, m_Ez_x_lo, m_Ey_x_hi, m_Ez_x_hi, +#endif +#if defined(WARPX_DIM_3D) + m_set_E_y_lo, m_set_E_y_hi, + m_Ex_y_lo, m_Ez_y_lo, m_Ex_y_hi, m_Ez_y_hi, +#endif + m_set_E_z_lo, m_set_E_z_hi, + m_Ex_z_lo, m_Ey_z_lo, m_Ex_z_hi, m_Ey_z_hi); +} + + +void +PEC_Insulator::ApplyPEC_InsulatortoBfield ( + std::array Bfield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time) +{ + bool const E_like = false; + bool const split_pml_field = false; + ApplyPEC_InsulatortoField(Bfield, field_boundary_lo, field_boundary_hi, ng_fieldgather, geom, + lev, patch_type, ref_ratios, time, split_pml_field, + E_like, +#if (AMREX_SPACEDIM > 1) + m_set_B_x_lo, m_set_B_x_hi, + m_By_x_lo, m_Bz_x_lo, m_By_x_hi, m_Bz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + m_set_B_y_lo, m_set_B_y_hi, + m_Bx_y_lo, m_Bz_y_lo, m_Bx_y_hi, m_Bz_y_hi, +#endif + m_set_B_z_lo, m_set_B_z_hi, + m_Bx_z_lo, m_By_z_lo, m_Bx_z_hi, m_By_z_hi); +} + + +void +PEC_Insulator::ApplyPEC_InsulatortoField ( + std::array field, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field, + bool E_like, +#if (AMREX_SPACEDIM > 1) + bool set_F_x_lo, bool set_F_x_hi, + std::unique_ptr const & a_Fy_x_lo, std::unique_ptr const & a_Fz_x_lo, + std::unique_ptr const & a_Fy_x_hi, std::unique_ptr const & a_Fz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + bool set_F_y_lo, bool set_F_y_hi, + std::unique_ptr const & a_Fx_y_lo, std::unique_ptr const & a_Fz_y_lo, + std::unique_ptr const & a_Fx_y_hi, std::unique_ptr const & a_Fz_y_hi, +#endif + bool set_F_z_lo, bool set_F_z_hi, + std::unique_ptr const & a_Fx_z_lo, std::unique_ptr const & a_Fy_z_lo, + std::unique_ptr const & a_Fx_z_hi, std::unique_ptr const & a_Fy_z_hi) +{ + using namespace amrex::literals; + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); + } + amrex::IntVect const domain_lo = domain_box.smallEnd(); + amrex::IntVect const domain_hi = domain_box.bigEnd(); + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; + } + +#if (AMREX_SPACEDIM > 1) + amrex::ParserExecutor<2> const area_parsers_x_lo = m_insulator_area_lo[0]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_x_hi = m_insulator_area_hi[0]->compile<2>(); +#endif +#if defined(WARPX_DIM_3D) + amrex::ParserExecutor<2> const area_parsers_y_lo = m_insulator_area_lo[1]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_y_hi = m_insulator_area_hi[1]->compile<2>(); +#endif + amrex::ParserExecutor<2> const area_parsers_z_lo = m_insulator_area_lo[WARPX_ZINDEX]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_z_hi = m_insulator_area_hi[WARPX_ZINDEX]->compile<2>(); + +#if (AMREX_SPACEDIM > 1) + amrex::ParserExecutor<3> const Fy_x_lo_parser = a_Fy_x_lo->compile<3>(); + amrex::ParserExecutor<3> const Fz_x_lo_parser = a_Fz_x_lo->compile<3>(); + amrex::ParserExecutor<3> const Fy_x_hi_parser = a_Fy_x_hi->compile<3>(); + amrex::ParserExecutor<3> const Fz_x_hi_parser = a_Fz_x_hi->compile<3>(); +#endif +#if defined(WARPX_DIM_3D) + amrex::ParserExecutor<3> const Fx_y_lo_parser = a_Fx_y_lo->compile<3>(); + amrex::ParserExecutor<3> const Fz_y_lo_parser = a_Fz_y_lo->compile<3>(); + amrex::ParserExecutor<3> const Fx_y_hi_parser = a_Fx_y_hi->compile<3>(); + amrex::ParserExecutor<3> const Fz_y_hi_parser = a_Fz_y_hi->compile<3>(); +#endif + amrex::ParserExecutor<3> const Fx_z_lo_parser = a_Fx_z_lo->compile<3>(); + amrex::ParserExecutor<3> const Fy_z_lo_parser = a_Fy_z_lo->compile<3>(); + amrex::ParserExecutor<3> const Fx_z_hi_parser = a_Fx_z_hi->compile<3>(); + amrex::ParserExecutor<3> const Fy_z_hi_parser = a_Fy_z_hi->compile<3>(); + + amrex::IntVect const Fx_nodal = field[0]->ixType().toIntVect(); + amrex::IntVect const Fy_nodal = field[1]->ixType().toIntVect(); + amrex::IntVect const Fz_nodal = field[2]->ixType().toIntVect(); + // For each field multifab, apply boundary condition to ncomponents + // If not split field, the boundary condition is applied to the regular field used in Maxwell's eq. + // If split_pml_field is true, then boundary condition is applied to all the split field components. + int const nComp_x = field[0]->nComp(); + int const nComp_y = field[1]->nComp(); + int const nComp_z = field[2]->nComp(); + + std::array const & dx = WarpX::CellSize(lev); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*field[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + // Extract field data + amrex::Array4 const & Fx = field[0]->array(mfi); + amrex::Array4 const & Fy = field[1]->array(mfi); + amrex::Array4 const & Fz = field[2]->array(mfi); + + // Extract tileboxes for which to loop + // if split field, the box includes nodal flag + // For E-field used in Maxwell's update, nodal flag plus cells that particles + // gather fields from in the guard-cell region are included. + // Note that for simulations without particles or laser, ng_field_gather is 0 + // and the guard-cell values of the E-field multifab will not be modified. + amrex::Box const & tex = (split_pml_field) ? mfi.tilebox(field[0]->ixType().toIntVect()) + : mfi.tilebox(field[0]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const & tey = (split_pml_field) ? mfi.tilebox(field[1]->ixType().toIntVect()) + : mfi.tilebox(field[1]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const & tez = (split_pml_field) ? mfi.tilebox(field[2]->ixType().toIntVect()) + : mfi.tilebox(field[2]->ixType().toIntVect(), ng_fieldgather); + + const amrex::XDim3 xyzmin_x = WarpX::LowerCorner(tex, lev, 0._rt); + const amrex::XDim3 xyzmin_y = WarpX::LowerCorner(tey, lev, 0._rt); + const amrex::XDim3 xyzmin_z = WarpX::LowerCorner(tez, lev, 0._rt); + amrex::IntVect const lo_x = tex.smallEnd(); + amrex::IntVect const lo_y = tey.smallEnd(); + amrex::IntVect const lo_z = tez.smallEnd(); + + // loop over cells and update fields + amrex::ParallelFor( + tex, nComp_x, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const shiftx = (Fx_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fx_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1] + shifty)*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const shiftz = (Fx_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX] + shiftz)*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = 0._rt; // Will be unused + F_hi[0] = 0._rt; // Will be unused + set_field_lo[0] = 0; // Will be unused + set_field_hi[0] = 0; // Will be unused +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = (set_F_y_lo ? Fx_y_lo_parser(x, z, time) : 0._rt); + F_hi[1] = (set_F_y_hi ? Fx_y_hi_parser(x, z, time) : 0._rt); + set_field_lo[1] = set_F_y_lo; + set_field_hi[1] = set_F_y_hi; +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = (set_F_z_lo ? Fx_z_lo_parser(x, y, time) : 0._rt); + F_hi[WARPX_ZINDEX] = (set_F_z_hi ? Fx_z_hi_parser(x, y, time) : 0._rt); + set_field_lo[WARPX_ZINDEX] = set_F_z_lo; + set_field_hi[WARPX_ZINDEX] = set_F_z_hi; + + int const icomp = 0; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fx, E_like, Fx_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + }, + tey, nComp_y, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const shiftx = (Fy_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fy_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1] + shifty)*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const shiftz = (Fy_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX] + shiftz)*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = (set_F_x_lo ? Fy_x_lo_parser(y, z, time) : 0._rt); + F_hi[0] = (set_F_x_hi ? Fy_x_hi_parser(y, z, time) : 0._rt); + set_field_lo[0] = set_F_x_lo; + set_field_hi[0] = set_F_x_hi; +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = 0._rt; // Will be unused + F_hi[1] = 0._rt; // Will be unused + set_field_lo[1] = 0; // Will be unused + set_field_hi[1] = 0; // Will be unused +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = (set_F_z_lo ? Fy_z_lo_parser(x, y, time) : 0._rt); + F_hi[WARPX_ZINDEX] = (set_F_z_hi ? Fy_z_hi_parser(x, y, time) : 0._rt); + set_field_lo[WARPX_ZINDEX] = set_F_z_lo; + set_field_hi[WARPX_ZINDEX] = set_F_z_hi; + + int const icomp = 1; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fy, E_like, Fy_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + }, + tez, nComp_z, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const shiftx = (Fz_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fz_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1] + shifty)*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const shiftz = (Fz_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX] + shiftz)*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = (set_F_x_lo ? Fz_x_lo_parser(y, z, time) : 0._rt); + F_hi[0] = (set_F_x_hi ? Fz_x_hi_parser(y, z, time) : 0._rt); + set_field_lo[0] = set_F_x_lo; + set_field_hi[0] = set_F_x_hi; +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = (set_F_y_lo ? Fz_y_lo_parser(x, z, time) : 0._rt); + F_hi[1] = (set_F_y_hi ? Fz_y_hi_parser(x, z, time) : 0._rt); + set_field_lo[1] = set_F_y_lo; + set_field_hi[1] = set_F_y_hi; +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = 0._rt; // Will be unused + F_hi[WARPX_ZINDEX] = 0._rt; // Will be unused + set_field_lo[WARPX_ZINDEX] = 0; // Will be unused + set_field_hi[WARPX_ZINDEX] = 0; // Will be unused + + int const icomp = 2; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fz, E_like, Fz_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + } + ); + } +} diff --git a/Source/BoundaryConditions/PEC_Insulator_fwd.H b/Source/BoundaryConditions/PEC_Insulator_fwd.H new file mode 100644 index 00000000000..9b2c1b05307 --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator_fwd.H @@ -0,0 +1,13 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef PEC_INSULATOR_FWD_H +#define PEC_INSULATOR_FWD_H + +class PEC_Insulator; + +#endif /* PEC_INSULATOR_FWD_H */ diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 69fa14658d2..3b346e9f1c3 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -17,6 +17,8 @@ # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif +#include + #include #include #include @@ -82,7 +84,7 @@ class SigmaBoxFactory : public amrex::FabFactory { public: - SigmaBoxFactory (const amrex::BoxArray& grid_ba, const amrex::Real* dx, + SigmaBoxFactory (const amrex::BoxArray* grid_ba, const amrex::Real* dx, const amrex::IntVect& ncell, const amrex::IntVect& delta, const amrex::Box& regular_domain, const amrex::Real v_sigma_sb, const int do_cubic_sigma_pml, const amrex::Real pml_damping_strength) @@ -99,7 +101,7 @@ public: [[nodiscard]] SigmaBox* create (const amrex::Box& box, int /*ncomps*/, const amrex::FabInfo& /*info*/, int /*box_index*/) const final { - return new SigmaBox(box, m_grids, m_dx, m_ncell, m_delta, m_regdomain, m_v_sigma_sb, m_do_cubic_sigma_pml, m_pml_damping_strength); + return new SigmaBox(box, *m_grids, m_dx, m_ncell, m_delta, m_regdomain, m_v_sigma_sb, m_do_cubic_sigma_pml, m_pml_damping_strength); } void destroy (SigmaBox* fab) const final @@ -114,7 +116,7 @@ public: } private: - const amrex::BoxArray& m_grids; + const amrex::BoxArray* m_grids; const amrex::Real* m_dx; amrex::IntVect m_ncell; amrex::IntVect m_delta; @@ -129,7 +131,7 @@ class MultiSigmaBox { public: MultiSigmaBox(const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, - const amrex::BoxArray& grid_ba, const amrex::Real* dx, + const amrex::BoxArray* grid_ba, const amrex::Real* dx, const amrex::IntVect& ncell, const amrex::IntVect& delta, const amrex::Box& regular_domain, amrex::Real v_sigma_sb, const int do_cubic_sigma_pml, const amrex::Real pml_damping_strength); @@ -150,10 +152,12 @@ public: amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, ablastr::utils::enums::GridType grid_type, int do_moving_window, int pml_has_particles, int do_pml_in_domain, - int psatd_solution_type, int J_in_time, int rho_in_time, + PSATDSolutionType psatd_solution_type, + JInTime J_in_time, RhoInTime rho_in_time, bool do_pml_dive_cleaning, bool do_pml_divb_cleaning, const amrex::IntVect& fill_guards_fields, const amrex::IntVect& fill_guards_current, + bool eb_enabled, int max_guard_EB, amrex::Real v_sigma_sb, const int do_cubic_sigma_pml, const amrex::Real pml_damping_strength, amrex::IntVect do_pml_Lo = amrex::IntVect::TheUnitVector(), @@ -161,23 +165,6 @@ public: void ComputePMLFactors (amrex::Real dt); - std::array GetE_fp (); - std::array GetB_fp (); - std::array Getj_fp (); - std::array GetE_cp (); - std::array GetB_cp (); - std::array Getj_cp (); - std::array Get_edge_lengths (); - std::array Get_face_areas (); - - // Used when WarpX::do_pml_dive_cleaning = true - amrex::MultiFab* GetF_fp (); - amrex::MultiFab* GetF_cp (); - - // Used when WarpX::do_pml_divb_cleaning = true - amrex::MultiFab* GetG_fp (); - amrex::MultiFab* GetG_cp (); - [[nodiscard]] const MultiSigmaBox& GetMultiSigmaBox_fp () const { return *sigba_fp; @@ -189,35 +176,33 @@ public: } #ifdef WARPX_USE_FFT - void PushPSATD (int lev); + void PushPSATD (ablastr::fields::MultiFabRegister& fields, int lev); #endif - void CopyJtoPMLs (const std::array& j_fp, - const std::array& j_cp); + void CopyJtoPMLs (ablastr::fields::MultiFabRegister& fields, int lev); - void Exchange (const std::array& mf_pml, - const std::array& mf, + void Exchange (ablastr::fields::VectorField mf_pml, + ablastr::fields::VectorField mf, + const PatchType& patch_type, + int do_pml_in_domain); + void Exchange (amrex::MultiFab* mf_pml, + amrex::MultiFab* mf, const PatchType& patch_type, int do_pml_in_domain); - void CopyJtoPMLs (PatchType patch_type, - const std::array& jp); - - void ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp, int do_pml_in_domain); - void ExchangeF (PatchType patch_type, amrex::MultiFab* Fp, int do_pml_in_domain); - - void ExchangeG (amrex::MultiFab* G_fp, amrex::MultiFab* G_cp, int do_pml_in_domain); - void ExchangeG (PatchType patch_type, amrex::MultiFab* Gp, int do_pml_in_domain); + void CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int lev + ); - void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryF (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryG (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundary (ablastr::fields::VectorField mf_pml, PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundary (amrex::MultiFab & mf_pml, PatchType patch_type, std::optional nodal_sync=std::nullopt); [[nodiscard]] bool ok () const { return m_ok; } - void CheckPoint (const std::string& dir) const; - void Restart (const std::string& dir); + void CheckPoint (ablastr::fields::MultiFabRegister& fields, const std::string& dir) const; + void Restart (ablastr::fields::MultiFabRegister& fields, const std::string& dir); static void Exchange (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom, int do_pml_in_domain); @@ -227,30 +212,12 @@ private: bool m_dive_cleaning; bool m_divb_cleaning; - const amrex::IntVect m_fill_guards_fields; - const amrex::IntVect m_fill_guards_current; + amrex::IntVect m_fill_guards_fields; + amrex::IntVect m_fill_guards_current; const amrex::Geometry* m_geom; const amrex::Geometry* m_cgeom; - std::array,3> pml_E_fp; - std::array,3> pml_B_fp; - std::array,3> pml_j_fp; - - std::array,3> pml_edge_lengths; - - std::array,3> pml_E_cp; - std::array,3> pml_B_cp; - std::array,3> pml_j_cp; - - // Used when WarpX::do_pml_dive_cleaning = true - std::unique_ptr pml_F_fp; - std::unique_ptr pml_F_cp; - - // Used when WarpX::do_pml_divb_cleaning = true - std::unique_ptr pml_G_fp; - std::unique_ptr pml_G_cp; - std::unique_ptr sigba_fp; std::unique_ptr sigba_cp; @@ -268,7 +235,7 @@ private: } #ifdef AMREX_USE_EB - amrex::EBFArrayBoxFactory const& fieldEBFactory () const noexcept { + [[nodiscard]] amrex::EBFArrayBoxFactory const& fieldEBFactory () const noexcept { return static_cast(*pml_field_factory); } #endif @@ -299,13 +266,15 @@ private: }; #ifdef WARPX_USE_FFT -void PushPMLPSATDSinglePatch( int lev, +void PushPMLPSATDSinglePatch ( + int lev, SpectralSolver& solver, - std::array,3>& pml_E, - std::array,3>& pml_B, - std::unique_ptr& pml_F, - std::unique_ptr& pml_G, - const amrex::IntVect& fill_guards); + ablastr::fields::VectorField& pml_E, + ablastr::fields::VectorField& pml_B, + ablastr::fields::ScalarField pml_F, + ablastr::fields::ScalarField pml_G, + const amrex::IntVect& fill_guards +); #endif #endif diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 6471a0ce050..c1628a0743e 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -10,7 +10,10 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/EmbeddedBoundaryInit.H" +#endif #ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif @@ -57,7 +60,7 @@ #endif using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -531,7 +534,7 @@ SigmaBox::ComputePMLFactorsE (const Real* a_dx, Real dt) } MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, - const BoxArray& grid_ba, const Real* dx, + const BoxArray* grid_ba, const Real* dx, const IntVect& ncell, const IntVect& delta, const amrex::Box& regular_domain, const amrex::Real v_sigma_sb, const int do_cubic_sigma_pml, const amrex::Real pml_damping_strength) : FabArray(ba,dm,1,0,MFInfo(), @@ -577,10 +580,12 @@ PML::PML (const int lev, const BoxArray& grid_ba, Real dt, int nox_fft, int noy_fft, int noz_fft, ablastr::utils::enums::GridType grid_type, int do_moving_window, int /*pml_has_particles*/, int do_pml_in_domain, - const int psatd_solution_type, const int J_in_time, const int rho_in_time, + const PSATDSolutionType psatd_solution_type, + const JInTime J_in_time, const RhoInTime rho_in_time, const bool do_pml_dive_cleaning, const bool do_pml_divb_cleaning, const amrex::IntVect& fill_guards_fields, const amrex::IntVect& fill_guards_current, + bool eb_enabled, int max_guard_EB, const amrex::Real v_sigma_sb, const int do_cubic_sigma_pml, amrex::Real pml_damping_strength, const amrex::IntVect do_pml_Lo, const amrex::IntVect do_pml_Hi) @@ -591,6 +596,12 @@ PML::PML (const int lev, const BoxArray& grid_ba, m_geom(geom), m_cgeom(cgeom) { +#ifndef AMREX_USE_EB + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, "PML: eb_enabled is true but was not compiled in."); +#endif + + using ablastr::fields::Direction; + // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain or fine patch(es) // (instead of extending `ncell` outside of the physical domain or fine patch(es)) // In order to implement this, we define a new reduced Box Array ensuring that it does not @@ -699,54 +710,67 @@ PML::PML (const int lev, const BoxArray& grid_ba, } #ifdef AMREX_USE_EB - pml_field_factory = amrex::makeEBFabFactory(*geom, ba, dm, - {max_guard_EB, max_guard_EB, max_guard_EB}, - amrex::EBSupport::full); -#else - amrex::ignore_unused(max_guard_EB); - pml_field_factory = std::make_unique(); + if (eb_enabled) { + pml_field_factory = amrex::makeEBFabFactory( + *geom, + ba, + dm, + {max_guard_EB, max_guard_EB, max_guard_EB}, + amrex::EBSupport::full + ); + } else #endif + { + amrex::ignore_unused(max_guard_EB); + pml_field_factory = std::make_unique(); + } // Allocate diagonal components (xx,yy,zz) only with divergence cleaning const int ncompe = (m_dive_cleaning) ? 3 : 2; const int ncompb = (m_divb_cleaning) ? 3 : 2; - const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); - - const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); + auto& warpx = WarpX::GetInstance(); + using ablastr::fields::Direction; + + const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt, false, false); + + const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{0}, lev, ba_Bx, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{1}, lev, ba_By, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{2}, lev, ba_Bz, dm, ncompb, ngb, 0.0_rt, false, false); + + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{0}, lev, ba_jx, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{1}, lev, ba_jy, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{2}, lev, ba_jz, dm, 1, ngb, 0.0_rt, false, false); #ifdef AMREX_USE_EB - const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); - WarpX::AllocInitMultiFab(pml_edge_lengths[0], ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[1], ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[2], ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[z]", 0.0_rt); + if (eb_enabled) { + const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{0}, lev, ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{1}, lev, ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{2}, lev, ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || + WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - auto const eb_fact = fieldEBFactory(); + auto const eb_fact = fieldEBFactory(); - WarpX::ComputeEdgeLengths(pml_edge_lengths, eb_fact); - WarpX::ScaleEdges(pml_edge_lengths, WarpX::CellSize(lev)); + ablastr::fields::VectorField t_pml_edge_lengths = warpx.m_fields.get_alldirs(FieldType::pml_edge_lengths, lev); + warpx::embedded_boundary::ComputeEdgeLengths(t_pml_edge_lengths, eb_fact); + warpx::embedded_boundary::ScaleEdges(t_pml_edge_lengths, WarpX::CellSize(lev)); + } } #endif @@ -754,7 +778,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, if (m_dive_cleaning) { const amrex::BoxArray ba_F_nodal = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - WarpX::AllocInitMultiFab(pml_F_fp, ba_F_nodal, dm, 3, ngf, lev, "pml_F_fp", 0.0_rt); + warpx.m_fields.alloc_init(FieldType::pml_F_fp, lev, ba_F_nodal, dm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -764,12 +788,12 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray ba_G_nodal = amrex::convert(ba, G_nodal_flag); - WarpX::AllocInitMultiFab(pml_G_fp, ba_G_nodal, dm, 3, ngf, lev, "pml_G_fp", 0.0_rt); + warpx.m_fields.alloc_init(FieldType::pml_G_fp, lev, ba_G_nodal, dm, 3, ngf, 0.0_rt, false, false); } Box single_domain_box = is_single_box_domain ? domain0 : Box(); // Empty box (i.e., Box()) means it's not a single box domain. - sigba_fp = std::make_unique(ba, dm, grid_ba_reduced, geom->CellSize(), + sigba_fp = std::make_unique(ba, dm, &grid_ba_reduced, geom->CellSize(), IntVect(ncell), IntVect(delta), single_domain_box, v_sigma_sb, do_cubic_sigma_pml, pml_damping_strength); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -846,24 +870,24 @@ PML::PML (const int lev, const BoxArray& grid_ba, cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,0).ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{0}, lev, cba_Ex, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{1}, lev, cba_Ey, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{2}, lev, cba_Ez, cdm, ncompe, nge, 0.0_rt, false, false); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,0).ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{0}, lev, cba_Bx, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{1}, lev, cba_By, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{2}, lev, cba_Bz, cdm, ncompb, ngb, 0.0_rt, false, false); if (m_dive_cleaning) { const amrex::BoxArray cba_F_nodal = amrex::convert(cba, amrex::IntVect::TheNodeVector()); - WarpX::AllocInitMultiFab(pml_F_cp, cba_F_nodal, cdm, 3, ngf, lev, "pml_F_cp", 0.0_rt); + warpx.m_fields.alloc_init(FieldType::pml_F_cp, lev, cba_F_nodal, cdm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -873,18 +897,18 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray cba_G_nodal = amrex::convert(cba, G_nodal_flag); - WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); + warpx.m_fields.alloc_init(FieldType::pml_G_cp, lev, cba_G_nodal, cdm, 3, ngf, 0.0_rt, false, false); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,0).ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,1).ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{0}, lev, cba_jx, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{1}, lev, cba_jy, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{2}, lev, cba_jz, cdm, 1, ngb, 0.0_rt, false, false); single_domain_box = is_single_box_domain ? cdomain : Box(); - sigba_cp = std::make_unique(cba, cdm, grid_cba_reduced, cgeom->CellSize(), + sigba_cp = std::make_unique(cba, cdm, &grid_cba_reduced, cgeom->CellSize(), cncells, cdelta, single_domain_box, v_sigma_sb, do_cubic_sigma_pml, pml_damping_strength); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -1056,96 +1080,32 @@ PML::ComputePMLFactors (amrex::Real dt) } } -std::array -PML::GetE_fp () -{ - return {pml_E_fp[0].get(), pml_E_fp[1].get(), pml_E_fp[2].get()}; -} - -std::array -PML::GetB_fp () -{ - return {pml_B_fp[0].get(), pml_B_fp[1].get(), pml_B_fp[2].get()}; -} - -std::array -PML::Getj_fp () -{ - return {pml_j_fp[0].get(), pml_j_fp[1].get(), pml_j_fp[2].get()}; -} - -std::array -PML::GetE_cp () -{ - return {pml_E_cp[0].get(), pml_E_cp[1].get(), pml_E_cp[2].get()}; -} - -std::array -PML::GetB_cp () -{ - return {pml_B_cp[0].get(), pml_B_cp[1].get(), pml_B_cp[2].get()}; -} - -std::array -PML::Getj_cp () -{ - return {pml_j_cp[0].get(), pml_j_cp[1].get(), pml_j_cp[2].get()}; -} - -std::array -PML::Get_edge_lengths() -{ - return {pml_edge_lengths[0].get(), pml_edge_lengths[1].get(), pml_edge_lengths[2].get()}; -} - - -MultiFab* -PML::GetF_fp () -{ - return pml_F_fp.get(); -} - -MultiFab* -PML::GetF_cp () -{ - return pml_F_cp.get(); -} - -MultiFab* -PML::GetG_fp () -{ - return pml_G_fp.get(); -} - -MultiFab* -PML::GetG_cp () +void +PML::CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int lev +) { - return pml_G_cp.get(); -} + using ablastr::fields::Direction; -void PML::Exchange (const std::array& mf_pml, - const std::array& mf, - const PatchType& patch_type, - const int do_pml_in_domain) -{ - const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } - if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } - if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } -} + bool const has_j_fp = fields.has_vector(FieldType::current_fp, lev); + bool const has_pml_j_fp = fields.has_vector(FieldType::pml_j_fp, lev); + bool const has_j_cp = fields.has_vector(FieldType::current_cp, lev); + bool const has_pml_j_cp = fields.has_vector(FieldType::pml_j_cp, lev); -void -PML::CopyJtoPMLs (PatchType patch_type, - const std::array& jp) -{ - if (patch_type == PatchType::fine && pml_j_fp[0] && jp[0]) + if (patch_type == PatchType::fine && has_pml_j_fp && has_j_fp) { + ablastr::fields::VectorField pml_j_fp = fields.get_alldirs(FieldType::pml_j_fp, lev); + ablastr::fields::VectorField jp = fields.get_alldirs(FieldType::current_fp, lev); CopyToPML(*pml_j_fp[0], *jp[0], *m_geom); CopyToPML(*pml_j_fp[1], *jp[1], *m_geom); CopyToPML(*pml_j_fp[2], *jp[2], *m_geom); } - else if (patch_type == PatchType::coarse && pml_j_cp[0] && jp[0]) + else if (patch_type == PatchType::coarse && has_j_cp && has_pml_j_cp) { + ablastr::fields::VectorField pml_j_cp = fields.get_alldirs(FieldType::pml_j_cp, lev); + ablastr::fields::VectorField jp = fields.get_alldirs(FieldType::current_cp, lev); CopyToPML(*pml_j_cp[0], *jp[0], *m_cgeom); CopyToPML(*pml_j_cp[1], *jp[1], *m_cgeom); CopyToPML(*pml_j_cp[2], *jp[2], *m_cgeom); @@ -1153,46 +1113,33 @@ PML::CopyJtoPMLs (PatchType patch_type, } void -PML::CopyJtoPMLs (const std::array& j_fp, - const std::array& j_cp) -{ - CopyJtoPMLs(PatchType::fine, j_fp); - CopyJtoPMLs(PatchType::coarse, j_cp); -} - -void -PML::ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp, int do_pml_in_domain) +PML::CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + int lev +) { - ExchangeF(PatchType::fine, F_fp, do_pml_in_domain); - ExchangeF(PatchType::coarse, F_cp, do_pml_in_domain); + CopyJtoPMLs(fields, PatchType::fine, lev); + CopyJtoPMLs(fields, PatchType::coarse, lev); } -void -PML::ExchangeF (PatchType patch_type, amrex::MultiFab* Fp, int do_pml_in_domain) -{ - if (patch_type == PatchType::fine && pml_F_fp && Fp) { - Exchange(*pml_F_fp, *Fp, *m_geom, do_pml_in_domain); - } else if (patch_type == PatchType::coarse && pml_F_cp && Fp) { - Exchange(*pml_F_cp, *Fp, *m_cgeom, do_pml_in_domain); - } -} - -void PML::ExchangeG (amrex::MultiFab* G_fp, amrex::MultiFab* G_cp, int do_pml_in_domain) +void PML::Exchange (ablastr::fields::VectorField mf_pml, + ablastr::fields::VectorField mf, + const PatchType& patch_type, + const int do_pml_in_domain) { - ExchangeG(PatchType::fine, G_fp, do_pml_in_domain); - ExchangeG(PatchType::coarse, G_cp, do_pml_in_domain); + const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } -void PML::ExchangeG (PatchType patch_type, amrex::MultiFab* Gp, int do_pml_in_domain) +void PML::Exchange (amrex::MultiFab* mf_pml, + amrex::MultiFab* mf, + const PatchType& patch_type, + const int do_pml_in_domain) { - if (patch_type == PatchType::fine && pml_G_fp && Gp) - { - Exchange(*pml_G_fp, *Gp, *m_geom, do_pml_in_domain); - } - else if (patch_type == PatchType::coarse && pml_G_cp && Gp) - { - Exchange(*pml_G_cp, *Gp, *m_cgeom, do_pml_in_domain); - } + const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; + if (mf_pml && mf) { Exchange(*mf_pml, *mf, geom, do_pml_in_domain); } } void @@ -1286,74 +1233,40 @@ PML::CopyToPML (MultiFab& pml, MultiFab& reg, const Geometry& geom) } void -PML::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) +PML::FillBoundary (ablastr::fields::VectorField mf_pml, PatchType patch_type, std::optional nodal_sync) { - if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - const Vector mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } -} + const auto& period = + (patch_type == PatchType::fine) ? + m_geom->periodicity() : + m_cgeom->periodicity(); -void -PML::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_B_fp[0]) - { - const auto& period = m_geom->periodicity(); - const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_B_cp[0]) - { - const auto& period = m_cgeom->periodicity(); - const Vector mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } + const Vector mf{mf_pml[0], mf_pml[1], mf_pml[2]}; + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } void -PML::FillBoundaryF (PatchType patch_type, std::optional nodal_sync) +PML::FillBoundary (amrex::MultiFab & mf_pml, PatchType patch_type, std::optional nodal_sync) { - if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_fp, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_cp, WarpX::do_single_precision_comms, period, nodal_sync); - } -} + const auto& period = + (patch_type == PatchType::fine) ? + m_geom->periodicity() : + m_cgeom->periodicity(); -void -PML::FillBoundaryG (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_G_fp && pml_G_fp->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_fp, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_G_cp && pml_G_cp->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_cp, WarpX::do_single_precision_comms, period, nodal_sync); - } + ablastr::utils::communication::FillBoundary(mf_pml, WarpX::do_single_precision_comms, period, nodal_sync); } void -PML::CheckPoint (const std::string& dir) const +PML::CheckPoint ( + ablastr::fields::MultiFabRegister& fields, + const std::string& dir +) const { - if (pml_E_fp[0]) + using ablastr::fields::Direction; + + if (fields.has_vector(FieldType::pml_E_fp, 0)) { + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, 0); VisMF::AsyncWrite(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::AsyncWrite(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::AsyncWrite(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1362,8 +1275,10 @@ PML::CheckPoint (const std::string& dir) const VisMF::AsyncWrite(*pml_B_fp[2], dir+"_Bz_fp"); } - if (pml_E_cp[0]) + if (fields.has_vector(FieldType::pml_E_cp, 0)) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, 0); VisMF::AsyncWrite(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::AsyncWrite(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::AsyncWrite(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1374,10 +1289,17 @@ PML::CheckPoint (const std::string& dir) const } void -PML::Restart (const std::string& dir) +PML::Restart ( + ablastr::fields::MultiFabRegister& fields, + const std::string& dir +) { - if (pml_E_fp[0]) + using ablastr::fields::Direction; + + if (fields.has_vector(FieldType::pml_E_fp, 0)) { + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, 0); VisMF::Read(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::Read(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::Read(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1386,8 +1308,10 @@ PML::Restart (const std::string& dir) VisMF::Read(*pml_B_fp[2], dir+"_Bz_fp"); } - if (pml_E_cp[0]) + if (fields.has_vector(FieldType::pml_E_cp, 0)) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, 0); VisMF::Read(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::Read(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::Read(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1399,11 +1323,20 @@ PML::Restart (const std::string& dir) #ifdef WARPX_USE_FFT void -PML::PushPSATD (const int lev) { +PML::PushPSATD (ablastr::fields::MultiFabRegister& fields, const int lev) +{ + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, lev); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, lev); + ablastr::fields::ScalarField pml_F_fp = (fields.has(FieldType::pml_F_fp, lev)) ? fields.get(FieldType::pml_F_fp, lev) : nullptr; + ablastr::fields::ScalarField pml_G_fp = (fields.has(FieldType::pml_G_fp, lev)) ? fields.get(FieldType::pml_G_fp, lev) : nullptr; // Update the fields on the fine and coarse patch PushPMLPSATDSinglePatch(lev, *spectral_solver_fp, pml_E_fp, pml_B_fp, pml_F_fp, pml_G_fp, m_fill_guards_fields); if (spectral_solver_cp) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, lev); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, lev); + ablastr::fields::ScalarField pml_F_cp = (fields.has(FieldType::pml_F_cp, lev)) ? fields.get(FieldType::pml_F_cp, lev) : nullptr; + ablastr::fields::ScalarField pml_G_cp = (fields.has(FieldType::pml_G_cp, lev)) ? fields.get(FieldType::pml_G_cp, lev) : nullptr; PushPMLPSATDSinglePatch(lev, *spectral_solver_cp, pml_E_cp, pml_B_cp, pml_F_cp, pml_G_cp, m_fill_guards_fields); } } @@ -1412,10 +1345,10 @@ void PushPMLPSATDSinglePatch ( const int lev, SpectralSolver& solver, - std::array,3>& pml_E, - std::array,3>& pml_B, - std::unique_ptr& pml_F, - std::unique_ptr& pml_G, + ablastr::fields::VectorField& pml_E, + ablastr::fields::VectorField& pml_B, + ablastr::fields::ScalarField pml_F, + ablastr::fields::ScalarField pml_G, const amrex::IntVect& fill_guards) { const SpectralFieldIndex& Idx = solver.m_spectral_index; diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index c908681d8e5..5508836a171 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -16,6 +16,8 @@ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif +#include + #include #include #include @@ -30,43 +32,38 @@ class PML_RZ { public: - PML_RZ (int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, - const amrex::Geometry* geom, int ncell, int do_pml_in_domain); + PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm, + amrex::Geometry const* geom, int ncell, int do_pml_in_domain); void ApplyDamping(amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::MultiFab* Bt_fp, amrex::MultiFab* Bz_fp, - amrex::Real dt); - - std::array GetE_fp (); - std::array GetB_fp (); + amrex::Real dt, ablastr::fields::MultiFabRegister& fields); #ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif - void FillBoundaryE (); - void FillBoundaryB (); - void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryE (ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryB (ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, std::optional nodal_sync=std::nullopt); - void CheckPoint (const std::string& dir) const; - void Restart (const std::string& dir); + void CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const; + void Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir); private: - const int m_ncell; - const int m_do_pml_in_domain; + int m_ncell; + int m_do_pml_in_domain; const amrex::Geometry* m_geom; - // Only contains Er and Et, and Br and Bt - std::array,2> pml_E_fp; - std::array,2> pml_B_fp; + // The MultiFabs pml_E_fp and pml_B_fp are setup using the registry. + // They hold Er, Et, and Br, Bt. #ifdef WARPX_USE_FFT - void PushPMLPSATDSinglePatchRZ ( int lev, + void PushPMLPSATDSinglePatchRZ (int lev, SpectralSolverRZ& solver, - std::array,2>& pml_E, - std::array,2>& pml_B); + ablastr::fields::MultiFabRegister& fields); #endif }; diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 78f3cf24987..8fd6a1869ae 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -8,7 +8,7 @@ #include "PML_RZ.H" #include "BoundaryConditions/PML_RZ.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif @@ -33,43 +33,55 @@ #include #include -using namespace amrex; -using namespace warpx::fields; +using namespace amrex::literals; +using warpx::fields::FieldType; +using ablastr::fields::Direction; -PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, - const amrex::Geometry* geom, const int ncell, const int do_pml_in_domain) +PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm, + amrex::Geometry const* geom, int ncell, int do_pml_in_domain) : m_ncell(ncell), m_do_pml_in_domain(do_pml_in_domain), m_geom(geom) { - - const amrex::MultiFab & Er_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,0); - const amrex::MultiFab & Et_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,1); - const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); - const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); - - const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); - const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); - const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); - const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[1], ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), lev, "pml_B_fp[1]", 0.0_rt); + auto & warpx = WarpX::GetInstance(); + + bool const remake = false; + bool const redistribute_on_remake = false; + + amrex::MultiFab const& Er_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + amrex::MultiFab const& Et_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + amrex::BoxArray const ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); + amrex::BoxArray const ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + + amrex::MultiFab const& Br_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{0},lev); + amrex::MultiFab const& Bt_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{1},lev); + amrex::BoxArray const ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); + amrex::BoxArray const ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{0}, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{1}, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); } void PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::MultiFab* Bt_fp, amrex::MultiFab* Bz_fp, - amrex::Real dt) + amrex::Real dt, ablastr::fields::MultiFabRegister& fields) { - const amrex::Real dr = m_geom->CellSize(0); - const amrex::Real cdt_over_dr = PhysConst::c*dt/dr; + amrex::Real const dr = m_geom->CellSize(0); + amrex::Real const cdt_over_dr = PhysConst::c*dt/dr; + + amrex::MultiFab* pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); + amrex::MultiFab* pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for ( amrex::MFIter mfi(*Et_fp, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -78,8 +90,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::Array4 const& Bt_arr = Bt_fp->array(mfi); amrex::Array4 const& Bz_arr = Bz_fp->array(mfi); - amrex::Array4 const& pml_Et_arr = pml_E_fp[1]->array(mfi); - amrex::Array4 const& pml_Bt_arr = pml_B_fp[1]->array(mfi); + amrex::Array4 const& pml_Et_arr = pml_Et->array(mfi); + amrex::Array4 const& pml_Bt_arr = pml_Bt->array(mfi); // Get the tileboxes from Efield and Bfield so that they include the guard cells // They are all the same, cell centered @@ -87,19 +99,19 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, // Box for the whole simulation domain amrex::Box const& domain = m_geom->Domain(); - const int nr_domain = domain.bigEnd(0); + int const nr_domain = domain.bigEnd(0); // Set tilebox to only include the upper radial cells - const int nr_damp = m_ncell; - const int nr_damp_min = (m_do_pml_in_domain)?(nr_domain - nr_damp):(nr_domain); + int const nr_damp = m_ncell; + int const nr_damp_min = (m_do_pml_in_domain)?(nr_domain - nr_damp):(nr_domain); tilebox.setSmall(0, nr_damp_min + 1); amrex::ParallelFor( tilebox, Et_fp->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int icomp) { - const auto rr = static_cast(i - nr_damp_min); - const amrex::Real wr = rr/nr_damp; - const amrex::Real damp_factor = std::exp( -4._rt * cdt_over_dr * wr*wr ); + auto const rr = static_cast(i - nr_damp_min); + amrex::Real const wr = rr/nr_damp; + amrex::Real const damp_factor = std::exp( -4._rt * cdt_over_dr * wr*wr ); // Substract the theta PML fields from the regular theta fields Et_arr(i,j,k,icomp) -= pml_Et_arr(i,j,k,icomp); @@ -117,105 +129,88 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, } } -std::array -PML_RZ::GetE_fp () -{ - return {pml_E_fp[0].get(), pml_E_fp[1].get()}; -} - -std::array -PML_RZ::GetB_fp () -{ - return {pml_B_fp[0].get(), pml_B_fp[1].get()}; -} - void -PML_RZ::FillBoundaryE () +PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional nodal_sync) { - FillBoundaryE(PatchType::fine); -} + amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); -void -PML_RZ::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) + if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { - const amrex::Periodicity& period = m_geom->periodicity(); - const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get()}; + amrex::Periodicity const& period = m_geom->periodicity(); + const amrex::Vector mf = {pml_Er, pml_Et}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML_RZ::FillBoundaryB () +PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional nodal_sync) { - FillBoundaryB(PatchType::fine); -} - -void -PML_RZ::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_B_fp[0]) + if (patch_type == PatchType::fine) { - const amrex::Periodicity& period = m_geom->periodicity(); - const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get()}; + amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); + + amrex::Periodicity const& period = m_geom->periodicity(); + const amrex::Vector mf = {pml_Br, pml_Bt}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML_RZ::CheckPoint (const std::string& dir) const +PML_RZ::CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const { - if (pml_E_fp[0]) - { - VisMF::AsyncWrite(*pml_E_fp[0], dir+"_Er_fp"); - VisMF::AsyncWrite(*pml_E_fp[1], dir+"_Et_fp"); - VisMF::AsyncWrite(*pml_B_fp[0], dir+"_Br_fp"); - VisMF::AsyncWrite(*pml_B_fp[1], dir+"_Bt_fp"); + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp"); } } void -PML_RZ::Restart (const std::string& dir) +PML_RZ::Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir) { - if (pml_E_fp[0]) - { - VisMF::Read(*pml_E_fp[0], dir+"_Er_fp"); - VisMF::Read(*pml_E_fp[1], dir+"_Et_fp"); - VisMF::Read(*pml_B_fp[0], dir+"_Br_fp"); - VisMF::Read(*pml_B_fp[1], dir+"_Bt_fp"); + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { + amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp"); } } #ifdef WARPX_USE_FFT void -PML_RZ::PushPSATD (const int lev) +PML_RZ::PushPSATD (int lev) { // Update the fields on the fine and coarse patch WarpX& warpx = WarpX::GetInstance(); SpectralSolverRZ& solver = warpx.get_spectral_solver_fp(lev); - PushPMLPSATDSinglePatchRZ(lev, solver, pml_E_fp, pml_B_fp); + PushPMLPSATDSinglePatchRZ(lev, solver, warpx.m_fields); } void PML_RZ::PushPMLPSATDSinglePatchRZ ( - const int lev, + int lev, SpectralSolverRZ& solver, - std::array,2>& pml_E, - std::array,2>& pml_B) + ablastr::fields::MultiFabRegister& fields) { - const SpectralFieldIndex& Idx = solver.m_spectral_index; + SpectralFieldIndex const& Idx = solver.m_spectral_index; + amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); + amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); // Perform forward Fourier transforms - solver.ForwardTransform(lev, *pml_E[0], Idx.Er_pml, *pml_E[1], Idx.Et_pml); - solver.ForwardTransform(lev, *pml_B[0], Idx.Br_pml, *pml_B[1], Idx.Bt_pml); + solver.ForwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml); + solver.ForwardTransform(lev, *pml_Br, Idx.Br_pml, *pml_Bt, Idx.Bt_pml); // Advance fields in spectral space - const bool doing_pml = true; + bool const doing_pml = true; solver.pushSpectralFields(doing_pml); // Perform backward Fourier transforms - solver.BackwardTransform(lev, *pml_E[0], Idx.Er_pml, *pml_E[1], Idx.Et_pml); - solver.BackwardTransform(lev, *pml_B[0], Idx.Br_pml, *pml_B[1], Idx.Bt_pml); + solver.BackwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml); + solver.BackwardTransform(lev, *pml_Br, Idx.Br_pml, *pml_Bt, Idx.Bt_pml); } #endif diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 90b2274d9c5..9c78367a370 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -11,10 +11,14 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "PML_current.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX_PML_kernels.H" +#include + #ifdef AMREX_USE_SENSEI_INSITU # include #endif @@ -62,9 +66,13 @@ WarpX::DampPML (const int lev, PatchType patch_type) WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->ApplyDamping(Efield_fp[lev][1].get(), Efield_fp[lev][2].get(), - Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), - dt[lev]); + using ablastr::fields::Direction; + using warpx::fields::FieldType; + pml_rz[lev]->ApplyDamping( m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), + dt[lev], m_fields); } #endif if (pml[lev]) { @@ -80,12 +88,11 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) const bool abc_in_pml = WarpX::do_abc_in_pml; if (pml[lev]->ok()) { - const auto& pml_E = (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); - const auto& pml_B = (patch_type == PatchType::fine) ? pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); - const auto& pml_F = (patch_type == PatchType::fine) ? pml[lev]->GetF_fp() : pml[lev]->GetF_cp(); - const auto& pml_G = (patch_type == PatchType::fine) ? pml[lev]->GetG_fp() : pml[lev]->GetG_cp(); - const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() - : pml[lev]->GetMultiSigmaBox_cp(); + using warpx::fields::FieldType; + + const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_E_fp, lev) : m_fields.get_alldirs(FieldType::pml_E_cp, lev); + const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_B_fp, lev) : m_fields.get_alldirs(FieldType::pml_B_cp, lev); + const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); const amrex::IntVect Ex_stag = pml_E[0]->ixType().toIntVect(); const amrex::IntVect Ey_stag = pml_E[1]->ixType().toIntVect(); @@ -96,12 +103,16 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) const amrex::IntVect Bz_stag = pml_B[2]->ixType().toIntVect(); amrex::IntVect F_stag; - if (pml_F) { + if (m_fields.has(FieldType::pml_F_fp, lev)) { + amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? + m_fields.get(FieldType::pml_F_fp, lev) : m_fields.get(FieldType::pml_F_cp, lev); F_stag = pml_F->ixType().toIntVect(); } amrex::IntVect G_stag; - if (pml_G) { + if (m_fields.has(FieldType::pml_G_fp, lev)) { + amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? + m_fields.get(FieldType::pml_G_fp, lev) : m_fields.get(FieldType::pml_G_cp, lev); G_stag = pml_G->ixType().toIntVect(); } @@ -192,7 +203,9 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) // For warpx_damp_pml_F(), mfi.nodaltilebox is used in the ParallelFor loop and here we // use mfi.tilebox. However, it does not matter because in damp_pml, where nodaltilebox // is used, only a simple multiplication is performed. - if (pml_F) { + if (m_fields.has(FieldType::pml_F_fp, lev)) { + amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? + m_fields.get(FieldType::pml_F_fp, lev) : m_fields.get(FieldType::pml_F_cp, lev); const Box& tnd = mfi.nodaltilebox(); auto const& pml_F_fab = pml_F->array(mfi); amrex::ParallelFor(tnd, [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -203,7 +216,10 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) } // Damp G when WarpX::do_divb_cleaning = true - if (pml_G) { + if (m_fields.has(FieldType::pml_G_fp, lev)) { + amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? + m_fields.get(FieldType::pml_G_fp, lev) : m_fields.get(FieldType::pml_G_cp, lev); + const Box& tb = mfi.tilebox(G_stag); auto const& pml_G_fab = pml_G->array(mfi); amrex::ParallelFor(tb, [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -242,8 +258,9 @@ WarpX::DampJPML (int lev, PatchType patch_type) if (pml[lev]->ok()) { + using warpx::fields::FieldType; - const auto& pml_j = (patch_type == PatchType::fine) ? pml[lev]->Getj_fp() : pml[lev]->Getj_cp(); + const auto& pml_j = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_j_fp, lev) : m_fields.get_alldirs(FieldType::pml_j_cp, lev); const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); @@ -269,13 +286,17 @@ WarpX::DampJPML (int lev, PatchType patch_type) const Real* sigma_star_cumsum_fac_j_z = sigba[mfi].sigma_star_cumsum_fac[1].data(); #endif -#ifdef AMREX_USE_EB - const auto& pml_edge_lenghts = pml[lev]->Get_edge_lengths(); + // Skip the field update if this gridpoint is inside the embedded boundary + amrex::Array4 eb_lxfab, eb_lyfab, eb_lzfab; + if (EB::enabled()) { + const auto &pml_edge_lenghts = m_fields.get_alldirs(FieldType::pml_edge_lengths, lev); - auto const& pml_lxfab = pml_edge_lenghts[0]->array(mfi); - auto const& pml_lyfab = pml_edge_lenghts[1]->array(mfi); - auto const& pml_lzfab = pml_edge_lenghts[2]->array(mfi); -#endif + eb_lxfab = pml_edge_lenghts[0]->array(mfi); + eb_lyfab = pml_edge_lenghts[1]->array(mfi); + eb_lzfab = pml_edge_lenghts[2]->array(mfi); + } else { + amrex::ignore_unused(eb_lxfab, eb_lyfab, eb_lzfab); + } const Box& tjx = mfi.tilebox( pml_j[0]->ixType().toIntVect() ); const Box& tjy = mfi.tilebox( pml_j[1]->ixType().toIntVect() ); @@ -301,27 +322,21 @@ WarpX::DampJPML (int lev, PatchType patch_type) amrex::ParallelFor( tjx, tjy, tjz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lxfab(i, j, k) <= 0) return; -#endif + if (eb_lxfab && eb_lxfab(i, j, k) <= 0) { return; } damp_jx_pml(i, j, k, pml_jxfab, sigma_star_cumsum_fac_j_x, sigma_cumsum_fac_j_y, sigma_cumsum_fac_j_z, xs_lo,y_lo, z_lo); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lyfab(i, j, k) <= 0) return; -#endif + if (eb_lyfab && eb_lyfab(i, j, k) <= 0) { return; } damp_jy_pml(i, j, k, pml_jyfab, sigma_cumsum_fac_j_x, sigma_star_cumsum_fac_j_y, sigma_cumsum_fac_j_z, x_lo,ys_lo, z_lo); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lzfab(i, j, k)<=0) return; -#endif + if (eb_lzfab && eb_lzfab(i, j, k) <= 0) { return; } damp_jz_pml(i, j, k, pml_jzfab, sigma_cumsum_fac_j_x, sigma_cumsum_fac_j_y, sigma_star_cumsum_fac_j_z, @@ -339,15 +354,12 @@ WarpX::DampJPML (int lev, PatchType patch_type) void WarpX::CopyJPML () { + using ablastr::fields::Direction; + for (int lev = 0; lev <= finest_level; ++lev) { if (pml[lev] && pml[lev]->ok()){ - pml[lev]->CopyJtoPMLs({ current_fp[lev][0].get(), - current_fp[lev][1].get(), - current_fp[lev][2].get() }, - { current_cp[lev][0].get(), - current_cp[lev][1].get(), - current_cp[lev][2].get() }); + pml[lev]->CopyJtoPMLs(m_fields, lev); } } } diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 2b063b99a15..6217eb04a33 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -1,4 +1,5 @@ #include "WarpX.H" +#include "BoundaryConditions/PEC_Insulator.H" #include "BoundaryConditions/PML.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" @@ -18,15 +19,15 @@ using namespace amrex; using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { /** Returns true if any field boundary is set to FieldBoundaryType FT, else returns false.*/ template [[nodiscard]] - bool isAnyBoundary (const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi) + bool isAnyBoundary (const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi) { const auto isFT = [](const auto& b){ return b == FT;}; @@ -37,8 +38,8 @@ namespace /** Returns true if any particle boundary is set to ParticleBoundaryType PT, else returns false.*/ template [[nodiscard]] - bool isAnyBoundary (const amrex::Vector& particle_boundary_lo, - const amrex::Vector& particle_boundary_hi) + bool isAnyBoundary (const amrex::Array& particle_boundary_lo, + const amrex::Array& particle_boundary_hi) { const auto isPT = [](const auto& b){ return b == PT;}; @@ -48,43 +49,117 @@ namespace } -void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) +void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type, amrex::Real time) { + using ablastr::fields::Direction; + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoEfield( - {getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2)}, - field_boundary_lo, field_boundary_hi, + m_fields.get_alldirs(FieldType::Efield_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( - pml[lev]->GetE_fp(), - field_boundary_lo, field_boundary_hi, + m_fields.get_alldirs(FieldType::pml_E_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio, split_pml_field); } } else { PEC::ApplyPECtoEfield( - {getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2)}, + m_fields.get_alldirs(FieldType::Efield_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + PEC::ApplyPECtoEfield( + m_fields.get_alldirs(FieldType::pml_E_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); + } + } + } + + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + if (patch_type == PatchType::fine) { + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::Efield_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::pml_E_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); + } + } else { + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::Efield_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::pml_E_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); + } + } + } + + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + if (patch_type == PatchType::fine) { + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + {m_fields.get(FieldType::Efield_fp,Direction{0},lev), + m_fields.get(FieldType::Efield_fp,Direction{1},lev), + m_fields.get(FieldType::Efield_fp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, time); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + m_fields.get_alldirs(FieldType::pml_E_fp, lev), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, time, + split_pml_field); + } + } else { + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + {m_fields.get(FieldType::Efield_cp,Direction{0},lev), + m_fields.get(FieldType::Efield_cp,Direction{1},lev), + m_fields.get(FieldType::Efield_cp,Direction{2},lev)}, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), - lev, patch_type, ref_ratio); + lev, patch_type, ref_ratio, time); if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( - pml[lev]->GetE_cp(), + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + m_fields.get_alldirs(FieldType::pml_E_cp, lev), field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), - lev, patch_type, ref_ratio, + lev, patch_type, ref_ratio, time, split_pml_field); } } @@ -92,45 +167,81 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev), lev); } else { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + m_fields.get(FieldType::Efield_cp, Direction{2}, lev), lev); } #endif } -void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) +void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type, amrex::Real time) { + using ablastr::fields::Direction; + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { - getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2) }, - field_boundary_lo, field_boundary_hi, + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); } else { - PEC::ApplyPECtoBfield( { - getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2)}, - field_boundary_lo, field_boundary_hi, + PEC::ApplyPECtoBfield( + m_fields.get_alldirs(FieldType::Bfield_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PEC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + } + } + + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + if (patch_type == PatchType::fine) { + PEC::ApplyPECtoEfield( + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + } else { + PEC::ApplyPECtoEfield( + m_fields.get_alldirs(FieldType::Bfield_cp, lev), + field_boundary_lo, field_boundary_hi, FieldBoundaryType::PMC, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); } } + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + if (patch_type == PatchType::fine) { + pec_insulator_boundary->ApplyPEC_InsulatortoBfield( + {m_fields.get(FieldType::Bfield_fp,Direction{0},lev), + m_fields.get(FieldType::Bfield_fp,Direction{1},lev), + m_fields.get(FieldType::Bfield_fp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, time); + } else { + pec_insulator_boundary->ApplyPEC_InsulatortoBfield( + {m_fields.get(FieldType::Bfield_cp,Direction{0},lev), + m_fields.get(FieldType::Bfield_cp,Direction{1},lev), + m_fields.get(FieldType::Bfield_cp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, time); + } + } + // Silver-Mueller boundaries are only applied on the first half-push of B // This is because the formula used for Silver-Mueller assumes that // E and B are staggered in time, which is only true after the first half-push if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ + auto Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], @@ -141,13 +252,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Bfield_fp,Direction{0},lev), + m_fields.get(FieldType::Bfield_fp,Direction{1},lev), + m_fields.get(FieldType::Bfield_fp,Direction{2},lev), lev); } else { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Bfield_cp,Direction{0},lev), + m_fields.get(FieldType::Bfield_cp,Direction{1},lev), + m_fields.get(FieldType::Bfield_cp,Direction{2},lev), lev); } #endif } @@ -157,7 +268,8 @@ void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, { if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || - ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) + ::isAnyBoundary(field_boundary_lo, field_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { PEC::ApplyReflectiveBoundarytoRhofield(rho, field_boundary_lo, field_boundary_hi, @@ -172,7 +284,8 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, { if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || - ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) + ::isAnyBoundary(field_boundary_lo, field_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, field_boundary_lo, field_boundary_hi, @@ -268,8 +381,9 @@ void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { + ablastr::fields::ScalarField electron_pressure_fp = m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); PEC::ApplyPECtoElectronPressure( - m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), + electron_pressure_fp, field_boundary_lo, field_boundary_hi, Geom(lev), lev, patch_type, ref_ratio); } else { diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index a6af894beb4..e3fd804b62c 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -31,8 +31,9 @@ namespace PEC { */ void ApplyPECtoEfield ( std::array Efield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + FieldBoundaryType bc_type, const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, int lev, PatchType patch_type, const amrex::Vector& ref_ratios, bool split_pml_field = false); @@ -52,10 +53,12 @@ namespace PEC { */ void ApplyPECtoBfield ( std::array Bfield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + FieldBoundaryType bc_type, const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, - int lev, PatchType patch_type, const amrex::Vector& ref_ratios); + int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + bool split_pml_field = false); /** * \brief Reflects charge density deposited over the PEC boundary back into @@ -73,10 +76,10 @@ namespace PEC { */ void ApplyReflectiveBoundarytoRhofield( amrex::MultiFab* rho, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, - const amrex::Vector& particle_boundary_lo, - const amrex::Vector& particle_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + const amrex::Array& particle_boundary_lo, + const amrex::Array& particle_boundary_hi, const amrex::Geometry& geom, int lev, PatchType patch_type, const amrex::Vector& ref_ratios); @@ -97,10 +100,10 @@ namespace PEC { void ApplyReflectiveBoundarytoJfield( amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, - const amrex::Vector& particle_boundary_lo, - const amrex::Vector& particle_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + const amrex::Array& particle_boundary_lo, + const amrex::Array& particle_boundary_hi, const amrex::Geometry& geom, int lev, PatchType patch_type, const amrex::Vector& ref_ratios); @@ -117,8 +120,8 @@ namespace PEC { */ void ApplyPECtoElectronPressure ( amrex::MultiFab* Pefield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, const amrex::Geometry& geom, int lev, PatchType patch_type, const amrex::Vector& ref_ratios); } diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0067f54d3ff..a3b75791582 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -121,7 +121,8 @@ namespace amrex::Array4 const& Efield, const amrex::IntVect& is_nodal, amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) + amrex::GpuArray const& fbndry_hi, + FieldBoundaryType bc_type) { // Tangential Efield components in guard cells set equal and opposite to cells // in the mirror locations across the PEC boundary, whereas normal E-field @@ -136,8 +137,8 @@ namespace // Loop over sides, iside = 0 (lo), iside = 1 (hi) for (int iside = 0; iside < 2; ++iside) { const bool isPECBoundary = ( (iside == 0) - ? fbndry_lo[idim] == FieldBoundaryType::PEC - : fbndry_hi[idim] == FieldBoundaryType::PEC ); + ? fbndry_lo[idim] == bc_type + : fbndry_hi[idim] == bc_type ); #if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), // icomp=1 is tangential to both x and z boundaries @@ -260,7 +261,8 @@ namespace amrex::Array4 const& Bfield, const amrex::IntVect & is_nodal, amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) + amrex::GpuArray const& fbndry_hi, + FieldBoundaryType bc_type) { amrex::IntVect ijk_mirror = ijk_vec; bool OnPECBoundary = false; @@ -271,8 +273,8 @@ namespace // Loop over sides, iside = 0 (lo), iside = 1 (hi) for (int iside = 0; iside < 2; ++iside) { const bool isPECBoundary = ( (iside == 0) - ? fbndry_lo[idim] == FieldBoundaryType::PEC - : fbndry_hi[idim] == FieldBoundaryType::PEC ); + ? fbndry_lo[idim] == bc_type + : fbndry_hi[idim] == bc_type ); if (isPECBoundary) { #if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) // For 2D : for icomp==1, (By in XZ, Btheta in RZ), @@ -357,7 +359,7 @@ namespace amrex::Array4 const& field, amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, amrex::GpuArray, AMREX_SPACEDIM> const& psign, - amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, + amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, amrex::GpuArray const& tangent_to_bndy, amrex::Box const& fabbox) { @@ -374,11 +376,11 @@ namespace amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) { - field(ijk_vec, n) = 0._rt; - // otherwise update the internal cell if the mirror guard cell exists + // Update the cell if the mirror guard cell exists + if (ijk_vec == iv_mirror && is_reflective[idim][iside] == 1) { + field(ijk_vec,n) = 0._rt; } else if (fabbox.contains(iv_mirror)) { + // Note that this includes the cells on the boundary for PMC field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); } } @@ -457,8 +459,9 @@ namespace void PEC::ApplyPECtoEfield ( std::array Efield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + FieldBoundaryType bc_type, const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, const int lev, PatchType patch_type, const amrex::Vector& ref_ratios, const bool split_pml_field) @@ -510,42 +513,27 @@ PEC::ApplyPECtoEfield ( amrex::ParallelFor( tex, nComp_x, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - Ex, Ex_nodal, fbndry_lo, fbndry_hi); + Ex, Ex_nodal, fbndry_lo, fbndry_hi, bc_type); }, tey, nComp_y, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - Ey, Ey_nodal, fbndry_lo, fbndry_hi); + Ey, Ey_nodal, fbndry_lo, fbndry_hi, bc_type); }, tez, nComp_z, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - Ez, Ez_nodal, fbndry_lo, fbndry_hi); + Ez, Ez_nodal, fbndry_lo, fbndry_hi, bc_type); } ); } @@ -555,10 +543,12 @@ PEC::ApplyPECtoEfield ( void PEC::ApplyPECtoBfield ( std::array Bfield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + FieldBoundaryType bc_type, const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, - const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + const bool split_pml_field) { amrex::Box domain_box = geom.Domain(); if (patch_type == PatchType::coarse && (lev > 0)) { @@ -594,50 +584,38 @@ PEC::ApplyPECtoBfield ( // gather fields from in the guard-cell region are included. // Note that for simulations without particles or laser, ng_field_gather is 0 // and the guard-cell values of the B-field multifab will not be modified. - amrex::Box const& tbx = mfi.tilebox(Bfield[0]->ixType().toIntVect(), ng_fieldgather); - amrex::Box const& tby = mfi.tilebox(Bfield[1]->ixType().toIntVect(), ng_fieldgather); - amrex::Box const& tbz = mfi.tilebox(Bfield[2]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const& tbx = (split_pml_field) ? mfi.tilebox(Bfield[0]->ixType().toIntVect()) + : mfi.tilebox(Bfield[0]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const& tby = (split_pml_field) ? mfi.tilebox(Bfield[1]->ixType().toIntVect()) + : mfi.tilebox(Bfield[1]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const& tbz = (split_pml_field) ? mfi.tilebox(Bfield[2]->ixType().toIntVect()) + : mfi.tilebox(Bfield[2]->ixType().toIntVect(), ng_fieldgather); // loop over cells and update fields amrex::ParallelFor( tbx, nComp_x, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - Bx, Bx_nodal, fbndry_lo, fbndry_hi); + Bx, Bx_nodal, fbndry_lo, fbndry_hi, bc_type); }, tby, nComp_y, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - By, By_nodal, fbndry_lo, fbndry_hi); + By, By_nodal, fbndry_lo, fbndry_hi, bc_type); }, tbz, nComp_z, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, - Bz, Bz_nodal, fbndry_lo, fbndry_hi); + Bz, Bz_nodal, fbndry_lo, fbndry_hi, bc_type); } ); } @@ -657,10 +635,10 @@ PEC::ApplyPECtoBfield ( void PEC::ApplyReflectiveBoundarytoRhofield ( amrex::MultiFab* rho, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, - const amrex::Vector& particle_boundary_lo, - const amrex::Vector& particle_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + const amrex::Array& particle_boundary_lo, + const amrex::Array& particle_boundary_hi, const amrex::Geometry& geom, const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { @@ -680,7 +658,7 @@ PEC::ApplyReflectiveBoundarytoRhofield ( // cells for boundaries that are NOT PEC amrex::Box grown_domain_box = domain_box; - amrex::GpuArray, AMREX_SPACEDIM> is_reflective; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM> psign; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; @@ -688,9 +666,11 @@ PEC::ApplyReflectiveBoundarytoRhofield ( is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + if (field_boundary_lo[idim] == FieldBoundaryType::PMC) { is_reflective[idim][0] = 2; } is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); + if (field_boundary_hi[idim] == FieldBoundaryType::PMC) { is_reflective[idim][1] = 2; } if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -699,10 +679,12 @@ PEC::ApplyReflectiveBoundarytoRhofield ( is_tangent_to_bndy[idim] = true; psign[idim][0] = ((particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_lo[idim] == FieldBoundaryType::PMC)) ? 1._rt : -1._rt; psign[idim][1] = ((particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_hi[idim] == FieldBoundaryType::PMC)) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -727,11 +709,7 @@ PEC::ApplyReflectiveBoundarytoRhofield ( // Loop over valid cells (i.e. cells inside the domain) amrex::ParallelFor(mfi.validbox(), nComp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); @@ -747,10 +725,10 @@ PEC::ApplyReflectiveBoundarytoRhofield ( void PEC::ApplyReflectiveBoundarytoJfield( amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, - const amrex::Vector& particle_boundary_lo, - const amrex::Vector& particle_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, + const amrex::Array& particle_boundary_lo, + const amrex::Array& particle_boundary_hi, const amrex::Geometry& geom, const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { @@ -780,17 +758,21 @@ PEC::ApplyReflectiveBoundarytoJfield( // directions of the current density multifab const amrex::IntVect ng_fieldgather = Jx->nGrowVect(); - amrex::GpuArray, AMREX_SPACEDIM> is_reflective; + amrex::GpuArray, AMREX_SPACEDIM> is_reflective; amrex::GpuArray, 3> is_tangent_to_bndy; amrex::GpuArray, AMREX_SPACEDIM>, 3> psign; amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) - || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC) + || ( field_boundary_lo[idim] == FieldBoundaryType::PMC); + if (field_boundary_lo[idim] == FieldBoundaryType::PMC) { is_reflective[idim][0] = 2; } is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) - || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC) + || ( field_boundary_hi[idim] == FieldBoundaryType::PMC); + if (field_boundary_hi[idim] == FieldBoundaryType::PMC) { is_reflective[idim][1] = 2; } if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -812,18 +794,22 @@ PEC::ApplyReflectiveBoundarytoJfield( if (is_tangent_to_bndy[icomp][idim]){ psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_lo[idim] == FieldBoundaryType::PMC)) ? 1._rt : -1._rt; psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_hi[idim] == FieldBoundaryType::PMC)) ? 1._rt : -1._rt; } else { psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_lo[idim] == FieldBoundaryType::PMC)) ? -1._rt : 1._rt; psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + ||(field_boundary_hi[idim] == FieldBoundaryType::PMC)) ? -1._rt : 1._rt; } } @@ -859,11 +845,7 @@ PEC::ApplyReflectiveBoundarytoJfield( // Loop over valid cells (i.e. cells inside the domain) amrex::ParallelFor(mfi.validbox(), Jx->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); @@ -894,11 +876,7 @@ PEC::ApplyReflectiveBoundarytoJfield( // Loop over valid cells (i.e. cells inside the domain) amrex::ParallelFor(mfi.validbox(), Jy->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); @@ -929,11 +907,7 @@ PEC::ApplyReflectiveBoundarytoJfield( // Loop over valid cells (i.e. cells inside the domain) amrex::ParallelFor(mfi.validbox(), Jz->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); @@ -948,8 +922,8 @@ PEC::ApplyReflectiveBoundarytoJfield( void PEC::ApplyPECtoElectronPressure ( amrex::MultiFab* Pefield, - const amrex::Vector& field_boundary_lo, - const amrex::Vector& field_boundary_hi, + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi, const amrex::Geometry& geom, const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { @@ -1000,11 +974,7 @@ PEC::ApplyPECtoElectronPressure ( // Loop over valid cells (i.e. cells inside the domain) amrex::ParallelFor(mfi.validbox(), nComp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif (defined WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index d5dd67226b7..f4f118892a8 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -28,7 +28,7 @@ class BTDiagnostics final : public Diagnostics { public: - BTDiagnostics (int i, const std::string& name); + BTDiagnostics (int i, const std::string& name, DiagTypes diag_type); private: /** Whether to plot raw (i.e., NOT cell-centered) fields */ @@ -161,6 +161,7 @@ private: * in z-direction for both 2D and 3D simulations in the Cartesian frame of reference. */ int m_moving_window_dir; + amrex::Real m_moving_window_beta; /** Number of back-transformed snapshots in the lab-frame requested by the user */ int m_num_snapshots_lab = std::numeric_limits::lowest(); @@ -241,7 +242,7 @@ private: * will be used by all snapshots to obtain lab-frame data at the respective * z slice location. */ - amrex::Vector > m_cell_centered_data; + std::string m_cell_centered_data_name; /** Vector of pointers to compute cell-centered data, per level, per component * using the coarsening-ratio provided by the user. */ @@ -346,7 +347,7 @@ private: * \param[in] i_buffer snapshot index */ void SetSnapshotFullStatus (int i_buffer); - /** Vector of field-data stored in the cell-centered multifab, m_cell_centered_data. + /** Vector of field-data stored in the cell-centered MultiFab. * All the fields are stored regardless of the specific fields to plot selected * by the user. */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 1cee9909226..cae2d2bbc03 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -14,13 +14,14 @@ #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/Algorithms/IsIn.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include #include @@ -47,15 +48,16 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { constexpr int permission_flag_rwxrxrx = 0755; } -BTDiagnostics::BTDiagnostics (int i, const std::string& name) - : Diagnostics{i, name} +BTDiagnostics::BTDiagnostics (int i, const std::string& name, DiagTypes diag_type) + : Diagnostics{i, name, diag_type}, + m_cell_centered_data_name("BTD_cell_centered_data_" + name) { ReadParameters(); } @@ -67,6 +69,7 @@ void BTDiagnostics::DerivedInitData () m_gamma_boost = WarpX::gamma_boost; m_beta_boost = std::sqrt( 1._rt - 1._rt/( m_gamma_boost * m_gamma_boost) ); m_moving_window_dir = WarpX::moving_window_dir; + m_moving_window_beta = WarpX::moving_window_v/PhysConst::c; // Currently, for BTD, all the data is averaged+coarsened to coarsest level // and then sliced+back-transformed+filled_to_buffer. // The number of levels to be output is nlev_output. @@ -82,7 +85,6 @@ void BTDiagnostics::DerivedInitData () m_old_z_boost.resize(m_num_buffers); m_buffer_counter.resize(m_num_buffers); m_snapshot_ncells_lab.resize(m_num_buffers); - m_cell_centered_data.resize(nmax_lev); m_cell_center_functors.resize(nmax_lev); m_max_buffer_multifabs.resize(m_num_buffers); m_buffer_flush_counter.resize(m_num_buffers); @@ -137,7 +139,7 @@ void BTDiagnostics::DerivedInitData () const int lev = 0; const amrex::Real dt_boosted_frame = warpx.getdt(lev); const int moving_dir = WarpX::moving_window_dir; - const amrex::Real Lz_lab = warpx.Geom(lev).ProbLength(moving_dir) / WarpX::gamma_boost / (1._rt+WarpX::beta_boost); + const amrex::Real Lz_lab = warpx.Geom(lev).ProbLength(moving_dir) * WarpX::gamma_boost * (1._rt - WarpX::beta_boost*m_moving_window_beta); const int ref_ratio = 1; const amrex::Real dz_snapshot_grid = dz_lab(dt_boosted_frame, ref_ratio); // Need enough buffers so the snapshot length is longer than the lab frame length @@ -148,40 +150,39 @@ void BTDiagnostics::DerivedInitData () // the final snapshot starts filling when the // right edge of the moving window intersects the final snapshot // time of final snapshot : t_sn = t0 + i*dt_snapshot - // where t0 is the time of first BTD snapshot, t0 = zmax / c * beta / (1-beta) + // where t0 is the time of first BTD snapshot, t0 = zmax / c * beta / (1-beta*beta_mw) // // the right edge of the moving window at the time of the final snapshot // has space time coordinates - // time t_intersect = t_sn, position z_intersect=zmax + c*t_sn + // time t_intersect = t_sn, position z_intersect=zmax + v_mw*t_sn // the boosted time of this space time pair is // t_intersect_boost = gamma * (t_intersect - beta * z_intersect_boost/c) - // = gamma * (t_sn * (1 - beta) - beta * zmax / c) - // = gamma * (zmax*beta/c + i*dt_snapshot*(1-beta) - beta*zmax/c) - // = gamma * i * dt_snapshot * (1-beta) - // = i * dt_snapshot / gamma / (1+beta) + // = gamma * (t_sn * (1 - beta*beta_mw) - beta * zmax / c) + // = gamma * (zmax*beta/c + i*dt_snapshot*(1-beta*beta_mw) - beta*zmax/c) + // = gamma * (1-beta*beta_mw) * i * dt_snapshot // // if j = final snapshot starting step, then we want to solve - // j dt_boosted_frame >= t_intersect_boost = i * dt_snapshot / gamma / (1+beta) - // j >= i / gamma / (1+beta) * dt_snapshot / dt_boosted_frame - const int final_snapshot_starting_step = static_cast(std::ceil(final_snapshot_iteration / WarpX::gamma_boost / (1._rt+WarpX::beta_boost) * m_dt_snapshots_lab / dt_boosted_frame)); + // j dt_boosted_frame >= t_intersect_boost = i * gamma * (1-beta*beta_mw) * dt_snapshot + // j >= i * gamma * (1-beta*beta_mw) * dt_snapshot / dt_boosted_frame + const int final_snapshot_starting_step = static_cast(std::ceil(final_snapshot_iteration * WarpX::gamma_boost * (1._rt - WarpX::beta_boost*m_moving_window_beta) * m_dt_snapshots_lab / dt_boosted_frame)); const int final_snapshot_fill_iteration = final_snapshot_starting_step + num_buffers * m_buffer_size - 1; const amrex::Real final_snapshot_fill_time = final_snapshot_fill_iteration * dt_boosted_frame; if (WarpX::compute_max_step_from_btd) { if (final_snapshot_fill_iteration > warpx.maxStep()) { warpx.updateMaxStep(final_snapshot_fill_iteration); amrex::Print()<<"max_step insufficient to fill all BTD snapshots. Automatically increased to: " - << final_snapshot_fill_iteration << std::endl; + << final_snapshot_fill_iteration << "\n"; } if (final_snapshot_fill_time > warpx.stopTime()) { warpx.updateStopTime(final_snapshot_fill_time); amrex::Print()<<"stop_time insufficient to fill all BTD snapshots. Automatically increased to: " - << final_snapshot_fill_time << std::endl; + << final_snapshot_fill_time << "\n"; } if (warpx.maxStep() == std::numeric_limits::max() && warpx.stopTime() == std::numeric_limits::max()) { amrex::Print()<<"max_step unspecified and stop time unspecified. Setting max step to " - <(m_cellcenter_varnames.size()); #endif - WarpX::AllocInitMultiFab(m_cell_centered_data[lev], ba, dmap, ncomps, amrex::IntVect(ngrow), lev, "cellcentered_BTD", 0._rt); + bool const remake = false; + bool const redistribute_on_remake = false; + warpx.m_fields.alloc_init(m_cell_centered_data_name, lev, ba, dmap, ncomps, amrex::IntVect(ngrow), 0.0_rt, + remake, redistribute_on_remake); } void BTDiagnostics::InitializeFieldFunctors (int lev) { + using ablastr::fields::Direction; + // Initialize fields functors only if do_back_transformed_fields is selected if (!m_do_back_transformed_fields) { return; } @@ -537,12 +545,14 @@ BTDiagnostics::InitializeFieldFunctors (int lev) #else auto & warpx = WarpX::GetInstance(); + auto & fields = warpx.m_fields; + // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point // to the correct field-data pointers m_all_field_functors[lev].clear(); // For back-transformed data, all the components are cell-centered and stored - // in a single multifab, m_cell_centered_data. + // in a single multifab. // Therefore, size of functors at all levels is 1. const int num_BT_functors = 1; m_all_field_functors[lev].resize(num_BT_functors); @@ -551,11 +561,11 @@ BTDiagnostics::InitializeFieldFunctors (int lev) // Create an object of class BackTransformFunctor for (int i = 0; i < num_BT_functors; ++i) { - // coarsening ratio is not provided since the source MultiFab, m_cell_centered_data + // coarsening ratio is not provided since the source MultiFab // is coarsened based on the user-defined m_crse_ratio const int nvars = static_cast(m_varnames.size()); m_all_field_functors[lev][i] = std::make_unique( - m_cell_centered_data[lev].get(), lev, + fields.get(m_cell_centered_data_name, lev), lev, nvars, m_num_buffers, m_varnames, m_varnames_fields); } @@ -567,23 +577,23 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp,Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp,Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp,Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -598,7 +608,9 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); + auto & fields = warpx.m_fields; + using ablastr::fields::Direction; + const int ncomp_multimodefab = fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -656,22 +668,25 @@ void BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; + auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); + auto & fields = warpx.m_fields; + const int ncomp_multimodefab = fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point // to the correct field-data pointers m_all_field_functors[lev].clear(); // For back-transformed data, all the components are cell-centered and stored - // in a single multifab, m_cell_centered_data. + // in a single MultiFab. // Therefore, size of functors at all levels is 1 const int num_BT_functors = 1; m_all_field_functors[lev].resize(num_BT_functors); for (int i = 0; i < num_BT_functors; ++i) { const int nvars = static_cast(m_varnames.size()); m_all_field_functors[lev][i] = std::make_unique( - m_cell_centered_data[lev].get(), lev, + fields.get(m_cell_centered_data_name, lev), lev, nvars, m_num_buffers, m_varnames, m_varnames_fields); } @@ -683,23 +698,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(fields.get(FieldType::current_fp, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } @@ -789,6 +804,8 @@ BTDiagnostics::PrepareFieldDataForOutput () if (!m_do_back_transformed_fields) { return; } auto & warpx = WarpX::GetInstance(); + auto & fields = warpx.m_fields; + // In this function, we will get cell-centered data for every level, lev, // using the cell-center functors and their respective operators() // Call m_cell_center_functors->operator @@ -798,21 +815,23 @@ BTDiagnostics::PrepareFieldDataForOutput () for (int icomp = 0; icompoperator()(*m_cell_centered_data[lev], icomp_dst); + // stores it in cell-centered MultiFab. + m_cell_center_functors[lev][icomp]->operator()(*fields.get(m_cell_centered_data_name, lev), icomp_dst); icomp_dst += m_cell_center_functors[lev][icomp]->nComp(); } // Check that the proper number of user-requested components are cell-centered AMREX_ALWAYS_ASSERT( icomp_dst == m_cellcenter_varnames.size() ); // fill boundary call is required to average_down (flatten) data to // the coarsest level. - ablastr::utils::communication::FillBoundary(*m_cell_centered_data[lev], WarpX::do_single_precision_comms, + ablastr::utils::communication::FillBoundary(*fields.get(m_cell_centered_data_name, lev), + WarpX::do_single_precision_comms, warpx.Geom(lev).periodicity()); } // Flattening out MF over levels for (int lev = warpx.finestLevel(); lev > 0; --lev) { - ablastr::coarsen::sample::Coarsen(*m_cell_centered_data[lev - 1], *m_cell_centered_data[lev], 0, 0, + ablastr::coarsen::sample::Coarsen(*fields.get(m_cell_centered_data_name, lev - 1), + *fields.get(m_cell_centered_data_name, lev), 0, 0, static_cast(m_cellcenter_varnames.size()), 0, WarpX::RefRatio(lev-1) ); } @@ -982,12 +1001,15 @@ BTDiagnostics::GetZSliceInDomainFlag (const int i_buffer, const int lev) { auto & warpx = WarpX::GetInstance(); const amrex::RealBox& boost_domain = warpx.Geom(lev).ProbDomain(); + const amrex::Real boost_cellsize = warpx.Geom(lev).CellSize(m_moving_window_dir); const amrex::Real buffer_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); const amrex::Real buffer_zmax_lab = m_snapshot_domain_lab[i_buffer].hi( m_moving_window_dir ); + // Exclude 0.5*boost_cellsize from the edge, to avoid that the interpolation to + // cell centers uses data from the guard cells. const bool slice_not_in_domain = - ( m_current_z_boost[i_buffer] <= boost_domain.lo(m_moving_window_dir) ) || - ( m_current_z_boost[i_buffer] >= boost_domain.hi(m_moving_window_dir) ) || + ( m_current_z_boost[i_buffer] <= boost_domain.lo(m_moving_window_dir) + 0.5_rt*boost_cellsize) || + ( m_current_z_boost[i_buffer] >= boost_domain.hi(m_moving_window_dir) - 0.5_rt*boost_cellsize) || ( m_current_z_lab[i_buffer] <= buffer_zmin_lab ) || ( m_current_z_lab[i_buffer] >= buffer_zmax_lab ); @@ -1069,7 +1091,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), labtime, m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, - m_plot_raw_fields, m_plot_raw_fields_guards, + m_plot_raw_fields, m_plot_raw_fields_guards, m_verbose, use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), m_max_buffer_multifabs.at(i_buffer), m_geom_snapshot.at(i_buffer).at(0), isLastBTDFlush); @@ -1440,6 +1462,17 @@ BTDiagnostics::InitializeParticleBuffer () m_totalParticles_in_buffer[i][isp] = 0; m_particles_buffer[i][isp] = std::make_unique(WarpX::GetInstance().GetParGDB()); const int idx = mpc.getSpeciesID(m_output_species_names[isp]); + + // SoA component names + { + auto &pc = mpc.GetParticleContainer(idx); + auto rn = pc.GetRealSoANames(); + rn.resize(WarpXParticleContainer::NArrayReal); // strip runtime comps + auto in = pc.GetRealSoANames(); + in.resize(WarpXParticleContainer::NArrayInt); // strip runtime comps + m_particles_buffer[i][isp]->SetSoACompileTimeNames(rn, in); + } + m_output_species[i].push_back(ParticleDiag(m_diag_name, m_output_species_names[isp], mpc.GetParticleContainerPtr(idx), diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 3e5fc1f19eb..f78e7b4574b 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -23,7 +23,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - BoundaryScrapingDiagnostics (int i, const std::string& name); + BoundaryScrapingDiagnostics (int i, const std::string& name, DiagTypes diag_type); private: /** Read relevant parameters for BoundaryScraping */ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index da1e5fdcc00..bcccda48c18 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -6,6 +6,7 @@ */ #include "BoundaryScrapingDiagnostics.H" +#include "EmbeddedBoundary/Enabled.H" #include "ComputeDiagFunctors/ComputeDiagFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" @@ -21,8 +22,8 @@ using namespace amrex::literals; -BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name) - : Diagnostics{i, name} +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name, DiagTypes diag_type) + : Diagnostics{i, name, diag_type} { ReadParameters(); } @@ -39,11 +40,11 @@ BoundaryScrapingDiagnostics::ReadParameters () // num_buffers corresponds to the number of boundaries // (upper/lower domain boundary in each dimension) - // + the EB boundary if available m_num_buffers = AMREX_SPACEDIM*2; -#ifdef AMREX_USE_EB - m_num_buffers += 1; -#endif + + // + the EB boundary if available + bool const eb_enabled = EB::enabled(); + if (eb_enabled) { m_num_buffers += 1; } // Do a few checks #ifndef WARPX_USE_OPENPMD @@ -152,7 +153,7 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer, bool /* force_flush */) warpx.gett_new(0), m_output_species.at(i_buffer), nlev_output, file_prefix, - m_file_min_digits, false, false, use_pinned_pc, isBTD, + m_file_min_digits, false, false, m_verbose, use_pinned_pc, isBTD, warpx.getistep(0), bufferID, numBTDBuffers, geom, isLastBTD); diff --git a/Source/Diagnostics/CMakeLists.txt b/Source/Diagnostics/CMakeLists.txt index 376487dc94a..d899bd5e155 100644 --- a/Source/Diagnostics/CMakeLists.txt +++ b/Source/Diagnostics/CMakeLists.txt @@ -7,7 +7,6 @@ foreach(D IN LISTS WarpX_DIMS) FullDiagnostics.cpp MultiDiagnostics.cpp ParticleIO.cpp - SliceDiagnostic.cpp WarpXIO.cpp WarpXOpenPMD.cpp BTDiagnostics.cpp diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index bef40ae1ce0..c4410b0a722 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -100,11 +100,11 @@ public: amrex::Real beta_boost) const; private: /** pointer to source multifab (cell-centered multi-component multifab) */ - amrex::MultiFab const * const m_mf_src = nullptr; + const amrex::MultiFab* m_mf_src = nullptr; /** level at which m_mf_src is defined */ - int const m_lev; + int m_lev; /** Number of buffers or snapshots */ - int const m_num_buffers; + int m_num_buffers; /** Vector of amrex::Box with index-space in the lab-frame */ amrex::Vector m_buffer_box; /** Vector of current z co-ordinate in the boosted-frame for each buffer */ diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H index dd5bb239ecf..6f0818b180e 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H @@ -36,7 +36,7 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab const * const m_mf_src = nullptr; + const amrex::MultiFab* m_mf_src = nullptr; int m_lev; /**< level on which mf_src is defined (used in cylindrical) */ /**< (for cylindrical) whether to average all modes into 1 comp */ bool m_convertRZmodes2cartesian; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H index 3d04a56742b..347c40e0338 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H @@ -3,6 +3,8 @@ #include "ComputeDiagFunctor.H" +#include + #include #include @@ -22,8 +24,13 @@ public: * (summing over modes) * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ - DivBFunctor(std::array arr_mf_src, int lev, amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian=true, int ncomp=1); + DivBFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1 + ); /** \brief Compute DivB directly into mf_dst. * @@ -34,8 +41,8 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ - std::array m_arr_mf_src; - int const m_lev; /**< level on which mf_src is defined (used in cylindrical) */ + ablastr::fields::VectorField m_arr_mf_src; + int m_lev; /**< level on which mf_src is defined (used in cylindrical) */ /**< (for cylindrical) whether to average all modes into 1 comp */ bool m_convertRZmodes2cartesian; }; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp index b5782e76ae6..224b74ba372 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp @@ -7,8 +7,13 @@ #include #include -DivBFunctor::DivBFunctor(const std::array arr_mf_src, const int lev, const amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian, const int ncomp) +DivBFunctor::DivBFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + const int lev, + const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian, + const int ncomp +) : ComputeDiagFunctor(ncomp, crse_ratio), m_arr_mf_src(arr_mf_src), m_lev(lev), m_convertRZmodes2cartesian(convertRZmodes2cartesian) {} diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H index 312ccaa5cd6..3874ebeb6c6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H @@ -3,6 +3,8 @@ #include "ComputeDiagFunctor.H" +#include + #include #include @@ -21,8 +23,13 @@ public: * \param[in] convertRZmodes2cartesian if true, all RZ modes are averaged into one component * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ - DivEFunctor(std::array arr_mf_src, int lev, amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian=true, int ncomp=1); + DivEFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1 + ); /** \brief Compute DivE directly into mf_dst. * @@ -33,8 +40,8 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ - std::array m_arr_mf_src; - int const m_lev; /**< level on which mf_src is defined (used in cylindrical) */ + ablastr::fields::VectorField m_arr_mf_src; + int m_lev; /**< level on which mf_src is defined (used in cylindrical) */ /**< (for cylindrical) whether to average all modes into 1 comp */ bool m_convertRZmodes2cartesian; }; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp index 62801cd431a..e2c4d98c708 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp @@ -13,9 +13,13 @@ #include #include -DivEFunctor::DivEFunctor(const std::array arr_mf_src, const int lev, - const amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian, const int ncomp) +DivEFunctor::DivEFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + const int lev, + const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian, + const int ncomp +) : ComputeDiagFunctor(ncomp, crse_ratio), m_arr_mf_src(arr_mf_src), m_lev(lev), m_convertRZmodes2cartesian(convertRZmodes2cartesian) { diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index d9f9a1e82e0..21e0d3f5034 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -39,7 +39,7 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** direction of the current density to save */ - const int m_dir; + int m_dir; /** level on which mf_src is defined */ int m_lev; /** (for cylindrical) whether to average all modes into 1 comp */ diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index ebaec47b2f1..df25bf7ff03 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -6,16 +6,18 @@ #include "JFunctor.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/MultiParticleContainer.H" #include "WarpX.H" +#include + #include #include #include #include -using namespace warpx::fields; +using warpx::fields::FieldType; JFunctor::JFunctor (const int dir, int lev, amrex::IntVect crse_ratio, @@ -29,30 +31,19 @@ JFunctor::JFunctor (const int dir, int lev, void JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + amrex::MultiFab* m_mf_src = warpx.m_fields.get(FieldType::current_fp,Direction{m_dir},m_lev); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) { // allocate temporary multifab to deposit current density into - amrex::Vector, 3 > > current_fp_temp; - current_fp_temp.resize(1); - - const auto& current_fp_x = warpx.getField(FieldType::current_fp, m_lev,0); - current_fp_temp[0][0] = std::make_unique( - current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() - ); - - const auto& current_fp_y = warpx.getField(FieldType::current_fp, m_lev,1); - current_fp_temp[0][1] = std::make_unique( - current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() - ); - const auto& current_fp_z = warpx.getField(FieldType::current_fp, m_lev,2); - current_fp_temp[0][2] = std::make_unique( - current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() - ); + ablastr::fields::MultiLevelVectorField current_fp_temp { + warpx.m_fields.get_alldirs(FieldType::current_fp, m_lev) + }; auto& mypc = warpx.GetPartContainer(); mypc.DepositCurrent(current_fp_temp, warpx.getdt(m_lev), 0.0); diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index aac5869da65..e06f90b5f0c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -1,12 +1,15 @@ -/* This file is part of Warpx. +/* Copyright 2023-2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Avigdor Veksler (TAE Technologies) * - * Authors: Avigdor Veksler * License: BSD-3-Clause-LBNL -*/ + */ #include "JdispFunctor.H" #include "WarpX.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" @@ -16,7 +19,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; JdispFunctor::JdispFunctor (int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) @@ -27,18 +30,20 @@ JdispFunctor::JdispFunctor (int dir, int lev, void JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + amrex::MultiFab* mf_j = warpx.m_fields.get(FieldType::current_fp, Direction{m_dir}, m_lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, "Displacement current diagnostic is only implemented for the HybridPICModel."); AMREX_ASSUME(hybrid_pic_model != nullptr); /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - amrex::MultiFab* mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir);; + amrex::MultiFab* mf_curlB = warpx.m_fields.get(FieldType::hybrid_current_fp_plasma, Direction{m_dir}, m_lev); //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to @@ -61,51 +66,6 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff -1, *mf_j, 0, 0, 1, Jdisp.nGrowVect() ); - if (hybrid_pic_model) { - // Subtract the interpolated j_external value from j_displacement. - /** pointer to external currents (Jext) multifab */ - amrex::MultiFab* mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); - - // Index type required for interpolating Jext from their respective - // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement - // locations. The staggering of J_displacement is the same as the - // staggering for J, so we use J_stag as the interpolation map. - // For interp to work below, the indices of the undefined dimensions - // must match. We set them as (1,1,1). - amrex::GpuArray Jext_IndexType = {1, 1, 1}; - amrex::GpuArray J_IndexType = {1, 1, 1}; - amrex::IntVect Jext_stag = mf_j_external->ixType().toIntVect(); - amrex::IntVect J_stag = mf_j->ixType().toIntVect(); - - // Index types for the dimensions simulated are overwritten. - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - Jext_IndexType[idim] = Jext_stag[idim]; - J_IndexType[idim] = J_stag[idim]; - } - - // Parameters for `interp` that maps from Jext to J. - // The "coarsening is just 1 i.e. no coarsening" - amrex::GpuArray const& coarsen = {1, 1, 1}; - - // Loop through the grids, and over the tiles within each grid to - // subtract the interpolated Jext from J_displacement. -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { - - Array4 const& Jdisp_arr = Jdisp.array(mfi); - Array4 const& Jext = mf_j_external->const_array(mfi); - - // Loop over cells and update the Jdisp MultiFab - amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Interpolate Jext to the staggering of J - auto const jext_interp = ablastr::coarsen::sample::Interp(Jext, Jext_IndexType, J_IndexType, coarsen, i, j, k, 0); - Jdisp_arr(i, j, k, 0) -= jext_interp; - }); - } - } - InterpolateMFForDiag(mf_dst, Jdisp, dcomp, warpx.DistributionMap(m_lev), m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H index 1b8785af7b7..491cd2cfe37 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H @@ -30,7 +30,7 @@ public: */ void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: - int const m_lev; /**< level on which mf_src is defined */ + int m_lev; /**< level on which mf_src is defined */ }; #endif // WARPX_PARTPERCELLFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H index 9718c9c7163..b0c3f28ab90 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H @@ -30,7 +30,7 @@ public: */ void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: - int const m_lev; /**< level on which mf_src is defined */ + int m_lev; /**< level on which mf_src is defined */ }; #endif // WARPX_PARTPERGRIDFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 7f141e95b32..5f3b3a86097 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -43,11 +43,11 @@ public: */ void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: - int const m_lev; /**< level on which mf_src is defined */ - int const m_ispec; /**< index of species to average over */ - bool const m_do_average; /**< Whether to calculate the average of the data */ - bool const m_do_filter; /**< whether to apply #m_filter_fn */ - /** Parser function to be averaged by the functor. Arguments: x, y, z, ux, uy, uz, upstream */ + int m_lev; /**< level on which mf_src is defined */ + int m_ispec; /**< index of species to average over */ + bool m_do_average; /**< Whether to calculate the average of the data */ + bool m_do_filter; /**< whether to apply #m_filter_fn */ + /** Parser function to be averaged by the functor. Arguments: x, y, z, ux, uy, uz */ std::unique_ptr m_map_fn_parser; /** Parser function to filter particles before pass to map. Arguments: x, y, z, ux, uy, uz, upstream */ std::unique_ptr m_filter_fn_parser; diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 1f7a8fc181b..d6bdd8d49e7 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -77,20 +77,7 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, // Ideally this will be replaced with the AMReX NGP interpolator // Always do x direction. No RZ case because it's not implemented, and code // will have aborted - int ii = 0, jj = 0, kk = 0; - const amrex::ParticleReal x = p.pos(0); - const amrex::Real lx = (x - plo[0]) * dxi[0]; - ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - const amrex::ParticleReal y = p.pos(1); - const amrex::Real ly = (y - plo[1]) * dxi[1]; - jj = static_cast(amrex::Math::floor(ly)); -#endif -#if defined(WARPX_DIM_3D) - const amrex::ParticleReal z = p.pos(2); - const amrex::Real lz = (z - plo[2]) * dxi[2]; - kk = static_cast(amrex::Math::floor(lz)); -#endif + const auto [ii, jj, kk] = amrex::getParticleCell(p, plo, dxi).dim3(); // Fix dimensions since parser assumes u = gamma * v / c const amrex::ParticleReal ux = p.rdata(PIdx::ux) / PhysConst::c; @@ -121,20 +108,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, // Ideally this will be replaced with the AMReX NGP interpolator // Always do x direction. No RZ case because it's not implemented, and code // will have aborted - int ii = 0, jj = 0, kk = 0; - const amrex::ParticleReal x = p.pos(0); - const amrex::Real lx = (x - plo[0]) * dxi[0]; - ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) - const amrex::ParticleReal y = p.pos(1); - const amrex::Real ly = (y - plo[1]) * dxi[1]; - jj = static_cast(amrex::Math::floor(ly)); -#endif -#if defined(WARPX_DIM_3D) - const amrex::ParticleReal z = p.pos(2); - const amrex::Real lz = (z - plo[2]) * dxi[2]; - kk = static_cast(amrex::Math::floor(lz)); -#endif + const auto [ii, jj, kk] = amrex::getParticleCell(p, plo, dxi).dim3(); + // Fix dimensions since parser assumes u = gamma * v / c const amrex::ParticleReal ux = p.rdata(PIdx::ux) / PhysConst::c; const amrex::ParticleReal uy = p.rdata(PIdx::uy) / PhysConst::c; diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H index bc0c8b9f270..073e6476110 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -43,13 +43,13 @@ public: private: // Level on which source MultiFab mf_src is defined in RZ geometry - int const m_lev; + int m_lev; // Whether to apply k-space filtering of charge density in the diagnostics output in RZ PSATD bool m_apply_rz_psatd_filter; // Species index to dump rho per species - const int m_species_index; + int m_species_index; // Whether to average all modes into one component in RZ geometry bool m_convertRZmodes2cartesian; diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 32e11903778..e7f572dd681 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -47,7 +47,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i rho = mypc.GetChargeDensity(m_lev, true); if (warpx.DoFluidSpecies()) { auto& myfl = warpx.GetFluidContainer(); - myfl.DepositCharge(m_lev, *rho); + myfl.DepositCharge(warpx.m_fields, *rho, m_lev); } } // Dump rho per species diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H index f6c425e74d5..1716ab61652 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H @@ -28,8 +28,8 @@ public: */ void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: - int const m_lev; /**< level on which mf_src is defined */ - int const m_ispec; /**< index of species to average over */ + int m_lev; /**< level on which mf_src is defined */ + int m_ispec; /**< index of species to average over */ }; #endif // WARPX_TEMPERATUREFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp index c42f8970d5e..cdf86c3b63d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp @@ -53,22 +53,7 @@ TemperatureFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::GpuArray const& dxi) { // Get position in AMReX convention to calculate corresponding index. - // Ideally this will be replaced with the AMReX NGP interpolator - // Always do x direction. - int ii = 0, jj = 0, kk = 0; - const amrex::ParticleReal x = p.pos(0); - const amrex::Real lx = (x - plo[0]) * dxi[0]; - ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - const amrex::ParticleReal y = p.pos(1); - const amrex::Real ly = (y - plo[1]) * dxi[1]; - jj = static_cast(amrex::Math::floor(ly)); -#endif -#if defined(WARPX_DIM_3D) - const amrex::ParticleReal z = p.pos(2); - const amrex::Real lz = (z - plo[2]) * dxi[2]; - kk = static_cast(amrex::Math::floor(lz)); -#endif + const auto [ii, jj, kk] = amrex::getParticleCell(p, plo, dxi).dim3(); const amrex::ParticleReal w = p.rdata(PIdx::w); const amrex::ParticleReal ux = p.rdata(PIdx::ux); @@ -103,34 +88,20 @@ TemperatureFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const for (WarpXParIter pti(pc, m_lev); pti.isValid(); ++pti) { const long np = pti.numParticles(); + auto& tile = pti.GetParticleTile(); + auto ptd = tile.getParticleTileData(); amrex::ParticleReal* wp = pti.GetAttribs(PIdx::w).dataPtr(); amrex::ParticleReal* uxp = pti.GetAttribs(PIdx::ux).dataPtr(); amrex::ParticleReal* uyp = pti.GetAttribs(PIdx::uy).dataPtr(); amrex::ParticleReal* uzp = pti.GetAttribs(PIdx::uz).dataPtr(); - auto const GetPosition = GetParticlePosition(pti); - amrex::Array4 const& out_array = sum_mf.array(pti); amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE (long ip) { - // --- Get particle quantities - amrex::ParticleReal xp, yp, zp; - GetPosition.AsStored(ip, xp, yp, zp); - // Get position in AMReX convention to calculate corresponding index. - int ii = 0, jj = 0, kk = 0; - const amrex::Real lx = (xp - plo[0]) * dxi[0]; - ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real lz = (zp - plo[1]) * dxi[1]; - jj = static_cast(amrex::Math::floor(lz)); -#elif defined(WARPX_DIM_3D) - const amrex::Real ly = (yp - plo[1]) * dxi[1]; - jj = static_cast(amrex::Math::floor(ly)); - const amrex::Real lz = (zp - plo[2]) * dxi[2]; - kk = static_cast(amrex::Math::floor(lz)); -#endif + const auto p = WarpXParticleContainer::ParticleType(ptd, ip); + const auto [ii, jj, kk] = getParticleCell(p, plo, dxi).dim3(); const amrex::ParticleReal w = wp[ip]; const amrex::ParticleReal ux = uxp[ip] - out_array(ii, jj, kk, 1); diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 20550364fb7..a078bab3597 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -21,6 +21,8 @@ #include #include +/** All types of diagnostics. */ +enum struct DiagTypes {Full, BackTransformed, BoundaryScraping, TimeAveraged}; /** * \brief base class for diagnostics. * Contains main routines to filter, compute and flush diagnostics. @@ -35,7 +37,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - Diagnostics (int i, std::string name); + Diagnostics (int i, std::string name, DiagTypes diag_type); /** Virtual Destructor to handle clean destruction of derived classes */ virtual ~Diagnostics (); @@ -45,6 +47,8 @@ public: Diagnostics(Diagnostics&& ) = default; Diagnostics& operator=(Diagnostics&& ) = default; + /** Stores the diag type */ + DiagTypes m_diag_type; /** Pack (stack) all fields in the cell-centered output MultiFab m_mf_output. * * Fields are computed (e.g., cell-centered or back-transformed) @@ -186,6 +190,8 @@ public: protected: /** Read Parameters of the base Diagnostics class */ bool BaseReadParameters (); + /** Whether to output a message when diagnostics are output */ + int m_verbose = 2; /** Initialize member variables of the base Diagnostics class. */ void InitBaseData (); /** Initialize m_mf_output vectors and data required to construct the buffers @@ -266,6 +272,12 @@ protected: * The second vector is loops over the total number of levels. */ amrex::Vector< amrex::Vector< amrex::MultiFab > > m_mf_output; + /** summation multifab, where all fields (computed, cell-centered, and stacked) + * are summed for every step in a time averaging period. + * The first vector is for total number of snapshots. (=1 for FullDiagnostics) + * The second vector is loops over the total number of levels. + */ + amrex::Vector< amrex::Vector > m_sum_mf_output; /** Geometry that defines the domain attributes corresponding to output multifab. * Specifically, the user-defined physical co-ordinates for the diagnostics diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 96da344c805..eb331e3dc65 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -4,6 +4,7 @@ #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FlushFormats/FlushFormatCatalyst.H" #include "FlushFormats/FlushFormatAscent.H" #include "FlushFormats/FlushFormatCheckpoint.H" #ifdef WARPX_USE_OPENPMD @@ -37,8 +38,8 @@ using namespace amrex::literals; -Diagnostics::Diagnostics (int i, std::string name) - : m_diag_name(std::move(name)), m_diag_index(i) +Diagnostics::Diagnostics (int i, std::string name, DiagTypes diag_type) + : m_diag_type(diag_type), m_diag_name(std::move(name)), m_diag_index(i) { } @@ -61,6 +62,9 @@ Diagnostics::BaseReadParameters () std::string dims; pp_geometry.get("dims", dims); + const amrex::ParmParse pp_warpx("warpx"); + pp_warpx.query("verbose", m_verbose); + // Query list of grid fields to write to output const bool varnames_specified = pp_diag_name.queryarr("fields_to_plot", m_varnames_fields); if (!varnames_specified){ @@ -76,8 +80,9 @@ Diagnostics::BaseReadParameters () if (utils::algorithms::is_in(m_varnames_fields, "phi")){ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( warpx.electrostatic_solver_id==ElectrostaticSolverAlgo::LabFrame || - warpx.electrostatic_solver_id==ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic, - "plot phi only works if do_electrostatic = labframe or do_electrostatic = labframe-electromagnetostatic"); + warpx.electrostatic_solver_id==ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic || + warpx.electrostatic_solver_id==ElectrostaticSolverAlgo::LabFrameEffectivePotential, + "plot phi only works if do_electrostatic = labframe, do_electrostatic = labframe-electromagnetostatic or do_electrostatic = labframe-effective-potential"); } // Sanity check if user requests to plot A @@ -228,8 +233,9 @@ Diagnostics::BaseReadParameters () if (WarpX::boost_direction[ dim_map[WarpX::moving_window_dir] ] == 1) { // Convert user-defined lo and hi for diagnostics to account for boosted-frame // simulations with moving window - const amrex::Real convert_factor = 1._rt/(WarpX::gamma_boost * (1._rt - WarpX::beta_boost) ); - // Assuming that the window travels with speed c + const amrex::Real beta_window = WarpX::moving_window_v / PhysConst::c; + const amrex::Real convert_factor = 1._rt/( + WarpX::gamma_boost * (1._rt - WarpX::beta_boost * beta_window) ); m_lo[WarpX::moving_window_dir] *= convert_factor; m_hi[WarpX::moving_window_dir] *= convert_factor; } @@ -505,7 +511,9 @@ Diagnostics::InitBaseData () m_flush_format = std::make_unique() ; } else if (m_format == "ascent"){ m_flush_format = std::make_unique(); - } else if (m_format == "sensei"){ + } else if (m_format == "catalyst") { + m_flush_format = std::make_unique(); + } else if (m_format == "sensei") { #ifdef AMREX_USE_SENSEI_INSITU m_flush_format = std::make_unique( dynamic_cast(const_cast(&warpx)), @@ -532,6 +540,14 @@ Diagnostics::InitBaseData () m_mf_output[i].resize( nmax_lev ); } + // allocate vector of buffers and vector of levels for each buffer for summation multifab for TimeAveragedDiagnostics + if (m_diag_type == DiagTypes::TimeAveraged) { + m_sum_mf_output.resize(m_num_buffers); + for (int i = 0; i < m_num_buffers; ++i) { + m_sum_mf_output[i].resize( nmax_lev ); + } + } + // allocate vector of geometry objects corresponding to each output multifab. m_geom_output.resize( m_num_buffers ); for (int i = 0; i < m_num_buffers; ++i) { @@ -571,6 +587,15 @@ Diagnostics::ComputeAndPack () // Check that the proper number of components of mf_avg were updated. AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); + if (m_diag_type == DiagTypes::TimeAveraged) { + + const amrex::Real real_a = 1.0; + // Compute m_sum_mf_output += real_a*m_mf_output + amrex::MultiFab::Saxpy( + m_sum_mf_output[i_buffer][lev], real_a, m_mf_output[i_buffer][lev], + 0, 0, m_mf_output[i_buffer][lev].nComp(), m_mf_output[i_buffer][lev].nGrowVect()); + } + // needed for contour plots of rho, i.e. ascent/sensei if (m_format == "sensei" || m_format == "ascent") { ablastr::utils::communication::FillBoundary(m_mf_output[i_buffer][lev], WarpX::do_single_precision_comms, diff --git a/Source/Diagnostics/FlushFormats/CMakeLists.txt b/Source/Diagnostics/FlushFormats/CMakeLists.txt index fea3124302a..f124b400a2b 100644 --- a/Source/Diagnostics/FlushFormats/CMakeLists.txt +++ b/Source/Diagnostics/FlushFormats/CMakeLists.txt @@ -2,6 +2,8 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + FlushFormatInSitu.cpp + FlushFormatCatalyst.cpp FlushFormatAscent.cpp FlushFormatCheckpoint.cpp FlushFormatPlotfile.cpp diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index be1322bd61b..f0a83f7a24b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -19,6 +19,7 @@ public: std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose = 2, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index 9d8d3fcd7d2..86ad4b09cfb 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -1,7 +1,7 @@ #ifndef WARPX_FLUSHFORMATASCENT_H_ #define WARPX_FLUSHFORMATASCENT_H_ -#include "FlushFormat.H" +#include "FlushFormatInSitu.H" #include "Diagnostics/ParticleDiag/ParticleDiag_fwd.H" @@ -24,7 +24,7 @@ * In particular, function WriteToFile takes fields and particles as input arguments, * and calls amrex functions to do the in-situ visualization. */ -class FlushFormatAscent : public FlushFormat +class FlushFormatAscent : public FlushFormatInSitu { public: /** Do in-situ visualization for field and particle data */ @@ -37,20 +37,12 @@ public: std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose = 2, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false ) const override; - -#ifdef AMREX_USE_ASCENT - /** \brief Do in-situ visualization for particle data. - * \param[in] particle_diags Each element of this vector handles output of 1 species. - * \param[out] a_bp_mesh blueprint mesh generated from the container - * Only compile if AMREX_USE_ASCENT because we need to pass a conduit class - */ - void WriteParticles(const amrex::Vector& particle_diags, conduit::Node& a_bp_mesh) const; -#endif + bool isLastBTDFlush = false) const override; FlushFormatAscent () = default; ~FlushFormatAscent() override = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 0024122ebf9..5a405568aa7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -18,6 +18,7 @@ FlushFormatAscent::WriteToFile ( const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose, const bool /*use_pinned_pc*/, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, @@ -32,7 +33,9 @@ FlushFormatAscent::WriteToFile ( "In-situ visualization is not currently supported for back-transformed diagnostics."); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); - amrex::Print() << Utils::TextMsg::Info("Writing Ascent file " + filename); + if (verbose > 0) { + amrex::Print() << Utils::TextMsg::Info("Writing Ascent file " + filename); + } // wrap mesh data WARPX_PROFILE_VAR("FlushFormatAscent::WriteToFile::MultiLevelToBlueprint", prof_ascent_mesh_blueprint); @@ -67,57 +70,7 @@ FlushFormatAscent::WriteToFile ( #else amrex::ignore_unused(varnames, mf, geom, iteration, time, - particle_diags, nlev, file_min_digits, isBTD); + particle_diags, nlev, file_min_digits, verbose, isBTD); #endif // AMREX_USE_ASCENT amrex::ignore_unused(prefix, plot_raw_fields, plot_raw_fields_guards); } - -#ifdef AMREX_USE_ASCENT -void -FlushFormatAscent::WriteParticles(const amrex::Vector& particle_diags, conduit::Node& a_bp_mesh) const -{ - WARPX_PROFILE("FlushFormatAscent::WriteParticles()"); - - // wrap particle data for each species - // we prefix the fields with "particle_{species_name}" b/c we - // want to to uniquely name all the fields that can be plotted - - for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { - Vector particle_varnames; - Vector particle_int_varnames; - std::string prefix = "particle_" + particle_diags[i].getSpeciesName(); - - // Get pc for species - // auto& pc = mypc->GetParticleContainer(i); - WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); - - // get names of real comps - std::map real_comps_map = pc->getParticleComps(); - - // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs - // not an efficient search, but N is small... - for(int j = 0; j < PIdx::nattribs; ++j) - { - auto rvn_it = real_comps_map.begin(); - for (; rvn_it != real_comps_map.end(); ++rvn_it) - if (rvn_it->second == j) - break; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - rvn_it != real_comps_map.end(), - "Ascent: SoA real attribute not found"); - std::string varname = rvn_it->first; - particle_varnames.push_back(prefix + "_" + varname); - } - // WarpXParticleContainer compile-time extra SoA attributes (int): 0 - - // WarpXParticleContainer "runtime" SoA attributes (Real), e.g QED: to do - - // wrap pc for current species into a blueprint topology - amrex::ParticleContainerToBlueprint(*pc, - particle_varnames, - particle_int_varnames, - a_bp_mesh, - prefix); - } -} -#endif // AMREX_USE_ASCENT diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.H b/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.H new file mode 100644 index 00000000000..c5f3b9148c1 --- /dev/null +++ b/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.H @@ -0,0 +1,65 @@ +#ifndef WARPX_FLUSHFORMATCATALYST_H_ +#define WARPX_FLUSHFORMATCATALYST_H_ + +#include "FlushFormatInSitu.H" + +#include "Diagnostics/ParticleDiag/ParticleDiag_fwd.H" + +#ifdef AMREX_USE_CONDUIT +# include +#endif +#include +#include + +#include + +#ifdef AMREX_USE_CATALYST +# include +#endif + +#include + +/** + * \brief This class aims at dumping performing in-situ analysis and visualization + * with Catalyst. Catalyst initialize and finalize are called in the constructor + * and destructor respectivelyThe function WriteToFile takes in the particles, + * meshes, and fields, and formats them to be used by given pipeline scripts. All + * exports are defined and executed externally through the given catalyst pipeline + * and implementation. + */ +class FlushFormatCatalyst : public FlushFormatInSitu +{ +public: + // Initialize + FlushFormatCatalyst(); + + /** Send particle, mesh, and field information through the catalyst pipeline */ + void WriteToFile ( + const amrex::Vector& varnames, + const amrex::Vector& mf, + amrex::Vector& geom, + amrex::Vector iteration, double time, + const amrex::Vector& particle_diags, int nlev, + std::string prefix, int file_min_digits, + bool plot_raw_fields, + bool plot_raw_fields_guards, + int verbose = 2, + bool use_pinned_pc = false, + bool isBTD = false, int snapshotID = -1, + int bufferID = 1, int numBuffers = 1, + const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), + bool isLastBTDFlush = false) const override; + +#ifdef AMREX_USE_CATALYST + ~FlushFormatCatalyst() override; +#else + ~FlushFormatCatalyst() override = default; +#endif + + FlushFormatCatalyst ( FlushFormatCatalyst const &) = default; + FlushFormatCatalyst& operator= ( FlushFormatCatalyst const & ) = default; + FlushFormatCatalyst ( FlushFormatCatalyst&& ) = default; + FlushFormatCatalyst& operator= ( FlushFormatCatalyst&& ) = default; +}; + +#endif diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.cpp new file mode 100644 index 00000000000..5e5f3634e8f --- /dev/null +++ b/Source/Diagnostics/FlushFormats/FlushFormatCatalyst.cpp @@ -0,0 +1,212 @@ +#include "FlushFormatCatalyst.H" + +#include "WarpX.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + +#include +#include + +#ifdef AMREX_USE_CATALYST + #include "conduit_cpp_to_c.hpp" +#endif + +#ifdef AMREX_USE_CATALYST +namespace internal { + +void EmptyParticleData ( + const std::vector& real_fields, + conduit::Node& node) +{ + const std::string topology_name = "particles"; + + // make a dummy node for catalyst + const std::string& patch_name = amrex::Concatenate("domain_x", 0, 6); + conduit::Node &patch = node[patch_name]; + + patch["state/domain_id"] = 0; + + std::string coordset_name = topology_name + "+coords"; + conduit::Node &n_coords = patch["coordsets"][coordset_name]; + n_coords["type"] = "explicit"; + + // create an explicit points topology + conduit::Node &n_topo = patch["topologies"][topology_name]; + n_topo["coordset"] = coordset_name; + n_topo["type"] = "unstructured"; + n_topo["elements/shape"] = "point"; + n_topo["elements/connectivity"].set(conduit::DataType::c_int(0)); + + n_coords["values/x"].set(conduit::DataType::c_int(0)); + n_coords["values/y"].set(conduit::DataType::c_int(0)); + n_coords["values/z"].set(conduit::DataType::c_int(0)); + + conduit::Node &n_fields = patch["fields"]; + + // id field + conduit::Node &n_f_id = n_fields[topology_name + "_id"]; + n_f_id["topology"] = topology_name; + n_f_id["association"] = "element"; + n_f_id["values"].set(conduit::DataType::c_int(0)); + + // cpu field + conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + n_f_cpu["topology"] = topology_name; + n_f_cpu["association"] = "element"; + n_f_cpu["values"].set(conduit::DataType::c_int(0)); + + for (auto& rfield : real_fields) + { + conduit::Node & n_f = n_fields[rfield]; + n_f["topology"] = topology_name; + n_f["association"] = "element"; + n_f["values"].set(conduit::DataType::c_int(0)); + } +} + +} // namespace internal +#endif + +using namespace amrex; + +FlushFormatCatalyst::FlushFormatCatalyst() { + ParmParse const pp_catalyst("catalyst"); + std::string scriptPaths; + std::string implementation {"paraview"}; + std::string searchPaths; + pp_catalyst.query("script_paths", scriptPaths); + pp_catalyst.query("implementation", implementation); + pp_catalyst.query("implementation_search_paths", searchPaths); + +#ifdef AMREX_USE_CATALYST + conduit::Node node; + + // Loop over all given paths and load all the scripts. Delimiters are ';' and ':' + size_t scriptNumber = 0; + size_t pos = 0; + std::string subpath; + while (scriptPaths.find(':') != std::string::npos || scriptPaths.find(';') != std::string::npos) { + pos = std::min(scriptPaths.find(':'), scriptPaths.find(';')); + subpath = scriptPaths.substr(0, pos); + + node["catalyst/scripts/script" + std::to_string(scriptNumber)].set_string(subpath); + + scriptNumber++; + scriptPaths.erase(0, pos + 1); + } + // Prevent empty end paths + if (scriptPaths.length() != 0) { + node["catalyst/scripts/script" + std::to_string(scriptNumber)].set_string(scriptPaths); + } + + node["catalyst_load/implementation"].set_string(implementation); + node["catalyst_load/search_paths/" + implementation].set_string(searchPaths); + + catalyst_status err = catalyst_initialize(conduit::c_node(&node)); + if (err != catalyst_status_ok) + { + std::string message = " Error: Failed to initialize Catalyst!\n"; + std::cerr << message << err << "\n"; + amrex::Print() << message; + amrex::Abort(message); + } + +#endif // AMREX_USE_CATALYST +} + +void +FlushFormatCatalyst::WriteToFile ( + const amrex::Vector& varnames, + const amrex::Vector& mf, + amrex::Vector& geom, + amrex::Vector iteration, double time, + const amrex::Vector& particle_diags, int nlev, + const std::string prefix, int file_min_digits, bool plot_raw_fields, + bool plot_raw_fields_guards, + int /*verbose*/, + bool /*use_pinned_pc*/, + bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, + const amrex::Geometry& /*full_BTD_snapshot*/, + bool /*isLastBTDFlush*/) const +{ +#ifdef AMREX_USE_CATALYST + amrex::Print() << Utils::TextMsg::Info("Running Catalyst pipeline scripts..."); + + WARPX_PROFILE("FlushFormatCatalyst::WriteToFile()"); + auto & warpx = WarpX::GetInstance(); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !isBTD, + "In-situ visualization is not currently supported for back-transformed diagnostics."); + + // Mesh data + WARPX_PROFILE_VAR("FlushFormatCatalyst::WriteToFile::MultiLevelToBlueprint", prof_catalyst_mesh_blueprint); + conduit::Node node; + auto & state = node["catalyst/state"]; + state["timestep"].set(iteration[0]); + state["time"].set(time); + + auto& meshChannel = node["catalyst/channels/mesh"]; + meshChannel["type"].set_string("multimesh"); + auto& meshData = meshChannel["data"]; + + amrex::MultiLevelToBlueprint( + nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, time, iteration, warpx.refRatio(), meshData); + WARPX_PROFILE_VAR_STOP(prof_catalyst_mesh_blueprint); + + // Particle data + WARPX_PROFILE_VAR("FlushFormatCatalyst::WriteToFile::WriteParticles", prof_catalyst_particles); + auto& particleChannel = node["catalyst/channels/particles"]; + particleChannel["type"].set_string("multimesh"); + auto& particleData = particleChannel["data"]; + + conduit::Node amrexParticles; + WriteParticles(particle_diags, amrexParticles); + particleData.update(amrexParticles); + if(!particleData.dtype().is_object()) + { + internal::EmptyParticleData(varnames, particleData); + } + + WARPX_PROFILE_VAR_STOP(prof_catalyst_particles); + + // Execution + WARPX_PROFILE_VAR("FlushFormatCatalyst::WriteToFile::execute", prof_catalyst_execute); + catalyst_status err = catalyst_execute(conduit::c_node(&node)); + if (err != catalyst_status_ok) + { + std::string message = " Error: Failed to execute Catalyst!\n"; + std::cerr << message << err << "\n"; + amrex::Print() << message; + } + WARPX_PROFILE_VAR_STOP(prof_catalyst_execute); + +#else + amrex::ignore_unused(varnames, mf, geom, iteration, time, + particle_diags, nlev, file_min_digits, isBTD); +#endif // AMREX_USE_CATALYST + amrex::ignore_unused(prefix, plot_raw_fields, file_min_digits, plot_raw_fields_guards); +} + +#ifdef AMREX_USE_CATALYST +FlushFormatCatalyst::~FlushFormatCatalyst() { + + conduit::Node node; + catalyst_status err = catalyst_finalize(conduit::c_node(&node)); + if (err != catalyst_status_ok) + { + std::string message = " Error: Failed to finalize Catalyst!\n"; + std::cerr << message << err << "\n"; + amrex::Print() << message; + amrex::Abort(message); + } else { + // Temporary, remove for final + std::cout << "Successfully finalized Catalyst\n"; + } + +} +#endif // AMREX_USE_CATALYST diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index 5c26ac97f61..e2cd28f9e1c 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -24,6 +24,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose = 2, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, @@ -34,6 +35,8 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile const amrex::Vector& particle_diags) const; void WriteDMaps (const std::string& dir, int nlev) const; + + void WriteReducedDiagsData (std::string const & dir) const; }; #endif // WARPX_FLUSHFORMATCHECKPOINT_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 1a3318ae0d8..ba371464782 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -5,12 +5,15 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" -#include "FieldSolver/Fields.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Fields.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + #include #include #include @@ -20,7 +23,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -37,12 +40,15 @@ FlushFormatCheckpoint::WriteToFile ( const std::string prefix, int file_min_digits, bool /*plot_raw_fields*/, bool /*plot_raw_fields_guards*/, + int verbose, const bool /*use_pinned_pc*/, bool /*isBTD*/, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { + using ablastr::fields::Direction; + WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()"); auto & warpx = WarpX::GetInstance(); @@ -52,8 +58,10 @@ FlushFormatCheckpoint::WriteToFile ( const std::string& checkpointname = amrex::Concatenate(prefix, iteration[0], file_min_digits); - amrex::Print() << Utils::TextMsg::Info( - "Writing checkpoint " + checkpointname); + if (verbose > 0) { + amrex::Print() << Utils::TextMsg::Info( + "Writing checkpoint " + checkpointname); + } // const int nlevels = finestLevel()+1; amrex::PreBuildDirectorHierarchy(checkpointname, default_level_prefix, nlev, true); @@ -64,85 +72,85 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getField(FieldType::current_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(warpx.getField(FieldType::current_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(warpx.getField(FieldType::current_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } @@ -150,11 +158,13 @@ FlushFormatCheckpoint::WriteToFile ( if (warpx.DoPML()) { if (warpx.GetPML(lev)) { warpx.GetPML(lev)->CheckPoint( + warpx.m_fields, amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml")); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (warpx.GetPML_RZ(lev)) { warpx.GetPML_RZ(lev)->CheckPoint( + warpx.m_fields, amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml_rz")); } #endif @@ -165,6 +175,8 @@ FlushFormatCheckpoint::WriteToFile ( WriteDMaps(checkpointname, nlev); + WriteReducedDiagsData(checkpointname); + VisMF::SetHeaderVersion(current_version); } @@ -179,31 +191,47 @@ FlushFormatCheckpoint::CheckpointParticles ( Vector real_names; Vector int_names; + Vector write_real_comps; + Vector write_int_comps; // note: positions skipped here, since we reconstruct a plotfile SoA from them - real_names.push_back("weight"); - real_names.push_back("momentum_x"); - real_names.push_back("momentum_y"); - real_names.push_back("momentum_z"); - + std::vector const fixed_names = {"weight", + "momentum_x", + "momentum_y", + "momentum_z" #ifdef WARPX_DIM_RZ - real_names.push_back("theta"); + ,"theta" #endif + }; + + for (auto const& name : fixed_names) { + real_names.push_back(name); + write_real_comps.push_back(1); + } - // get the names of the real comps - // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + // get the names of the extra real comps real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM); - auto runtime_rnames = pc->getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { - real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + write_real_comps.resize(pc->NumRealComps() - AMREX_SPACEDIM); + + // note, skip the required compnent names here + auto rnames = pc->GetRealSoANames(); + for (std::size_t index = PIdx::nattribs; index < rnames.size(); ++index) { + std::size_t const i = index - AMREX_SPACEDIM; + real_names[i] = rnames[index]; + write_real_comps[i] = pc->h_redistribute_real_comp[index]; } // and the int comps int_names.resize(pc->NumIntComps()); - auto runtime_inames = pc->getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) { int_names[x.second+0] = x.first; } + write_int_comps.resize(pc->NumIntComps()); + auto inames = pc->GetIntSoANames(); + for (std::size_t index = 0; index < inames.size(); ++index) { + int_names[index] = inames[index]; + write_int_comps[index] = pc->h_redistribute_int_comp[index]; + } - pc->Checkpoint(dir, part_diag.getSpeciesName(), true, + pc->Checkpoint(dir, part_diag.getSpeciesName(), + write_real_comps, write_int_comps, real_names, int_names); } } @@ -236,3 +264,12 @@ FlushFormatCheckpoint::WriteDMaps (const std::string& dir, int nlev) const } } } + +void +FlushFormatCheckpoint::WriteReducedDiagsData (std::string const & dir) const +{ + if (ParallelDescriptor::IOProcessor()) { + auto & warpx = WarpX::GetInstance(); + warpx.reduced_diags->WriteCheckpointData(dir); + } +} diff --git a/Source/Diagnostics/FlushFormats/FlushFormatInSitu.H b/Source/Diagnostics/FlushFormats/FlushFormatInSitu.H new file mode 100644 index 00000000000..b71fc79b510 --- /dev/null +++ b/Source/Diagnostics/FlushFormats/FlushFormatInSitu.H @@ -0,0 +1,48 @@ +#ifndef WARPX_FLUSHFORMATINSITU_H_ +#define WARPX_FLUSHFORMATINSITU_H_ + +#include "FlushFormat.H" + +#include "Diagnostics/ParticleDiag/ParticleDiag_fwd.H" + +#if defined(AMREX_USE_CONDUIT) || defined(AMREX_USE_ASCENT) +# include +#endif +#include +#include + +#include + +#include + +/** + * \brief This class is a wrapper for the two single-backend in-situ formats, those + * being Catalyst and Ascent. They both use the exact same code for writing particles + * and this class aims to reduce redundancy by defining the method only once. + */ +class FlushFormatInSitu : public FlushFormat +{ +public: + // May need to include separate exporters for particles, meshes, etc etc. Would probably also be useful + // to give some higher level debug functions (like in the plotfile flush) since it's useful sometimes. + // Could probably also include AMREX_USE_ASCENT or AMREX_USE_CONDUIT in here + #if defined(AMREX_USE_CONDUIT) || defined(AMREX_USE_ASCENT) + /** \brief Do in-situ visualization for particle data. + * \param[in] particle_diags Each element of this vector handles output of 1 species. + * \param[out] a_bp_mesh blueprint mesh generated from the container + * Only compile if AMREX_USE_CONDUIT or AMREX_USE_ASCENT because we need to pass a + * conduit class (conduit is required for catalyst so it does not need to be checked) + */ + void WriteParticles(const amrex::Vector& particle_diags, conduit::Node& a_bp_mesh) const; + #endif + + FlushFormatInSitu () = default; + ~FlushFormatInSitu() override = default; + + FlushFormatInSitu ( FlushFormatInSitu const &) = default; + FlushFormatInSitu& operator= ( FlushFormatInSitu const & ) = default; + FlushFormatInSitu ( FlushFormatInSitu&& ) = default; + FlushFormatInSitu& operator= ( FlushFormatInSitu&& ) = default; +}; + +#endif diff --git a/Source/Diagnostics/FlushFormats/FlushFormatInSitu.cpp b/Source/Diagnostics/FlushFormats/FlushFormatInSitu.cpp new file mode 100644 index 00000000000..af8f53df9b9 --- /dev/null +++ b/Source/Diagnostics/FlushFormats/FlushFormatInSitu.cpp @@ -0,0 +1,64 @@ +#include "FlushFormatInSitu.H" + +#include "WarpX.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + +#include +#include + +#if defined(AMREX_USE_CONDUIT) || defined(AMREX_USE_ASCENT) + #include "conduit_cpp_to_c.hpp" +#endif + +using namespace amrex; + +#if defined(AMREX_USE_CONDUIT) || defined(AMREX_USE_ASCENT) +void +FlushFormatInSitu::WriteParticles(const amrex::Vector& particle_diags, conduit::Node& a_bp_mesh) const +{ + WARPX_PROFILE("FlushFormatInSitu::WriteParticles()"); + + // wrap particle data for each species + // we prefix the fields with "particle_{species_name}" b/c we + // want to to uniquely name all the fields that can be plotted + + for (const auto & particle_diag : particle_diags) { + Vector particle_varnames; + Vector particle_int_varnames; + std::string prefix = "particle_" + particle_diag.getSpeciesName(); + + // Get pc for species + // auto& pc = mypc->GetParticleContainer(i); + WarpXParticleContainer* pc = particle_diag.getParticleContainer(); + + // get names of real comps + std::vector real_comps_map = pc->GetRealSoANames(); + + // WarpXParticleContainer compile-time extra AoS attributes (Real): 0 + // WarpXParticleContainer compile-time extra AoS attributes (int): 0 + + // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs + // not an efficient search, but N is small... + for(int j = 0; j < PIdx::nattribs; ++j) + { + std::string varname = real_comps_map.at(j); + particle_varnames.push_back(prefix + "_" + varname); + } + // WarpXParticleContainer compile-time extra SoA attributes (int): 0 + + // WarpXParticleContainer "runtime" SoA attributes (Real), e.g QED: to do + + // wrap pc for current species into a blueprint topology + amrex::ParticleContainerToBlueprint(*pc, + particle_varnames, + particle_int_varnames, + a_bp_mesh, + prefix); + } +} +#endif // defined(AMREX_USE_CONDUIT) || defined(AMREX_USE_ASCENT) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index 141760ac2a3..5666d85bf3a 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -36,11 +36,12 @@ public: std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false ) const override; + bool isLastBTDFlush = false) const override; ~FlushFormatOpenPMD () override = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index e0c8c4ef2d6..aeb26656b46 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -123,6 +123,7 @@ FlushFormatOpenPMD::WriteToFile ( const amrex::Vector& particle_diags, int output_levels, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose, const bool use_pinned_pc, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& full_BTD_snapshot, @@ -130,16 +131,18 @@ FlushFormatOpenPMD::WriteToFile ( { WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()"); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); - if (!isBTD) - { - amrex::Print() << Utils::TextMsg::Info("Writing openPMD file " + filename); - } else - { - amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) - + " to snapshot " + std::to_string(snapshotID) + " to openPMD BTD " + prefix); - if (isLastBTDFlush) + if (verbose > 0) { + if (!isBTD) + { + amrex::Print() << Utils::TextMsg::Info("Writing openPMD file " + filename); + } else { - amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in openPMD BTD " + prefix); + amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) + + " to snapshot " + std::to_string(snapshotID) + " to openPMD BTD " + prefix); + if (isLastBTDFlush) + { + amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in openPMD BTD " + prefix); + } } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index c62056b8907..18648c07e69 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -31,6 +31,7 @@ public: std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose = 2, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 69dfc740063..b4618604ea2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -1,6 +1,6 @@ #include "FlushFormatPlotfile.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/Filter/FilterFunctors.H" @@ -13,6 +13,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + #include #include #include @@ -48,7 +50,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -64,6 +66,7 @@ FlushFormatPlotfile::WriteToFile ( const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose, const bool /*use_pinned_pc*/, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& /*full_BTD_snapshot*/, @@ -72,17 +75,19 @@ FlushFormatPlotfile::WriteToFile ( WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()"); auto & warpx = WarpX::GetInstance(); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); - if (!isBTD) - { - amrex::Print() << Utils::TextMsg::Info("Writing plotfile " + filename); - } else - { - amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) - + " to snapshot " + std::to_string(snapshotID) + " in plotfile BTD " + prefix ); - if (isLastBTDFlush) - { - amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in plotfile BTD " + filename); - } + if (verbose > 0) { + if (!isBTD) + { + amrex::Print() << Utils::TextMsg::Info("Writing plotfile " + filename); + } else + { + amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) + + " to snapshot " + std::to_string(snapshotID) + " in plotfile BTD " + prefix ); + if (isLastBTDFlush) + { + amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in plotfile BTD " + filename); + } + } } Vector rfs; @@ -226,7 +231,7 @@ FlushFormatPlotfile::WriteJobInfo(const std::string& dir) const jobInfoFile << " Inputs File Parameters\n"; jobInfoFile << PrettyLine; - ParmParse::dumpTable(jobInfoFile, true); + ParmParse::prettyPrintTable(jobInfoFile); jobInfoFile.close(); } @@ -367,13 +372,13 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, real_names.push_back("theta"); #endif - // get the names of the real comps - - // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + // get the names of the extra real comps real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM); - auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { - real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + + // note, skip the required compnent names here + auto rnames = tmp.GetRealSoANames(); + for (std::size_t index = PIdx::nattribs; index < rnames.size(); ++index) { + real_names[index - AMREX_SPACEDIM] = rnames[index]; } // plot any "extra" fields by default @@ -385,8 +390,10 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, // and the names int_names.resize(tmp.NumIntComps()); - auto runtime_inames = tmp.getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) { int_names[x.second+0] = x.first; } + auto inames = tmp.GetIntSoANames(); + for (std::size_t index = 0; index < inames.size(); ++index) { + int_names[index] = inames[index]; + } // plot by default int_flags.resize(tmp.NumIntComps(), 1); @@ -556,6 +563,8 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { + using ablastr::fields::Direction; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) @@ -566,84 +575,103 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); - if (warpx.isFieldInitialized(FieldType::F_fp, lev)) + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), dm, raw_pltname, + default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), dm, raw_pltname, + default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), dm, raw_pltname, + default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{0}, lev), dm, raw_pltname, + default_level_prefix, "jx_fp", lev,plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{1}, lev), dm, raw_pltname, + default_level_prefix, "jy_fp", lev,plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{2}, lev), dm, raw_pltname, + default_level_prefix, "jz_fp", lev,plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), dm, raw_pltname, + default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), dm, raw_pltname, + default_level_prefix, "By_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), dm, raw_pltname, + default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards ); + if (warpx.m_fields.has(FieldType::F_fp, lev)) { - WriteRawMF(warpx.getField(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::F_fp, lev), dm, raw_pltname, + default_level_prefix, "F_fp", lev, plot_raw_fields_guards ); } - if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) + if (warpx.m_fields.has(FieldType::rho_fp, lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.getField(FieldType::rho_fp, lev).nComp() - WarpX::ncomps; - const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.m_fields.get(FieldType::rho_fp, lev)->nComp() - WarpX::ncomps; + const MultiFab rho_new(*warpx.m_fields.get(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.isFieldInitialized(FieldType::phi_fp, lev)) { - WriteRawMF(warpx.getField(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + if (warpx.m_fields.has(FieldType::phi_fp, lev)) { + WriteRawMF( *warpx.m_fields.get(FieldType::phi_fp, lev), dm, raw_pltname, + default_level_prefix, "phi_fp", lev, plot_raw_fields_guards ); } // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.getFieldPointer(FieldType::Efield_cp, lev, 0), warpx.getFieldPointer(FieldType::Efield_cp, lev, 1), warpx.getFieldPointer(FieldType::Efield_cp, lev, 2), - warpx.getFieldPointer(FieldType::Efield_fp, lev, 0), warpx.getFieldPointer(FieldType::Efield_fp, lev, 1), warpx.getFieldPointer(FieldType::Efield_fp, lev, 2), + warpx.m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), - warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.getFieldPointer(FieldType::current_cp, lev, 0), warpx.getFieldPointer(FieldType::current_cp, lev, 1), warpx.getFieldPointer(FieldType::current_cp, lev, 2), - warpx.getFieldPointer(FieldType::current_fp, lev, 0), warpx.getFieldPointer(FieldType::current_fp, lev, 1), warpx.getFieldPointer(FieldType::current_fp, lev, 2), + warpx.m_fields.get(FieldType::current_cp, Direction{0}, lev), warpx.m_fields.get(FieldType::current_cp, Direction{1}, lev), warpx.m_fields.get(FieldType::current_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::current_fp, Direction{0}, lev), warpx.m_fields.get(FieldType::current_fp, Direction{1}, lev), warpx.m_fields.get(FieldType::current_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) + if (warpx.m_fields.has(FieldType::F_fp, lev) && warpx.m_fields.has(FieldType::F_cp, lev)) { - WriteCoarseScalar("F", warpx.getFieldPointer(FieldType::F_cp, lev), warpx.getFieldPointer(FieldType::F_fp, lev), + WriteCoarseScalar("F", warpx.m_fields.get(FieldType::F_cp, lev), warpx.m_fields.get(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) + if (warpx.m_fields.has(FieldType::rho_fp, lev) && warpx.m_fields.has(FieldType::rho_cp, lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.getFieldPointer(FieldType::rho_cp, lev), warpx.getFieldPointer(FieldType::rho_fp, lev), + WriteCoarseScalar("rho", warpx.m_fields.get(FieldType::rho_cp, lev), warpx.m_fields.get(FieldType::rho_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 45ea40077e4..87ed00e539e 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -57,6 +57,7 @@ public: std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, + int verbose = 2, bool use_pinned_pc = false, bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index 468ed81ce18..b96c6d76f91 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -51,7 +51,7 @@ FlushFormatSensei::WriteToFile ( const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, - const bool use_pinned_pc, + int verbose, const bool use_pinned_pc, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { @@ -63,7 +63,7 @@ FlushFormatSensei::WriteToFile ( #ifndef AMREX_USE_SENSEI_INSITU amrex::ignore_unused(varnames, mf, iteration, time, particle_diags, - isBTD); + verbose, isBTD); #else WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !isBTD, @@ -71,7 +71,9 @@ FlushFormatSensei::WriteToFile ( WARPX_PROFILE("FlushFormatSensei::WriteToFile()"); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); - amrex::Print() << Utils::TextMsg::Info("Writing Sensei file " + filename); + if (verbose > 0) { + amrex::Print() << Utils::TextMsg::Info("Writing Sensei file " + filename); + } amrex::Vector *mf_ptr = const_cast*>(&mf); diff --git a/Source/Diagnostics/FlushFormats/Make.package b/Source/Diagnostics/FlushFormats/Make.package index 8b3a796007a..59c4525da2e 100644 --- a/Source/Diagnostics/FlushFormats/Make.package +++ b/Source/Diagnostics/FlushFormats/Make.package @@ -1,5 +1,7 @@ CEXE_sources += FlushFormatPlotfile.cpp CEXE_sources += FlushFormatCheckpoint.cpp +CEXE_sources += FlushFormatInSitu.cpp +CEXE_sources += FlushFormatCatalyst.cpp CEXE_sources += FlushFormatAscent.cpp CEXE_sources += FlushFormatSensei.cpp ifeq ($(USE_OPENPMD), TRUE) diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 1b999a9b361..61f63aa78e2 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -4,12 +4,22 @@ #include "Diagnostics.H" #include "Utils/Parser/IntervalsParser.H" +#include + #include class FullDiagnostics final : public Diagnostics { public: - FullDiagnostics (int i, const std::string& name); + FullDiagnostics (int i, const std::string& name, DiagTypes diag_type); + /** Type of time averaging for diagnostics (fields only) + * None corresponds to instantaneous diags + * Static corresponds to a fixed starting step for averaging, + * will average until the end, and dump out intermediate average results + * Dynamic corresponds to a moving period for averaging where the start step + * is as many steps before the output interval as the averaging period is long. + */ + enum struct TimeAverageType {None, Static, Dynamic}; private: /** Read user-requested parameters for full diagnostics */ void ReadParameters (); @@ -25,10 +35,20 @@ private: * before writing the diagnostic. */ bool m_solver_deposits_current = true; - /** Flush m_mf_output and particles to file for the i^th buffer */ + /** Whether the diagnostics are averaging data over time or not */ + TimeAverageType m_time_average_mode = TimeAverageType::None; + /** Period to average fields over: in steps */ + int m_average_period_steps = -1; + /** Period to average fields over: in seconds */ + amrex::Real m_average_period_time = -1.0; + /** Time step to start averaging */ + int m_average_start_step = -1; + /** Flush m_mf_output or m_sum_mf_output and particles to file for the i^th buffer */ void Flush (int i_buffer, bool /* force_flush */) override; /** Flush raw data */ void FlushRaw (); + /** Initialize Data required to compute TimeAveraged diagnostics */ + void DerivedInitData () override; /** whether to compute and pack cell-centered data in m_mf_output * \param[in] step current time step * \param[in] force_flush if true, return true for any step since output must be diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 8c3da11042a..ec932e8d13a 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -13,7 +13,7 @@ #include "ComputeDiagFunctors/ProcessorMapFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FlushFormats/FlushFormat.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/IsIn.H" @@ -21,6 +21,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "WarpX.H" +#include + #include #include #include @@ -44,10 +46,10 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; -FullDiagnostics::FullDiagnostics (int i, const std::string& name): - Diagnostics{i, name}, +FullDiagnostics::FullDiagnostics (int i, const std::string& name, DiagTypes diag_type): + Diagnostics{i, name, diag_type}, m_solver_deposits_current{ (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None) || (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic)} @@ -56,6 +58,27 @@ FullDiagnostics::FullDiagnostics (int i, const std::string& name): BackwardCompatibility(); } +void +FullDiagnostics::DerivedInitData() { + if (m_diag_type == DiagTypes::TimeAveraged) { + auto & warpx = WarpX::GetInstance(); + if (m_time_average_mode == TimeAverageType::Dynamic) { + + // already checked in ReadParameters that only one of the parameters is set + // calculate the other averaging period parameter from the other one, respectively + if (m_average_period_steps > 0) { + m_average_period_time = m_average_period_steps * warpx.getdt(0); + } else if (m_average_period_time > 0) { + m_average_period_steps = static_cast (std::round(m_average_period_time / warpx.getdt(0))); + } + amrex::Print() << Utils::TextMsg::Info( + "Initializing TimeAveragedDiagnostics " + m_diag_name + + " with an averaging period of " + std::to_string(m_average_period_steps) + " steps" + ); + } + } +} + void FullDiagnostics::InitializeParticleBuffer () { @@ -91,8 +114,8 @@ FullDiagnostics::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_format == "plotfile" || m_format == "openpmd" || m_format == "checkpoint" || m_format == "ascent" || - m_format == "sensei", - ".format must be plotfile or openpmd or checkpoint or ascent or sensei"); + m_format == "sensei" || m_format == "catalyst", + ".format must be plotfile or openpmd or checkpoint or ascent or catalyst or sensei"); std::vector intervals_string_vec = {"0"}; pp_diag_name.getarr("intervals", intervals_string_vec); m_intervals = utils::parser::IntervalsParser(intervals_string_vec); @@ -100,6 +123,92 @@ FullDiagnostics::ReadParameters () const bool plot_raw_fields_guards_specified = pp_diag_name.query("plot_raw_fields_guards", m_plot_raw_fields_guards); const bool raw_specified = plot_raw_fields_specified || plot_raw_fields_guards_specified; + if (m_diag_type == DiagTypes::TimeAveraged) { + std::string m_time_average_mode_str = "none"; + /** Whether the diagnostics are averaging data over time or not + * Valid options are "fixed_start" and "dynamic_start". + */ + pp_diag_name.get("time_average_mode", m_time_average_mode_str); + + const amrex::ParmParse pp_warpx("warpx"); + std::vector dt_interval_vec = {"-1"}; + const bool timestep_may_vary = pp_warpx.queryarr("dt_update_interval", dt_interval_vec); + amrex::Print() << Utils::TextMsg::Warn("Time step varies?" + std::to_string(timestep_may_vary)); + if (timestep_may_vary) { + WARPX_ABORT_WITH_MESSAGE( + "Time-averaged diagnostics (encountered in: " + + m_diag_name + ") are currently not supported with adaptive time-stepping" + ); + } + + if (m_time_average_mode_str == "fixed_start") { + m_time_average_mode = TimeAverageType::Static; + } else if (m_time_average_mode_str == "dynamic_start") { + m_time_average_mode = TimeAverageType::Dynamic; + } else if (m_time_average_mode_str == "none") { + m_time_average_mode = TimeAverageType::None; + } else { + WARPX_ABORT_WITH_MESSAGE( + "Unknown time averaging mode. Valid entries are: none, fixed_start, dynamic_start" + ); + } + + const bool averaging_period_steps_specified = pp_diag_name.query( + "average_period_steps", m_average_period_steps + ); + const bool averaging_period_time_specified = pp_diag_name.queryWithParser( + "average_period_time", m_average_period_time + ); + + if (m_time_average_mode == TimeAverageType::Static) { + // This fails if users do not specify a start. + pp_diag_name.get("average_start_step", m_average_start_step); + if (m_average_start_step == 0) { + WARPX_ABORT_WITH_MESSAGE( + "Static-start time-averaged diagnostic " + m_diag_name + " requires a positive (non-zero) value " + "for the 'average_start_step' parameter." + ); + } + + if (averaging_period_time_specified || averaging_period_steps_specified) { + const std::string period_spec_warn_msg = "An averaging period was specified for the 'static_start' averaging mode " \ + "but will be IGNORED. Averaging will be performed between step " \ + + std::to_string(m_average_start_step) \ + + " and the specified intervals."; + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + period_spec_warn_msg, + ablastr::warn_manager::WarnPriority::medium + ); + } + + } + + if (m_time_average_mode == TimeAverageType::Dynamic) { + // one of the two averaging period options must be set but neither none nor both + if ( + (averaging_period_steps_specified && averaging_period_time_specified) + || !(averaging_period_steps_specified || averaging_period_time_specified) + ) { + WARPX_ABORT_WITH_MESSAGE("Please specify either 'average_period_steps' or 'average_period_time', not both."); + } + + int unused_start_step = -1; + const bool averaging_start_on_dynamic_period_specified = pp_diag_name.query("average_start_step", unused_start_step); + if (averaging_start_on_dynamic_period_specified) { + const std::string start_spec_warn_msg = "An averaging start step was specified for the 'dynamic_start'" \ + "time-averaged diagnostic " + m_diag_name + " but will be IGNORED. " \ + "Averaging will begin with the first averaging period."; + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + start_spec_warn_msg, + ablastr::warn_manager::WarnPriority::medium + ); + } + } + } + + #ifdef WARPX_DIM_RZ pp_diag_name.query("dump_rz_modes", m_dump_rz_modes); #else @@ -137,11 +246,46 @@ FullDiagnostics::Flush ( int i_buffer, bool /* force_flush */ ) // is supported for BackTransformed Diagnostics, in BTDiagnostics class. auto & warpx = WarpX::GetInstance(); - m_flush_format->WriteToFile( - m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), - warpx.gett_new(0), - m_output_species.at(i_buffer), nlev_output, m_file_prefix, - m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); + // Get the time step on coarsest level. + const int step = warpx.getistep(0); + // For time-averaged diagnostics, we still write out an instantaneous diagnostic on step 0 + // to accommodate a user workflow that only uses that type of diagnostic. + // This allows for quicker turnaround in setup by avoiding having to set an additional instantaneous diagnostic. + if (m_diag_type == DiagTypes::TimeAveraged && step > 0) { + if (m_time_average_mode == TimeAverageType::Static || m_time_average_mode == TimeAverageType::Dynamic) { + // Loop over the output levels and divide by the number of steps in the averaging period + for (int lev = 0; lev < nlev_output; ++lev) { + m_sum_mf_output.at(i_buffer).at(lev).mult(1._rt/static_cast(m_average_period_steps)); + } + + m_flush_format->WriteToFile( + m_varnames, m_sum_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, + m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, + m_verbose); + + // Reset the values in the dynamic start time-averaged diagnostics after flush + if (m_time_average_mode == TimeAverageType::Dynamic) { + for (int lev = 0; lev < nlev_output; ++lev) { + m_sum_mf_output.at(i_buffer).at(lev).setVal(0.); + } + } + } + } else { + if (m_diag_type == DiagTypes::TimeAveraged && step == 0) { + // For both dynamic_start and fixed_start at step 0 we prepare an instantaneous output + amrex::Print() << Utils::TextMsg::Info("Time-averaged diagnostic " + m_diag_name + + " is preparing an instantaneous output during step " + std::to_string(step)); + } + + m_flush_format->WriteToFile( + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, + m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, + m_verbose); + } FlushRaw(); } @@ -164,26 +308,78 @@ FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) bool FullDiagnostics::DoComputeAndPack (int step, bool force_flush) { + // Start averaging at output step (from diag.intervals) - period + 1 + bool in_averaging_period = false; + if (m_diag_type == DiagTypes::TimeAveraged) { + + if (step > 0) { + + if (m_time_average_mode == TimeAverageType::Dynamic) { + m_average_start_step = m_intervals.nextContains(step) - m_average_period_steps; + // check that the periods do not overlap and that the start step is not negative + if (m_average_start_step > 0) { + // The start step cannot be on an interval step because then we would begin a new period and also output the old one + if (m_average_start_step < m_intervals.previousContains(step)) { + WARPX_ABORT_WITH_MESSAGE( + "Averaging periods may not overlap within a single diagnostic. " + "Please create a second diagnostic for overlapping time averaging periods " + "and account for the increased memory consumption." + ); + } + } else { + WARPX_ABORT_WITH_MESSAGE( + "The step to begin time averaging (" + + std::to_string(m_average_start_step) + + ") for diagnostic " + m_diag_name + " must be a positive number." + ); + } + + if (step >= m_average_start_step && step <= m_intervals.nextContains(step)) { + in_averaging_period = true; + + if (m_time_average_mode == TimeAverageType::Static) { + // Update time averaging period to current step + m_average_period_steps = step - m_average_start_step; + } + } + // Print information on when time-averaging is active + if ((m_verbose > 1) && in_averaging_period) { + if (step == m_average_start_step) { + amrex::Print() << Utils::TextMsg::Info( + "Begin time averaging for " + m_diag_name + " and output at step " + + std::to_string(m_intervals.nextContains(step)) + ); + } else { + amrex::Print() + << Utils::TextMsg::Info( + "Time-averaging during this step for diagnostic: " + m_diag_name); + } + } + } + } + } // Data must be computed and packed for full diagnostics // whenever the data needs to be flushed. - return (force_flush || m_intervals.contains(step+1)); + return (force_flush || m_intervals.contains(step+1) || in_averaging_period); + } void FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::current_fp, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -214,85 +410,43 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; + std::vector field_names = {"r", "t", "z"}; + // Fill vector of functors for all components except individual cylindrical modes. const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Er"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Et"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Ez"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Br"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + for (int idir=0; idir < 3; idir++) { + if ( m_varnames_fields[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("E"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("B"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, deposit_current, ncomp); + deposit_current = false; + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir]+"_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]+"_displacement"), ncomp); + } } - } else if ( m_varnames_fields[comp] == "rho" ){ + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp); @@ -315,19 +469,19 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique( warpx.m_fields.get(FieldType::G_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::phi_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -344,14 +498,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } } else if ( m_varnames_fields[comp] == "divB" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), + warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); @@ -394,29 +548,28 @@ void FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::current_fp, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Check if divE is requested // If so, all components will be written out - bool divE_requested = false; - for (int comp = 0; comp < m_varnames.size(); comp++) { - if ( m_varnames[comp] == "divE" ) { - divE_requested = true; - } - } + const bool divE_requested = std::any_of( + std::begin(m_varnames), + std::end(m_varnames), + [](const auto& varname) { return varname == "divE"; }); // If rho is requested, all components will be written out const bool rho_requested = utils::algorithms::is_in( m_varnames, "rho" ); @@ -441,19 +594,19 @@ FullDiagnostics::AddRZModesToDiags (int lev) for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim), lev, + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp()); + warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim), lev, + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.getFieldPointer(FieldType::Bfield_aux, 0, 0)->nComp()); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ @@ -462,12 +615,12 @@ FullDiagnostics::AddRZModesToDiags (int lev) dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.getFieldPointer(FieldType::current_fp, 0, 0)->nComp()); + warpx.m_fields.get(FieldType::current_fp,Direction{0},0)->nComp()); } // divE if (divE_requested) { m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } @@ -597,6 +750,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setHi( idim, warpx.Geom(lev).ProbLo(idim) + (ba.getCellCenteredBox( static_cast(ba.size())-1 ).bigEnd(idim) + 1) * warpx.Geom(lev).CellSize(idim)); } + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -611,6 +765,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { int const ncomp = static_cast(m_varnames.size()); m_mf_output[i_buffer][lev] = amrex::MultiFab(ba, dmap, ncomp, ngrow); + if (m_diag_type == DiagTypes::TimeAveraged) { + // Allocate MultiFab for cell-centered field output accumulation. The data will be averaged before flushing. + m_sum_mf_output[i_buffer][lev] = amrex::MultiFab(ba, dmap, ncomp, ngrow); + // Initialize to zero because we add data. + m_sum_mf_output[i_buffer][lev].setVal(0.); + } + if (lev == 0) { // The extent of the domain covered by the diag multifab, m_mf_output //default non-periodic geometry for diags @@ -659,21 +820,35 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; + using ablastr::fields::Direction; + +#if defined(WARPX_DIM_RZ) + std::vector field_names = {"r", "t", "z"}; +#else + std::vector field_names = {"x", "y", "z"}; +#endif + m_all_field_functors[lev].resize(ntot); // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jz_displacement" ) { - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "rho" ){ + for (int idir=0; idir < 3; idir++) { + if ( m_varnames[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true, deposit_current); + deposit_current = false; + } else if ( m_varnames[comp] == "j"+field_names[idir]+"_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true); + } else if ( m_varnames[comp] == "A"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{idir}, lev), lev, m_crse_ratio); + } + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); } else if ( m_varnames[comp].rfind("rho_", 0) == 0 ){ @@ -685,79 +860,24 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); i_T_species++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::G_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Efield_aux, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "proc_number" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); - } - else { - -#ifdef WARPX_DIM_RZ - if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if (m_varnames[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); - } else if (m_varnames[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); - } else { - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); - } -#else - // Valid transverse fields in Cartesian coordinates - if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jx" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jy" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jx_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jy_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); - } else { - std::cout << "Error on component " << m_varnames[comp] << std::endl; - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); - } -#endif + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio); + } else { + std::cout << "Error on component " << m_varnames[comp] << "\n"; + WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } } // Add functors for average particle data for each species diff --git a/Source/Diagnostics/Make.package b/Source/Diagnostics/Make.package index 75b41fba5e8..28afdb35290 100644 --- a/Source/Diagnostics/Make.package +++ b/Source/Diagnostics/Make.package @@ -4,7 +4,6 @@ CEXE_sources += FullDiagnostics.cpp CEXE_sources += WarpXIO.cpp CEXE_sources += ParticleIO.cpp CEXE_sources += FieldIO.cpp -CEXE_sources += SliceDiagnostic.cpp CEXE_sources += BTDiagnostics.cpp CEXE_sources += BoundaryScrapingDiagnostics.cpp CEXE_sources += BTD_Plotfile_Header_Impl.cpp diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index d220396ed12..a22e20b44da 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -11,8 +11,6 @@ #include #include -/** All types of diagnostics. */ -enum struct DiagTypes {Full, BackTransformed, BoundaryScraping}; /** * \brief This class contains a vector of all diagnostics in the simulation. diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index ea14919c713..2119ac276f9 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -21,12 +21,12 @@ MultiDiagnostics::MultiDiagnostics () */ alldiags.resize( ndiags ); for (int i=0; i(i, diags_names[i]); + if ( diags_types[i] == DiagTypes::Full || diags_types[i] == DiagTypes::TimeAveraged ){ + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else if ( diags_types[i] == DiagTypes::BackTransformed ){ - alldiags[i] = std::make_unique(i, diags_names[i]); + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else if ( diags_types[i] == DiagTypes::BoundaryScraping ){ - alldiags[i] = std::make_unique(i, diags_names[i]); + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else { WARPX_ABORT_WITH_MESSAGE("Unknown diagnostic type"); } @@ -68,9 +68,10 @@ MultiDiagnostics::ReadParameters () std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", - ".diag_type must be Full or BackTransformed or BoundaryScraping"); + diag_type_str == "Full" || diag_type_str == "TimeAveraged" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", + ".diag_type must be Full, TimeAveraged, BackTransformed or BoundaryScraping"); if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "TimeAveraged") { diags_types[i] = DiagTypes::TimeAveraged; } if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 3672557ede6..b78b44e3b05 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -36,26 +36,23 @@ ParticleDiag::ParticleDiag ( std::fill(m_plot_flags.begin(), m_plot_flags.end(), 0); bool contains_positions = false; if (variables[0] != "none"){ - std::map existing_variable_names = pc->getParticleComps(); + for (auto& var : variables){ #ifdef WARPX_DIM_RZ - // we reconstruct to Cartesian x,y,z for RZ particle output - existing_variable_names["y"] = PIdx::theta; + // we reconstruct to Cartesian x,y,z for RZ particle output + if (var == "y") { var = "theta"; } #endif - for (const auto& var : variables){ if (var == "phi") { // User requests phi on particle. This is *not* part of the variables that // the particle container carries, and is only added to particles during output. // Therefore, this case needs to be treated specifically. m_plot_phi = true; } else { - const auto search = existing_variable_names.find(var); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - search != existing_variable_names.end(), + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(pc->HasRealComp(var), "variables argument '" + var +"' is not an existing attribute for this species"); - m_plot_flags[existing_variable_names.at(var)] = 1; + m_plot_flags[pc->GetRealCompIndex(var)] = 1; - if (var == "x" || var == "y" || var == "z") { + if (var == "x" || var == "y" || var == "z" || var == "theta") { contains_positions = true; } } @@ -75,7 +72,7 @@ ParticleDiag::ParticleDiag ( // Always write out theta, whether or not it's requested, // to be consistent with always writing out r and z. // TODO: openPMD does a reconstruction to Cartesian, so we can now skip force-writing this - m_plot_flags[pc->getParticleComps().at("theta")] = 1; + m_plot_flags[pc->GetRealCompIndex("theta")] = 1; #endif // build filter functors diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index bfb1867e741..62a5e126558 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -7,7 +7,7 @@ * License: BSD-3-Clause-LBNL */ -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/ParticleIO.H" #include "Particles/MultiParticleContainer.H" #include "Particles/PhysicalParticleContainer.H" @@ -43,7 +43,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; void LaserParticleContainer::ReadHeader (std::istream& is) @@ -153,21 +153,24 @@ MultiParticleContainer::Restart (const std::string& dir) real_comp_names.push_back(comp_name); } - for (auto const& comp : pc->getParticleRuntimeComps()) { - auto search = std::find(real_comp_names.begin(), real_comp_names.end(), comp.first); + int n_rc = 0; + for (auto const& comp : pc->GetRealSoANames()) { + // skip compile-time components + if (n_rc < WarpXParticleContainer::NArrayReal) { continue; } + n_rc++; + + auto search = std::find(real_comp_names.begin(), real_comp_names.end(), comp); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( search != real_comp_names.end(), "Species " + species_names[i] - + "needs runtime real component " + comp.first + + " needs runtime real component " + comp + ", but it was not found in the checkpoint file." ); } for (int j = PIdx::nattribs-AMREX_SPACEDIM; j < nr; ++j) { const auto& comp_name = real_comp_names[j]; - auto current_comp_names = pc->getParticleComps(); - auto search = current_comp_names.find(comp_name); - if (search == current_comp_names.end()) { + if (!pc->HasRealComp(comp_name)) { amrex::Print() << Utils::TextMsg::Info( "Runtime real component " + comp_name + " was found in the checkpoint file, but it has not been added yet. " @@ -187,20 +190,23 @@ MultiParticleContainer::Restart (const std::string& dir) int_comp_names.push_back(comp_name); } - for (auto const& comp : pc->getParticleRuntimeiComps()) { - auto search = std::find(int_comp_names.begin(), int_comp_names.end(), comp.first); + int n_ic = 0; + for (auto const& comp : pc->GetIntSoANames()) { + // skip compile-time components + if (n_ic < WarpXParticleContainer::NArrayInt) { continue; } + n_ic++; + + auto search = std::find(int_comp_names.begin(), int_comp_names.end(), comp); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( search != int_comp_names.end(), - "Species " + species_names[i] + "needs runtime int component " + comp.first + "Species " + species_names[i] + " needs runtime int component " + comp + ", but it was not found in the checkpoint file." ); } for (int j = 0; j < ni; ++j) { const auto& comp_name = int_comp_names[j]; - auto current_comp_names = pc->getParticleiComps(); - auto search = current_comp_names.find(comp_name); - if (search == current_comp_names.end()) { + if (!pc->HasIntComp(comp_name)) { amrex::Print()<< Utils::TextMsg::Info( "Runtime int component " + comp_name + " was found in the checkpoint file, but it has not been added yet. " @@ -241,7 +247,7 @@ MultiParticleContainer::WriteHeader (std::ostream& os) const void storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, - int electrostatic_solver_id, bool is_full_diagnostic ) { + ElectrostaticSolverAlgo electrostatic_solver_id, bool is_full_diagnostic ) { using PinnedParIter = typename PinnedMemoryParticleContainer::ParIterType; @@ -259,17 +265,17 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, "Output of the electrostatic potential (phi) on the particles was requested, " "but this is only available with `diag_type = Full`."); tmp.AddRealComp("phi"); - int const phi_index = tmp.getParticleComps().at("phi"); + int const phi_index = tmp.GetRealCompIndex("phi"); auto& warpx = WarpX::GetInstance(); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif for (int lev=0; lev<=warpx.finestLevel(); lev++) { const amrex::Geometry& geom = warpx.Geom(lev); auto plo = geom.ProbLoArray(); auto dxi = geom.InvCellSizeArray(); - amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + amrex::MultiFab const& phi = *warpx.m_fields.get(FieldType::phi_fp, lev); +#ifdef AMREX_USE_OMP + #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { auto phi_grid = phi[pti].array(); diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index 9b5a28a3516..af18ab508a9 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -109,7 +109,7 @@ BeamRelevant::BeamRelevant (const std::string& rd_name) ofs << "[" << c++ << "]alpha_y()"; ofs << m_sep; ofs << "[" << c++ << "]beta_x(m)"; ofs << m_sep; ofs << "[" << c++ << "]beta_y(m)"; ofs << m_sep; - ofs << "[" << c++ << "]charge(C)"; ofs << std::endl; + ofs << "[" << c++ << "]charge(C)"; ofs << "\n"; #elif (defined WARPX_DIM_XZ) int c = 0; ofs << "#"; @@ -131,7 +131,7 @@ BeamRelevant::BeamRelevant (const std::string& rd_name) ofs << "[" << c++ << "]emittance_z(m)"; ofs << m_sep; ofs << "[" << c++ << "]alpha_x()"; ofs << m_sep; ofs << "[" << c++ << "]beta_x(m)"; ofs << m_sep; - ofs << "[" << c++ << "]charge(C)"; ofs << std::endl; + ofs << "[" << c++ << "]charge(C)"; ofs << "\n"; #elif (defined WARPX_DIM_1D_Z) int c = 0; ofs << "#"; @@ -148,7 +148,7 @@ BeamRelevant::BeamRelevant (const std::string& rd_name) ofs << "[" << c++ << "]pz_rms(kg*m/s)"; ofs << m_sep; ofs << "[" << c++ << "]gamma_rms()"; ofs << m_sep; ofs << "[" << c++ << "]emittance_z(m)"; ofs << m_sep; - ofs << "[" << c++ << "]charge(C)"; ofs << std::endl; + ofs << "[" << c++ << "]charge(C)"; ofs << "\n"; #endif // close file ofs.close(); diff --git a/Source/Diagnostics/ReducedDiags/CMakeLists.txt b/Source/Diagnostics/ReducedDiags/CMakeLists.txt index e63764bc24f..c548553b875 100644 --- a/Source/Diagnostics/ReducedDiags/CMakeLists.txt +++ b/Source/Diagnostics/ReducedDiags/CMakeLists.txt @@ -3,25 +3,29 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BeamRelevant.cpp + ChargeOnEB.cpp ColliderRelevant.cpp + DifferentialLuminosity.cpp + DifferentialLuminosity2D.cpp FieldEnergy.cpp + FieldMaximum.cpp + FieldMomentum.cpp + FieldPoyntingFlux.cpp FieldProbe.cpp FieldProbeParticleContainer.cpp - FieldMomentum.cpp + FieldReduction.cpp + FieldProbe.cpp LoadBalanceCosts.cpp LoadBalanceEfficiency.cpp MultiReducedDiags.cpp ParticleEnergy.cpp - ParticleMomentum.cpp + ParticleExtrema.cpp ParticleHistogram.cpp ParticleHistogram2D.cpp + ParticleMomentum.cpp + ParticleNumber.cpp ReducedDiags.cpp - FieldMaximum.cpp - ParticleExtrema.cpp RhoMaximum.cpp - ParticleNumber.cpp - FieldReduction.cpp - FieldProbe.cpp - ChargeOnEB.cpp + Timestep.cpp ) endforeach() diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 2991831420e..050b18d3b9d 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -8,7 +8,8 @@ #include "ChargeOnEB.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/Parser/ParserUtils.H" @@ -24,10 +25,11 @@ #include #include +#include #include using namespace amrex; -using namespace warpx::fields; + // constructor ChargeOnEB::ChargeOnEB (const std::string& rd_name) @@ -44,6 +46,10 @@ ChargeOnEB::ChargeOnEB (const std::string& rd_name) "ChargeOnEB reduced diagnostics only works when compiling with EB support"); #endif + if (!EB::enabled()) { + throw std::runtime_error("ChargeOnEB reduced diagnostics only works when EBs are enabled at runtime"); + } + // resize data array m_data.resize(1, 0.0_rt); @@ -72,8 +78,7 @@ ChargeOnEB::ChargeOnEB (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]time(s)"; ofs << m_sep; - ofs << "[" << c++ << "]Charge (C)"; - ofs << std::endl; + ofs << "[" << c++ << "]Charge (C)\n"; // close file ofs.close(); } @@ -87,7 +92,12 @@ void ChargeOnEB::ComputeDiags (const int step) // Judge whether the diags should be done if (!m_intervals.contains(step+1)) { return; } + if (!EB::enabled()) { + throw std::runtime_error("ChargeOnEB::ComputeDiags only works when EBs are enabled at runtime"); + } #if ((defined WARPX_DIM_3D) && (defined AMREX_USE_EB)) + using ablastr::fields::Direction; + // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -95,9 +105,10 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_fp, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_fp, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_fp, lev,2); + using warpx::fields::FieldType; + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); @@ -132,9 +143,9 @@ void ChargeOnEB::ComputeDiags (const int step) // Skip boxes that do not intersect with the embedded boundary // (i.e. either fully covered or fully regular) - amrex::FabType fab_type = eb_flag[mfi].getType(box); - if (fab_type == amrex::FabType::regular) continue; - if (fab_type == amrex::FabType::covered) continue; + const amrex::FabType fab_type = eb_flag[mfi].getType(box); + if (fab_type == amrex::FabType::regular) { continue; } + if (fab_type == amrex::FabType::covered) { continue; } // Extract data for electric field const amrex::Array4 & Ex_arr = Ex.array(mfi); @@ -153,7 +164,7 @@ void ChargeOnEB::ComputeDiags (const int step) [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Only cells that are partially covered do contribute to the integral - if (eb_flag_arr(i,j,k).isRegular() || eb_flag_arr(i,j,k).isCovered()) return; + if (eb_flag_arr(i,j,k).isRegular() || eb_flag_arr(i,j,k).isCovered()) { return; } // Find nodal point which is outside of the EB // (eb_normal points towards the *interior* of the EB) @@ -164,14 +175,14 @@ void ChargeOnEB::ComputeDiags (const int step) // Find cell-centered point which is outside of the EB // (eb_normal points towards the *interior* of the EB) int i_c = i; - if ((eb_bnd_normal_arr(i,j,k,0)>0) && (eb_bnd_cent_arr(i,j,k,0)<=0)) i_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,0)<0) && (eb_bnd_cent_arr(i,j,k,0)>=0)) i_c += 1; + if ((eb_bnd_normal_arr(i,j,k,0)>0) && (eb_bnd_cent_arr(i,j,k,0)<=0)) { i_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,0)<0) && (eb_bnd_cent_arr(i,j,k,0)>=0)) { i_c += 1; } int j_c = j; - if ((eb_bnd_normal_arr(i,j,k,1)>0) && (eb_bnd_cent_arr(i,j,k,1)<=0)) j_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,1)<0) && (eb_bnd_cent_arr(i,j,k,1)>=0)) j_c += 1; + if ((eb_bnd_normal_arr(i,j,k,1)>0) && (eb_bnd_cent_arr(i,j,k,1)<=0)) { j_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,1)<0) && (eb_bnd_cent_arr(i,j,k,1)>=0)) { j_c += 1; } int k_c = k; - if ((eb_bnd_normal_arr(i,j,k,2)>0) && (eb_bnd_cent_arr(i,j,k,2)<=0)) k_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,2)<0) && (eb_bnd_cent_arr(i,j,k,2)>=0)) k_c += 1; + if ((eb_bnd_normal_arr(i,j,k,2)>0) && (eb_bnd_cent_arr(i,j,k,2)<=0)) { k_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,2)<0) && (eb_bnd_cent_arr(i,j,k,2)>=0)) { k_c += 1; } // Compute contribution to the surface integral $\int dS \cdot E$) amrex::Real local_integral_contribution = 0; @@ -182,9 +193,9 @@ void ChargeOnEB::ComputeDiags (const int step) // Add weighting if requested by user if (do_parser_weighting) { // Get the 3D position of the centroid of surface element - amrex::Real x = (i + 0.5_rt + eb_bnd_cent_arr(i,j,k,0))*dx[0] + real_box.lo(0); - amrex::Real y = (j + 0.5_rt + eb_bnd_cent_arr(i,j,k,1))*dx[1] + real_box.lo(1); - amrex::Real z = (k + 0.5_rt + eb_bnd_cent_arr(i,j,k,2))*dx[2] + real_box.lo(2); + const amrex::Real x = (i + 0.5_rt + eb_bnd_cent_arr(i,j,k,0))*dx[0] + real_box.lo(0); + const amrex::Real y = (j + 0.5_rt + eb_bnd_cent_arr(i,j,k,1))*dx[1] + real_box.lo(1); + const amrex::Real z = (k + 0.5_rt + eb_bnd_cent_arr(i,j,k,2))*dx[2] + real_box.lo(2); // Apply weighting local_integral_contribution *= fun_weightingparser(x, y, z); } diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 1b0c74cf7fa..dfd64fe5af9 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -8,7 +8,7 @@ #include "ColliderRelevant.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif @@ -59,7 +59,7 @@ #include using namespace amrex; -using namespace warpx::fields; + ColliderRelevant::ColliderRelevant (const std::string& rd_name) : ReducedDiags{rd_name} @@ -181,7 +181,7 @@ ColliderRelevant::ColliderRelevant (const std::string& rd_name) const auto& el = m_headers_indices[name]; ofs << m_sep << "[" << el.idx + off << "]" << el.header; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } @@ -429,6 +429,8 @@ void ColliderRelevant::ComputeDiags (int step) amrex::Real chimax_f = 0.0_rt; amrex::Real chiave_f = 0.0_rt; + using ablastr::fields::Direction; + if (myspc.DoQED()) { // define variables in preparation for field gatheeduce_data.value()ring @@ -441,13 +443,14 @@ void ColliderRelevant::ComputeDiags (int step) const int lev = 0; // define variables in preparation for field gathering + using warpx::fields::FieldType; const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.H b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.H new file mode 100644 index 00000000000..13c1dca55f9 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.H @@ -0,0 +1,62 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY_H_ + +#include "ReducedDiags.H" +#include + +#include +#include +#include + +/** + * This class contains the differential luminosity diagnostics. + */ +class DifferentialLuminosity : public ReducedDiags +{ +public: + + /** + * constructor + * @param[in] rd_name reduced diags names + */ + DifferentialLuminosity(const std::string& rd_name); + + /// name of the two colliding species + std::vector m_beam_name; + + /// number of bins + int m_bin_num; + + /// max and min bin values + amrex::Real m_bin_max; + amrex::Real m_bin_min; + + /// bin size + amrex::Real m_bin_size; + + void ComputeDiags(int step) final; + +private: + /// auxiliary structure to store headers and indices of the reduced diagnostics + struct aux_header_index + { + std::string header; + int idx; + }; + + /// map to store header texts and indices of the reduced diagnostics + std::map m_headers_indices; + + // Array in which to accumulate the luminosity across timesteps + amrex::Gpu::DeviceVector< amrex::Real > d_data; +}; + +#endif // WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY_H_ diff --git a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.cpp b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.cpp new file mode 100644 index 00000000000..ef5e0da6014 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity.cpp @@ -0,0 +1,299 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti, Yinjian Zhao + * License: BSD-3-Clause-LBNL + */ +#include "DifferentialLuminosity.H" + +#include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/SpeciesPhysicalProperties.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/ParticleUtils.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/WarpXConst.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using ParticleType = WarpXParticleContainer::ParticleType; +using ParticleTileType = WarpXParticleContainer::ParticleTileType; +using ParticleTileDataType = ParticleTileType::ParticleTileDataType; +using ParticleBins = amrex::DenseBins; +using index_type = ParticleBins::index_type; + +using namespace amrex; + +DifferentialLuminosity::DifferentialLuminosity (const std::string& rd_name) +: ReducedDiags{rd_name} +{ + // read colliding species names - must be 2 + const amrex::ParmParse pp_rd_name(m_rd_name); + pp_rd_name.getarr("species", m_beam_name); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_beam_name.size() == 2u, + "DifferentialLuminosity diagnostics must involve exactly two species"); + + // RZ coordinate is not supported +#if (defined WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE( + "DifferentialLuminosity diagnostics do not work in RZ geometry."); +#endif + + // read bin parameters + int bin_num = 0; + amrex::Real bin_max = 0.0_rt, bin_min = 0.0_rt; + utils::parser::getWithParser(pp_rd_name, "bin_number", bin_num); + utils::parser::getWithParser(pp_rd_name, "bin_max", bin_max); + utils::parser::getWithParser(pp_rd_name, "bin_min", bin_min); + m_bin_num = bin_num; + m_bin_max = bin_max; + m_bin_min = bin_min; + m_bin_size = (bin_max - bin_min) / bin_num; + + // resize and zero-out data array + m_data.resize(m_bin_num,0.0_rt); + d_data.resize(m_bin_num,0.0_rt); + + if (amrex::ParallelDescriptor::IOProcessor()) + { + if ( m_write_header ) + { + // open file + std::ofstream ofs; + ofs.open(m_path + m_rd_name + "." + m_extension, + std::ofstream::out | std::ofstream::app); + // write header row + int off = 0; + ofs << "#"; + ofs << "[" << off++ << "]step()"; + ofs << m_sep; + ofs << "[" << off++ << "]time(s)"; + for (int i = 0; i < m_bin_num; ++i) + { + ofs << m_sep; + ofs << "[" << off++ << "]"; + const Real b = m_bin_min + m_bin_size*(Real(i)+0.5_rt); + ofs << "bin" << 1+i << "=" << b << "(eV)"; + } + ofs << "\n"; + // close file + ofs.close(); + } + } +} + +void DifferentialLuminosity::ComputeDiags (int step) +{ +#if defined(WARPX_DIM_RZ) + amrex::ignore_unused(step); +#else + WARPX_PROFILE("DifferentialLuminosity::ComputeDiags"); + + // Since this diagnostic *accumulates* the luminosity in the + // array d_data, we add contributions at *each timestep*, but + // we only write the data to file at intervals specified by the user. + const Real c_sq = PhysConst::c*PhysConst::c; + const Real c_over_qe = PhysConst::c/PhysConst::q_e; + + // get a reference to WarpX instance + auto& warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(0); + // get cell volume + Geometry const & geom = warpx.Geom(0); + const Real dV = AMREX_D_TERM(geom.CellSize(0), *geom.CellSize(1), *geom.CellSize(2)); + + // declare local variables + auto const num_bins = m_bin_num; + Real const bin_min = m_bin_min; + Real const bin_size = m_bin_size; + + // get MultiParticleContainer class object + const MultiParticleContainer& mypc = warpx.GetPartContainer(); + + auto& species_1 = mypc.GetParticleContainerFromName(m_beam_name[0]); + auto& species_2 = mypc.GetParticleContainerFromName(m_beam_name[1]); + + const ParticleReal m1 = species_1.getMass(); + const ParticleReal m2 = species_2.getMass(); + + amrex::Real* const AMREX_RESTRICT dptr_data = d_data.dataPtr(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + int const nlevs = std::max(0, species_1.finestLevel()+1); // species_1 ? + for (int lev = 0; lev < nlevs; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + + for (amrex::MFIter mfi = species_1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + ParticleBins bins_1 = ParticleUtils::findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = ParticleUtils::findParticlesInEachCell( lev, mfi, ptile_2 ); + + // Species + const auto soa_1 = ptile_1.getParticleTileData(); + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + + // Particle data in the tile/box + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; // v*gamma=p/m + amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + bool const species1_is_photon = species_1.AmIA(); + + const auto soa_2 = ptile_2.getParticleTileData(); + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + bool const species2_is_photon = species_2.AmIA(); + + // Extract low-level data + auto const n_cells = static_cast(bins_1.numBins()); + + // Loop over cells + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + + for(index_type i_1=cell_start_1; i_1=num_bins ) { continue; } // discard if out-of-range + + Real const inv_p1t = 1.0_rt/p1t; + Real const inv_p2t = 1.0_rt/p2t; + + Real const beta1_sq = (p1x*p1x + p1y*p1y + p1z*p1z) * inv_p1t*inv_p1t; + Real const beta2_sq = (p2x*p2x + p2y*p2y + p2z*p2z) * inv_p2t*inv_p2t; + Real const beta1_dot_beta2 = (p1x*p2x + p1y*p2y + p1z*p2z) * inv_p1t*inv_p2t; + + // Here we use the fact that: + // (v1 - v2)^2 = v1^2 + v2^2 - 2 v1.v2 + // and (v1 x v2)^2 = v1^2 v2^2 - (v1.v2)^2 + // we also use beta=v/c instead of v + + Real const radicand = beta1_sq + beta2_sq - 2*beta1_dot_beta2 - beta1_sq*beta2_sq + beta1_dot_beta2*beta1_dot_beta2; + + Real const dL_dEcom = PhysConst::c * std::sqrt( radicand ) * w1[j_1] * w2[j_2] / dV / bin_size * dt; // m^-2 eV^-1 + + amrex::HostDevice::Atomic::Add(&dptr_data[bin], dL_dEcom); + + } // particles species 2 + } // particles species 1 + }); // cells + } // boxes + } // levels + + // Only write to file at intervals specified by the user. + // At these intervals, the data needs to ready on the CPU, + // so we copy it from the GPU to the CPU and reduce across MPI ranks. + if (m_intervals.contains(step+1)) { + // blocking copy from device to host + amrex::Gpu::copy(amrex::Gpu::deviceToHost, + d_data.begin(), d_data.end(), m_data.begin()); + + // reduced sum over mpi ranks + ParallelDescriptor::ReduceRealSum + (m_data.data(), static_cast(m_data.size()), ParallelDescriptor::IOProcessorNumber()); + } + +#endif // not RZ + +} diff --git a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.H b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.H new file mode 100644 index 00000000000..7ffefec324e --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.H @@ -0,0 +1,70 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti, Remi Lehe + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY2D_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY2D_H_ + +#include "ReducedDiags.H" +#include +#include + +#include +#include +#include + +/** + * This class contains the differential luminosity diagnostics. + */ +class DifferentialLuminosity2D : public ReducedDiags +{ +public: + + /** + * constructor + * @param[in] rd_name reduced diags names + */ + DifferentialLuminosity2D(const std::string& rd_name); + + /// File type + std::string m_openpmd_backend {"default"}; + + /// minimum number of digits for file suffix (file-based only supported for now) */ + int m_file_min_digits = 6; + + /// name of the two colliding species + std::vector m_beam_name; + + /// number of bins for the c.o.m. energy of the 2 species + int m_bin_num_1; + int m_bin_num_2; + + /// max and min bin values + amrex::Real m_bin_max_1; + amrex::Real m_bin_min_1; + amrex::Real m_bin_max_2; + amrex::Real m_bin_min_2; + + /// bin size + amrex::Real m_bin_size_1; + amrex::Real m_bin_size_2; + + /// output data + amrex::TableData m_h_data_2D; + + void ComputeDiags(int step) final; + + void WriteToFile (int step) const final; + +private: + + /// output table in which to accumulate the luminosity across timesteps + amrex::TableData m_d_data_2D; + +}; + +#endif // WARPX_DIAGNOSTICS_REDUCEDDIAGS_DIFFERENTIALLUMINOSITY2D_H_ diff --git a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp new file mode 100644 index 00000000000..b3968b9fb02 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp @@ -0,0 +1,401 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Arianna Formenti, Yinjian Zhao, Remi Lehe + * License: BSD-3-Clause-LBNL + */ +#include "DifferentialLuminosity2D.H" + +#include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "Diagnostics/OpenPMDHelpFunction.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/SpeciesPhysicalProperties.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/ParticleUtils.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/WarpXConst.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef WARPX_USE_OPENPMD +# include +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using ParticleType = WarpXParticleContainer::ParticleType; +using ParticleTileType = WarpXParticleContainer::ParticleTileType; +using ParticleTileDataType = ParticleTileType::ParticleTileDataType; +using ParticleBins = amrex::DenseBins; +using index_type = ParticleBins::index_type; + +#ifdef WARPX_USE_OPENPMD +namespace io = openPMD; +#endif + +using namespace amrex; + +DifferentialLuminosity2D::DifferentialLuminosity2D (const std::string& rd_name) +: ReducedDiags{rd_name} +{ + // RZ coordinate is not supported +#if (defined WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE( + "DifferentialLuminosity2D diagnostics does not work in RZ geometry."); +#endif + + // read colliding species names - must be 2 + amrex::ParmParse pp_rd_name(m_rd_name); + pp_rd_name.getarr("species", m_beam_name); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_beam_name.size() == 2u, + "DifferentialLuminosity2D diagnostics must involve exactly two species"); + + pp_rd_name.query("openpmd_backend", m_openpmd_backend); + pp_rd_name.query("file_min_digits", m_file_min_digits); + // pick first available backend if default is chosen + if( m_openpmd_backend == "default" ) { + m_openpmd_backend = WarpXOpenPMDFileType(); + } + pp_rd_name.add("openpmd_backend", m_openpmd_backend); + + // read bin parameters for species 1 + int bin_num_1 = 0; + amrex::Real bin_max_1 = 0.0_rt, bin_min_1 = 0.0_rt; + utils::parser::getWithParser(pp_rd_name, "bin_number_1", bin_num_1); + utils::parser::getWithParser(pp_rd_name, "bin_max_1", bin_max_1); + utils::parser::getWithParser(pp_rd_name, "bin_min_1", bin_min_1); + m_bin_num_1 = bin_num_1; + m_bin_max_1 = bin_max_1; + m_bin_min_1 = bin_min_1; + m_bin_size_1 = (bin_max_1 - bin_min_1) / bin_num_1; + + // read bin parameters for species 2 + int bin_num_2 = 0; + amrex::Real bin_max_2 = 0.0_rt, bin_min_2 = 0.0_rt; + utils::parser::getWithParser(pp_rd_name, "bin_number_2", bin_num_2); + utils::parser::getWithParser(pp_rd_name, "bin_max_2", bin_max_2); + utils::parser::getWithParser(pp_rd_name, "bin_min_2", bin_min_2); + m_bin_num_2 = bin_num_2; + m_bin_max_2 = bin_max_2; + m_bin_min_2 = bin_min_2; + m_bin_size_2 = (bin_max_2 - bin_min_2) / bin_num_2; + + // resize data array on the host + Array tlo{0,0}; // lower bounds + Array thi{m_bin_num_1-1, m_bin_num_2-1}; // inclusive upper bounds + m_h_data_2D.resize(tlo, thi, The_Pinned_Arena()); + + auto const& h_table_data = m_h_data_2D.table(); + // initialize data on the host + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + h_table_data(i,j) = 0.0_rt; + } + } + + // resize data on the host + m_d_data_2D.resize(tlo, thi); + // copy data from host to device + m_d_data_2D.copy(m_h_data_2D); + Gpu::streamSynchronize(); +} // end constructor + +void DifferentialLuminosity2D::ComputeDiags (int step) +{ +#if defined(WARPX_DIM_RZ) + amrex::ignore_unused(step); +#else + + WARPX_PROFILE("DifferentialLuminosity2D::ComputeDiags"); + + // Since this diagnostic *accumulates* the luminosity in the + // table m_d_data_2D, we add contributions at *each timestep*, but + // we only write the data to file at intervals specified by the user. + const Real c_sq = PhysConst::c*PhysConst::c; + const Real c_over_qe = PhysConst::c/PhysConst::q_e; + + // output table data + auto d_table = m_d_data_2D.table(); + + // get a reference to WarpX instance + auto& warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(0); + // get cell volume + Geometry const & geom = warpx.Geom(0); + const Real dV = AMREX_D_TERM(geom.CellSize(0), *geom.CellSize(1), *geom.CellSize(2)); + + // declare local variables + auto const num_bins_1 = m_bin_num_1; + Real const bin_min_1 = m_bin_min_1; + Real const bin_size_1 = m_bin_size_1; + auto const num_bins_2 = m_bin_num_2; + Real const bin_min_2 = m_bin_min_2; + Real const bin_size_2 = m_bin_size_2; + + // get MultiParticleContainer class object + const MultiParticleContainer& mypc = warpx.GetPartContainer(); + + auto& species_1 = mypc.GetParticleContainerFromName(m_beam_name[0]); + auto& species_2 = mypc.GetParticleContainerFromName(m_beam_name[1]); + + const ParticleReal m1 = species_1.getMass(); + const ParticleReal m2 = species_2.getMass(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + int const nlevs = std::max(0, species_1.finestLevel()+1); // species_1 ? + for (int lev = 0; lev < nlevs; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + + for (amrex::MFIter mfi = species_1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + ParticleBins bins_1 = ParticleUtils::findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = ParticleUtils::findParticlesInEachCell( lev, mfi, ptile_2 ); + + // species 1 + const auto soa_1 = ptile_1.getParticleTileData(); + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + + // extract particle data of species 1 in the current tile/box + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; // u=v*gamma=p/m + amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + bool const species1_is_photon = species_1.AmIA(); + + // same for species 2 + const auto soa_2 = ptile_2.getParticleTileData(); + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + bool const species2_is_photon = species_2.AmIA(); + + // Extract low-level (cell-level) data + auto const n_cells = static_cast(bins_1.numBins()); + + // Loop over cells + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + + for(index_type i_1=cell_start_1; i_1=num_bins_1 ) { continue; } // discard if out-of-range + + // determine energy bin of particle 2 + int const bin_2 = int(Math::floor((E_2-bin_min_2)/bin_size_2)); + if ( bin_2<0 || bin_2>=num_bins_2 ) { continue; } // discard if out-of-range + + Real const inv_p1t = 1.0_rt/p1t; + Real const inv_p2t = 1.0_rt/p2t; + + Real const beta1_sq = (p1x*p1x + p1y*p1y + p1z*p1z) * inv_p1t*inv_p1t; + Real const beta2_sq = (p2x*p2x + p2y*p2y + p2z*p2z) * inv_p2t*inv_p2t; + Real const beta1_dot_beta2 = (p1x*p2x + p1y*p2y + p1z*p2z) * inv_p1t*inv_p2t; + + // Here we use the fact that: + // (v1 - v2)^2 = v1^2 + v2^2 - 2 v1.v2 + // and (v1 x v2)^2 = v1^2 v2^2 - (v1.v2)^2 + // we also use beta=v/c instead of v + Real const radicand = beta1_sq + beta2_sq - 2*beta1_dot_beta2 - beta1_sq*beta2_sq + beta1_dot_beta2*beta1_dot_beta2; + + Real const d2L_dE1_dE2 = PhysConst::c * std::sqrt( radicand ) * w1[j_1] * w2[j_2] / (dV * bin_size_1 * bin_size_2) * dt; // m^-2 eV^-2 + + amrex::Real &data = d_table(bin_1, bin_2); + amrex::HostDevice::Atomic::Add(&data, d2L_dE1_dE2); + + } // particles species 2 + } // particles species 1 + }); // cells + } // boxes + } // levels + + // Only write to file at intervals specified by the user. + // At these intervals, the data needs to ready on the CPU, + // so we copy it from the GPU to the CPU and reduce across MPI ranks. + if (m_intervals.contains(step+1)) { + + // Copy data from GPU memory + m_h_data_2D.copy(m_d_data_2D); + + // reduced sum over mpi ranks + const int size = static_cast (m_d_data_2D.size()); + ParallelDescriptor::ReduceRealSum + (m_h_data_2D.table().p, size, ParallelDescriptor::IOProcessorNumber()); + } + + // Return for all that are not IO processor + if ( !ParallelDescriptor::IOProcessor() ) { return; } + +#endif // not RZ +} // end void DifferentialLuminosity2D::ComputeDiags + +void DifferentialLuminosity2D::WriteToFile (int step) const +{ + // Judge if the diags should be done at this step + if (!m_intervals.contains(step+1)) { return; } + +#ifdef WARPX_USE_OPENPMD + // only IO processor writes + if ( !ParallelDescriptor::IOProcessor() ) { return; } + + // TODO: support different filename templates + std::string filename = "openpmd"; + // TODO: support also group-based encoding + const std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T"); + filename = filename.append(fileSuffix).append(".").append(m_openpmd_backend); + + // transform paths for Windows + #ifdef _WIN32 + const std::string filepath = openPMD::auxiliary::replace_all( + m_path + m_rd_name + "/" + filename, "/", "\\"); + #else + const std::string filepath = m_path + m_rd_name + "/" + filename; + #endif + + // Create the OpenPMD series + auto series = io::Series( + filepath, + io::Access::CREATE); + auto i = series.iterations[step + 1]; + // record + auto f_mesh = i.meshes["d2L_dE1_dE2"]; // m^-2 eV^-2 + f_mesh.setUnitDimension({ + {io::UnitDimension::L, -6}, + {io::UnitDimension::M, -2}, + {io::UnitDimension::T, 4} + }); + + // record components + auto data = f_mesh[io::RecordComponent::SCALAR]; + + // meta data + f_mesh.setAxisLabels({"E2", "E1"}); // eV, eV + std::vector< double > const& gridGlobalOffset = {m_bin_min_2, m_bin_min_1}; + f_mesh.setGridGlobalOffset(gridGlobalOffset); + f_mesh.setGridSpacing({m_bin_size_2, m_bin_size_1}); + + data.setPosition({0.5, 0.5}); + + auto dataset = io::Dataset( + io::determineDatatype(), + {static_cast(m_bin_num_2), static_cast(m_bin_num_1)}); + data.resetDataset(dataset); + + // Get time at level 0 + auto & warpx = WarpX::GetInstance(); + auto const time = warpx.gett_new(0); + i.setTime(time); + + auto const& h_table_data = m_h_data_2D.table(); + data.storeChunkRaw( + h_table_data.p, + {0, 0}, + {static_cast(m_bin_num_2), static_cast(m_bin_num_1)}); + + series.flush(); + i.close(); + series.close(); +#else + amrex::ignore_unused(step); + WARPX_ABORT_WITH_MESSAGE("DifferentialLuminosity2D: Needs openPMD-api compiled into WarpX, but was not found!"); +#endif +} diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.H b/Source/Diagnostics/ReducedDiags/FieldEnergy.H index 40de174526e..fe17f15f071 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.H +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.H @@ -40,13 +40,14 @@ public: void ComputeDiags(int step) final; /** - * \brief Calculate the integral of the field squared in RZ + * \brief Calculate the integral of the field squared, taking into + * account the fraction of the cell volume within the domain. * * \param field The MultiFab to be integrated * \param lev The refinement level * \return The integral */ - amrex::Real ComputeNorm2RZ(const amrex::MultiFab& field, int lev); + amrex::Real ComputeNorm2(const amrex::MultiFab& field, int lev); }; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 9c0fac101a2..d16319c37e8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -7,12 +7,14 @@ #include "FieldEnergy.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -28,8 +30,8 @@ #include #include -using namespace amrex; -using namespace warpx::fields; +using namespace amrex::literals; +using warpx::fields::FieldType; // constructor FieldEnergy::FieldEnergy (const std::string& rd_name) @@ -38,7 +40,7 @@ FieldEnergy::FieldEnergy (const std::string& rd_name) // read number of levels int nLevel = 0; - const ParmParse pp_amr("amr"); + amrex::ParmParse const pp_amr("amr"); pp_amr.query("max_level", nLevel); nLevel += 1; @@ -46,7 +48,7 @@ FieldEnergy::FieldEnergy (const std::string& rd_name) // resize data array m_data.resize(noutputs*nLevel, 0.0_rt); - if (ParallelDescriptor::IOProcessor()) + if (amrex::ParallelDescriptor::IOProcessor()) { if ( m_write_header ) { @@ -67,7 +69,7 @@ FieldEnergy::FieldEnergy (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]B_lev" + std::to_string(lev) + "(J)"; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } @@ -82,51 +84,40 @@ void FieldEnergy::ComputeDiags (int step) if (!m_intervals.contains(step+1)) { return; } // get a reference to WarpX instance - auto & warpx = WarpX::GetInstance(); + auto const & warpx = WarpX::GetInstance(); // get number of level - const auto nLevel = warpx.finestLevel() + 1; + int const nLevel = warpx.finestLevel() + 1; + + using ablastr::fields::Direction; // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + amrex::MultiFab const & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + amrex::MultiFab const & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + amrex::MultiFab const & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + amrex::MultiFab const & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + amrex::MultiFab const & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + amrex::MultiFab const & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // get cell volume - const std::array &dx = WarpX::CellSize(lev); - const amrex::Real dV = dx[0]*dx[1]*dx[2]; - -#if defined(WARPX_DIM_RZ) - amrex::Real const tmpEx = ComputeNorm2RZ(Ex, lev); - amrex::Real const tmpEy = ComputeNorm2RZ(Ey, lev); - amrex::Real const tmpEz = ComputeNorm2RZ(Ez, lev); - amrex::Real const Es = tmpEx + tmpEy + tmpEz; - - amrex::Real const tmpBx = ComputeNorm2RZ(Bx, lev); - amrex::Real const tmpBy = ComputeNorm2RZ(By, lev); - amrex::Real const tmpBz = ComputeNorm2RZ(Bz, lev); - amrex::Real const Bs = tmpBx + tmpBy + tmpBz; -#else - Geometry const & geom = warpx.Geom(lev); + std::array const &dx = WarpX::CellSize(lev); + amrex::Real const dV = dx[0]*dx[1]*dx[2]; // compute E squared - Real const tmpEx = Ex.norm2(0,geom.periodicity()); - Real const tmpEy = Ey.norm2(0,geom.periodicity()); - Real const tmpEz = Ez.norm2(0,geom.periodicity()); - Real const Es = tmpEx*tmpEx + tmpEy*tmpEy + tmpEz*tmpEz; + amrex::Real const tmpEx = ComputeNorm2(Ex, lev); + amrex::Real const tmpEy = ComputeNorm2(Ey, lev); + amrex::Real const tmpEz = ComputeNorm2(Ez, lev); // compute B squared - Real const tmpBx = Bx.norm2(0,geom.periodicity()); - Real const tmpBy = By.norm2(0,geom.periodicity()); - Real const tmpBz = Bz.norm2(0,geom.periodicity()); - Real const Bs = tmpBx*tmpBx + tmpBy*tmpBy + tmpBz*tmpBz; -#endif + amrex::Real const tmpBx = ComputeNorm2(Bx, lev); + amrex::Real const tmpBy = ComputeNorm2(By, lev); + amrex::Real const tmpBz = ComputeNorm2(Bz, lev); + + amrex::Real const Es = tmpEx + tmpEy + tmpEz; + amrex::Real const Bs = tmpBx + tmpBy + tmpBz; constexpr int noutputs = 3; // total energy, E-field energy and B-field energy constexpr int index_total = 0; @@ -152,15 +143,13 @@ void FieldEnergy::ComputeDiags (int step) } // end void FieldEnergy::ComputeDiags -// Function that computes the sum of the field squared in RZ +// Function that computes the sum of the field squared. +// This takes into account the fraction of the cell volumes within the domain +// and the cell volumes in cylindrical coordinates. amrex::Real -FieldEnergy::ComputeNorm2RZ(const amrex::MultiFab& field, const int lev) +FieldEnergy::ComputeNorm2(amrex::MultiFab const& field, [[maybe_unused]]int lev) { - // get a reference to WarpX instance - auto & warpx = WarpX::GetInstance(); - - Geometry const & geom = warpx.Geom(lev); - const amrex::Real dr = geom.CellSize(0); + amrex::IntVect const is_nodal = field.ixType().toIntVect(); amrex::ReduceOps reduce_ops; amrex::ReduceData reduce_data(reduce_ops); @@ -174,45 +163,63 @@ FieldEnergy::ComputeNorm2RZ(const amrex::MultiFab& field, const int lev) amrex::Array4 const& field_arr = field.array(mfi); - const amrex::Box tilebox = mfi.tilebox(); - amrex::Box tb = convert(tilebox, field.ixType().toIntVect()); + amrex::Box const tilebox = mfi.tilebox(); + amrex::Box const tb = convert(tilebox, is_nodal); + amrex::IntVect const tb_lo = tb.smallEnd(); + amrex::IntVect const tb_hi = tb.bigEnd(); +#if defined(WARPX_DIM_RZ) // Lower corner of tile box physical domain - const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); - const Dim3 lo = lbound(tilebox); - const Dim3 hi = ubound(tilebox); - const Real rmin = xyzmin.x + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); - const int irmin = lo.x; - const int irmax = hi.x; + auto const & warpx = WarpX::GetInstance(); + amrex::Geometry const & geom = warpx.Geom(lev); + amrex::Real const dr = geom.CellSize(0); + amrex::XDim3 const xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + amrex::Real const rmin = xyzmin.x + (is_nodal[0] ? 0._rt : 0.5_rt*dr); +#endif - int const ncomp = field.nComp(); + // On the boundaries, if the grid is nodal, use half of the volume. + // This applies to all boundary conditions, and to the overlap of + // boxes within the domain. + // Previously, the code used MultiFab::norm2, but that does not do + // the half-volume scaling for the domain boundaries when not periodic. + + auto volume_factor = [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + amrex::ignore_unused(i,j,k,n); +#if defined WARPX_DIM_RZ + amrex::Real const r = rmin + (i - tb_lo[0])*dr; + amrex::Real v_factor = 2._rt*r; + if (i == tb_lo[0] && is_nodal[0]) { v_factor = r + dr/4._rt; } + if (i == tb_hi[0] && is_nodal[0]) { v_factor = r - dr/4._rt; } + if (j == tb_lo[1] && is_nodal[1]) { v_factor *= 0.5_rt; } + if (j == tb_hi[1] && is_nodal[1]) { v_factor *= 0.5_rt; } + amrex::Real const theta_integral = (n == 0 ? 1._rt : 0.5_rt); + return MathConst::pi*v_factor*theta_integral; +#else + amrex::Real v_factor = 1._rt; + AMREX_D_TERM( + if (i == tb_lo[0] && is_nodal[0]) { v_factor *= 0.5_rt; }, + if (j == tb_lo[1] && is_nodal[1]) { v_factor *= 0.5_rt; }, + if (k == tb_lo[2] && is_nodal[2]) { v_factor *= 0.5_rt; }) + AMREX_D_TERM( + if (i == tb_hi[0] && is_nodal[0]) { v_factor *= 0.5_rt; }, + if (j == tb_hi[1] && is_nodal[1]) { v_factor *= 0.5_rt; }, + if (k == tb_hi[2] && is_nodal[2]) { v_factor *= 0.5_rt; }) + return v_factor; +#endif + }; - for (int idir=0 ; idir < AMREX_SPACEDIM ; idir++) { - if (WarpX::field_boundary_hi[idir] == FieldBoundaryType::Periodic) { - // For periodic boundaries, do not include the data in the nodes - // on the upper edge of the domain - tb.enclosedCells(idir); - } - } + int const ncomp = field.nComp(); reduce_ops.eval(tb, ncomp, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) -> ReduceTuple { - const amrex::Real r = rmin + (i - irmin)*dr; - amrex::Real volume_factor = r; - if (r == 0._rt) { - volume_factor = dr/8._rt; - } else if (rmin == 0._rt && i == irmax) { - volume_factor = r/2._rt - dr/8._rt; - } - const amrex::Real theta_integral = (n == 0 ? 2._rt : 1._rt); - return theta_integral*field_arr(i,j,k,n)*field_arr(i,j,k,n)*volume_factor; + return field_arr(i,j,k,n)*field_arr(i,j,k,n)*volume_factor(i,j,k,n); }); } - const amrex::Real field_sum = amrex::get<0>(reduce_data.value()); - const amrex::Real result = MathConst::pi*field_sum; + amrex::Real result = amrex::get<0>(reduce_data.value()); + amrex::ParallelDescriptor::ReduceRealSum(result); + return result; } -// end Real FieldEnergy::ComputeNorm2RZ diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 24d9dda3ea6..8c7eb6b4dec 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -7,10 +7,11 @@ #include "FieldMaximum.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" +#include #include #include @@ -39,7 +40,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor FieldMaximum::FieldMaximum (const std::string& rd_name) @@ -92,7 +93,7 @@ FieldMaximum::FieldMaximum (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]max_|B|_lev" + std::to_string(lev) + "(T)"; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } @@ -112,16 +113,18 @@ void FieldMaximum::ComputeDiags (int step) // get number of level const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 7eb16efecff..764e9874c39 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -7,11 +7,12 @@ #include "FieldMomentum.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include @@ -38,7 +39,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} @@ -87,7 +88,7 @@ FieldMomentum::FieldMomentum (const std::string& rd_name) ofs << "momentum_z_lev" << lev << "(kg*m/s)"; } - ofs << std::endl; + ofs << "\n"; ofs.close(); } } @@ -104,16 +105,18 @@ void FieldMomentum::ComputeDiags (int step) // Get number of refinement levels const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // Loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev, 0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev, 1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev, 2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev, 1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.H b/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.H new file mode 100644 index 00000000000..3a45bd6c789 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.H @@ -0,0 +1,63 @@ +/* Copyright 2019-2020 + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDPOYTINGFLUX_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDPOYTINGFLUX_H_ + +#include "ReducedDiags.H" + +#include + +/** + * \brief This class mainly contains a function that computes the field Poynting flux, + * S = E cross B, integrated over each face of the domain. + */ +class FieldPoyntingFlux : public ReducedDiags +{ +public: + + /** + * \brief Constructor + * + * \param[in] rd_name reduced diags names + */ + FieldPoyntingFlux (const std::string& rd_name); + + /** + * \brief Call the routine to compute the Poynting flux if needed + * + * \param[in] step current time step + */ + void ComputeDiags (int step) final; + + /** + * \brief Call the routine to compute the Poynting flux at the mid step time level + * + * \param[in] step current time step + */ + void ComputeDiagsMidStep (int step) final; + + /** + * \brief This function computes the electromagnetic Poynting flux, + * obtained by integrating the electromagnetic Poynting flux density g = eps0 * (E x B) + * on the surface of the domain. + * + * \param[in] step current time step + */ + void ComputePoyntingFlux (); + + void WriteCheckpointData (std::string const & dir) final; + + void ReadCheckpointData (std::string const & dir) final; + +private: + + bool use_mid_step_value = false; + +}; + +#endif diff --git a/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.cpp b/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.cpp new file mode 100644 index 00000000000..f760516f2b9 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.cpp @@ -0,0 +1,333 @@ +/* Copyright 2019-2020 + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "FieldPoyntingFlux.H" + +#include "Fields.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXConst.H" +#include "WarpX.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace amrex::literals; + +FieldPoyntingFlux::FieldPoyntingFlux (const std::string& rd_name) + : ReducedDiags{rd_name} +{ + // Resize data array + // lo and hi is 2 + // space dims is AMREX_SPACEDIM + // instantaneous and integrated is 2 + // The order will be outward flux for low faces, then high faces, + // energy loss for low faces, then high faces + m_data.resize(2*AMREX_SPACEDIM*2, 0.0_rt); + + if (amrex::ParallelDescriptor::IOProcessor()) + { + if (m_write_header) + { + // Open file + std::ofstream ofs{m_path + m_rd_name + "." + m_extension, std::ofstream::out}; + + int c = 0; + + // Write header row + ofs << "#"; + ofs << "[" << c++ << "]step()"; + ofs << m_sep; + ofs << "[" << c++ << "]time(s)"; + + std::vector sides = {"lo", "hi"}; + +#if defined(WARPX_DIM_3D) + std::vector space_coords = {"x", "y", "z"}; +#elif defined(WARPX_DIM_XZ) + std::vector space_coords = {"x", "z"}; +#elif defined(WARPX_DIM_1D_Z) + std::vector space_coords = {"z"}; +#elif defined(WARPX_DIM_RZ) + std::vector space_coords = {"r", "z"}; +#endif + + // Only on level 0 + for (int iside = 0; iside < 2; iside++) { + for (int ic = 0; ic < AMREX_SPACEDIM; ic++) { + ofs << m_sep; + ofs << "[" << c++ << "]outward_power_" + sides[iside] + "_" + space_coords[ic] +"(W)"; + }} + for (int iside = 0; iside < 2; iside++) { + for (int ic = 0; ic < AMREX_SPACEDIM; ic++) { + ofs << m_sep; + ofs << "[" << c++ << "]integrated_energy_loss_" + sides[iside] + "_" + space_coords[ic] +"(J)"; + }} + + ofs << "\n"; + ofs.close(); + } + } +} + +void FieldPoyntingFlux::ComputeDiags (int /*step*/) +{ + // This will be called at the end of the time step. Only calculate the + // flux if it had not already been calculated mid step. + if (!use_mid_step_value) { + ComputePoyntingFlux(); + } +} + +void FieldPoyntingFlux::ComputeDiagsMidStep (int /*step*/) +{ + // If this is called, always use the value calculated here. + use_mid_step_value = true; + ComputePoyntingFlux(); +} + +void FieldPoyntingFlux::ComputePoyntingFlux () +{ + using warpx::fields::FieldType; + using ablastr::fields::Direction; + + // Note that this is calculated every step to get the + // full resolution on the integrated data + + int const lev = 0; + + // Get a reference to WarpX instance + auto & warpx = WarpX::GetInstance(); + + // RZ coordinate only working with one mode +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(warpx.n_rz_azimuthal_modes == 1, + "FieldPoyntingFlux reduced diagnostics only implemented in RZ geometry for one mode"); +#endif + + amrex::Box domain_box = warpx.Geom(lev).Domain(); + domain_box.surroundingNodes(); + + // Get MultiFab data at given refinement level + amrex::MultiFab const & Ex = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + amrex::MultiFab const & Ey = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + amrex::MultiFab const & Ez = *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev); + amrex::MultiFab const & Bx = *warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev); + amrex::MultiFab const & By = *warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev); + amrex::MultiFab const & Bz = *warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev); + + // Coarsening ratio (no coarsening) + amrex::GpuArray const cr{1,1,1}; + + // Reduction component (fourth component in Array4) + constexpr int comp = 0; + + // Index type (staggering) of each MultiFab + // (with third component set to zero in 2D) + amrex::GpuArray Ex_stag{0,0,0}; + amrex::GpuArray Ey_stag{0,0,0}; + amrex::GpuArray Ez_stag{0,0,0}; + amrex::GpuArray Bx_stag{0,0,0}; + amrex::GpuArray By_stag{0,0,0}; + amrex::GpuArray Bz_stag{0,0,0}; + for (int i = 0; i < AMREX_SPACEDIM; ++i) + { + Ex_stag[i] = Ex.ixType()[i]; + Ey_stag[i] = Ey.ixType()[i]; + Ez_stag[i] = Ez.ixType()[i]; + Bx_stag[i] = Bx.ixType()[i]; + By_stag[i] = By.ixType()[i]; + Bz_stag[i] = Bz.ixType()[i]; + } + + for (amrex::OrientationIter face; face; ++face) { + + int const face_dir = face().coordDir(); + + if (face().isHigh() && WarpX::field_boundary_hi[face_dir] == FieldBoundaryType::Periodic) { + // For upper periodic boundaries, copy the lower value instead of regenerating it. + int const iu = int(face()); + int const il = int(face().flip()); + m_data[iu] = -m_data[il]; + m_data[iu + 2*AMREX_SPACEDIM] = -m_data[il + 2*AMREX_SPACEDIM]; + continue; + } + + amrex::Box const boundary = amrex::bdryNode(domain_box, face()); + + // Get cell area + amrex::Real const *dx = warpx.Geom(lev).CellSize(); + std::array dxtemp = {AMREX_D_DECL(dx[0], dx[1], dx[2])}; + dxtemp[face_dir] = 1._rt; + amrex::Real const dA = AMREX_D_TERM(dxtemp[0], *dxtemp[1], *dxtemp[2]); + + // Node-centered in the face direction, Cell-centered in other directions + amrex::GpuArray cc{0,0,0}; + cc[face_dir] = 1; + + // Only calculate the ExB term that is normal to the surface. + // normal_dir is the normal direction relative to the WarpX coordinates +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : it is either 0, or 2 + int const normal_dir = 2*face_dir; +#elif (defined WARPX_DIM_1D_Z) + // For 1D : it is always 2 + int const normal_dir = 2; +#else + // For 3D : it is the same as the face direction + int const normal_dir = face_dir; +#endif + + amrex::ReduceOps reduce_ops; + amrex::ReduceData reduce_data(reduce_ops); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + // Loop over boxes, interpolate E,B data to cell face centers + // and compute sum over cells of (E x B) components + for (amrex::MFIter mfi(Ex, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Array4 const & Ex_arr = Ex[mfi].array(); + amrex::Array4 const & Ey_arr = Ey[mfi].array(); + amrex::Array4 const & Ez_arr = Ez[mfi].array(); + amrex::Array4 const & Bx_arr = Bx[mfi].array(); + amrex::Array4 const & By_arr = By[mfi].array(); + amrex::Array4 const & Bz_arr = Bz[mfi].array(); + + amrex::Box box = enclosedCells(mfi.nodaltilebox()); + box.surroundingNodes(face_dir); + + // Find the intersection with the boundary + // boundary needs to have the same type as box + amrex::Box const boundary_matched = amrex::convert(boundary, box.ixType()); + box &= boundary_matched; + +#if defined(WARPX_DIM_RZ) + // Lower corner of box physical domain + amrex::XDim3 const xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + amrex::Dim3 const lo = amrex::lbound(box); + amrex::Real const dr = warpx.Geom(lev).CellSize(lev); + amrex::Real const rmin = xyzmin.x; + int const irmin = lo.x; +#endif + + auto area_factor = [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::ignore_unused(i,j,k); +#if defined WARPX_DIM_RZ + amrex::Real r; + if (normal_dir == 0) { + r = rmin + (i - irmin)*dr; + } else { + r = rmin + (i + 0.5_rt - irmin)*dr; + } + return 2._rt*MathConst::pi*r; +#else + return 1._rt; +#endif + }; + + // Compute E x B + reduce_ops.eval(box, reduce_data, + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::GpuTuple + { + amrex::Real Ex_cc = 0._rt, Ey_cc = 0._rt, Ez_cc = 0._rt; + amrex::Real Bx_cc = 0._rt, By_cc = 0._rt, Bz_cc = 0._rt; + + if (normal_dir == 1 || normal_dir == 2) { + Ex_cc = ablastr::coarsen::sample::Interp(Ex_arr, Ex_stag, cc, cr, i, j, k, comp); + Bx_cc = ablastr::coarsen::sample::Interp(Bx_arr, Bx_stag, cc, cr, i, j, k, comp); + } + + if (normal_dir == 0 || normal_dir == 2) { + Ey_cc = ablastr::coarsen::sample::Interp(Ey_arr, Ey_stag, cc, cr, i, j, k, comp); + By_cc = ablastr::coarsen::sample::Interp(By_arr, By_stag, cc, cr, i, j, k, comp); + } + if (normal_dir == 0 || normal_dir == 1) { + Ez_cc = ablastr::coarsen::sample::Interp(Ez_arr, Ez_stag, cc, cr, i, j, k, comp); + Bz_cc = ablastr::coarsen::sample::Interp(Bz_arr, Bz_stag, cc, cr, i, j, k, comp); + } + + amrex::Real const af = area_factor(i,j,k); + if (normal_dir == 0) { return af*(Ey_cc * Bz_cc - Ez_cc * By_cc); } + else if (normal_dir == 1) { return af*(Ez_cc * Bx_cc - Ex_cc * Bz_cc); } + else { return af*(Ex_cc * By_cc - Ey_cc * Bx_cc); } + }); + } + + int const sign = (face().isLow() ? -1 : 1); + auto r = reduce_data.value(); + int const ii = int(face()); + m_data[ii] = sign*amrex::get<0>(r)/PhysConst::mu0*dA; + + } + + amrex::ParallelDescriptor::ReduceRealSum(m_data.data(), 2*AMREX_SPACEDIM); + + amrex::Real const dt = warpx.getdt(lev); + for (int ii=0 ; ii < 2*AMREX_SPACEDIM ; ii++) { + m_data[ii + 2*AMREX_SPACEDIM] += m_data[ii]*dt; + } + +} + +void +FieldPoyntingFlux::WriteCheckpointData (std::string const & dir) +{ + // Write out the current values of the time integrated data + std::ofstream chkfile{dir + "/FieldPoyntingFlux_data.txt", std::ofstream::out}; + if (!chkfile.good()) { + WARPX_ABORT_WITH_MESSAGE("FieldPoyntingFlux::WriteCheckpointData: could not open file for writing checkpoint data"); + } + + chkfile.precision(17); + + for (int i=0; i < 2*AMREX_SPACEDIM; i++) { + chkfile << m_data[2*AMREX_SPACEDIM + i] << "\n"; + } +} + +void +FieldPoyntingFlux::ReadCheckpointData (std::string const & dir) +{ + // Read in the current values of the time integrated data + std::ifstream chkfile{dir + "/FieldPoyntingFlux_data.txt", std::ifstream::in}; + if (!chkfile.good()) { + WARPX_ABORT_WITH_MESSAGE("FieldPoyntingFlux::ReadCheckpointData: could not open file for reading checkpoint data"); + } + + for (int i=0; i < 2*AMREX_SPACEDIM; i++) { + amrex::Real data; + if (chkfile >> data) { + m_data[2*AMREX_SPACEDIM + i] = data; + } else { + WARPX_ABORT_WITH_MESSAGE("FieldPoyntingFlux::ReadCheckpointData: could not read in time integrated data"); + } + } +} diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 1113075f4cd..923ae727d08 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -7,7 +7,7 @@ #include "FieldProbe.H" #include "FieldProbeParticleContainer.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/UpdatePosition.H" @@ -17,6 +17,7 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include @@ -45,7 +46,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor @@ -225,7 +226,7 @@ FieldProbe::FieldProbe (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]part_S_lev" + std::to_string(lev) + u_map[FieldProbePIdx::S]; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); @@ -381,6 +382,8 @@ void FieldProbe::ComputeDiags (int step) // get number of mesh-refinement levels const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { @@ -398,12 +401,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = warpx.getField(FieldType::Efield_aux, lev, 0); - const amrex::MultiFab &Ey = warpx.getField(FieldType::Efield_aux, lev, 1); - const amrex::MultiFab &Ez = warpx.getField(FieldType::Efield_aux, lev, 2); - const amrex::MultiFab &Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); - const amrex::MultiFab &By = warpx.getField(FieldType::Bfield_aux, lev, 1); - const amrex::MultiFab &Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); + const amrex::MultiFab &Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab &Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab &Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab &Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab &By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab &Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); /* * Prepare interpolation of field components to probe_position @@ -678,7 +681,7 @@ void FieldProbe::WriteToFile (int step) const ofs << m_sep; ofs << sorted_data[i * noutputs + k]; } - ofs << std::endl; + ofs << "\n"; } // end loop over data size // close file ofs.close(); diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index f467499ad56..d2c6dc6f6da 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -9,7 +9,7 @@ #define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDREDUCTION_H_ #include "ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "WarpX.H" #include @@ -73,7 +73,7 @@ private: std::unique_ptr m_parser; // Type of reduction (e.g. Maximum, Minimum or Sum) - int m_reduction_type; + ReductionType m_reduction_type; public: @@ -87,7 +87,9 @@ public: template void ComputeFieldReduction() { + using ablastr::fields::Direction; using namespace amrex::literals; + using warpx::fields::FieldType; // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -99,15 +101,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); - const amrex::MultiFab & jx = warpx.getField(warpx::fields::FieldType::current_fp, lev,0); - const amrex::MultiFab & jy = warpx.getField(warpx::fields::FieldType::current_fp, lev,1); - const amrex::MultiFab & jz = warpx.getField(warpx::fields::FieldType::current_fp, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); + const amrex::MultiFab & jx = *warpx.m_fields.get(FieldType::current_fp, Direction{0},lev); + const amrex::MultiFab & jy = *warpx.m_fields.get(FieldType::current_fp, Direction{1},lev); + const amrex::MultiFab & jz = *warpx.m_fields.get(FieldType::current_fp, Direction{2},lev); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp index 683ae1921d6..fbf67be4e4a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp @@ -60,9 +60,7 @@ FieldReduction::FieldReduction (const std::string& rd_name) parser_string = std::regex_replace(parser_string, std::regex("\n\\s*"), " "); // read reduction type - std::string reduction_type_string; - pp_rd_name.get("reduction_type", reduction_type_string); - m_reduction_type = GetAlgorithmInteger (pp_rd_name, "reduction_type"); + pp_rd_name.get_enum_sloppy("reduction_type", m_reduction_type, "-_"); if (amrex::ParallelDescriptor::IOProcessor()) { @@ -77,9 +75,9 @@ FieldReduction::FieldReduction (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]time(s)"; ofs << m_sep; - ofs << "[" << c++ << "]" + reduction_type_string + " of " + parser_string + " (SI units)"; + ofs << "[" << c++ << "]" + amrex::getEnumNameString(m_reduction_type) + " of " + parser_string + " (SI units)"; - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H index cfba804d0f0..b30d6b0bb6e 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H @@ -40,9 +40,9 @@ public: * (cost, processor, level, i_low, j_low, k_low, gpu_ID [if GPU run], num_cells, num_macro_particles * note: the hostname per box is stored separately (in m_data_string) */ #ifdef AMREX_USE_GPU - const int m_nDataFields = 9; + static const int m_nDataFields = 9; #else - const int m_nDataFields = 8; + static const int m_nDataFields = 8; #endif /** used to keep track of max number of boxes over all timesteps; this allows diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 882c94b12eb..c496300c54e 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -7,12 +7,14 @@ #include "LoadBalanceCosts.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "WarpX.H" +#include + #include #include #include @@ -36,7 +38,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -123,11 +125,13 @@ void LoadBalanceCosts::ComputeDiags (int step) // shift index for m_data int shift_m_data = 0; + using ablastr::fields::Direction; + // save data for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); @@ -275,7 +279,7 @@ void LoadBalanceCosts::WriteToFile (int step) const // end loop over data size // end line - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); @@ -329,7 +333,7 @@ void LoadBalanceCosts::WriteToFile (int step) const ofstmp << m_sep; ofstmp << "[" << c++ << "]hostname_box_" + std::to_string(boxNumber) + "()"; } - ofstmp << std::endl; + ofstmp << "\n"; // open the data-containing file const std::string fileDataName = m_path + m_rd_name + "." + m_extension; @@ -360,7 +364,7 @@ void LoadBalanceCosts::WriteToFile (int step) const { ofstmp << m_sep << "NaN"; } - ofstmp << std::endl; + ofstmp << "\n"; } // close files diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp index b6f3a58e670..6c1f46a6c54 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp @@ -50,7 +50,7 @@ LoadBalanceEfficiency::LoadBalanceEfficiency (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]lev" + std::to_string(lev); } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); diff --git a/Source/Diagnostics/ReducedDiags/Make.package b/Source/Diagnostics/ReducedDiags/Make.package index a1f08a24da3..98fa093e2df 100644 --- a/Source/Diagnostics/ReducedDiags/Make.package +++ b/Source/Diagnostics/ReducedDiags/Make.package @@ -1,23 +1,26 @@ CEXE_sources += MultiReducedDiags.cpp CEXE_sources += ReducedDiags.cpp -CEXE_sources += ParticleEnergy.cpp -CEXE_sources += ParticleMomentum.cpp +CEXE_sources += BeamRelevant.cpp +CEXE_sources += ChargeOnEB.cpp +CEXE_sources += ColliderRelevant.cpp +CEXE_sources += DifferentialLuminosity.cpp +CEXE_sources += DifferentialLuminosity2D.cpp CEXE_sources += FieldEnergy.cpp +CEXE_sources += FieldMaximum.cpp +CEXE_sources += FieldMomentum.cpp +CEXE_sources += FieldPoyntingFlux.cpp CEXE_sources += FieldProbe.cpp CEXE_sources += FieldProbeParticleContainer.cpp -CEXE_sources += FieldMomentum.cpp -CEXE_sources += BeamRelevant.cpp -CEXE_sources += ColliderRelevant.cpp +CEXE_sources += FieldReduction.cpp CEXE_sources += LoadBalanceCosts.cpp CEXE_sources += LoadBalanceEfficiency.cpp +CEXE_sources += ParticleEnergy.cpp +CEXE_sources += ParticleExtrema.cpp CEXE_sources += ParticleHistogram.cpp CEXE_sources += ParticleHistogram2D.cpp -CEXE_sources += FieldMaximum.cpp -CEXE_sources += FieldProbe.cpp -CEXE_sources += ParticleExtrema.cpp -CEXE_sources += RhoMaximum.cpp +CEXE_sources += ParticleMomentum.cpp CEXE_sources += ParticleNumber.cpp -CEXE_sources += FieldReduction.cpp -CEXE_sources += ChargeOnEB.cpp +CEXE_sources += RhoMaximum.cpp +CEXE_sources += Timestep.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ReducedDiags diff --git a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.H b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.H index 1a2f51794c6..5a782db7118 100644 --- a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.H @@ -49,10 +49,21 @@ public: * @param[in] step current iteration time */ void ComputeDiags (int step); + /** Loop over all ReducedDiags and call their ComputeDiagsMidStep + * @param[in] step current iteration time */ + void ComputeDiagsMidStep (int step); + /** Loop over all ReducedDiags and call their WriteToFile * @param[in] step current iteration time */ void WriteToFile (int step); + /** \brief Loop over all ReducedDiags and call their WriteCheckpointData + * @param[in] dir checkpoint directory */ + void WriteCheckpointData (std::string const & dir); + + /** \brief Loop over all ReducedDiags and call their ReadCheckpointData + * @param[in] dir checkpoint directory */ + void ReadCheckpointData (std::string const & dir); }; #endif diff --git a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp index 73e63aeb4d3..e4c982f7323 100644 --- a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp @@ -9,10 +9,13 @@ #include "BeamRelevant.H" #include "ChargeOnEB.H" #include "ColliderRelevant.H" +#include "DifferentialLuminosity.H" +#include "DifferentialLuminosity2D.H" #include "FieldEnergy.H" #include "FieldMaximum.H" -#include "FieldProbe.H" #include "FieldMomentum.H" +#include "FieldPoyntingFlux.H" +#include "FieldProbe.H" #include "FieldReduction.H" #include "LoadBalanceCosts.H" #include "LoadBalanceEfficiency.H" @@ -23,6 +26,7 @@ #include "ParticleMomentum.H" #include "ParticleNumber.H" #include "RhoMaximum.H" +#include "Timestep.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" @@ -51,23 +55,27 @@ MultiReducedDiags::MultiReducedDiags () using CS = const std::string& ; const auto reduced_diags_dictionary = std::map(CS)>>{ + {"BeamRelevant", [](CS s){return std::make_unique(s);}}, + {"ChargeOnEB", [](CS s){return std::make_unique(s);}}, + {"ColliderRelevant", [](CS s){return std::make_unique(s);}}, + {"DifferentialLuminosity",[](CS s){return std::make_unique(s);}}, + {"DifferentialLuminosity2D",[](CS s){return std::make_unique(s);}}, {"ParticleEnergy", [](CS s){return std::make_unique(s);}}, + {"ParticleExtrema", [](CS s){return std::make_unique(s);}}, + {"ParticleHistogram", [](CS s){return std::make_unique(s);}}, + {"ParticleHistogram2D", [](CS s){return std::make_unique(s);}}, {"ParticleMomentum", [](CS s){return std::make_unique(s);}}, + {"ParticleNumber", [](CS s){return std::make_unique(s);}}, {"FieldEnergy", [](CS s){return std::make_unique(s);}}, - {"FieldMomentum", [](CS s){return std::make_unique(s);}}, {"FieldMaximum", [](CS s){return std::make_unique(s);}}, + {"FieldMomentum", [](CS s){return std::make_unique(s);}}, + {"FieldPoyntingFlux", [](CS s){return std::make_unique(s);}}, {"FieldProbe", [](CS s){return std::make_unique(s);}}, {"FieldReduction", [](CS s){return std::make_unique(s);}}, - {"RhoMaximum", [](CS s){return std::make_unique(s);}}, - {"BeamRelevant", [](CS s){return std::make_unique(s);}}, - {"ColliderRelevant", [](CS s){return std::make_unique(s);}}, {"LoadBalanceCosts", [](CS s){return std::make_unique(s);}}, {"LoadBalanceEfficiency", [](CS s){return std::make_unique(s);}}, - {"ParticleHistogram", [](CS s){return std::make_unique(s);}}, - {"ParticleHistogram2D", [](CS s){return std::make_unique(s);}}, - {"ParticleNumber", [](CS s){return std::make_unique(s);}}, - {"ParticleExtrema", [](CS s){return std::make_unique(s);}}, - {"ChargeOnEB", [](CS s){return std::make_unique(s);}} + {"RhoMaximum", [](CS s){return std::make_unique(s);}}, + {"Timestep", [](CS s){return std::make_unique(s);}} }; // loop over all reduced diags and fill m_multi_rd with requested reduced diags std::transform(m_rd_names.begin(), m_rd_names.end(), std::back_inserter(m_multi_rd), @@ -120,6 +128,20 @@ void MultiReducedDiags::ComputeDiags (int step) } // end void MultiReducedDiags::ComputeDiags +// call functions to compute diags at the mid step time level +void MultiReducedDiags::ComputeDiagsMidStep (int step) +{ + WARPX_PROFILE("MultiReducedDiags::ComputeDiagsMidStep()"); + + // loop over all reduced diags + for (int i_rd = 0; i_rd < static_cast(m_rd_names.size()); ++i_rd) + { + m_multi_rd[i_rd] -> ComputeDiagsMidStep(step); + } + // end loop over all reduced diags +} +// end void MultiReducedDiags::ComputeDiagsMidStep + // function to write data void MultiReducedDiags::WriteToFile (int step) { @@ -138,3 +160,26 @@ void MultiReducedDiags::WriteToFile (int step) // end loop over all reduced diags } // end void MultiReducedDiags::WriteToFile + +void MultiReducedDiags::WriteCheckpointData (std::string const & dir) +{ + // Only the I/O rank does + if ( !ParallelDescriptor::IOProcessor() ) { return; } + + // loop over all reduced diags + for (int i_rd = 0; i_rd < static_cast(m_rd_names.size()); ++i_rd) + { + m_multi_rd[i_rd]->WriteCheckpointData(dir); + } + // end loop over all reduced diags +} + +void MultiReducedDiags::ReadCheckpointData (std::string const & dir) +{ + // loop over all reduced diags + for (int i_rd = 0; i_rd < static_cast(m_rd_names.size()); ++i_rd) + { + m_multi_rd[i_rd]->ReadCheckpointData(dir); + } + // end loop over all reduced diags +} diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index 9446af5e20c..41b58ea8fe3 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -78,7 +78,7 @@ ParticleEnergy::ParticleEnergy (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]" << species_names[i] + "_mean(J)"; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 811fee7a9de..c82b060b67c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -11,7 +11,7 @@ #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" #include "Particles/MultiParticleContainer.H" @@ -21,6 +21,8 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -52,7 +54,7 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor ParticleExtrema::ParticleExtrema (const std::string& rd_name) @@ -140,7 +142,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) const auto& el = m_headers_indices[name]; ofs << m_sep << "[" << el.idx + off << "]" << el.header; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } @@ -260,18 +262,20 @@ void ParticleExtrema::ComputeDiags (int step) const bool galerkin_interpolation = WarpX::galerkin_interpolation; const amrex::IntVect ngEB = warpx.getngEB(); + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev <= level_number; ++lev) { // define variables in preparation for field gathering const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // declare reduce_op amrex::ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index 5343de2ba68..95f27e311d5 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -146,7 +146,7 @@ ParticleHistogram::ParticleHistogram (const std::string& rd_name) ofs << "bin" + std::to_string(1+i) + "=" + std::to_string(b) + "()"; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 47d1ec2cdd4..8cd4a9bc00c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -107,7 +107,7 @@ ParticleMomentum::ParticleMomentum (const std::string& rd_name) ofs << species_names[i] + "_mean_z(kg*m/s)"; } - ofs << std::endl; + ofs << "\n"; ofs.close(); } } diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index bfdf1daffe3..83dc3d118f6 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -75,7 +75,7 @@ ParticleNumber::ParticleNumber (const std::string& rd_name) ofs << m_sep; ofs << "[" << c++ << "]" << species_names[i] + "_weight()"; } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 2c942e1df6d..a32de30cc6f 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -83,6 +83,13 @@ public: */ virtual void ComputeDiags (int step) = 0; + /** + * function to compute diags at the mid step time level + * + * @param[in] step current time step + */ + virtual void ComputeDiagsMidStep (int step); + /** * write to file function * @@ -90,6 +97,20 @@ public: */ virtual void WriteToFile (int step) const; + /** + * \brief Write out checkpoint related data + * + * \param[in] dir Directory where checkpoint data is written + */ + virtual void WriteCheckpointData (std::string const & dir); + + /** + * \brief Read in checkpoint related data + * + * \param[in] dir Directory where checkpoint data was written + */ + virtual void ReadCheckpointData (std::string const & dir); + /** * This function queries deprecated input parameters and aborts * the run if one of them is specified. diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index e3835dde416..b0e20584a12 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -28,12 +28,15 @@ m_rd_name{rd_name} { BackwardCompatibility(); + const ParmParse pp_rd("reduced_diags"); const ParmParse pp_rd_name(m_rd_name); // read path + pp_rd.query("path", m_path); pp_rd_name.query("path", m_path); // read extension + pp_rd.query("extension", m_extension); pp_rd_name.query("extension", m_extension); // check if it is a restart run @@ -61,13 +64,16 @@ m_rd_name{rd_name} // read reduced diags intervals std::vector intervals_string_vec = {"1"}; - pp_rd_name.getarr("intervals", intervals_string_vec); + pp_rd.queryarr("intervals", intervals_string_vec); + pp_rd_name.queryarr("intervals", intervals_string_vec); m_intervals = utils::parser::IntervalsParser(intervals_string_vec); // read separator + pp_rd.query("separator", m_sep); pp_rd_name.query("separator", m_sep); // precision of data in the output file + utils::parser::queryWithParser(pp_rd, "precision", m_precision); utils::parser::queryWithParser(pp_rd_name, "precision", m_precision); } // end constructor @@ -86,6 +92,27 @@ void ReducedDiags::LoadBalance () // load balancing operations } +void ReducedDiags::ComputeDiagsMidStep (int /*step*/) +{ + // Defines an empty function ComputeDiagsMidStep() to be overwritten if needed. + // Function used to calculate the diagnostic at the mid step time leve + // (instead of at the end of the step). +} + +void ReducedDiags::WriteCheckpointData (std::string const & /*dir*/) +{ + // Defines an empty function WriteCheckpointData() to be overwritten if needed. + // Function used to write out and data needed by the diagnostic in + // the checkpoint. +} + +void ReducedDiags::ReadCheckpointData (std::string const & /*dir*/) +{ + // Defines an empty function ReadCheckpointData() to be overwritten if needed. + // Function used to read in any data that was written out in the checkpoint + // when doing a restart. +} + void ReducedDiags::BackwardCompatibility () const { const amrex::ParmParse pp_rd_name(m_rd_name); @@ -121,7 +148,7 @@ void ReducedDiags::WriteToFile (int step) const // end loop over data size // end line - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 9902415d9a9..66460613a56 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -113,7 +113,7 @@ RhoMaximum::RhoMaximum (const std::string& rd_name) + "_|rho|_lev" + std::to_string(lev) + "(C/m^3)"; } } - ofs << std::endl; + ofs << "\n"; // close file ofs.close(); } diff --git a/Source/Diagnostics/ReducedDiags/Timestep.H b/Source/Diagnostics/ReducedDiags/Timestep.H new file mode 100644 index 00000000000..bcf4fe6452f --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/Timestep.H @@ -0,0 +1,35 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Thomas Marks + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ + +#include "ReducedDiags.H" +#include + +/** + * This class contains a function for retrieving the current simulation timestep as a diagnostic. + * Useful mainly for simulations using adaptive timestepping. + */ +class Timestep: public ReducedDiags { +public: + /** + * constructor + * @param[in] rd_name reduced diags name + */ + Timestep (const std::string& rd_name); + + /** + * This function gets the current physical timestep of the simulation at all refinement levels. + * @param[in] step current time step + */ + void ComputeDiags (int step) final; +}; + +#endif //WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ diff --git a/Source/Diagnostics/ReducedDiags/Timestep.cpp b/Source/Diagnostics/ReducedDiags/Timestep.cpp new file mode 100644 index 00000000000..e74f22c27ec --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/Timestep.cpp @@ -0,0 +1,72 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Thomas Marks + * + * License: BSD-3-Clause-LBNL + */ + +#include "Timestep.H" + +#include "WarpX.H" + +#include +#include +#include // TODO: remove this +#include + +#include + +using namespace amrex::literals; + +// constructor +Timestep::Timestep (const std::string& rd_name) +:ReducedDiags{rd_name} +{ + const auto& warpx = WarpX::GetInstance(); + const auto max_level = warpx.maxLevel(); + + // data size should be equal to the number of refinement levels + m_data.resize(max_level + 1, 0.0_rt); + + if (amrex::ParallelDescriptor::IOProcessor() && m_write_header) { + // open file + std::ofstream ofs{m_path + m_rd_name + "." + m_extension, std::ofstream::out}; + + // write header row + int c = 0; + ofs << "#"; + ofs << "[" << c++ << "]step()"; + ofs << m_sep; + ofs << "[" << c++ << "]time(s)"; + ofs << m_sep; + + for (int lev = 0; lev <= max_level; lev++) { + ofs << "[" << c++ << "]timestep[" << lev << "](s)"; + if (lev < max_level) { + ofs << m_sep; + } + } + + // close file + ofs << "\n"; + ofs.close(); + } +} +// end constructor + +// function to get current simulation timestep at all refinement levels +void Timestep::ComputeDiags (int step) { + // Check if diagnostic should be done + if (!m_intervals.contains(step+1)) { return; } + + const auto& warpx = WarpX::GetInstance(); + const auto max_level = warpx.maxLevel(); + const auto dt = warpx.getdt(); + + for (int lev = 0; lev <= max_level; lev++) { + m_data[lev] = dt[lev]; + } +} +// end Timestep::ComputeDiags diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H deleted file mode 100644 index 570f86d5384..00000000000 --- a/Source/Diagnostics/SliceDiagnostic.H +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2019 Revathi Jambunathan - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_SliceDiagnostic_H_ -#define WARPX_SliceDiagnostic_H_ - -#include - -#include - -#include - -std::unique_ptr CreateSlice( const amrex::MultiFab& mf, - const amrex::Vector &dom_geom, - amrex::RealBox &slice_realbox, - amrex::IntVect &slice_cr_ratio ); - -void CheckSliceInput( amrex::RealBox real_box, - amrex::RealBox &slice_cc_nd_box, amrex::RealBox &slice_realbox, - amrex::IntVect &slice_cr_ratio, amrex::Vector dom_geom, - amrex::IntVect SliceType, amrex::IntVect &slice_lo, - amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); - -void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - const amrex::Vector& geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, - amrex::IntVect SliceType, amrex::RealBox real_box); - -void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, - amrex::IntVect slice_lo, amrex::Vector geom, - int idir, amrex::IntVect IndType, amrex::RealBox slice_realbox, - int srccomp, int ncomp, int nghost, amrex::RealBox real_box); - -#endif diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp deleted file mode 100644 index 97af967f2be..00000000000 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ /dev/null @@ -1,521 +0,0 @@ -/* Copyright 2019-2020 Luca Fedeli, Revathi Jambunathan, Weiqun Zhang - * - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "SliceDiagnostic.H" - -#include "FieldSolver/Fields.H" -#include "Utils/TextMsg.H" -#include "WarpX.H" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -using namespace amrex; -using namespace warpx::fields; - -/* \brief - * The functions creates the slice for diagnostics based on the user-input. - * The slice can be 1D, 2D, or 3D and it inherits the index type of the underlying data. - * The implementation assumes that the slice is aligned with the coordinate axes. - * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. - * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) - * The values are copied from src mf to dst smf using amrex::ParallelCopy - * If interpolation is required, then on the smf, using data points stored in the ghost cells, the data in interpolated. - * If coarsening is required, then a coarse slice multifab is generated (cs_mf) and the - * values of the refined slice (smf) is averaged down to obtain the coarse slice. - * \param mf is the source multifab containing the field data - * \param dom_geom is the geometry of the domain and used in the function to obtain the - * CellSize of the underlying grid. - * \param slice_realbox defines the extent of the slice - * \param slice_cr_ratio provides the coarsening ratio for diagnostics - */ - -std::unique_ptr -CreateSlice( const MultiFab& mf, const Vector &dom_geom, - RealBox &slice_realbox, IntVect &slice_cr_ratio ) -{ - std::unique_ptr smf; - std::unique_ptr cs_mf; - - Vector slice_ncells(AMREX_SPACEDIM); - const int nghost = 1; - auto nlevels = static_cast(dom_geom.size()); - const int ncomp = (mf).nComp(); - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( nlevels==1, - "Slice diagnostics does not work with mesh refinement yet (TO DO)."); - - const auto conversionType = (mf).ixType(); - IntVect SliceType(AMREX_D_DECL(0,0,0)); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) - { - SliceType[idim] = conversionType.nodeCentered(idim); - } - - const RealBox& real_box = dom_geom[0].ProbDomain(); - RealBox slice_cc_nd_box; - const int default_grid_size = 32; - int slice_grid_size = default_grid_size; - - bool interpolate = false; - bool coarsen = false; - - // same index space as domain // - IntVect slice_lo(AMREX_D_DECL(0,0,0)); - IntVect slice_hi(AMREX_D_DECL(1,1,1)); - IntVect interp_lo(AMREX_D_DECL(0,0,0)); - - CheckSliceInput(real_box, slice_cc_nd_box, slice_realbox, slice_cr_ratio, - dom_geom, SliceType, slice_lo, - slice_hi, interp_lo); - int configuration_dim = 0; - // Determine if interpolation is required and number of cells in slice // - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - - // Flag for interpolation if required // - if ( interp_lo[idim] == 1) { - interpolate = true; - } - - // For the case when a dimension is reduced // - if ( ( slice_hi[idim] - slice_lo[idim]) == 1) { - slice_ncells[idim] = 1; - } - else { - slice_ncells[idim] = ( slice_hi[idim] - slice_lo[idim] + 1 ) - / slice_cr_ratio[idim]; - - const int refined_ncells = slice_hi[idim] - slice_lo[idim] + 1 ; - if ( slice_cr_ratio[idim] > 1) { - coarsen = true; - - // modify slice_grid_size if >= refines_cells // - if ( slice_grid_size >= refined_ncells ) { - slice_grid_size = refined_ncells - 1; - } - - } - configuration_dim += 1; - } - } - if (configuration_dim==1) { - ablastr::warn_manager::WMRecordWarning("Diagnostics", - "The slice configuration is 1D and cannot be visualized using yt."); - } - - // Slice generation with index type inheritance // - const Box slice(slice_lo, slice_hi); - - Vector sba(1); - sba[0].define(slice); - sba[0].maxSize(slice_grid_size); - - // Distribution mapping for slice can be different from that of domain // - Vector sdmap(1); - sdmap[0] = DistributionMapping{sba[0]}; - - smf = std::make_unique(amrex::convert(sba[0],SliceType), sdmap[0], - ncomp, nghost); - - // Copy data from domain to slice that has same cell size as that of // - // the domain mf. src and dst have the same number of ghost cells // - const amrex::IntVect nghost_vect(AMREX_D_DECL(nghost, nghost, nghost)); - ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); - - // interpolate if required on refined slice // - if (interpolate) { - InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, - ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); - } - - - if (!coarsen) { - return smf; - } - else { - Vector crse_ba(1); - crse_ba[0] = sba[0]; - crse_ba[0].coarsen(slice_cr_ratio); - - AMREX_ALWAYS_ASSERT(crse_ba[0].size() == sba[0].size()); - - cs_mf = std::make_unique(amrex::convert(crse_ba[0],SliceType), - sdmap[0], ncomp,nghost); - - const MultiFab& mfSrc = *smf; - MultiFab& mfDst = *cs_mf; - - MFIter mfi_dst(mfDst); - for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { - - Array4 const& Src_fabox = mfSrc.const_array(mfi); - - const Box& Dst_bx = mfi_dst.validbox(); - Array4 const& Dst_fabox = mfDst.array(mfi_dst); - - const int scomp = 0; - const int dcomp = 0; - - const IntVect cctype(AMREX_D_DECL(0,0,0)); - if( SliceType==cctype ) { - amrex::amrex_avgdown(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, - ncomp, slice_cr_ratio); - } - const IntVect ndtype(AMREX_D_DECL(1,1,1)); - if( SliceType == ndtype ) { - amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,0).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 0); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,1).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 1); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,2).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 2); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,0).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 0); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,1).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 1); - } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,2).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 2); - } - - if ( mfi_dst.isValid() ) { - ++mfi_dst; - } - - } - return cs_mf; - } -} - - -/* \brief - * This function modifies the slice input parameters under certain conditions. - * The coarsening ratio, slice_cr_ratio is modified if the input is not an exponent of 2. - * for example, if the coarsening ratio is 3, 5 or 6, which is not an exponent of 2, - * then the value of coarsening ratio is modified to the nearest exponent of 2. - * The default value for coarsening ratio is 1. - * slice_realbox.lo and slice_realbox.hi are set equal to the simulation domain lo and hi - * if for the user-input for the slice lo and hi coordinates are outside the domain. - * If the slice_realbox.lo and slice_realbox.hi coordinates do not align with the data - * points and the number of cells in that dimension is greater than 1, and if the extent of - * the slice in that dimension is not coarsenable, then the value lo and hi coordinates are - * shifted to the nearest coarsenable point to include some extra data points in the slice. - * If slice_realbox.lo==slice_realbox.hi, then that dimension has only one cell and no - * modifications are made to the value. If the lo and hi do not align with a data point, - * then it is flagged for interpolation. - * \param real_box a Real box defined for the underlying domain. - * \param slice_realbox a Real box for defining the slice dimension. - * \param slice_cc_nd_box a Real box for defining the modified lo and hi of the slice - * such that the coordinates align with the underlying data points. - * If the dimension is reduced to have only one cell, the slice_realbox is not modified and * instead the values are interpolated to the coordinate from the nearest data points. - * \param slice_cr_ratio contains values of the coarsening ratio which may be modified - * if the input values do not satisfy coarsenability conditions. - * \param slice_lo and slice_hi are the index values of the slice - * \param interp_lo are set to 0 or 1 if they are flagged for interpolation. - * The slice shares the same index space as that of the simulation domain. - */ - - -void -CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, - RealBox &slice_realbox, IntVect &slice_cr_ratio, - Vector dom_geom, IntVect const SliceType, - IntVect &slice_lo, IntVect &slice_hi, IntVect &interp_lo) -{ - - IntVect slice_lo2(AMREX_D_DECL(0,0,0)); - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - // Modify coarsening ratio if the input value is not an exponent of 2 for AMR // - if ( slice_cr_ratio[idim] > 0 ) { - const int log_cr_ratio = - static_cast(floor ( log2( double(slice_cr_ratio[idim])))); - slice_cr_ratio[idim] = - static_cast (exp2( double(log_cr_ratio) )); - } - - //// Default coarsening ratio is 1 // - // Modify lo if input is out of bounds // - if ( slice_realbox.lo(idim) < real_box.lo(idim) ) { - slice_realbox.setLo( idim, real_box.lo(idim)); - std::stringstream warnMsg; - warnMsg << " slice lo is out of bounds. " << - " Modified it in dimension " << idim << - " to be aligned with the domain box."; - ablastr::warn_manager::WMRecordWarning("Diagnostics", - warnMsg.str(), ablastr::warn_manager::WarnPriority::low); - } - - // Modify hi if input in out od bounds // - if ( slice_realbox.hi(idim) > real_box.hi(idim) ) { - slice_realbox.setHi( idim, real_box.hi(idim)); - std::stringstream warnMsg; - warnMsg << " slice hi is out of bounds. " << - " Modified it in dimension " << idim << - " to be aligned with the domain box."; - ablastr::warn_manager::WMRecordWarning("Diagnostics", - warnMsg.str(), ablastr::warn_manager::WarnPriority::low); - } - - const auto very_small_number = 1E-10; - - // Factor to ensure index values computation depending on index type // - const double fac = ( 1.0 - SliceType[idim] )*dom_geom[0].CellSize(idim)*0.5; - // if dimension is reduced to one cell length // - if ( slice_realbox.hi(idim) - slice_realbox.lo(idim) <= 0) - { - slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); - slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - - if ( slice_cr_ratio[idim] > 1) { slice_cr_ratio[idim] = 1; } - - // check for interpolation -- compute index lo with floor and ceil - if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { - slice_lo[idim] = static_cast( - floor( ( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) + fac ) ) - / dom_geom[0].CellSize(idim)) + fac * very_small_number) ); - slice_lo2[idim] = static_cast( - ceil( ( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) + fac) ) - / dom_geom[0].CellSize(idim)) - fac * very_small_number) ); - } - else { - slice_lo[idim] = static_cast( - std::round( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) ) ) - / dom_geom[0].CellSize(idim)) ); - slice_lo2[idim] = static_cast( - std::ceil((slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) ) ) - / dom_geom[0].CellSize(idim) ) ); - } - - // flag for interpolation -- if reduced dimension location // - // does not align with data point // - if ( slice_lo[idim] == slice_lo2[idim]) { - if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) < fac ) { - interp_lo[idim] = 1; - } - } - else { - interp_lo[idim] = 1; - } - - // ncells = 1 if dimension is reduced // - slice_hi[idim] = slice_lo[idim] + 1; - } - else - { - // moving realbox.lo and realbox.hi to nearest coarsenable grid point // - auto index_lo = static_cast(floor(((slice_realbox.lo(idim) + very_small_number - - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))) ); - auto index_hi = static_cast(ceil(((slice_realbox.hi(idim) - very_small_number - - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))) ); - - bool modify_cr = true; - - while ( modify_cr ) { - int lo_new = index_lo; - int hi_new = index_hi; - const int mod_lo = index_lo % slice_cr_ratio[idim]; - const int mod_hi = index_hi % slice_cr_ratio[idim]; - modify_cr = false; - - // To ensure that the index.lo is coarsenable // - if ( mod_lo > 0) { - lo_new = index_lo - mod_lo; - } - // To ensure that the index.hi is coarsenable // - if ( mod_hi > 0) { - hi_new = index_hi + (slice_cr_ratio[idim] - mod_hi); - } - - // If modified index.hi is > baselinebox.hi, move the point // - // to the previous coarsenable point // - const auto small_number = 0.01; - if ( (hi_new * dom_geom[0].CellSize(idim)) - > real_box.hi(idim) - real_box.lo(idim) + dom_geom[0].CellSize(idim)*small_number) - { - hi_new = index_hi - mod_hi; - } - - if ( (hi_new - lo_new) == 0 ){ - std::stringstream warnMsg; - warnMsg << " Coarsening ratio " << slice_cr_ratio[idim] << " in dim "<< idim << - "is leading to zero cells for slice." << " Thus reducing cr_ratio by half.\n"; - - ablastr::warn_manager::WMRecordWarning("Diagnostics", - warnMsg.str()); - - slice_cr_ratio[idim] = slice_cr_ratio[idim]/2; - modify_cr = true; - } - - if ( !modify_cr ) { - index_lo = lo_new; - index_hi = hi_new; - } - slice_lo[idim] = index_lo; - slice_hi[idim] = index_hi - 1; // since default is cell-centered - } - slice_realbox.setLo( idim, index_lo * dom_geom[0].CellSize(idim) - + real_box.lo(idim) ); - slice_realbox.setHi( idim, index_hi * dom_geom[0].CellSize(idim) - + real_box.lo(idim) ); - slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) + Real(fac) ); - slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) - Real(fac) ); - } - } -} - - -/* \brief - * This function is called if the coordinates of the slice do not align with data points - * \param interp_lo is an IntVect which is flagged as 1, if interpolation - is required in the dimension. - */ -void -InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, - const Vector& geom, int ncomp, int nghost, - IntVect slice_lo, IntVect /*slice_hi*/, IntVect SliceType, - const RealBox real_box) -{ - for (MFIter mfi(smf); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); - FArrayBox& fabox = smf[mfi]; - - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( interp_lo[idim] == 1 ) { - InterpolateLo( bx, fabox, slice_lo, geom, idim, SliceType, - slice_realbox, 0, ncomp, nghost, real_box); - } - } - } - -} - -void -InterpolateLo(const Box& bx, FArrayBox &fabox, IntVect slice_lo, - Vector geom, int idir, IntVect IndType, - RealBox slice_realbox, int srccomp, int ncomp, - int /*nghost*/, const RealBox real_box ) -{ - auto fabarr = fabox.array(); - const auto lo = amrex::lbound(bx); - const auto hi = amrex::ubound(bx); - const double fac = ( 1.0-IndType[idir] )*geom[0].CellSize(idir) * 0.5; - const int imin = slice_lo[idir]; - const double minpos = imin*geom[0].CellSize(idir) + fac + real_box.lo(idir); - const double maxpos = (imin+1)*geom[0].CellSize(idir) + fac + real_box.lo(idir); - const double slice_minpos = slice_realbox.lo(idir) ; - - switch (idir) { - case 0: - { - if ( imin >= lo.x && imin <= lo.x) { - for (int n = srccomp; n < srccomp + ncomp; ++n) { - for (int k = lo.z; k <= hi.z; ++k) { - for (int j = lo.y; j <= hi.y; ++j) { - for (int i = lo.x; i <= hi.x; ++i) { - const double minval = fabarr(i,j,k,n); - const double maxval = fabarr(i+1,j,k,n); - const double ratio = (maxval - minval) / (maxpos - minpos); - const double xdiff = slice_minpos - minpos; - const double newval = minval + xdiff * ratio; - fabarr(i,j,k,n) = static_cast(newval); - } - } - } - } - } - break; - } - case 1: - { - if ( imin >= lo.y && imin <= lo.y) { - for (int n = srccomp; n < srccomp+ncomp; ++n) { - for (int k = lo.z; k <= hi.z; ++k) { - for (int j = lo.y; j <= hi.y; ++j) { - for (int i = lo.x; i <= hi.x; ++i) { - const double minval = fabarr(i,j,k,n); - const double maxval = fabarr(i,j+1,k,n); - const double ratio = (maxval - minval) / (maxpos - minpos); - const double xdiff = slice_minpos - minpos; - const double newval = minval + xdiff * ratio; - fabarr(i,j,k,n) = static_cast(newval); - } - } - } - } - } - break; - } - case 2: - { - if ( imin >= lo.z && imin <= lo.z) { - for (int n = srccomp; n < srccomp+ncomp; ++n) { - for (int k = lo.z; k <= hi.z; ++k) { - for (int j = lo.y; j <= hi.y; ++j) { - for (int i = lo.x; i <= hi.x; ++i) { - const double minval = fabarr(i,j,k,n); - const double maxval = fabarr(i,j,k+1,n); - const double ratio = (maxval - minval) / (maxpos - minpos); - const double xdiff = slice_minpos - minpos; - const double newval = minval + xdiff * ratio; - fabarr(i,j,k,n) = static_cast(newval); - } - } - } - } - } - break; - } - - } - -} diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index a17abe04178..e90ae98eb17 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -11,12 +11,15 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "FieldIO.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" #include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include #include @@ -87,6 +90,9 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in void WarpX::InitFromCheckpoint () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + WARPX_PROFILE("WarpX::InitFromCheckpoint()"); amrex::Print()<< Utils::TextMsg::Info( @@ -278,101 +284,101 @@ WarpX::InitFromCheckpoint () for (int lev = 0; lev < nlevs; ++lev) { for (int i = 0; i < 3; ++i) { - current_fp[lev][i]->setVal(0.0); - Efield_fp[lev][i]->setVal(0.0); - Bfield_fp[lev][i]->setVal(0.0); + m_fields.get(FieldType::current_fp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Efield_fp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_fp, Direction{i}, lev)->setVal(0.0); } if (lev > 0) { for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i]->setVal(0.0); - Bfield_aux[lev][i]->setVal(0.0); + m_fields.get(FieldType::Efield_aux, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_aux, Direction{i}, lev)->setVal(0.0); - current_cp[lev][i]->setVal(0.0); - Efield_cp[lev][i]->setVal(0.0); - Bfield_cp[lev][i]->setVal(0.0); + m_fields.get(FieldType::current_cp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Efield_cp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_cp, Direction{i}, lev)->setVal(0.0); } } - VisMF::Read(*Efield_fp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_fp")); - VisMF::Read(*Efield_fp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_fp")); - VisMF::Read(*Efield_fp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_fp")); - VisMF::Read(*Bfield_fp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_fp")); - VisMF::Read(*Bfield_fp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_fp")); - VisMF::Read(*Bfield_fp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Read(*Efield_avg_fp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_fp")); - VisMF::Read(*Efield_avg_fp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_fp")); - VisMF::Read(*Efield_avg_fp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_fp")); } if (is_synchronized) { - VisMF::Read(*current_fp[lev][0], + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jx_fp")); - VisMF::Read(*current_fp[lev][1], + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jy_fp")); - VisMF::Read(*current_fp[lev][2], + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Read(*Efield_cp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_cp")); - VisMF::Read(*Efield_cp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_cp")); - VisMF::Read(*Efield_cp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_cp")); - VisMF::Read(*Bfield_cp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_cp")); - VisMF::Read(*Bfield_cp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_cp")); - VisMF::Read(*Bfield_cp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Read(*Efield_avg_cp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_cp")); - VisMF::Read(*Efield_avg_cp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_cp")); - VisMF::Read(*Efield_avg_cp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][0], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][1], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][2], + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_cp")); } if (is_synchronized) { - VisMF::Read(*current_cp[lev][0], + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jx_cp")); - VisMF::Read(*current_cp[lev][1], + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jy_cp")); - VisMF::Read(*current_cp[lev][2], + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jz_cp")); } } @@ -383,20 +389,30 @@ WarpX::InitFromCheckpoint () { for (int lev = 0; lev < nlevs; ++lev) { if (pml[lev]) { - pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + pml[lev]->Restart(m_fields, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + pml_rz[lev]->Restart(m_fields, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); } #endif } } - InitializeEBGridData(maxLevel()); + if (EB::enabled()) { InitializeEBGridData(maxLevel()); } + + reduced_diags->ReadCheckpointData(restart_chkfile); // Initialize particles mypc->AllocData(); mypc->Restart(restart_chkfile); + if (m_implicit_solver) { + + m_implicit_solver->Define(this); + m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme, + particle_tol_in_implicit_scheme ); + m_implicit_solver->CreateParticleAttributes(); + } + } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index e3eaef33c44..63991ccfe37 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -10,7 +10,6 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "FieldIO.H" #include "Particles/Filter/FilterFunctors.H" -#include "Particles/NamedComponentParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/RelativeCellPosition.H" @@ -466,7 +465,7 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) std::string const filename = GetFileName(filepath); std::ofstream pv_helper_file(m_dirPrefix + "/paraview.pmd"); - pv_helper_file << filename << std::endl; + pv_helper_file << filename << "\n"; pv_helper_file.close(); } } @@ -531,10 +530,10 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& part { WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); -for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { +for (const auto & particle_diag : particle_diags) { - WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); - PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); + WarpXParticleContainer* pc = particle_diag.getParticleContainer(); + PinnedMemoryParticleContainer* pinned_pc = particle_diag.getPinnedParticleContainer(); if (isBTD || use_pinned_pc) { if (!pinned_pc->isDefined()) { continue; // Skip to the next particle container @@ -548,17 +547,17 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { auto rtmap = pc->getParticleComps(); const auto mass = pc->AmIA() ? PhysConst::m_e : pc->getMass(); - RandomFilter const random_filter(particle_diags[i].m_do_random_filter, - particle_diags[i].m_random_fraction); - UniformFilter const uniform_filter(particle_diags[i].m_do_uniform_filter, - particle_diags[i].m_uniform_stride); - ParserFilter parser_filter(particle_diags[i].m_do_parser_filter, + RandomFilter const random_filter(particle_diag.m_do_random_filter, + particle_diag.m_random_fraction); + UniformFilter const uniform_filter(particle_diag.m_do_uniform_filter, + particle_diag.m_uniform_stride); + ParserFilter parser_filter(particle_diag.m_do_parser_filter, utils::parser::compileParser (particle_diags[i].m_particle_filter_parser.get()), pc->getMass(), time, rtmap); parser_filter.m_units = InputUnits::SI; - GeometryFilter const geometry_filter(particle_diags[i].m_do_geom_filter, - particle_diags[i].m_diag_domain); + GeometryFilter const geometry_filter(particle_diag.m_do_geom_filter, + particle_diag.m_diag_domain); if (isBTD || use_pinned_pc) { particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); @@ -589,54 +588,62 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { } // Gather the electrostatic potential (phi) on the macroparticles - if ( particle_diags[i].m_plot_phi ) { + if ( particle_diag.m_plot_phi ) { storePhiOnParticles( tmp, WarpX::electrostatic_solver_id, !use_pinned_pc ); } - // names of amrex::Real and int particle attributes in SoA data + // names of amrex::ParticleReal and int particle attributes in SoA data + auto const rn = tmp.GetRealSoANames(); + auto const in = tmp.GetIntSoANames(); amrex::Vector real_names; - amrex::Vector int_names; - amrex::Vector int_flags; - amrex::Vector real_flags; - // see openPMD ED-PIC extension for namings - // note: an underscore separates the record name from its component - // for non-scalar records - // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX + amrex::Vector int_names(in.begin(), in.end()); + + // transform names to openPMD, separated by underscores + { + // see openPMD ED-PIC extension for namings + // note: an underscore separates the record name from its component + // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX #if !defined (WARPX_DIM_1D_Z) - real_names.push_back("position_x"); + real_names.push_back("position_x"); #endif #if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - real_names.push_back("position_y"); + real_names.push_back("position_y"); #endif - real_names.push_back("position_z"); - real_names.push_back("weighting"); - real_names.push_back("momentum_x"); - real_names.push_back("momentum_y"); - real_names.push_back("momentum_z"); - // get the names of the real comps - real_names.resize(tmp.NumRealComps()); - auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) + real_names.push_back("position_z"); + real_names.push_back("weighting"); + real_names.push_back("momentum_x"); + real_names.push_back("momentum_y"); + real_names.push_back("momentum_z"); + } + for (size_t i = real_names.size(); i < rn.size(); ++i) { - real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); + real_names.push_back(rn[i]); } + + for (size_t i = PIdx::nattribs; i < rn.size(); ++i) + { + real_names[i] = detail::snakeToCamel(rn[i]); + } + // plot any "extra" fields by default - real_flags = particle_diags[i].m_plot_flags; + amrex::Vector real_flags = particle_diag.m_plot_flags; real_flags.resize(tmp.NumRealComps(), 1); - // and the names - int_names.resize(tmp.NumIntComps()); - auto runtime_inames = tmp.getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) + + // and the int names + for (size_t i = 0; i < in.size(); ++i) { - int_names[x.second+0] = detail::snakeToCamel(x.first); + int_names[i] = detail::snakeToCamel(in[i]); } + // plot by default + amrex::Vector int_flags; int_flags.resize(tmp.NumIntComps(), 1); // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. DumpToFile(&tmp, - particle_diags.at(i).getSpeciesName(), + particle_diag.getSpeciesName(), m_CurrentStep, real_flags, int_flags, @@ -1197,6 +1204,8 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, if (WarpX::do_dive_cleaning) { meshes.setAttribute("chargeCorrectionParameters", "period=1"); } + // TODO set meta-data information for time-averaged quantities + // but we need information of the specific diagnostic in here } diff --git a/Source/Diagnostics/requirements.txt b/Source/Diagnostics/requirements.txt index d9f5cb553ee..9c85c8a621d 100644 --- a/Source/Diagnostics/requirements.txt +++ b/Source/Diagnostics/requirements.txt @@ -5,4 +5,4 @@ # License: BSD-3-Clause-LBNL # keep this entry for GitHub's dependency graph -openPMD-api>=0.15.1 +openPMD-api>=0.16.1 diff --git a/Source/EmbeddedBoundary/CMakeLists.txt b/Source/EmbeddedBoundary/CMakeLists.txt index 3fa0ea0228f..909886bbad6 100644 --- a/Source/EmbeddedBoundary/CMakeLists.txt +++ b/Source/EmbeddedBoundary/CMakeLists.txt @@ -2,6 +2,8 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + EmbeddedBoundaryInit.cpp + Enabled.cpp WarpXInitEB.cpp WarpXFaceExtensions.cpp WarpXFaceInfoBox.H diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index 7ee47c1172c..0b27fd054cd 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -14,8 +14,6 @@ #include #include -#ifdef AMREX_USE_EB - namespace DistanceToEB { @@ -28,7 +26,8 @@ amrex::Real dot_product (const amrex::RealVect& a, const amrex::RealVect& b) noe AMREX_GPU_HOST_DEVICE AMREX_INLINE void normalize (amrex::RealVect& a) noexcept { - amrex::Real inv_norm = 1.0/std::sqrt(dot_product(a,a)); + using namespace amrex::literals; + amrex::Real const inv_norm = 1.0_rt / std::sqrt(dot_product(a, a)); AMREX_D_DECL(a[0] *= inv_norm, a[1] *= inv_norm, a[2] *= inv_norm); @@ -46,6 +45,7 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP amrex::Array4 const& phi, amrex::GpuArray const& dxi) noexcept { + using namespace amrex::literals; #if (defined WARPX_DIM_3D) amrex::RealVect normal{0.0, 0.0, 0.0}; @@ -53,11 +53,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int kk = 0; kk < 2; ++kk) { for (int jj=0; jj< 2; ++jj) { for (int ii = 0; ii < 2; ++ii) { - int icstart = ic + iic; - amrex::Real sign = (ii%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(jj%2); - int w2comp = static_cast(kk%2); + int const icstart = ic + iic; + amrex::Real const sign = (ii%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(jj%2); + int const w2comp = static_cast(kk%2); normal[0] += sign * phi(icstart + ii, j + jj, k + kk) * dxi[0] * Wc[0][wccomp] * W[1][w1comp] * W[2][w2comp]; } } @@ -67,11 +67,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int kk = 0; kk < 2; ++kk) { for (int ii=0; ii< 2; ++ii) { for (int jj = 0; jj < 2; ++jj) { - int jcstart = jc + iic; - amrex::Real sign = (jj%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); - int w2comp = static_cast(kk%2); + int const jcstart = jc + iic; + amrex::Real const sign = (jj%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); + int const w2comp = static_cast(kk%2); normal[1] += sign * phi(i + ii, jcstart + jj, k + kk) * dxi[1] * W[0][w1comp] * Wc[1][wccomp] * W[2][w2comp]; } } @@ -81,11 +81,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int jj = 0; jj < 2; ++jj) { for (int ii=0; ii< 2; ++ii) { for (int kk = 0; kk < 2; ++kk) { - int kcstart = kc + iic; - amrex::Real sign = (kk%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); - int w2comp = static_cast(jj%2); + int const kcstart = kc + iic; + amrex::Real const sign = (kk%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); + int const w2comp = static_cast(jj%2); normal[2] += sign * phi(i + ii, j + jj, kcstart + kk) * dxi[2] * W[0][w1comp] * W[1][w2comp] * Wc[2][wccomp]; } } @@ -97,10 +97,10 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int iic = 0; iic < 2; ++iic) { for (int jj=0; jj< 2; ++jj) { for (int ii = 0; ii < 2; ++ii) { - int icstart = ic + iic; - amrex::Real sign = (ii%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(jj%2); + int const icstart = ic + iic; + amrex::Real const sign = (ii%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(jj%2); normal[0] += sign * phi(icstart + ii, j + jj, k) * dxi[0] * Wc[0][wccomp] * W[1][w1comp]; } } @@ -108,10 +108,10 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int iic = 0; iic < 2; ++iic) { for (int ii=0; ii< 2; ++ii) { for (int jj = 0; jj < 2; ++jj) { - int jcstart = jc + iic; - amrex::Real sign = (jj%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); + int const jcstart = jc + iic; + amrex::Real const sign = (jj%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); normal[1] += sign * phi(i + ii, jcstart + jj, k) * dxi[1] * W[0][w1comp] * Wc[1][wccomp]; } } @@ -120,13 +120,17 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP #else amrex::ignore_unused(i, j, k, ic, jc, kc, W, Wc, phi, dxi); - amrex::RealVect normal{0.0, 0.0}; - WARPX_ABORT_WITH_MESSAGE("Error: interp_distance not yet implemented in 1D"); + amrex::RealVect normal(0.0); + + AMREX_IF_ON_DEVICE(( + AMREX_DEVICE_ASSERT(0); + )) + AMREX_IF_ON_HOST(( + WARPX_ABORT_WITH_MESSAGE("Error: interp_normal not yet implemented in 1D"); + )) #endif return normal; } } - -#endif // AMREX_USE_EB #endif // WARPX_DISTANCETOEB_H_ diff --git a/Source/EmbeddedBoundary/EmbeddedBoundaryInit.H b/Source/EmbeddedBoundary/EmbeddedBoundaryInit.H new file mode 100644 index 00000000000..ed29fe5b688 --- /dev/null +++ b/Source/EmbeddedBoundary/EmbeddedBoundaryInit.H @@ -0,0 +1,141 @@ +/* Copyright 2021-2025 Lorenzo Giacomel, Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_EMBEDDED_BOUNDARY_EMBEDDED_BOUNDARY_INIT_H_ +#define WARPX_EMBEDDED_BOUNDARY_EMBEDDED_BOUNDARY_INIT_H_ + +#include "Enabled.H" + +#ifdef AMREX_USE_EB + +#include + +#include +#include +#include +#include + +#include + +namespace warpx::embedded_boundary +{ + + /** \brief Set a flag to indicate in which cells a particle should deposit charge/current + * with a reduced, order 1 shape. + * + * More specifically, the flag is set to 1 if any of the neighboring cells over which the + * particle shape might extend are either partially or fully covered by an embedded boundary. + * This ensures that a particle in this cell deposits with an order 1 shape, which in turn + * makes sure that the particle never deposits any charge in a partially or fully covered cell. + * + * \param[in] eb_reduce_particle_shape multifab to be filled with 1s and 0s + * \param[in] eb_fact EB factory + * \param[in] particle_shape_order order of the particle shape function + * \param[in] periodicity TODO Geom(0).periodicity() + */ + void MarkReducedShapeCells ( + std::unique_ptr & eb_reduce_particle_shape, + amrex::EBFArrayBoxFactory const & eb_fact, + int particle_shape_order, + const amrex::Periodicity& periodicity); + + /** \brief Set a flag to indicate on which grid points the field `field` + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by all finite-difference solvers, except the + * ECT solver, which instead uses `MarkUpdateECellsECT` and `MarkUpdateBCellsECT`. + * It uses a stair-case approximation of the embedded boundary: + * If a grid point touches cells that are either partially or fully covered + * by the embedded boundary: the corresponding field is not updated. + * + * More specifically, this function fills the iMultiFabs in `eb_update` + * (which have the same indexType as the MultiFabs in `field`) with 1 + * or 0, depending on whether the grid point should be updated or not. + */ + void MarkUpdateCellsStairCase ( + std::array< std::unique_ptr,3> & eb_update, + ablastr::fields::VectorField const & field, + amrex::EBFArrayBoxFactory const & eb_fact ); + + /** \brief Set a flag to indicate on which grid points the E field + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by ECT solver. The E field is not updated if + * the edge on which it is defined is fully covered by the embedded boundary. + * + * More specifically, this function fills the iMultiFabs in `eb_update_E` + * (which have the same indexType as the E field) with 1 or 0, depending + * on whether the grid point should be updated or not. + */ + void MarkUpdateECellsECT ( + std::array< std::unique_ptr,3> & eb_update_E, + ablastr::fields::VectorField const& edge_lengths ); + + /** \brief Set a flag to indicate on which grid points the B field + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by ECT solver. The B field is not updated if + * the face on which it is defined is fully covered by the embedded boundary. + * + * More specifically, this function fills the iMultiFabs in `eb_update_B` + * (which have the same indexType as the B field) with 1 or 0, depending + * on whether the grid point should be updated or not. + */ + void MarkUpdateBCellsECT ( + std::array< std::unique_ptr,3> & eb_update_B, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& edge_lengths ); + + /** + * \brief Initialize information for cell extensions. + * The flags convention for m_flag_info_face is as follows + * - 0 for unstable cells + * - 1 for stable cells which have not been intruded + * - 2 for stable cells which have been intruded + * Here we cannot know if a cell is intruded or not so we initialize all stable cells with 1 + */ + void MarkExtensionCells( + const std::array& cell_size, + std::array< std::unique_ptr, 3 > & flag_info_face, + std::array< std::unique_ptr, 3 > & flag_ext_face, + const ablastr::fields::VectorField& b_field, + const ablastr::fields::VectorField& face_areas, + const ablastr::fields::VectorField& edge_lengths, + const ablastr::fields::VectorField& area_mod); + + /** + * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. + * An edge of length 0 is fully covered. + */ + void ComputeEdgeLengths ( + ablastr::fields::VectorField& edge_lengths, + const amrex::EBFArrayBoxFactory& eb_fact); + /** + * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. + * An edge of area 0 is fully covered. + */ + void ComputeFaceAreas ( + ablastr::fields::VectorField& face_areas, + const amrex::EBFArrayBoxFactory& eb_fact); + + /** + * \brief Scale the edges lengths by the mesh width to obtain the real lengths. + */ + void ScaleEdges ( + ablastr::fields::VectorField& edge_lengths, + const std::array& cell_size); + /** + * \brief Scale the edges areas by the mesh width to obtain the real areas. + */ + void ScaleAreas ( + ablastr::fields::VectorField& face_areas, + const std::array& cell_size); +} + +#endif + +#endif //WARPX_EMBEDDED_BOUNDARY_EMBEDDED_BOUNDARY_H_ diff --git a/Source/EmbeddedBoundary/EmbeddedBoundaryInit.cpp b/Source/EmbeddedBoundary/EmbeddedBoundaryInit.cpp new file mode 100644 index 00000000000..6a4caec2e99 --- /dev/null +++ b/Source/EmbeddedBoundary/EmbeddedBoundaryInit.cpp @@ -0,0 +1,614 @@ +/* Copyright 2021-2025 Lorenzo Giacomel, Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "Enabled.H" + +#ifdef AMREX_USE_EB + +#include "EmbeddedBoundaryInit.H" + +#include "Fields.H" +#include "Utils/TextMsg.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace web = warpx::embedded_boundary; + +void +web::MarkReducedShapeCells ( + std::unique_ptr & eb_reduce_particle_shape, + amrex::EBFArrayBoxFactory const & eb_fact, + int const particle_shape_order, + const amrex::Periodicity& periodicity) +{ + // Pre-fill array with 0, including in the ghost cells outside of the domain. + // (The guard cells in the domain will be updated by `FillBoundary` at the end of this function.) + eb_reduce_particle_shape->setVal(0, eb_reduce_particle_shape->nGrow()); + + // Extract structures for embedded boundaries + amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*eb_reduce_particle_shape); mfi.isValid(); ++mfi) { + + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const & eb_reduce_particle_shape_arr = eb_reduce_particle_shape->array(mfi); + + // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells + const amrex::Box eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); + amrex::FabType const fab_type = eb_flag[mfi].getType( eb_info_box ); + + if (fab_type == amrex::FabType::regular) { // All cells in the box are regular + + // Every cell in box is regular: do not reduce particle shape in any cell + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_reduce_particle_shape_arr(i, j, k) = 0; + }); + + } else if (fab_type == amrex::FabType::covered) { // All cells in the box are covered + + // Every cell in box is fully covered: reduce particle shape + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_reduce_particle_shape_arr(i, j, k) = 1; + }); + + } else { // The box contains a mix of covered and regular cells + + auto const & flag = eb_flag[mfi].array(); + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + + // Check if any of the neighboring cells over which the particle shape might extend + // are either partially or fully covered. In this case, set eb_reduce_particle_shape_arr + // to one for this cell, to indicate that the particle should use an order 1 shape + // (This ensures that the particle never deposits any charge in a partially or + // fully covered cell, even with higher-order shapes) + // Note: in the code below `particle_shape_order/2` corresponds to the number of neighboring cells + // over which the shape factor could extend, in each direction. + int const i_start = i-particle_shape_order/2; + int const i_end = i+particle_shape_order/2; +#if AMREX_SPACEDIM > 1 + int const j_start = j-particle_shape_order/2; + int const j_end = j+particle_shape_order/2; +#else + int const j_start = j; + int const j_end = j; +#endif +#if AMREX_SPACEDIM > 2 + int const k_start = k-particle_shape_order/2; + int const k_end = k+particle_shape_order/2; +#else + int const k_start = k; + int const k_end = k; +#endif + int reduce_shape = 0; + for (int i_cell = i_start; i_cell <= i_end; ++i_cell) { + for (int j_cell = j_start; j_cell <= j_end; ++j_cell) { + for (int k_cell = k_start; k_cell <= k_end; ++k_cell) { + // `isRegular` returns `false` if the cell is either partially or fully covered. + if ( !flag(i_cell, j_cell, k_cell).isRegular() ) { + reduce_shape = 1; + } + } + } + } + eb_reduce_particle_shape_arr(i, j, k) = reduce_shape; + }); + + } + + } + // FillBoundary to set the values in the guard cells + eb_reduce_particle_shape->FillBoundary(periodicity); +} + +void +web::MarkUpdateCellsStairCase ( + std::array< std::unique_ptr,3> & eb_update, + ablastr::fields::VectorField const& field, + amrex::EBFArrayBoxFactory const & eb_fact ) +{ + + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + // Extract structures for embedded boundaries + amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); + + for (int idim = 0; idim < 3; ++idim) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*field[idim]); mfi.isValid(); ++mfi) { + + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const & eb_update_arr = eb_update[idim]->array(mfi); + + // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells + const amrex::Box eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); + amrex::FabType const fab_type = eb_flag[mfi].getType( eb_info_box ); + + if (fab_type == amrex::FabType::regular) { // All cells in the box are regular + + // Every cell in box is regular: update field in every cell + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_update_arr(i, j, k) = 1; + }); + + } else if (fab_type == amrex::FabType::covered) { // All cells in the box are covered + + // Every cell in box is fully covered: do not update field + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_update_arr(i, j, k) = 0; + }); + + } else { // The box contains a mix of covered and regular cells + + auto const & flag = eb_flag[mfi].array(); + auto index_type = field[idim]->ixType(); + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + + // Stair-case approximation: If neighboring cells of this gridpoint + // are either partially or fully covered: do not update field + + // The number of cells that we need to check depend on the index type + // of the `eb_update_arr` in each direction. + // If `eb_update_arr` is nodal in a given direction, we need to check the cells + // to the left and right of this nodal gridpoint. + // For instance, if `eb_update_arr` is nodal in the first dimension, we need + // to check the cells at index i-1 and at index i, since, with AMReX indexing conventions, + // these are the neighboring cells for the nodal gripoint at index i. + // If `eb_update_arr` is cell-centerd in a given direction, we only need to check + // the cell at the same position (e.g., in the first dimension: the cell at index i). + int const i_start = ( index_type.nodeCentered(0) )? i-1 : i; +#if AMREX_SPACEDIM > 1 + int const j_start = ( index_type.nodeCentered(1) )? j-1 : j; +#else + int const j_start = j; +#endif +#if AMREX_SPACEDIM > 2 + int const k_start = ( index_type.nodeCentered(2) )? k-1 : k; +#else + int const k_start = k; +#endif + // Loop over neighboring cells + int eb_update_flag = 1; + for (int i_cell = i_start; i_cell <= i; ++i_cell) { + for (int j_cell = j_start; j_cell <= j; ++j_cell) { + for (int k_cell = k_start; k_cell <= k; ++k_cell) { + // If one of the neighboring is either partially or fully covered + // (i.e. if they are not regular cells), do not update field + // (`isRegular` returns `false` if the cell is either partially or fully covered.) + if ( !flag(i_cell, j_cell, k_cell).isRegular() ) { + eb_update_flag = 0; + } + } + } + } + eb_update_arr(i, j, k) = eb_update_flag; + }); + + } + + } + + } + +} + +void +web::MarkUpdateECellsECT ( + std::array< std::unique_ptr,3> & eb_update_E, + ablastr::fields::VectorField const& edge_lengths ) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*eb_update_E[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + const amrex::Box& tbx = mfi.tilebox( eb_update_E[0]->ixType().toIntVect(), eb_update_E[0]->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( eb_update_E[1]->ixType().toIntVect(), eb_update_E[1]->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( eb_update_E[2]->ixType().toIntVect(), eb_update_E[2]->nGrowVect() ); + + amrex::Array4 const & eb_update_Ex_arr = eb_update_E[0]->array(mfi); + amrex::Array4 const & eb_update_Ey_arr = eb_update_E[1]->array(mfi); + amrex::Array4 const & eb_update_Ez_arr = eb_update_E[2]->array(mfi); + + amrex::Array4 const & lx_arr = edge_lengths[0]->array(mfi); + amrex::Array4 const & lz_arr = edge_lengths[2]->array(mfi); +#if defined(WARPX_DIM_3D) + amrex::Array4 const & ly_arr = edge_lengths[1]->array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Dim3 const lx_lo = amrex::lbound(lx_arr); + amrex::Dim3 const lx_hi = amrex::ubound(lx_arr); + amrex::Dim3 const lz_lo = amrex::lbound(lz_arr); + amrex::Dim3 const lz_hi = amrex::ubound(lz_arr); +#endif + + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update Ex if the edge on which it lives is fully covered + eb_update_Ex_arr(i, j, k) = (lx_arr(i, j, k) == 0)? 0 : 1; + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: Do not update Ey if the edge on which it lives is fully covered + eb_update_Ey_arr(i, j, k) = (ly_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // In XZ and RZ: Ey is associated with a mesh node, + // so we need to check if the mesh node is covered + if((lx_arr(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)==0) + ||(lx_arr(std::max(i-1, lx_lo.x), std::min(j , lx_hi.y), k)==0) + ||(lz_arr(std::min(i , lz_hi.x), std::min(j , lz_hi.y), k)==0) + ||(lz_arr(std::min(i , lz_hi.x), std::max(j-1, lz_lo.y), k)==0)) { + eb_update_Ey_arr(i, j, k) = 0; + } else { + eb_update_Ey_arr(i, j, k) = 1; + } +#endif + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update Ez if the edge on which it lives is fully covered + eb_update_Ez_arr(i, j, k) = (lz_arr(i, j, k) == 0)? 0 : 1; + } + ); + + } +} + +void +web::MarkUpdateBCellsECT ( + std::array< std::unique_ptr,3> & eb_update_B, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& edge_lengths ) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*eb_update_B[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + const amrex::Box& tbx = mfi.tilebox( eb_update_B[0]->ixType().toIntVect(), eb_update_B[0]->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( eb_update_B[1]->ixType().toIntVect(), eb_update_B[1]->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( eb_update_B[2]->ixType().toIntVect(), eb_update_B[2]->nGrowVect() ); + + amrex::Array4 const & eb_update_Bx_arr = eb_update_B[0]->array(mfi); + amrex::Array4 const & eb_update_By_arr = eb_update_B[1]->array(mfi); + amrex::Array4 const & eb_update_Bz_arr = eb_update_B[2]->array(mfi); + +#ifdef WARPX_DIM_3D + amrex::Array4 const & Sx_arr = face_areas[0]->array(mfi); + amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); + amrex::Array4 const & Sz_arr = face_areas[2]->array(mfi); + amrex::ignore_unused(edge_lengths); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); + amrex::Array4 const & lx_arr = edge_lengths[0]->array(mfi); + amrex::Array4 const & lz_arr = edge_lengths[2]->array(mfi); +#endif + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: do not update Bx if the face on which it lives is fully covered + eb_update_Bx_arr(i, j, k) = (Sx_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + //In XZ and RZ, Bx lives on a z-edge ; do not update if fully covered + eb_update_Bx_arr(i, j, k) = (lz_arr(i, j, k) == 0)? 0 : 1; +#endif + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update By if the face on which it lives is fully covered + eb_update_By_arr(i, j, k) = (Sy_arr(i, j, k) == 0)? 0 : 1; + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: do not update Bz if the face on which it lives is fully covered + eb_update_Bz_arr(i, j, k) = (Sz_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + //In XZ and RZ, Bz lives on a x-edge ; do not update if fully covered + eb_update_Bz_arr(i, j, k) = (lx_arr(i, j, k) == 0)? 0 : 1; +#endif + } + ); + + } +} + +void +web::MarkExtensionCells ( + const std::array& cell_size, + std::array< std::unique_ptr, 3 > & flag_info_face, + std::array< std::unique_ptr, 3 > & flag_ext_face, + const ablastr::fields::VectorField& b_field, + const ablastr::fields::VectorField& face_areas, + const ablastr::fields::VectorField& edge_lengths, + const ablastr::fields::VectorField& area_mod) +{ + using ablastr::fields::Direction; + using warpx::fields::FieldType; + +#ifdef WARPX_DIM_RZ + amrex::ignore_unused(cell_size, flag_info_face, flag_ext_face, b_field, + face_areas, edge_lengths, area_mod); + +#elif !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) + + WARPX_ABORT_WITH_MESSAGE("MarkExtensionCells only implemented in 2D and 3D"); + +#else + + for (int idim = 0; idim < 3; ++idim) { + +# if defined(WARPX_DIM_XZ) + if (idim == 0 || idim == 2) { + flag_info_face[idim]->setVal(0.); + flag_ext_face[idim]->setVal(0.); + continue; + } +# endif + for (amrex::MFIter mfi(*b_field[idim]); mfi.isValid(); ++mfi) { + auto* face_areas_idim_max_lev = face_areas[idim]; + + const amrex::Box& box = mfi.tilebox(face_areas_idim_max_lev->ixType().toIntVect(), + face_areas_idim_max_lev->nGrowVect() ); + + auto const& S = face_areas_idim_max_lev->array(mfi); + auto const& flag_info_face_data = flag_info_face[idim]->array(mfi); + auto const& flag_ext_face_data = flag_ext_face[idim]->array(mfi); + auto const& lx = edge_lengths[0]->array(mfi); + auto const& ly = edge_lengths[1]->array(mfi); + auto const& lz = edge_lengths[2]->array(mfi); + auto const& mod_areas_dim_data = area_mod[idim]->array(mfi); + + const amrex::Real dx = cell_size[0]; + const amrex::Real dy = cell_size[1]; + const amrex::Real dz = cell_size[2]; + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Minimal area for this cell to be stable + mod_areas_dim_data(i, j, k) = S(i, j, k); + double S_stab; + if (idim == 0){ + S_stab = 0.5 * std::max({ly(i, j, k) * dz, ly(i, j, k + 1) * dz, + lz(i, j, k) * dy, lz(i, j + 1, k) * dy}); + }else if (idim == 1){ + +# if defined(WARPX_DIM_XZ) + S_stab = 0.5 * std::max({lx(i, j, k) * dz, lx(i, j + 1, k) * dz, + lz(i, j, k) * dx, lz(i + 1, j, k) * dx}); +# else + S_stab = 0.5 * std::max({lx(i, j, k) * dz, lx(i, j, k + 1) * dz, + lz(i, j, k) * dx, lz(i + 1, j, k) * dx}); +# endif + }else { + S_stab = 0.5 * std::max({lx(i, j, k) * dy, lx(i, j + 1, k) * dy, + ly(i, j, k) * dx, ly(i + 1, j, k) * dx}); + } + + // Does this face need to be extended? + // The difference between flag_info_face and flag_ext_face is that: + // - for every face flag_info_face contains a: + // * 0 if the face needs to be extended + // * 1 if the face is large enough to lend area to other faces + // * 2 if the face is actually intruded by other face + // Here we only take care of the first two cases. The entries corresponding + // to the intruded faces are going to be set in the function ComputeFaceExtensions + // - for every face flag_ext_face contains a: + // * 1 if the face needs to be extended + // * 0 otherwise + // In the function ComputeFaceExtensions, after the cells are extended, the + // corresponding entries in flag_ext_face are set to zero. This helps to keep + // track of which cells could not be extended + flag_ext_face_data(i, j, k) = int(S(i, j, k) < S_stab && S(i, j, k) > 0); + if(flag_ext_face_data(i, j, k)){ + flag_info_face_data(i, j, k) = 0; + } + // Is this face available to lend area to other faces? + // The criterion is that the face has to be interior and not already unstable itself + if(int(S(i, j, k) > 0 && !flag_ext_face_data(i, j, k))) { + flag_info_face_data(i, j, k) = 1; + } + }); + } + } +#endif +} + +void +web::ComputeEdgeLengths ( + ablastr::fields::VectorField& edge_lengths, + const amrex::EBFArrayBoxFactory& eb_fact) +{ + BL_PROFILE("ComputeEdgeLengths"); + +#if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) && !defined(WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE("ComputeEdgeLengths only implemented in 2D and 3D"); +#endif + + auto const &flags = eb_fact.getMultiEBCellFlagFab(); + auto const &edge_centroid = eb_fact.getEdgeCent(); + for (int idim = 0; idim < 3; ++idim){ +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + if (idim == 1) { + edge_lengths[1]->setVal(0.); + continue; + } +#endif + for (amrex::MFIter mfi(flags); mfi.isValid(); ++mfi){ + amrex::Box const box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), + edge_lengths[idim]->nGrowVect()); + amrex::FabType const fab_type = flags[mfi].getType(box); + auto const &edge_lengths_dim = edge_lengths[idim]->array(mfi); + + if (fab_type == amrex::FabType::regular) { + // every cell in box is all regular + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + edge_lengths_dim(i, j, k) = 1.; + }); + } else if (fab_type == amrex::FabType::covered) { + // every cell in box is all covered + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + edge_lengths_dim(i, j, k) = 0.; + }); + } else { +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + int idim_amrex = idim; + if (idim == 2) { idim_amrex = 1; } + auto const &edge_cent = edge_centroid[idim_amrex]->const_array(mfi); +#elif defined(WARPX_DIM_3D) + auto const &edge_cent = edge_centroid[idim]->const_array(mfi); +#endif + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (edge_cent(i, j, k) == amrex::Real(-1.0)) { + // This edge is all covered + edge_lengths_dim(i, j, k) = 0.; + } else if (edge_cent(i, j, k) == amrex::Real(1.0)) { + // This edge is all open + edge_lengths_dim(i, j, k) = 1.; + } else { + // This edge is cut. + edge_lengths_dim(i, j, k) = 1 - amrex::Math::abs(amrex::Real(2.0) + * edge_cent(i, j, k)); + } + + }); + } + } + } +} + + +void +web::ComputeFaceAreas ( + ablastr::fields::VectorField& face_areas, + const amrex::EBFArrayBoxFactory& eb_fact) +{ + BL_PROFILE("ComputeFaceAreas"); + +#if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) && !defined(WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE("ComputeFaceAreas only implemented in 2D and 3D"); +#endif + + auto const &flags = eb_fact.getMultiEBCellFlagFab(); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + //In 2D the volume frac is actually the area frac. + auto const &area_frac = eb_fact.getVolFrac(); +#elif defined(WARPX_DIM_3D) + auto const &area_frac = eb_fact.getAreaFrac(); +#endif + + for (int idim = 0; idim < 3; ++idim) { +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + if (idim == 0 || idim == 2) { + face_areas[idim]->setVal(0.); + continue; + } +#endif + for (amrex::MFIter mfi(flags); mfi.isValid(); ++mfi) { + amrex::Box const box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), + face_areas[idim]->nGrowVect()); + amrex::FabType const fab_type = flags[mfi].getType(box); + auto const &face_areas_dim = face_areas[idim]->array(mfi); + if (fab_type == amrex::FabType::regular) { + // every cell in box is all regular + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + face_areas_dim(i, j, k) = amrex::Real(1.); + }); + } else if (fab_type == amrex::FabType::covered) { + // every cell in box is all covered + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + face_areas_dim(i, j, k) = amrex::Real(0.); + }); + } else { +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + auto const &face = area_frac.const_array(mfi); +#elif defined(WARPX_DIM_3D) + auto const &face = area_frac[idim]->const_array(mfi); +#endif + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + face_areas_dim(i, j, k) = face(i, j, k); + }); + } + } + } +} + +void +web::ScaleEdges ( + ablastr::fields::VectorField& edge_lengths, + const std::array& cell_size) +{ + BL_PROFILE("ScaleEdges"); + +#if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) && !defined(WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE("ScaleEdges only implemented in 2D and 3D"); +#endif + + for (int idim = 0; idim < 3; ++idim){ +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + if (idim == 1) { continue; } +#endif + for (amrex::MFIter mfi(*edge_lengths[0]); mfi.isValid(); ++mfi) { + const amrex::Box& box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), + edge_lengths[idim]->nGrowVect() ); + auto const &edge_lengths_dim = edge_lengths[idim]->array(mfi); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + edge_lengths_dim(i, j, k) *= cell_size[idim]; + }); + } + } +} + + +void +web::ScaleAreas ( + ablastr::fields::VectorField& face_areas, + const std::array& cell_size) +{ + BL_PROFILE("ScaleAreas"); + +#if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) && !defined(WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE("ScaleAreas only implemented in 2D and 3D"); +#endif + + for (int idim = 0; idim < 3; ++idim) { +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + if (idim == 0 || idim == 2) { continue; } +#endif + for (amrex::MFIter mfi(*face_areas[0]); mfi.isValid(); ++mfi) { + const amrex::Box& box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), + face_areas[idim]->nGrowVect() ); + amrex::Real const full_area = cell_size[(idim+1)%3]*cell_size[(idim+2)%3]; + auto const &face_areas_dim = face_areas[idim]->array(mfi); + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + face_areas_dim(i, j, k) *= full_area; + }); + + } + } +} + +#endif diff --git a/Source/EmbeddedBoundary/Enabled.H b/Source/EmbeddedBoundary/Enabled.H new file mode 100644 index 00000000000..af01272e262 --- /dev/null +++ b/Source/EmbeddedBoundary/Enabled.H @@ -0,0 +1,18 @@ +/* Copyright 2024 Axel Huebl + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_EB_ENABLED_H_ +#define WARPX_EB_ENABLED_H_ + +#include + +namespace EB +{ + /** Are embedded boundaries enabled? */ + bool enabled (); + +} // namespace EB +#endif // WARPX_EB_ENABLED_H_ diff --git a/Source/EmbeddedBoundary/Enabled.cpp b/Source/EmbeddedBoundary/Enabled.cpp new file mode 100644 index 00000000000..935c51c7c0f --- /dev/null +++ b/Source/EmbeddedBoundary/Enabled.cpp @@ -0,0 +1,39 @@ +/* Copyright 2024 Axel Huebl + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "Enabled.H" + +#ifdef AMREX_USE_EB +#include +#endif +#if defined(AMREX_USE_EB) && defined(WARPX_DIM_RZ) +#include +#endif + + +namespace EB +{ + bool enabled () + { +#ifndef AMREX_USE_EB + return false; +#else + amrex::ParmParse const pp_warpx("warpx"); + amrex::ParmParse const pp_eb2("eb2"); + + // test various runtime options to enable EBs + std::string eb_implicit_function; + bool eb_enabled = pp_warpx.query("eb_implicit_function", eb_implicit_function); + + // https://amrex-codes.github.io/amrex/docs_html/EB.html + std::string eb_stl; + eb_enabled |= pp_eb2.query("geom_type", eb_stl); + + return eb_enabled; +#endif + } + +} // namespace EB diff --git a/Source/EmbeddedBoundary/Make.package b/Source/EmbeddedBoundary/Make.package index 6ed944d19bd..fd46932827d 100644 --- a/Source/EmbeddedBoundary/Make.package +++ b/Source/EmbeddedBoundary/Make.package @@ -1,8 +1,12 @@ +CEXE_headers += EmbeddedBoundaryInit.H +CEXE_headers += Enabled.H CEXE_headers += ParticleScraper.H CEXE_headers += ParticleBoundaryProcess.H CEXE_headers += DistanceToEB.H CEXE_headers += WarpXFaceInfoBox.H +CEXE_sources += EmbeddedBoundaryInit.cpp +CEXE_sources += Enabled.cpp CEXE_sources += WarpXInitEB.cpp CEXE_sources += WarpXFaceExtensions.cpp diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 1e915b39381..860541542be 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -65,7 +65,7 @@ */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, int lev, F&& f) { scrapeParticlesAtEB(pc, distance_to_eb, lev, lev, std::forward(f)); } @@ -108,7 +108,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, F&& f) +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, F&& f) { scrapeParticlesAtEB(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); } @@ -153,7 +153,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, int lev_min, int lev_max, F&& f) { BL_PROFILE("scrapeParticlesAtEB"); @@ -176,7 +176,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) return; + if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) { return; } amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); @@ -185,7 +185,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan amrex::Real W[AMREX_SPACEDIM][2]; ablastr::particles::compute_weights( xp, yp, zp, plo, dxi, i, j, k, W); - amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 21c13f23845..61009fb46e0 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -6,15 +6,21 @@ */ #include "WarpXFaceInfoBox.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" #include +#include #include #include #include +using namespace ablastr::fields; +using warpx::fields::FieldType; + /** * \brief Get the value of arr in the neighbor (i_n, j_n) on the plane with normal 'dim'. * @@ -163,43 +169,49 @@ ComputeSStab(const int i, const int j, const int k, amrex::Array1D -WarpX::CountExtFaces() { +WarpX::CountExtFaces () { amrex::Array1D sums{0, 0, 0}; #ifdef AMREX_USE_EB + if (EB::enabled()) { #ifndef WARPX_DIM_RZ #ifdef WARPX_DIM_XZ - // In 2D we change the extrema of the for loop so that we only have the case idim=1 - for(int idim = 1; idim < AMREX_SPACEDIM; ++idim) { + // In 2D we change the extrema of the for loop so that we only have the case idim=1 + for(int idim = 1; idim < AMREX_SPACEDIM; ++idim) { #elif defined(WARPX_DIM_3D) for(int idim = 0; idim < AMREX_SPACEDIM; ++idim) { #else - WARPX_ABORT_WITH_MESSAGE( - "CountExtFaces: Only implemented in 2D3V and 3D3V"); + WARPX_ABORT_WITH_MESSAGE( + "CountExtFaces: Only implemented in 2D3V and 3D3V"); #endif - amrex::ReduceOps reduce_ops; - amrex::ReduceData reduce_data(reduce_ops); - for (amrex::MFIter mfi(*m_flag_ext_face[maxLevel()][idim]); mfi.isValid(); ++mfi) { - amrex::Box const &box = mfi.validbox(); - auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - reduce_ops.eval(box, reduce_data, - [=] AMREX_GPU_DEVICE(int i, int j, int k) -> amrex::GpuTuple { - return flag_ext_face(i, j, k); - }); - } + amrex::ReduceOps reduce_ops; + amrex::ReduceData reduce_data(reduce_ops); + for (amrex::MFIter mfi(*m_flag_ext_face[maxLevel()][idim]); mfi.isValid(); ++mfi) { + amrex::Box const &box = mfi.validbox(); + auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); + reduce_ops.eval(box, reduce_data, + [=] AMREX_GPU_DEVICE(int i, int j, int k) -> amrex::GpuTuple { + return flag_ext_face(i, j, k); + }); + } - auto r = reduce_data.value(); - sums(idim) = amrex::get<0>(r); - } + auto r = reduce_data.value(); + sums(idim) = amrex::get<0>(r); + } - amrex::ParallelDescriptor::ReduceIntSum(&(sums(0)), AMREX_SPACEDIM); + amrex::ParallelDescriptor::ReduceIntSum(&(sums(0)), AMREX_SPACEDIM); #endif + } #endif return sums; } void -WarpX::ComputeFaceExtensions(){ +WarpX::ComputeFaceExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeFaceExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB amrex::Array1D N_ext_faces = CountExtFaces(); ablastr::warn_manager::WMRecordWarning("Embedded Boundary", @@ -276,7 +288,7 @@ WarpX::ComputeFaceExtensions(){ void WarpX::InitBorrowing() { int idim = 0; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_x = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_x.inds_pointer.resize(box); @@ -292,7 +304,7 @@ WarpX::InitBorrowing() { } idim = 1; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_y = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_y.inds_pointer.resize(box); @@ -305,7 +317,7 @@ WarpX::InitBorrowing() { } idim = 2; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_z = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_z.inds_pointer.resize(box); @@ -421,7 +433,11 @@ ComputeNBorrowEightFacesExtension(const amrex::Dim3 cell, const amrex::Real S_ex void -WarpX::ComputeOneWayExtensions() { +WarpX::ComputeOneWayExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeOneWayExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB #ifndef WARPX_DIM_RZ auto const eb_fact = fieldEBFactory(maxLevel()); @@ -442,27 +458,26 @@ WarpX::ComputeOneWayExtensions() { WARPX_ABORT_WITH_MESSAGE( "ComputeOneWayExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); + auto const &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; auto const &borrowing_inds_pointer = borrowing.inds_pointer.array(); auto const &borrowing_size = borrowing.size.array(); - amrex::Long ncells = box.numPts(); + amrex::Long const ncells = box.numPts(); int* borrowing_inds = borrowing.inds.data(); FaceInfoBox::Neighbours* borrowing_neigh_faces = borrowing.neigh_faces.data(); amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); + auto const &S_mod = m_fields.get(FieldType::area_mod, Direction{idim}, maxLevel())->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); + const auto &lx = m_fields.get(FieldType::edge_lengths, Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get(FieldType::edge_lengths, Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get(FieldType::edge_lengths, Direction{2}, maxLevel())->array(mfi); vecs_size = amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell) { @@ -503,7 +518,7 @@ WarpX::ComputeOneWayExtensions() { for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { //This if makes sure that we don't visit the "diagonal neighbours" - if( !(i_n == j_n || i_n == -j_n)){ + if (i_n != j_n && i_n != -j_n){ // Here a face is available if it doesn't need to be extended itself and if its // area exceeds Sz_ext. Here we need to take into account if the intruded face // has given away already some area, so we use Sz_red rather than Sz. @@ -545,8 +560,14 @@ WarpX::ComputeOneWayExtensions() { void -WarpX::ComputeEightWaysExtensions() { +WarpX::ComputeEightWaysExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeEightWaysExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB + using namespace amrex::literals; + #ifndef WARPX_DIM_RZ auto const &cell_size = CellSize(maxLevel()); @@ -564,26 +585,27 @@ WarpX::ComputeEightWaysExtensions() { WARPX_ABORT_WITH_MESSAGE( "ComputeEightWaysExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); + auto const &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; auto const &borrowing_inds_pointer = borrowing.inds_pointer.array(); auto const &borrowing_size = borrowing.size.array(); - amrex::Long ncells = box.numPts(); + amrex::Long const ncells = box.numPts(); int* borrowing_inds = borrowing.inds.data(); FaceInfoBox::Neighbours* borrowing_neigh_faces = borrowing.neigh_faces.data(); amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); + auto const &S_mod = m_fields.get(FieldType::area_mod, Direction{idim}, maxLevel())->array(mfi); + + const auto &lx = m_fields.get(FieldType::edge_lengths, Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get(FieldType::edge_lengths, Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get(FieldType::edge_lengths, Direction{2}, maxLevel())->array(mfi); vecs_size += amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell){ @@ -650,7 +672,7 @@ WarpX::ComputeEightWaysExtensions() { neg_face = false; for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { - if(local_avail(i_n + 1, j_n + 1)){ + if (local_avail(i_n + 1, j_n + 1) != 0_rt){ const amrex::Real patch = S_ext * GetNeigh(S, i, j, k, i_n, j_n, idim) / denom; if(GetNeigh(S_mod, i, j, k, i_n, j_n, idim) - patch <= 0) { neg_face = true; @@ -675,7 +697,7 @@ WarpX::ComputeEightWaysExtensions() { int count = 0; for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { - if(local_avail(i_n + 1, j_n + 1)){ + if(local_avail(i_n + 1, j_n + 1) != 0_rt){ const amrex::Real patch = S_ext * GetNeigh(S, i, j, k, i_n, j_n, idim) / denom; borrowing_inds[ps + count] = ps + count; FaceInfoBox::addConnectedNeighbor(i_n, j_n, ps + count, @@ -703,7 +725,11 @@ WarpX::ComputeEightWaysExtensions() { } void -WarpX::ApplyBCKCorrection(const int idim) { +WarpX::ApplyBCKCorrection (const int idim) +{ + if (!EB::enabled()) { + throw std::runtime_error("ApplyBCKCorrection only works when EBs are enabled at runtime"); + } #if defined(AMREX_USE_EB) and !defined(WARPX_DIM_RZ) const std::array &cell_size = CellSize(maxLevel()); @@ -711,15 +737,15 @@ WarpX::ApplyBCKCorrection(const int idim) { const amrex::Real dy = cell_size[1]; const amrex::Real dz = cell_size[2]; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.tilebox(); const amrex::Array4 &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); const amrex::Array4 &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); - const amrex::Array4 &S = m_face_areas[maxLevel()][idim]->array(mfi); - const amrex::Array4 &lx = m_face_areas[maxLevel()][0]->array(mfi); - const amrex::Array4 &ly = m_face_areas[maxLevel()][1]->array(mfi); - const amrex::Array4 &lz = m_face_areas[maxLevel()][2]->array(mfi); + const amrex::Array4 &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); + const amrex::Array4 &lx = m_fields.get(FieldType::face_areas, Direction{0}, maxLevel())->array(mfi);; + const amrex::Array4 &ly = m_fields.get(FieldType::face_areas, Direction{1}, maxLevel())->array(mfi);; + const amrex::Array4 &lz = m_fields.get(FieldType::face_areas, Direction{2}, maxLevel())->array(mfi);; amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { if (flag_ext_face(i, j, k)) { @@ -736,9 +762,10 @@ WarpX::ApplyBCKCorrection(const int idim) { } void -WarpX::ShrinkBorrowing() { +WarpX::ShrinkBorrowing () +{ for(int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing.inds.resize(borrowing.vecs_size); borrowing.neigh_faces.resize(borrowing.vecs_size); diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 655bec0dc29..8b7ad7b9d64 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -7,39 +7,29 @@ #include "WarpX.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef AMREX_USE_EB +# include "Fields.H" # include "Utils/Parser/ParserUtils.H" # include "Utils/TextMsg.H" -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include # include # include +using namespace ablastr::fields; + #endif #ifdef AMREX_USE_EB @@ -57,6 +47,8 @@ namespace { ParserIF& operator= (const ParserIF& rhs) = delete; ParserIF& operator= (ParserIF&& rhs) = delete; + ~ParserIF() = default; + AMREX_GPU_HOST_DEVICE inline amrex::Real operator() (AMREX_D_DECL(amrex::Real x, amrex::Real y, amrex::Real z)) const noexcept { @@ -80,6 +72,14 @@ namespace { void WarpX::InitEB () { + if (!EB::enabled()) { + throw std::runtime_error("InitEB only works when EBs are enabled at runtime"); + } + +#if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) && !defined(WARPX_DIM_RZ) + WARPX_ABORT_WITH_MESSAGE("EBs only implemented in 2D and 3D"); +#endif + #ifdef AMREX_USE_EB BL_PROFILE("InitEB"); @@ -88,7 +88,7 @@ WarpX::InitEB () pp_warpx.query("eb_implicit_function", impf); if (! impf.empty()) { auto eb_if_parser = utils::parser::makeParser(impf, {"x", "y", "z"}); - ParserIF pif(eb_if_parser.compile<3>()); + ParserIF const pif(eb_if_parser.compile<3>()); auto gshop = amrex::EB2::makeShop(pif, eb_if_parser); // The last argument of amrex::EB2::Build is the maximum coarsening level // to which amrex should try to coarsen the EB. It will stop after coarsening @@ -100,311 +100,29 @@ WarpX::InitEB () } else { amrex::ParmParse pp_eb2("eb2"); if (!pp_eb2.contains("geom_type")) { - std::string geom_type = "all_regular"; + std::string const geom_type = "all_regular"; pp_eb2.add("geom_type", geom_type); // use all_regular by default } // See the comment above on amrex::EB2::Build for the hard-wired number 20. amrex::EB2::Build(Geom(maxLevel()), maxLevel(), maxLevel()+20); } - #endif } -#ifdef AMREX_USE_EB void -WarpX::ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, - const amrex::EBFArrayBoxFactory& eb_fact) { - BL_PROFILE("ComputeEdgeLengths"); - - auto const &flags = eb_fact.getMultiEBCellFlagFab(); - auto const &edge_centroid = eb_fact.getEdgeCent(); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - edge_lengths[1]->setVal(0.); -#endif - for (amrex::MFIter mfi(flags); mfi.isValid(); ++mfi){ -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int idim = 0; idim < 3; ++idim){ - if(idim == 1) continue; -#elif defined(WARPX_DIM_3D) - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim){ -#else - WARPX_ABORT_WITH_MESSAGE( - "ComputeEdgeLengths: Only implemented in 2D3V and 3D3V"); -#endif - amrex::Box box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), - edge_lengths[idim]->nGrowVect()); - amrex::FabType fab_type = flags[mfi].getType(box); - auto const &edge_lengths_dim = edge_lengths[idim]->array(mfi); - - if (fab_type == amrex::FabType::regular) { - // every cell in box is all regular - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - edge_lengths_dim(i, j, k) = 1.; - }); - } else if (fab_type == amrex::FabType::covered) { - // every cell in box is all covered - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - edge_lengths_dim(i, j, k) = 0.; - }); - } else { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - int idim_amrex = idim; - if(idim == 2) idim_amrex = 1; - auto const &edge_cent = edge_centroid[idim_amrex]->const_array(mfi); -#elif defined(WARPX_DIM_3D) - auto const &edge_cent = edge_centroid[idim]->const_array(mfi); -#else - WARPX_ABORT_WITH_MESSAGE( - "ComputeEdgeLengths: Only implemented in 2D3V and 3D3V"); -#endif - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - if (edge_cent(i, j, k) == amrex::Real(-1.0)) { - // This edge is all covered - edge_lengths_dim(i, j, k) = 0.; - } else if (edge_cent(i, j, k) == amrex::Real(1.0)) { - // This edge is all open - edge_lengths_dim(i, j, k) = 1.; - } else { - // This edge is cut. - edge_lengths_dim(i, j, k) = 1 - amrex::Math::abs(amrex::Real(2.0) - * edge_cent(i, j, k)); - } - - }); - } - } - } -} - - -void -WarpX::ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, - const amrex::EBFArrayBoxFactory& eb_fact) { - BL_PROFILE("ComputeFaceAreas"); - - auto const &flags = eb_fact.getMultiEBCellFlagFab(); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In 2D the volume frac is actually the area frac. - auto const &area_frac = eb_fact.getVolFrac(); -#elif defined(WARPX_DIM_3D) - auto const &area_frac = eb_fact.getAreaFrac(); -#else - WARPX_ABORT_WITH_MESSAGE( - "ComputeFaceAreas: Only implemented in 2D3V and 3D3V"); -#endif - -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - face_areas[0]->setVal(0.); - face_areas[2]->setVal(0.); -#endif - for (amrex::MFIter mfi(flags); mfi.isValid(); ++mfi) { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - // In 2D we change the extrema of the for loop so that we only have the case idim=1 - for (int idim = 1; idim < AMREX_SPACEDIM; ++idim) { -#elif defined(WARPX_DIM_3D) - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { -#else - WARPX_ABORT_WITH_MESSAGE( - "ComputeFaceAreas: Only implemented in 2D3V and 3D3V"); -#endif - amrex::Box box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), - face_areas[idim]->nGrowVect()); - amrex::FabType fab_type = flags[mfi].getType(box); - auto const &face_areas_dim = face_areas[idim]->array(mfi); - if (fab_type == amrex::FabType::regular) { - // every cell in box is all regular - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - face_areas_dim(i, j, k) = amrex::Real(1.); - }); - } else if (fab_type == amrex::FabType::covered) { - // every cell in box is all covered - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - face_areas_dim(i, j, k) = amrex::Real(0.); - }); - } else { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto const &face = area_frac.const_array(mfi); -#elif defined(WARPX_DIM_3D) - auto const &face = area_frac[idim]->const_array(mfi); -#else - WARPX_ABORT_WITH_MESSAGE( - "ComputeFaceAreas: Only implemented in 2D3V and 3D3V"); -#endif - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - face_areas_dim(i, j, k) = face(i, j, k); - }); - } - } - } -} - - -void -WarpX::ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, - const std::array& cell_size) { - BL_PROFILE("ScaleEdges"); - - for (amrex::MFIter mfi(*edge_lengths[0]); mfi.isValid(); ++mfi) { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int idim = 0; idim < 3; ++idim){ - if(idim == 1) continue; -#elif defined(WARPX_DIM_3D) - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim){ -#else - WARPX_ABORT_WITH_MESSAGE( - "ScaleEdges: Only implemented in 2D3V and 3D3V"); -#endif - const amrex::Box& box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), - edge_lengths[idim]->nGrowVect() ); - auto const &edge_lengths_dim = edge_lengths[idim]->array(mfi); - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - edge_lengths_dim(i, j, k) *= cell_size[idim]; - }); - } - } -} - -void -WarpX::ScaleAreas(std::array< std::unique_ptr, 3 >& face_areas, - const std::array& cell_size) { - BL_PROFILE("ScaleAreas"); - - amrex::Real full_area; - - for (amrex::MFIter mfi(*face_areas[0]); mfi.isValid(); ++mfi) { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - // In 2D we change the extrema of the for loop so that we only have the case idim=1 - for (int idim = 1; idim < AMREX_SPACEDIM; ++idim) { -#elif defined(WARPX_DIM_3D) - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { -#else - WARPX_ABORT_WITH_MESSAGE( - "ScaleAreas: Only implemented in 2D3V and 3D3V"); -#endif - const amrex::Box& box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), - face_areas[idim]->nGrowVect() ); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - full_area = cell_size[0]*cell_size[2]; -#elif defined(WARPX_DIM_3D) - if (idim == 0) { - full_area = cell_size[1]*cell_size[2]; - } else if (idim == 1) { - full_area = cell_size[0]*cell_size[2]; - } else { - full_area = cell_size[0]*cell_size[1]; - } -#else - WARPX_ABORT_WITH_MESSAGE( - "ScaleAreas: Only implemented in 2D3V and 3D3V"); -#endif - auto const &face_areas_dim = face_areas[idim]->array(mfi); - - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - face_areas_dim(i, j, k) *= full_area; - }); - - } - } -} - - -void -WarpX::MarkCells(){ -#ifndef WARPX_DIM_RZ - auto const &cell_size = CellSize(maxLevel()); - -#ifdef WARPX_DIM_3D - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { -#elif defined(WARPX_DIM_XZ) - m_flag_info_face[maxLevel()][0]->setVal(0.); - m_flag_info_face[maxLevel()][2]->setVal(0.); - m_flag_ext_face[maxLevel()][0]->setVal(0.); - m_flag_ext_face[maxLevel()][2]->setVal(0.); - // In 2D we change the extrema of the for loop so that we only have the case idim=1 - for (int idim = 1; idim < AMREX_SPACEDIM; ++idim) { -#else - WARPX_ABORT_WITH_MESSAGE( - "MarkCells: Only implemented in 2D3V and 3D3V"); -#endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { - //amrex::Box const &box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect()); - const amrex::Box& box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect(), - m_face_areas[maxLevel()][idim]->nGrowVect() ); - - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); - auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); - auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); - auto const &mod_areas_dim = m_area_mod[maxLevel()][idim]->array(mfi); - - const amrex::Real dx = cell_size[0]; - const amrex::Real dy = cell_size[1]; - const amrex::Real dz = cell_size[2]; - - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // Minimal area for this cell to be stable - mod_areas_dim(i, j, k) = S(i, j, k); - double S_stab; - if(idim == 0){ - S_stab = 0.5 * std::max({ly(i, j, k) * dz, ly(i, j, k + 1) * dz, - lz(i, j, k) * dy, lz(i, j + 1, k) * dy}); - }else if(idim == 1){ -#ifdef WARPX_DIM_XZ - S_stab = 0.5 * std::max({lx(i, j, k) * dz, lx(i, j + 1, k) * dz, - lz(i, j, k) * dx, lz(i + 1, j, k) * dx}); -#elif defined(WARPX_DIM_3D) - S_stab = 0.5 * std::max({lx(i, j, k) * dz, lx(i, j, k + 1) * dz, - lz(i, j, k) * dx, lz(i + 1, j, k) * dx}); -#else - WARPX_ABORT_WITH_MESSAGE( - "MarkCells: Only implemented in 2D3V and 3D3V"); -#endif - }else { - S_stab = 0.5 * std::max({lx(i, j, k) * dy, lx(i, j + 1, k) * dy, - ly(i, j, k) * dx, ly(i + 1, j, k) * dx}); - } - - // Does this face need to be extended? - // The difference between flag_info_face and flag_ext_face is that: - // - for every face flag_info_face contains a: - // * 0 if the face needs to be extended - // * 1 if the face is large enough to lend area to other faces - // * 2 if the face is actually intruded by other face - // Here we only take care of the first two cases. The entries corresponding - // to the intruded faces are going to be set in the function ComputeFaceExtensions - // - for every face flag_ext_face contains a: - // * 1 if the face needs to be extended - // * 0 otherwise - // In the function ComputeFaceExtensions, after the cells are extended, the - // corresponding entries in flag_ext_face are set to zero. This helps to keep - // track of which cells could not be extended - flag_ext_face(i, j, k) = int(S(i, j, k) < S_stab && S(i, j, k) > 0); - if(flag_ext_face(i, j, k)){ - flag_info_face(i, j, k) = 0; - } - // Is this face available to lend area to other faces? - // The criterion is that the face has to be interior and not already unstable itself - if(int(S(i, j, k) > 0 && !flag_ext_face(i, j, k))) { - flag_info_face(i, j, k) = 1; - } - }); - } +WarpX::ComputeDistanceToEB () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeDistanceToEB only works when EBs are enabled at runtime"); } -#endif -} -#endif - -void -WarpX::ComputeDistanceToEB () { #ifdef AMREX_USE_EB BL_PROFILE("ComputeDistanceToEB"); + using warpx::fields::FieldType; const amrex::EB2::IndexSpace& eb_is = amrex::EB2::IndexSpace::top(); for (int lev=0; lev<=maxLevel(); lev++) { const amrex::EB2::Level& eb_level = eb_is.getLevel(Geom(lev)); auto const eb_fact = fieldEBFactory(lev); - amrex::FillSignedDistance(*m_distance_to_eb[lev], eb_level, eb_fact, 1); + amrex::FillSignedDistance(*m_fields.get(FieldType::distance_to_eb, lev), eb_level, eb_fact, 1); } #endif } diff --git a/Source/Evolve/WarpXComputeDt.cpp b/Source/Evolve/WarpXComputeDt.cpp index c1a87166920..f88b2044927 100644 --- a/Source/Evolve/WarpXComputeDt.cpp +++ b/Source/Evolve/WarpXComputeDt.cpp @@ -13,6 +13,7 @@ #else # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -27,29 +28,29 @@ #include #include +/** + * Compute the minimum of array x, where x has dimension AMREX_SPACEDIM + */ +AMREX_FORCE_INLINE amrex::Real +minDim (const amrex::Real* x) +{ + return std::min({AMREX_D_DECL(x[0], x[1], x[2])}); +} + /** * Determine the timestep of the simulation. */ void WarpX::ComputeDt () { // Handle cases where the timestep is not limited by the speed of light - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None || - electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - - std::stringstream errorMsg; - if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { - errorMsg << "warpx.const_dt must be specified with the electrostatic solver."; - } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - errorMsg << "warpx.const_dt must be specified with the hybrid-PIC solver."; - } else { - errorMsg << "warpx.const_dt must be specified when not using a field solver."; - } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), errorMsg.str()); - - for (int lev=0; lev<=max_level; lev++) { - dt[lev] = m_const_dt.value(); - } - return; + // and no constant timestep is provided + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), "warpx.const_dt must be specified with the hybrid-PIC solver."); + } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_const_dt.has_value() || m_dt_update_interval.isActivated(), + "warpx.const_dt must be specified with the electrostatic solver, or warpx.dt_update_interval must be > 0." + ); } // Determine the appropriate timestep as limited by the speed of light @@ -58,16 +59,17 @@ WarpX::ComputeDt () if (m_const_dt.has_value()) { deltat = m_const_dt.value(); + } else if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { + // Set dt for electrostatic algorithm + if (m_max_dt.has_value()) { + deltat = m_max_dt.value(); + } else { + deltat = cfl * minDim(dx) / PhysConst::c; + } } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Computation of dt for spectral algorithm // (determined by the minimum cell size in all directions) -#if defined(WARPX_DIM_1D_Z) - deltat = cfl * dx[0] / PhysConst::c; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - deltat = cfl * std::min(dx[0], dx[1]) / PhysConst::c; -#else - deltat = cfl * std::min(dx[0], std::min(dx[1], dx[2])) / PhysConst::c; -#endif + deltat = cfl * minDim(dx) / PhysConst::c; } else { // Computation of dt for FDTD algorithm #ifdef WARPX_DIM_RZ @@ -92,28 +94,43 @@ WarpX::ComputeDt () dt.resize(0); dt.resize(max_level+1,deltat); - if (do_subcycling) { + if (m_do_subcycling) { for (int lev = max_level-1; lev >= 0; --lev) { dt[lev] = dt[lev+1] * refRatio(lev)[0]; } } } +/** + * Determine the simulation timestep from the maximum speed of all particles + * Sets timestep so that a particle can only cross cfl*dx cells per timestep. + */ void -WarpX::PrintDtDxDyDz () +WarpX::UpdateDtFromParticleSpeeds () { - for (int lev=0; lev <= max_level; lev++) { - const amrex::Real* dx_lev = geom[lev].CellSize(); - amrex::Print() << "Level " << lev << ": dt = " << dt[lev] -#if defined(WARPX_DIM_1D_Z) - << " ; dz = " << dx_lev[0] << '\n'; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - << " ; dx = " << dx_lev[0] - << " ; dz = " << dx_lev[1] << '\n'; -#elif defined(WARPX_DIM_3D) - << " ; dx = " << dx_lev[0] - << " ; dy = " << dx_lev[1] - << " ; dz = " << dx_lev[2] << '\n'; -#endif + const amrex::Real* dx = geom[max_level].CellSize(); + const amrex::Real dx_min = minDim(dx); + + const amrex::ParticleReal max_v = mypc->maxParticleVelocity(); + amrex::Real deltat_new = 0.; + + // Protections from overly-large timesteps + if (max_v == 0) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_max_dt.has_value(), "Particles at rest and no constant or maximum timestep specified. Aborting."); + deltat_new = m_max_dt.value(); + } else { + deltat_new = cfl * dx_min / max_v; + } + + // Restrict to be less than user-specified maximum timestep, if present + if (m_max_dt.has_value()) { + deltat_new = std::min(deltat_new, m_max_dt.value()); + } + + // Update dt + dt[max_level] = deltat_new; + + for (int lev = max_level-1; lev >= 0; --lev) { + dt[lev] = dt[lev+1] * refRatio(lev)[0]; } } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 72ef84b8f75..8275f6c4d7f 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -13,7 +13,9 @@ #include "BoundaryConditions/PML.H" #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "EmbeddedBoundary/Enabled.H" #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ @@ -59,12 +61,60 @@ using namespace amrex; using ablastr::utils::SignalHandling; +namespace +{ + /** Print Unused Parameter Warnings after Step 1 + * + * Instead of waiting for a simulation to end, we already do an early "unused parameter check" + * after step 1 to inform users early of potential issues with their simulation setup. + */ + void checkEarlyUnusedParams () + { + amrex::Print() << "\n"; // better: conditional \n based on return value + amrex::ParmParse::QueryUnusedInputs(); + + // Print the warning list right after the first step. + amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); + } +} + +void +WarpX::Synchronize () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP( + lev, + 0.5_rt*dt[lev], + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) + ); + } + is_synchronized = true; +} + void WarpX::Evolve (int numsteps) { WARPX_PROFILE_REGION("WarpX::Evolve()"); WARPX_PROFILE("WarpX::Evolve()"); + using ablastr::fields::Direction; + Real cur_time = t_new[0]; // Note that the default argument is numsteps = -1 @@ -94,6 +144,18 @@ WarpX::Evolve (int numsteps) CheckLoadBalance(step); + // Update timestep for electrostatic solver if a constant dt is not provided + // This first synchronizes the position and velocity before setting the new timestep + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None && + !m_const_dt.has_value() && m_dt_update_interval.contains(step+1)) { + if (verbose) { + amrex::Print() << Utils::TextMsg::Info("updating timestep"); + } + Synchronize(); + UpdateDtFromParticleSpeeds(); + } + + // If position and velocity are synchronized, push velocity backward one half step if (evolve_scheme == EvolveScheme::Explicit) { ExplicitFillBoundaryEBUpdateAux(); @@ -142,7 +204,7 @@ WarpX::Evolve (int numsteps) OneStep_multiJ(cur_time); } // Electromagnetic case: no subcycling or no mesh refinement - else if ( !do_subcycling || (finest_level == 0)) + else if ( !m_do_subcycling || (finest_level == 0)) { OneStep_nosub(cur_time); // E: guard cells are up-to-date @@ -150,14 +212,14 @@ WarpX::Evolve (int numsteps) // F: guard cells are NOT up-to-date } // Electromagnetic case: subcycling with one level of mesh refinement - else if (do_subcycling && (finest_level == 1)) + else if (m_do_subcycling && (finest_level == 1)) { OneStep_sub1(cur_time); } else { WARPX_ABORT_WITH_MESSAGE( - "do_subcycling = " + std::to_string(do_subcycling) + "do_subcycling = " + std::to_string(m_do_subcycling) + " is an unsupported do_subcycling type."); } @@ -174,25 +236,9 @@ WarpX::Evolve (int numsteps) // TODO: move out if (evolve_scheme == EvolveScheme::Explicit) { + // At the end of last step, push p by 0.5*dt to synchronize if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); - } - is_synchronized = true; + Synchronize(); } } @@ -206,6 +252,7 @@ WarpX::Evolve (int numsteps) // sync up time for (int i = 0; i <= max_level; ++i) { + t_old[i] = t_new[i]; t_new[i] = cur_time; } multi_diags->FilterComputePackFlush( step, false, true ); @@ -280,7 +327,7 @@ WarpX::Evolve (int numsteps) // inputs: unused parameters (e.g. typos) check after step 1 has finished if (!early_params_checked) { - checkEarlyUnusedParams(); + ::checkEarlyUnusedParams(); early_params_checked = true; } @@ -360,7 +407,7 @@ WarpX::OneStep_nosub (Real cur_time) WarpX::Hybrid_QED_Push(dt); FillBoundaryE(guard_cells.ng_alloc_EB); } - PushPSATD(); + PushPSATD(cur_time); if (do_pml) { DampPML(); @@ -388,15 +435,15 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryF(guard_cells.ng_FieldSolverF); FillBoundaryG(guard_cells.ng_FieldSolverG); - EvolveB(0.5_rt * dt[0], DtType::FirstHalf); // We now have B^{n+1/2} + EvolveB(0.5_rt * dt[0], DtType::FirstHalf, cur_time); // We now have B^{n+1/2} FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - if (WarpX::em_solver_medium == MediumForEM::Vacuum) { + if (m_em_solver_medium == MediumForEM::Vacuum) { // vacuum medium - EvolveE(dt[0]); // We now have E^{n+1} - } else if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + EvolveE(dt[0], cur_time); // We now have E^{n+1} + } else if (m_em_solver_medium == MediumForEM::Macroscopic) { // macroscopic medium - MacroscopicEvolveE(dt[0]); // We now have E^{n+1} + MacroscopicEvolveE(dt[0], cur_time); // We now have E^{n+1} } else { WARPX_ABORT_WITH_MESSAGE("Medium for EM is unknown"); } @@ -404,7 +451,7 @@ WarpX::OneStep_nosub (Real cur_time) EvolveF(0.5_rt * dt[0], DtType::SecondHalf); EvolveG(0.5_rt * dt[0], DtType::SecondHalf); - EvolveB(0.5_rt * dt[0], DtType::SecondHalf); // We now have B^{n+1} + EvolveB(0.5_rt * dt[0], DtType::SecondHalf, cur_time + 0.5_rt * dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); @@ -416,7 +463,7 @@ WarpX::OneStep_nosub (Real cur_time) // E and B are up-to-date in the domain, but all guard cells are // outdated. - if (safe_guard_cells) { + if (m_safe_guard_cells) { FillBoundaryB(guard_cells.ng_alloc_EB); } } // !PSATD @@ -431,19 +478,13 @@ bool WarpX::checkStopSimulation (amrex::Real cur_time) m_exit_loop_due_to_interrupt_signal; } -void WarpX::checkEarlyUnusedParams () -{ - amrex::Print() << "\n"; // better: conditional \n based on return value - amrex::ParmParse::QueryUnusedInputs(); - - // Print the warning list right after the first step. - amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); -} - void WarpX::ExplicitFillBoundaryEBUpdateAux () { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, - "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux without Explicit evolve scheme set!"); + + using ablastr::fields::Direction; + using warpx::fields::FieldType; // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. @@ -459,9 +500,16 @@ void WarpX::ExplicitFillBoundaryEBUpdateAux () // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + mypc->PushP( + lev, + -0.5_rt*dt[lev], + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) + ); } is_synchronized = false; @@ -524,11 +572,13 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num } // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); - m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); - mypc->deleteInvalidParticles(); -#endif + if (EB::enabled()) { + using warpx::fields::FieldType; + mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels(FieldType::distance_to_eb, finest_level)); + m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries( + *mypc, m_fields.get_mr_levels(FieldType::distance_to_eb, finest_level)); + mypc->deleteInvalidParticles(); + } if (sort_intervals.contains(step+1)) { if (verbose) { @@ -540,23 +590,22 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num void WarpX::SyncCurrentAndRho () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (fft_periodic_single_box) { // With periodic single box, synchronize J and rho here, // even with current correction or Vay deposition - if (current_deposition_algo == CurrentDepositionAlgo::Vay) - { - // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - SyncCurrent(current_fp_vay, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); - } - else - { - SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); - } + std::string const current_fp_string = (current_deposition_algo == CurrentDepositionAlgo::Vay) + ? "current_fp_vay" : "current_fp"; + // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR + + SyncCurrent(current_fp_string); + SyncRho(); + } else // no periodic single box { @@ -566,42 +615,46 @@ void WarpX::SyncCurrentAndRho () if (!current_correction && current_deposition_algo != CurrentDepositionAlgo::Vay) { - SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncCurrent("current_fp"); + SyncRho(); } if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } + if (use_filter) { + ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); + } } } } else // FDTD { - SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncCurrent("current_fp"); + SyncRho(); } // Reflect charge and current density over PEC boundaries, if needed. for (int lev = 0; lev <= finest_level; ++lev) { - if (rho_fp[lev]) { - ApplyRhofieldBoundary(lev, rho_fp[lev].get(), PatchType::fine); + if (m_fields.has(FieldType::rho_fp, lev)) { + ApplyRhofieldBoundary(lev, m_fields.get(FieldType::rho_fp,lev), PatchType::fine); } - ApplyJfieldBoundary( - lev, current_fp[lev][0].get(), current_fp[lev][1].get(), - current_fp[lev][2].get(), PatchType::fine - ); + ApplyJfieldBoundary(lev, + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), + PatchType::fine); if (lev > 0) { - if (rho_cp[lev]) { - ApplyRhofieldBoundary(lev, rho_cp[lev].get(), PatchType::coarse); + if (m_fields.has(FieldType::rho_cp, lev)) { + ApplyRhofieldBoundary(lev, m_fields.get(FieldType::rho_cp,lev), PatchType::coarse); } - ApplyJfieldBoundary( - lev, current_cp[lev][0].get(), current_cp[lev][1].get(), - current_cp[lev][2].get(), PatchType::coarse - ); + ApplyJfieldBoundary(lev, + m_fields.get(FieldType::current_cp, Direction{0}, lev), + m_fields.get(FieldType::current_cp, Direction{1}, lev), + m_fields.get(FieldType::current_cp, Direction{2}, lev), + PatchType::coarse); } } } @@ -616,6 +669,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) "multi-J algorithm not implemented for FDTD" ); + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + const int rho_mid = spectral_solver_fp[0]->m_spectral_index.rho_mid; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; @@ -627,7 +684,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDForwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -636,31 +693,36 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) - if (rho_fp[0] && rho_in_time == RhoInTime::Linear) + if (m_fields.has(FieldType::rho_fp, 0) && rho_in_time == RhoInTime::Linear) { + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels(FieldType::rho_fp, finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; + // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) mypc->DepositCharge(rho_fp, -dt[0]); // Filter, exchange boundary, and interpolate across levels - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); // Forward FFT of rho - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_new); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_new); } // 4) Deposit J at relative time -dt with time step dt // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - auto& current = (do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], -dt[0]); + std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + mypc->DepositCurrent( m_fields.get_mr_levels_alldirs(current_string, finest_level), dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ("current_fp", "current_cp"); } // Number of depositions for multi-J scheme @@ -685,31 +747,36 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - auto& current = (do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], t_deposit_current); + std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + mypc->DepositCurrent( m_fields.get_mr_levels_alldirs(current_string, finest_level), dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ("current_fp", "current_cp"); // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) - if (rho_fp[0]) + if (m_fields.has(FieldType::rho_fp, 0)) { + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels(FieldType::rho_fp, finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; + // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } // Deposit rho at relative time t_deposit_charge mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); // Forward FFT of rho const int rho_idx = (rho_in_time == RhoInTime::Linear) ? rho_new : rho_mid; - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_idx); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_idx); } if (WarpX::current_correction) @@ -725,7 +792,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } @@ -736,7 +803,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) { // We summed the integral of the field over 2*dt PSATDScaleAverageFields(1._rt / (2._rt*dt[0])); - PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); + PSATDBackwardTransformEBavg( + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_lev0_coarse_patch), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_lev0_coarse_patch) + ); } // Evolve fields in PML @@ -744,12 +816,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) { if (do_pml && pml[lev]->ok()) { - pml[lev]->PushPSATD(lev); + pml[lev]->PushPSATD(m_fields, lev); } - ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } - ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } + ApplyEfieldBoundary(lev, PatchType::fine, cur_time + dt[0]); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse, cur_time + dt[0]); } + ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf, cur_time + dt[0]); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf, cur_time + dt[0]); } } // Damp fields in PML before exchanging guard cells @@ -804,25 +876,42 @@ WarpX::OneStep_sub1 (Real cur_time) const int fine_lev = 1; const int coarse_lev = 0; + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + // i) Push particles and fields on the fine patch (first fine step) PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); - RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); - RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } - SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); - ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); + RestrictCurrentFromFineToCoarsePatch( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch), fine_lev); + RestrictRhoFromFineToCoarsePatch(fine_lev); + if (use_filter) { + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + } + SumBoundaryJ( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + fine_lev, Geom(fine_lev).periodicity()); + + if (m_fields.has(FieldType::rho_fp, finest_level) && + m_fields.has(FieldType::rho_cp, finest_level)) { + ApplyFilterandSumBoundaryRho( + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch), + fine_lev, PatchType::fine, 0, 2*ncomps); + } - EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); + EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf, cur_time); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); + EvolveE(fine_lev, PatchType::fine, dt[fine_lev], cur_time); FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldGather); - EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf); + EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf, cur_time + 0.5_rt * dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf); if (do_pml) { @@ -838,25 +927,37 @@ WarpX::OneStep_sub1 (Real cur_time) // by only half a coarse step (first half) PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); - AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); - AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); + AddCurrentFromFineLevelandSumBoundary( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch), + m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_lev0_coarse_patch), coarse_lev); + + if (m_fields.has(FieldType::rho_fp, finest_level) && + m_fields.has(FieldType::rho_cp, finest_level) && + m_fields.has(FieldType::rho_buf, finest_level)) { + AddRhoFromFineLevelandSumBoundary( + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch), + m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_lev0_coarse_patch), + coarse_lev, 0, ncomps); + } - EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); + EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf, cur_time); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF); - EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); + EvolveE(fine_lev, PatchType::coarse, dt[fine_lev], cur_time); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); - EvolveB(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::FirstHalf); + EvolveB(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::FirstHalf, cur_time); EvolveF(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::FirstHalf); FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather, WarpX::sync_nodal_points); FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolverF, WarpX::sync_nodal_points); - EvolveE(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev]); + EvolveE(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], cur_time); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather); // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? @@ -867,22 +968,33 @@ WarpX::OneStep_sub1 (Real cur_time) // iv) Push particles and fields on the fine patch (second fine step) PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); - RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); - RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } - SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); - ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); + RestrictCurrentFromFineToCoarsePatch( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch), fine_lev); + RestrictRhoFromFineToCoarsePatch(fine_lev); + if (use_filter) { + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + } + SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity()); + + if (m_fields.has(FieldType::rho_fp, finest_level) && + m_fields.has(FieldType::rho_cp, finest_level)) { + ApplyFilterandSumBoundaryRho( + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch), + fine_lev, PatchType::fine, 0, ncomps); + } - EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); + EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf, cur_time + dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolverF); - EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); + EvolveE(fine_lev, PatchType::fine, dt[fine_lev], cur_time + dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf); + EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf, cur_time + 1.5_rt*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::SecondHalf); if (do_pml) { @@ -890,7 +1002,7 @@ WarpX::OneStep_sub1 (Real cur_time) FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } - if ( safe_guard_cells ) { + if ( m_safe_guard_cells ) { FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); @@ -898,14 +1010,27 @@ WarpX::OneStep_sub1 (Real cur_time) // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) RestoreCurrent(coarse_lev); - AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); - AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, ncomps, ncomps); + AddCurrentFromFineLevelandSumBoundary( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch), + m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_lev0_coarse_patch), + coarse_lev); + + if (m_fields.has(FieldType::rho_fp, finest_level) && + m_fields.has(FieldType::rho_cp, finest_level) && + m_fields.has(FieldType::rho_buf, finest_level)) { + AddRhoFromFineLevelandSumBoundary( + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch), + m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_lev0_coarse_patch), + coarse_lev, ncomps, ncomps); + } - EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); + EvolveE(fine_lev, PatchType::coarse, dt[fine_lev], cur_time + 0.5_rt * dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); + EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf, cur_time + 0.5_rt * dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); if (do_pml) { @@ -920,11 +1045,11 @@ WarpX::OneStep_sub1 (Real cur_time) FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF, WarpX::sync_nodal_points); - EvolveE(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev]); + EvolveE(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], cur_time + 0.5_rt*dt[coarse_lev]); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::SecondHalf); + EvolveB(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::SecondHalf, cur_time + 0.5_rt*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5_rt*dt[coarse_lev], DtType::SecondHalf); if (do_pml) { @@ -938,12 +1063,12 @@ WarpX::OneStep_sub1 (Real cur_time) WarpX::sync_nodal_points); } DampPML(coarse_lev, PatchType::fine); - if ( safe_guard_cells ) { + if ( m_safe_guard_cells ) { FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } } - if ( safe_guard_cells ) { + if ( m_safe_guard_cells ) { FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } @@ -952,35 +1077,41 @@ WarpX::OneStep_sub1 (Real cur_time) void WarpX::doFieldIonization () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + for (int lev = 0; lev <= finest_level; ++lev) { - doFieldIonization(lev); + mypc->doFieldIonization( + lev, + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) + ); } } -void -WarpX::doFieldIonization (int lev) -{ - mypc->doFieldIonization(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); -} - #ifdef WARPX_QED void WarpX::doQEDEvents () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + for (int lev = 0; lev <= finest_level; ++lev) { - doQEDEvents(lev); + mypc->doQedEvents( + lev, + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) + ); } } - -void -WarpX::doQEDEvents (int lev) -{ - mypc->doQedEvents(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); -} #endif void @@ -997,50 +1128,53 @@ void WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, PushType push_type) { - amrex::MultiFab* current_x = nullptr; - amrex::MultiFab* current_y = nullptr; - amrex::MultiFab* current_z = nullptr; + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + std::string current_fp_string; if (WarpX::do_current_centering) { - current_x = current_fp_nodal[lev][0].get(); - current_y = current_fp_nodal[lev][1].get(); - current_z = current_fp_nodal[lev][2].get(); + current_fp_string = "current_fp_nodal"; } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - // Note that Vay deposition is supported only for PSATD and the code currently aborts otherwise - current_x = current_fp_vay[lev][0].get(); - current_y = current_fp_vay[lev][1].get(); - current_z = current_fp_vay[lev][2].get(); + current_fp_string = "current_fp_vay"; } else { - current_x = current_fp[lev][0].get(); - current_y = current_fp[lev][1].get(); - current_z = current_fp[lev][2].get(); + current_fp_string = "current_fp"; } - mypc->Evolve(lev, - *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], - *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], - *current_x, *current_y, *current_z, - current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), - rho_fp[lev].get(), charge_buf[lev].get(), - Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), - Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_current, push_type); + mypc->Evolve( + m_fields, + lev, + current_fp_string, + cur_time, + dt[lev], + a_dt_type, + skip_current, + push_type + ); if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. - ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), lev); - if (current_buf[lev][0].get()) { - ApplyInverseVolumeScalingToCurrentDensity(current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), lev-1); + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), + lev); + if (m_fields.has_vector(FieldType::current_buf, lev)) { + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get(FieldType::current_buf, Direction{0}, lev), + m_fields.get(FieldType::current_buf, Direction{1}, lev), + m_fields.get(FieldType::current_buf, Direction{2}, lev), + lev-1); } - if (rho_fp[lev]) { - ApplyInverseVolumeScalingToChargeDensity(rho_fp[lev].get(), lev); - if (charge_buf[lev]) { - ApplyInverseVolumeScalingToChargeDensity(charge_buf[lev].get(), lev-1); + if (m_fields.has(FieldType::rho_fp, lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get(FieldType::rho_fp, lev), lev); + if (m_fields.has(FieldType::rho_buf, lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get(FieldType::rho_buf, lev), lev-1); } } // #else @@ -1059,10 +1193,12 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, } if (do_fluid_species) { - myfl->Evolve(lev, - *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], - *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], - rho_fp[lev].get(), *current_x, *current_y, *current_z, cur_time, skip_current); + myfl->Evolve(m_fields, + lev, + current_fp_string, + cur_time, + skip_current + ); } } } @@ -1075,17 +1211,19 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, void WarpX::applyMirrors (Real time) { + using ablastr::fields::Direction; + // something to do? - if (num_mirrors == 0) { + if (m_num_mirrors == 0) { return; } // Loop over the mirrors - for(int i_mirror=0; i_mirror1) @@ -1097,51 +1235,35 @@ WarpX::applyMirrors (Real time) // Loop over levels for(int lev=0; lev<=finest_level; lev++) { - // Mirror must contain at least mirror_z_npoints[i_mirror] cells + // Mirror must contain at least m_mirror_z_npoints[i_mirror] cells const amrex::Real dz = WarpX::CellSize(lev)[2]; - const amrex::Real z_max = std::max(z_max_tmp, z_min+mirror_z_npoints[i_mirror]*dz); - - // Get fine patch field MultiFabs - amrex::MultiFab& Ex = *Efield_fp[lev][0].get(); - amrex::MultiFab& Ey = *Efield_fp[lev][1].get(); - amrex::MultiFab& Ez = *Efield_fp[lev][2].get(); - amrex::MultiFab& Bx = *Bfield_fp[lev][0].get(); - amrex::MultiFab& By = *Bfield_fp[lev][1].get(); - amrex::MultiFab& Bz = *Bfield_fp[lev][2].get(); - - // Set each field to zero between z_min and z_max - NullifyMF(Ex, lev, z_min, z_max); - NullifyMF(Ey, lev, z_min, z_max); - NullifyMF(Ez, lev, z_min, z_max); - NullifyMF(Bx, lev, z_min, z_max); - NullifyMF(By, lev, z_min, z_max); - NullifyMF(Bz, lev, z_min, z_max); + const amrex::Real z_max = std::max(z_max_tmp, z_min+m_mirror_z_npoints[i_mirror]*dz); + + // Set each field on the fine patch to zero between z_min and z_max + NullifyMF(m_fields, "Efield_fp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp", Direction{2}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{2}, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } - if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } + NullifyMF(m_fields, "F_fp", lev, z_min, z_max); + NullifyMF(m_fields, "G_fp", lev, z_min, z_max); if (lev>0) { - // Get coarse patch field MultiFabs - amrex::MultiFab& cEx = *Efield_cp[lev][0].get(); - amrex::MultiFab& cEy = *Efield_cp[lev][1].get(); - amrex::MultiFab& cEz = *Efield_cp[lev][2].get(); - amrex::MultiFab& cBx = *Bfield_cp[lev][0].get(); - amrex::MultiFab& cBy = *Bfield_cp[lev][1].get(); - amrex::MultiFab& cBz = *Bfield_cp[lev][2].get(); - - // Set each field to zero between z_min and z_max - NullifyMF(cEx, lev, z_min, z_max); - NullifyMF(cEy, lev, z_min, z_max); - NullifyMF(cEz, lev, z_min, z_max); - NullifyMF(cBx, lev, z_min, z_max); - NullifyMF(cBy, lev, z_min, z_max); - NullifyMF(cBz, lev, z_min, z_max); + // Set each field on the coarse patch to zero between z_min and z_max + NullifyMF(m_fields, "Efield_cp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp", Direction{2}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{2}, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } - if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } + NullifyMF(m_fields, "F_cp", lev, z_min, z_max); + NullifyMF(m_fields, "G_cp", lev, z_min, z_max); } } } diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index 859117eb214..896b3f04fc2 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -2,13 +2,14 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - ElectrostaticSolver.cpp WarpXPushFieldsEM.cpp WarpXPushFieldsHybridPIC.cpp WarpX_QED_Field_Pushers.cpp + WarpXSolveFieldsES.cpp ) endforeach() +add_subdirectory(ElectrostaticSolvers) add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) add_subdirectory(ImplicitSolvers) diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H deleted file mode 100644 index c5c2ce8b6eb..00000000000 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ /dev/null @@ -1,125 +0,0 @@ -/* Copyright 2021 Modern Electron - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_ELECTROSTATICSOLVER_H_ -#define WARPX_ELECTROSTATICSOLVER_H_ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - - -namespace ElectrostaticSolver { - - struct PhiCalculatorEB { - - amrex::Real t; - amrex::ParserExecutor<4> potential_eb; - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - amrex::Real operator()(const amrex::Real x, const amrex::Real z) const noexcept { - using namespace amrex::literals; - return potential_eb(x, 0.0_rt, z, t); - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - amrex::Real operator()(const amrex::Real x, const amrex::Real y, const amrex::Real z) const noexcept { - return potential_eb(x, y, z, t); - } - }; - - class PoissonBoundaryHandler { - public: - - amrex::Array lobc, hibc; - bool bcs_set = false; - std::array dirichlet_flag; - bool has_non_periodic = false; - bool phi_EB_only_t = true; - - void definePhiBCs (const amrex::Geometry& geom); - - void buildParsers (); - void buildParsersEB (); - - /* \brief Sets the EB potential string and updates the function parser - * - * \param [in] potential The string value of the potential - */ - void setPotentialEB(const std::string& potential) { - potential_eb_str = potential; - buildParsersEB(); - } - - [[nodiscard]] PhiCalculatorEB - getPhiEB(amrex::Real t) const noexcept - { - return PhiCalculatorEB{t, potential_eb}; - } - - // set default potentials to zero in order for current tests to pass - // but forcing the user to specify a potential might be better - std::string potential_xlo_str = "0"; - std::string potential_xhi_str = "0"; - std::string potential_ylo_str = "0"; - std::string potential_yhi_str = "0"; - std::string potential_zlo_str = "0"; - std::string potential_zhi_str = "0"; - std::string potential_eb_str = "0"; - - amrex::ParserExecutor<1> potential_xlo; - amrex::ParserExecutor<1> potential_xhi; - amrex::ParserExecutor<1> potential_ylo; - amrex::ParserExecutor<1> potential_yhi; - amrex::ParserExecutor<1> potential_zlo; - amrex::ParserExecutor<1> potential_zhi; - amrex::ParserExecutor<1> potential_eb_t; - amrex::ParserExecutor<4> potential_eb; - - private: - - amrex::Parser potential_xlo_parser; - amrex::Parser potential_xhi_parser; - amrex::Parser potential_ylo_parser; - amrex::Parser potential_yhi_parser; - amrex::Parser potential_zlo_parser; - amrex::Parser potential_zhi_parser; - amrex::Parser potential_eb_parser; - }; - - /** use amrex to directly calculate the electric field since with EB's the - * - * simple finite difference scheme in WarpX::computeE sometimes fails - */ - class EBCalcEfromPhiPerLevel { - private: - amrex::Vector< - amrex::Array - > m_e_field; - - public: - EBCalcEfromPhiPerLevel(amrex::Vector > e_field) - : m_e_field(std::move(e_field)) {} - - void operator()(amrex::MLMG & mlmg, int const lev) { - using namespace amrex::literals; - - mlmg.getGradSolution({m_e_field[lev]}); - for (auto &field: m_e_field[lev]) { - field->mult(-1._rt); - } - } - }; -} // namespace ElectrostaticSolver - -#endif // WARPX_ELECTROSTATICSOLVER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp deleted file mode 100644 index 189f2f2bb0a..00000000000 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ /dev/null @@ -1,1145 +0,0 @@ -/* Copyright 2019 Remi Lehe - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX.H" - -#include "FieldSolver/ElectrostaticSolver.H" -#include "Fluids/MultiFluidContainer.H" -#include "Fluids/WarpXFluidContainer.H" -#include "Parallelization/GuardCellManager.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/WarpXParticleContainer.H" -#include "Python/callbacks.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/WarpXAlgorithmSelection.H" -#include "Utils/WarpXConst.H" -#include "Utils/TextMsg.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef AMREX_USE_EB -# include -#endif - -#include -#include -#include - -using namespace amrex; -using namespace warpx::fields; - -void -WarpX::ComputeSpaceChargeField (bool const reset_fields) -{ - WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); - if (reset_fields) { - // Reset all E and B fields to 0, before calculating space-charge fields - WARPX_PROFILE("WarpX::ComputeSpaceChargeField::reset_fields"); - for (int lev = 0; lev <= max_level; lev++) { - for (int comp=0; comp<3; comp++) { - Efield_fp[lev][comp]->setVal(0); - Bfield_fp[lev][comp]->setVal(0); - } - } - } - - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || - electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - AddSpaceChargeFieldLabFrame(); - } - else { - // Loop over the species and add their space-charge contribution to E and B. - // Note that the fields calculated here does not include the E field - // due to simulation boundary potentials - for (int ispecies=0; ispeciesnSpecies(); ispecies++){ - WarpXParticleContainer& species = mypc->GetParticleContainer(ispecies); - if (species.initialize_self_fields || - (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic)) { - AddSpaceChargeField(species); - } - } - - // Add the field due to the boundary potentials - if (m_boundary_potential_specified || - (electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic)){ - AddBoundaryField(); - } - } -} - -/* Compute the potential `phi` by solving the Poisson equation with the - simulation specific boundary conditions and boundary values, then add the - E field due to that `phi` to `Efield_fp`. -*/ -void -WarpX::AddBoundaryField () -{ - WARPX_PROFILE("WarpX::AddBoundaryField"); - - // Store the boundary conditions for the field solver if they haven't been - // stored yet - if (!m_poisson_boundary_handler.bcs_set) { - m_poisson_boundary_handler.definePhiBCs(Geom(0)); - } - - // Allocate fields for charge and potential - const int num_levels = max_level + 1; - Vector > rho(num_levels); - Vector > phi(num_levels); - // Use number of guard cells used for local deposition of rho - const amrex::IntVect ng = guard_cells.ng_depos_rho; - for (int lev = 0; lev <= max_level; lev++) { - BoxArray nba = boxArray(lev); - nba.surroundingNodes(); - rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); - rho[lev]->setVal(0.); - phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); - phi[lev]->setVal(0.); - } - - // Set the boundary potentials appropriately - setPhiBC(phi); - - // beta is zero for boundaries - const std::array beta = {0._rt}; - - // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, self_fields_required_precision, - self_fields_absolute_tolerance, self_fields_max_iters, - self_fields_verbosity ); - - // Compute the corresponding electric and magnetic field, from the potential phi. - computeE( Efield_fp, phi, beta ); - computeB( Bfield_fp, phi, beta ); -} - -void -WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) -{ - WARPX_PROFILE("WarpX::AddSpaceChargeField"); - - if (pc.getCharge() == 0) { - return; - } - - // Store the boundary conditions for the field solver if they haven't been - // stored yet - if (!m_poisson_boundary_handler.bcs_set) { - m_poisson_boundary_handler.definePhiBCs(Geom(0)); - } - -#ifdef WARPX_DIM_RZ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "Error: RZ electrostatic only implemented for a single mode"); -#endif - - // Allocate fields for charge and potential - const int num_levels = max_level + 1; - Vector > rho(num_levels); - Vector > rho_coarse(num_levels); // Used in order to interpolate between levels - Vector > phi(num_levels); - // Use number of guard cells used for local deposition of rho - const amrex::IntVect ng = guard_cells.ng_depos_rho; - for (int lev = 0; lev <= max_level; lev++) { - BoxArray nba = boxArray(lev); - nba.surroundingNodes(); - rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); - rho[lev]->setVal(0.); - phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); - phi[lev]->setVal(0.); - if (lev > 0) { - // For MR levels: allocated the coarsened version of rho - BoxArray cba = nba; - cba.coarsen(refRatio(lev-1)); - rho_coarse[lev] = std::make_unique(cba, DistributionMap(lev), 1, ng); - rho_coarse[lev]->setVal(0.); - } - } - - // Deposit particle charge density (source of Poisson solver) - // The options below are identical to those in MultiParticleContainer::DepositCharge - bool const local = true; - bool const reset = false; - bool const apply_boundary_and_scale_volume = true; - bool const interpolate_across_levels = false; - if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, - interpolate_across_levels); - } - for (int lev = 0; lev <= max_level; lev++) { - if (lev > 0) { - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); - } - } - } - SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels - - // Get the particle beta vector - bool const local_average = false; // Average across all MPI ranks - std::array beta_pr = pc.meanParticleVelocity(local_average); - std::array beta; - for (int i=0 ; i < static_cast(beta.size()) ; i++) { - beta[i] = beta_pr[i]/PhysConst::c; // Normalize - } - - // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, pc.self_fields_required_precision, - pc.self_fields_absolute_tolerance, pc.self_fields_max_iters, - pc.self_fields_verbosity ); - - // Compute the corresponding electric and magnetic field, from the potential phi - computeE( Efield_fp, phi, beta ); - computeB( Bfield_fp, phi, beta ); - -} - -void -WarpX::AddSpaceChargeFieldLabFrame () -{ - WARPX_PROFILE("WarpX::AddSpaceChargeFieldLabFrame"); - - // Store the boundary conditions for the field solver if they haven't been - // stored yet - if (!m_poisson_boundary_handler.bcs_set) { - m_poisson_boundary_handler.definePhiBCs(Geom(0)); - } - -#ifdef WARPX_DIM_RZ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "Error: RZ electrostatic only implemented for a single mode"); -#endif - - // Deposit particle charge density (source of Poisson solver) - mypc->DepositCharge(rho_fp, 0.0_rt); - if (do_fluid_species) { - int const lev = 0; - myfl->DepositCharge( lev, *rho_fp[lev] ); - } - for (int lev = 0; lev <= max_level; lev++) { - if (lev > 0) { - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); - } - } - } - SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels -#ifndef WARPX_DIM_RZ - for (int lev = 0; lev <= finestLevel(); lev++) { - // Reflect density over PEC boundaries, if needed. - ApplyRhofieldBoundary(lev, rho_fp[lev].get(), PatchType::fine); - } -#endif - - // beta is zero in lab frame - // Todo: use simpler finite difference form with beta=0 - const std::array beta = {0._rt}; - - // set the boundary potentials appropriately - setPhiBC(phi_fp); - - // Compute the potential phi, by solving the Poisson equation - if (IsPythonCallbackInstalled("poissonsolver")) { - - // Use the Python level solver (user specified) - ExecutePythonCallback("poissonsolver"); - - } else { - -#if defined(WARPX_DIM_1D_Z) - // Use the tridiag solver with 1D - computePhiTriDiagonal(rho_fp, phi_fp); -#else - // Use the AMREX MLMG or the FFT (IGF) solver otherwise - computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, - self_fields_absolute_tolerance, self_fields_max_iters, - self_fields_verbosity); -#endif - - } - - // Compute the electric field. Note that if an EB is used the electric - // field will be calculated in the computePhi call. -#ifndef AMREX_USE_EB - computeE( Efield_fp, phi_fp, beta ); -#else - if ( IsPythonCallbackInstalled("poissonsolver") ) computeE( Efield_fp, phi_fp, beta ); -#endif - - // Compute the magnetic field - computeB( Bfield_fp, phi_fp, beta ); -} - -/* Compute the potential `phi` by solving the Poisson equation with `rho` as - a source, assuming that the source moves at a constant speed \f$\vec{\beta}\f$. - This uses the amrex solver. - - More specifically, this solves the equation - \f[ - \vec{\nabla}^2 r \phi - (\vec{\beta}\cdot\vec{\nabla})^2 r \phi = -\frac{r \rho}{\epsilon_0} - \f] - - \param[in] rho The charge density a given species - \param[out] phi The potential to be computed by this function - \param[in] beta Represents the velocity of the source of `phi` - \param[in] required_precision The relative convergence threshold for the MLMG solver - \param[in] absolute_tolerance The absolute convergence threshold for the MLMG solver - \param[in] max_iters The maximum number of iterations allowed for the MLMG solver - \param[in] verbosity The verbosity setting for the MLMG solver -*/ -void -WarpX::computePhi (const amrex::Vector >& rho, - amrex::Vector >& phi, - std::array const beta, - Real const required_precision, - Real absolute_tolerance, - int const max_iters, - int const verbosity) const -{ - // create a vector to our fields, sorted by level - amrex::Vector sorted_rho; - amrex::Vector sorted_phi; - for (int lev = 0; lev <= finest_level; ++lev) { - sorted_rho.emplace_back(rho[lev].get()); - sorted_phi.emplace_back(phi[lev].get()); - } - -#if defined(AMREX_USE_EB) - - std::optional post_phi_calculation; - - // EB: use AMReX to directly calculate the electric field since with EB's the - // simple finite difference scheme in WarpX::computeE sometimes fails - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || - electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - // TODO: maybe make this a helper function or pass Efield_fp directly - amrex::Vector< - amrex::Array - > e_field; - for (int lev = 0; lev <= finest_level; ++lev) { - e_field.push_back( -# if defined(WARPX_DIM_1D_Z) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 2) - } -# elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 2) - } -# elif defined(WARPX_DIM_3D) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2) - } -# endif - ); - } - post_phi_calculation = ElectrostaticSolver::EBCalcEfromPhiPerLevel(e_field); - } - - std::optional > eb_farray_box_factory; - amrex::Vector< - amrex::EBFArrayBoxFactory const * - > factories; - for (int lev = 0; lev <= finest_level; ++lev) { - factories.push_back(&WarpX::fieldEBFactory(lev)); - } - eb_farray_box_factory = factories; -#else - const std::optional post_phi_calculation; - const std::optional > eb_farray_box_factory; -#endif - - bool const is_solver_igf_on_lev0 = - WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; - - ablastr::fields::computePhi( - sorted_rho, - sorted_phi, - beta, - required_precision, - absolute_tolerance, - max_iters, - verbosity, - this->geom, - this->dmap, - this->grids, - WarpX::grid_type, - this->m_poisson_boundary_handler, - is_solver_igf_on_lev0, - WarpX::do_single_precision_comms, - this->ref_ratio, - post_phi_calculation, - gett_new(0), - eb_farray_box_factory - ); - -} - - -/* \brief Set Dirichlet boundary conditions for the electrostatic solver. - - The given potential's values are fixed on the boundaries of the given - dimension according to the desired values from the simulation input file, - boundary.potential_lo and boundary.potential_hi. - - \param[inout] phi The electrostatic potential - \param[in] idim The dimension for which the Dirichlet boundary condition is set -*/ -void -WarpX::setPhiBC ( amrex::Vector>& phi ) const -{ - // check if any dimension has non-periodic boundary conditions - if (!m_poisson_boundary_handler.has_non_periodic) { return; } - - // get the boundary potentials at the current time - amrex::Array phi_bc_values_lo; - amrex::Array phi_bc_values_hi; - phi_bc_values_lo[WARPX_ZINDEX] = m_poisson_boundary_handler.potential_zlo(gett_new(0)); - phi_bc_values_hi[WARPX_ZINDEX] = m_poisson_boundary_handler.potential_zhi(gett_new(0)); -#ifndef WARPX_DIM_1D_Z - phi_bc_values_lo[0] = m_poisson_boundary_handler.potential_xlo(gett_new(0)); - phi_bc_values_hi[0] = m_poisson_boundary_handler.potential_xhi(gett_new(0)); -#endif -#if defined(WARPX_DIM_3D) - phi_bc_values_lo[1] = m_poisson_boundary_handler.potential_ylo(gett_new(0)); - phi_bc_values_hi[1] = m_poisson_boundary_handler.potential_yhi(gett_new(0)); -#endif - - auto dirichlet_flag = m_poisson_boundary_handler.dirichlet_flag; - - // loop over all mesh refinement levels and set the boundary values - for (int lev=0; lev <= max_level; lev++) { - - amrex::Box domain = Geom(lev).Domain(); - domain.surroundingNodes(); - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { - // Extract the potential - auto phi_arr = phi[lev]->array(mfi); - // Extract tileboxes for which to loop - const Box& tb = mfi.tilebox( phi[lev]->ixType().toIntVect() ); - - // loop over dimensions - for (int idim=0; idim, 3> >& E, - const amrex::Vector >& phi, - std::array const beta ) const -{ - for (int lev = 0; lev <= max_level; lev++) { - - const Real* dx = Geom(lev).CellSize(); - -#ifdef AMREX_USE_OMP -# pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { -#if defined(WARPX_DIM_3D) - const Real inv_dx = 1._rt/dx[0]; - const Real inv_dy = 1._rt/dx[1]; - const Real inv_dz = 1._rt/dx[2]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real inv_dx = 1._rt/dx[0]; - const Real inv_dz = 1._rt/dx[1]; -#else - const Real inv_dz = 1._rt/dx[0]; -#endif - const amrex::IntVect ex_type = E[lev][0]->ixType().toIntVect(); - const amrex::IntVect ey_type = E[lev][1]->ixType().toIntVect(); - const amrex::IntVect ez_type = E[lev][2]->ixType().toIntVect(); - - const amrex::Box& tbx = mfi.tilebox(ex_type); - const amrex::Box& tby = mfi.tilebox(ey_type); - const amrex::Box& tbz = mfi.tilebox(ez_type); - - const auto& phi_arr = phi[lev]->array(mfi); - const auto& Ex_arr = (*E[lev][0])[mfi].array(); - const auto& Ey_arr = (*E[lev][1])[mfi].array(); - const auto& Ez_arr = (*E[lev][2])[mfi].array(); - - const Real beta_x = beta[0]; - const Real beta_y = beta[1]; - const Real beta_z = beta[2]; - - // Calculate the electric field - // Use discretized derivative that matches the staggering of the grid. - // Nodal solver - if (ex_type == amrex::IntVect::TheNodeVector() && - ey_type == amrex::IntVect::TheNodeVector() && - ez_type == amrex::IntVect::TheNodeVector()) - { -#if defined(WARPX_DIM_3D) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ex_arr(i,j,k) += - +(beta_x*beta_x-1._rt)*0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) - + beta_x*beta_y *0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) - + beta_x*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ey_arr(i,j,k) += - + beta_y*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) - +(beta_y*beta_y-1._rt)*0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) - + beta_y*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - + beta_z*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) - + beta_z*beta_y *0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) - +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); - } - ); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ex_arr(i,j,k) += - +(beta_x*beta_x-1._rt)*0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) - + beta_x*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ey_arr(i,j,k) += - +beta_x*beta_y*0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) - +beta_y*beta_z*0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - + beta_z*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) - +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); - } - ); -#else - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ex_arr(i,j,k) += - +(beta_x*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ey_arr(i,j,k) += - +beta_y*beta_z*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); - } - ); -#endif - } - else // Staggered solver - { -#if defined(WARPX_DIM_3D) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ex_arr(i,j,k) += - +(beta_x*beta_x-1._rt) *inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i ,j ,k )) - + beta_x*beta_y*0.25_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k ) - + phi_arr(i+1,j+1,k )-phi_arr(i+1,j-1,k )) - + beta_x*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1) - + phi_arr(i+1,j ,k+1)-phi_arr(i+1,j ,k-1)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ey_arr(i,j,k) += - + beta_y*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k ) - + phi_arr(i+1,j+1,k )-phi_arr(i-1,j+1,k )) - +(beta_y*beta_y-1._rt) *inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j ,k )) - + beta_y*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1) - + phi_arr(i ,j+1,k+1)-phi_arr(i ,j+1,k-1)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - + beta_z*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k ) - + phi_arr(i+1,j ,k+1)-phi_arr(i-1,j ,k+1)) - + beta_z*beta_y*0.25_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k ) - + phi_arr(i ,j+1,k+1)-phi_arr(i ,j-1,k+1)) - +(beta_z*beta_z-1._rt) *inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k )); - } - ); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ParallelFor( tbx, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ex_arr(i,j,k) += - +(beta_x*beta_x-1._rt)*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k)) - +beta_x*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k) - + phi_arr(i+1,j+1,k)-phi_arr(i+1,j-1,k)); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - +beta_z*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k) - + phi_arr(i+1,j+1,k)-phi_arr(i-1,j+1,k)) - +(beta_z*beta_z-1._rt)*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k)); - } - ); - ignore_unused(beta_y); -#else - amrex::ParallelFor( tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Ez_arr(i,j,k) += - +(beta_z*beta_z-1._rt)*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k)); - } - ); - ignore_unused(beta_x,beta_y); -#endif - } - } - } -} - - -/* \brief Compute the magnetic field that corresponds to `phi`, and - add it to the set of MultiFab `B`. - - The magnetic field is calculated by assuming that the source that - produces the `phi` potential is moving with a constant speed \f$\vec{\beta}\f$: - \f[ - \vec{B} = -\frac{1}{c}\vec{\beta}\times\vec{\nabla}\phi - \f] - (this represents the term \f$\vec{\nabla} \times \vec{A}\f$, in the case of a moving source) - - \param[inout] E Electric field on the grid - \param[in] phi The potential from which to compute the electric field - \param[in] beta Represents the velocity of the source of `phi` -*/ -void -WarpX::computeB (amrex::Vector, 3> >& B, - const amrex::Vector >& phi, - std::array const beta ) const -{ - // return early if beta is 0 since there will be no B-field - if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } - - for (int lev = 0; lev <= max_level; lev++) { - - const Real* dx = Geom(lev).CellSize(); - -#ifdef AMREX_USE_OMP -# pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { -#if defined(WARPX_DIM_3D) - const Real inv_dx = 1._rt/dx[0]; - const Real inv_dy = 1._rt/dx[1]; - const Real inv_dz = 1._rt/dx[2]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real inv_dx = 1._rt/dx[0]; - const Real inv_dz = 1._rt/dx[1]; -#else - const Real inv_dz = 1._rt/dx[0]; -#endif - const amrex::IntVect bx_type = B[lev][0]->ixType().toIntVect(); - const amrex::IntVect by_type = B[lev][1]->ixType().toIntVect(); - const amrex::IntVect bz_type = B[lev][2]->ixType().toIntVect(); - - const amrex::Box& tbx = mfi.tilebox(bx_type); - const amrex::Box& tby = mfi.tilebox(by_type); - const amrex::Box& tbz = mfi.tilebox(bz_type); - - const auto& phi_arr = phi[lev]->array(mfi); - const auto& Bx_arr = (*B[lev][0])[mfi].array(); - const auto& By_arr = (*B[lev][1])[mfi].array(); - const auto& Bz_arr = (*B[lev][2])[mfi].array(); - - const Real beta_x = beta[0]; - const Real beta_y = beta[1]; - const Real beta_z = beta[2]; - - constexpr Real inv_c = 1._rt/PhysConst::c; - - // Calculate the magnetic field - // Use discretized derivative that matches the staggering of the grid. - // Nodal solver - if (bx_type == amrex::IntVect::TheNodeVector() && - by_type == amrex::IntVect::TheNodeVector() && - bz_type == amrex::IntVect::TheNodeVector()) - { -#if defined(WARPX_DIM_3D) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*0.5_rt*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k-1)) - +beta_z*inv_dy*0.5_rt*(phi_arr(i,j+1,k )-phi_arr(i,j-1,k ))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j,k )-phi_arr(i-1,j,k )) - +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k-1))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bz_arr(i,j,k) += inv_c * ( - -beta_x*inv_dy*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)) - +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k))); - } - ); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*0.5_rt*(phi_arr(i,j+1,k)-phi_arr(i,j-1,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) - +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bz_arr(i,j,k) += inv_c * ( - +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k))); - } - ); -#else - amrex::ParallelFor( tbx, tby, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - +beta_x*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); - } - ); - ignore_unused(beta_z,tbz,Bz_arr); -#endif - } - else // Staggered solver - { -#if defined(WARPX_DIM_3D) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*0.5_rt*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k ) - + phi_arr(i,j+1,k+1)-phi_arr(i,j+1,k )) - +beta_z*inv_dy*0.5_rt*(phi_arr(i,j+1,k )-phi_arr(i,j ,k ) - + phi_arr(i,j+1,k+1)-phi_arr(i,j ,k+1))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j,k )-phi_arr(i ,j,k ) - + phi_arr(i+1,j,k+1)-phi_arr(i ,j,k+1)) - +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k ) - + phi_arr(i+1,j,k+1)-phi_arr(i+1,j,k ))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bz_arr(i,j,k) += inv_c * ( - -beta_x*inv_dy*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k) - + phi_arr(i+1,j+1,k)-phi_arr(i+1,j ,k)) - +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k) - + phi_arr(i+1,j+1,k)-phi_arr(i ,j+1,k))); - } - ); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ParallelFor( tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*(phi_arr(i,j+1,k)-phi_arr(i,j,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k) - + phi_arr(i+1,j+1,k)-phi_arr(i ,j+1,k)) - +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k) - + phi_arr(i+1,j+1,k)-phi_arr(i+1,j ,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bz_arr(i,j,k) += inv_c * ( - +beta_y*inv_dx*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); - } - ); -#else - amrex::ParallelFor( tbx, tby, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - Bx_arr(i,j,k) += inv_c * ( - -beta_y*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - By_arr(i,j,k) += inv_c * ( - +beta_x*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); - } - ); - ignore_unused(beta_z,tbz,Bz_arr); -#endif - } - } - } -} - -/* \brief Compute the potential by solving Poisson's equation with - a 1D tridiagonal solve. - - \param[in] rho The charge density a given species - \param[out] phi The potential to be computed by this function -*/ -void -WarpX::computePhiTriDiagonal (const amrex::Vector >& rho, - amrex::Vector >& phi) const -{ - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "The tridiagonal solver cannot be used with mesh refinement"); - - const int lev = 0; - - const amrex::Real* dx = Geom(lev).CellSize(); - const amrex::Real xmin = Geom(lev).ProbLo(0); - const amrex::Real xmax = Geom(lev).ProbHi(0); - const int nx_full_domain = static_cast( (xmax - xmin)/dx[0] + 0.5_rt ); - - int nx_solve_min = 1; - int nx_solve_max = nx_full_domain - 1; - - auto field_boundary_lo0 = WarpX::field_boundary_lo[0]; - auto field_boundary_hi0 = WarpX::field_boundary_hi[0]; - if (field_boundary_lo0 == FieldBoundaryType::Neumann || field_boundary_lo0 == FieldBoundaryType::Periodic) { - // Neumann or periodic boundary condition - // Solve for the point on the lower boundary - nx_solve_min = 0; - } - if (field_boundary_hi0 == FieldBoundaryType::Neumann || field_boundary_hi0 == FieldBoundaryType::Periodic) { - // Neumann or periodic boundary condition - // Solve for the point on the upper boundary - nx_solve_max = nx_full_domain; - } - - // Create a 1-D MultiFab that covers all of x. - // The tridiag solve will be done in this MultiFab and then copied out afterwards. - const amrex::IntVect lo_full_domain(AMREX_D_DECL(0,0,0)); - const amrex::IntVect hi_full_domain(AMREX_D_DECL(nx_full_domain,0,0)); - const amrex::Box box_full_domain_node(lo_full_domain, hi_full_domain, amrex::IntVect::TheNodeVector()); - const BoxArray ba_full_domain_node(box_full_domain_node); - const amrex::Vector pmap = {0}; // The data will only be on processor 0 - const amrex::DistributionMapping dm_full_domain(pmap); - - // Put the data in the pinned arena since the tridiag solver will be done on the CPU, but have - // the data readily accessible from the GPU. - auto phi1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); - auto zwork1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); - auto rho1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); - - if (field_boundary_lo0 == FieldBoundaryType::PEC || field_boundary_hi0 == FieldBoundaryType::PEC) { - // Copy from phi to get the boundary values - phi1d_mf.ParallelCopy(*phi[lev], 0, 0, 1); - } - rho1d_mf.ParallelCopy(*rho[lev], 0, 0, 1); - - // Multiplier on the charge density - const amrex::Real norm = dx[0]*dx[0]/PhysConst::ep0; - rho1d_mf.mult(norm); - - // Use the MFIter loop since when parallel, only process zero has a FAB. - // This skips the loop on all other processors. - for (MFIter mfi(phi1d_mf); mfi.isValid(); ++mfi) { - - const auto& phi1d_arr = phi1d_mf[mfi].array(); - const auto& zwork1d_arr = zwork1d_mf[mfi].array(); - const auto& rho1d_arr = rho1d_mf[mfi].array(); - - // The loops are always performed on the CPU - - amrex::Real diag = 2._rt; - - // The initial values depend on the boundary condition - if (field_boundary_lo0 == FieldBoundaryType::PEC) { - - phi1d_arr(1,0,0) = (phi1d_arr(0,0,0) + rho1d_arr(1,0,0))/diag; - - } else if (field_boundary_lo0 == FieldBoundaryType::Neumann) { - - // Neumann boundary condition - phi1d_arr(0,0,0) = rho1d_arr(0,0,0)/diag; - - zwork1d_arr(1,0,0) = 2._rt/diag; - diag = 2._rt - zwork1d_arr(1,0,0); - phi1d_arr(1,0,0) = (rho1d_arr(1,0,0) - (-1._rt)*phi1d_arr(1-1,0,0))/diag; - - } else if (field_boundary_lo0 == FieldBoundaryType::Periodic) { - - phi1d_arr(0,0,0) = rho1d_arr(0,0,0)/diag; - - zwork1d_arr(1,0,0) = 1._rt/diag; - diag = 2._rt - zwork1d_arr(1,0,0); - phi1d_arr(1,0,0) = (rho1d_arr(1,0,0) - (-1._rt)*phi1d_arr(1-1,0,0))/diag; - - } - - // Loop upward, calculating the Gaussian elimination multipliers and right hand sides - for (int i_up = 2 ; i_up < nx_solve_max ; i_up++) { - - zwork1d_arr(i_up,0,0) = 1._rt/diag; - diag = 2._rt - zwork1d_arr(i_up,0,0); - phi1d_arr(i_up,0,0) = (rho1d_arr(i_up,0,0) - (-1._rt)*phi1d_arr(i_up-1,0,0))/diag; - - } - - // The last value depend on the boundary condition - amrex::Real zwork_product = 1.; // Needed for parallel boundaries - if (field_boundary_hi0 == FieldBoundaryType::PEC) { - - int const nxm1 = nx_full_domain - 1; - zwork1d_arr(nxm1,0,0) = 1._rt/diag; - diag = 2._rt - zwork1d_arr(nxm1,0,0); - phi1d_arr(nxm1,0,0) = (phi1d_arr(nxm1+1,0,0) + rho1d_arr(nxm1,0,0) - (-1._rt)*phi1d_arr(nxm1-1,0,0))/diag; - - } else if (field_boundary_hi0 == FieldBoundaryType::Neumann) { - - // Neumann boundary condition - zwork1d_arr(nx_full_domain,0,0) = 1._rt/diag; - diag = 2._rt - 2._rt*zwork1d_arr(nx_full_domain,0,0); - if (diag == 0._rt) { - // This happens if the lower boundary is also Neumann. - // It this case, the potential is relative to an arbitrary constant, - // so set the upper boundary to zero to force a value. - phi1d_arr(nx_full_domain,0,0) = 0.; - } else { - phi1d_arr(nx_full_domain,0,0) = (rho1d_arr(nx_full_domain,0,0) - (-1._rt)*phi1d_arr(nx_full_domain-1,0,0))/diag; - } - - } else if (field_boundary_hi0 == FieldBoundaryType::Periodic) { - - zwork1d_arr(nx_full_domain,0,0) = 1._rt/diag; - - for (int i = 1 ; i <= nx_full_domain ; i++) { - zwork_product *= zwork1d_arr(i,0,0); - } - - diag = 2._rt - zwork1d_arr(nx_full_domain,0,0) - zwork_product; - // Note that rho1d_arr(0,0,0) is used to ensure that the same value is used - // on both boundaries. - phi1d_arr(nx_full_domain,0,0) = (rho1d_arr(0,0,0) - (-1._rt)*phi1d_arr(nx_full_domain-1,0,0))/diag; - - } - - // Loop downward to calculate the phi - if (field_boundary_lo0 == FieldBoundaryType::Periodic) { - - // With periodic, the right hand column adds an extra term for all rows - for (int i_down = nx_full_domain-1 ; i_down >= 0 ; i_down--) { - zwork_product /= zwork1d_arr(i_down+1,0,0); - phi1d_arr(i_down,0,0) = phi1d_arr(i_down,0,0) + zwork1d_arr(i_down+1,0,0)*phi1d_arr(i_down+1,0,0) + zwork_product*phi1d_arr(nx_full_domain,0,0); - } - - } else { - - for (int i_down = nx_solve_max-1 ; i_down >= nx_solve_min ; i_down--) { - phi1d_arr(i_down,0,0) = phi1d_arr(i_down,0,0) + zwork1d_arr(i_down+1,0,0)*phi1d_arr(i_down+1,0,0); - } - - } - - } - - // Copy phi1d to phi - phi[lev]->ParallelCopy(phi1d_mf, 0, 0, 1); -} - -void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geometry& geom) -{ -#ifdef WARPX_DIM_RZ - if (geom.ProbLo(0) == 0){ - lobc[0] = LinOpBCType::Neumann; - dirichlet_flag[0] = false; - - // handle the r_max boundary explicitly - if (WarpX::field_boundary_hi[0] == FieldBoundaryType::PEC) { - hibc[0] = LinOpBCType::Dirichlet; - dirichlet_flag[1] = true; - } - else if (WarpX::field_boundary_hi[0] == FieldBoundaryType::Neumann) { - hibc[0] = LinOpBCType::Neumann; - dirichlet_flag[1] = false; - } - else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary condition at the outer radius must be either PEC or neumann " - "when using the electrostatic solver" - ); - } - } - const int dim_start = 1; -#else - const int dim_start = 0; - amrex::ignore_unused(geom); -#endif - for (int idim=dim_start; idim(); - potential_xhi = potential_xhi_parser.compile<1>(); - potential_ylo = potential_ylo_parser.compile<1>(); - potential_yhi = potential_yhi_parser.compile<1>(); - potential_zlo = potential_zlo_parser.compile<1>(); - potential_zhi = potential_zhi_parser.compile<1>(); - - buildParsersEB(); -} - -void ElectrostaticSolver::PoissonBoundaryHandler::buildParsersEB () -{ - potential_eb_parser = utils::parser::makeParser(potential_eb_str, {"x", "y", "z", "t"}); - - // check if the EB potential is a function of space or only of time - const std::set eb_symbols = potential_eb_parser.symbols(); - if ((eb_symbols.count("x") != 0) || (eb_symbols.count("y") != 0) - || (eb_symbols.count("z") != 0)) { - potential_eb = potential_eb_parser.compile<4>(); - phi_EB_only_t = false; - } - else { - potential_eb_parser = utils::parser::makeParser(potential_eb_str, {"t"}); - potential_eb_t = potential_eb_parser.compile<1>(); - } -} diff --git a/Source/FieldSolver/ElectrostaticSolvers/CMakeLists.txt b/Source/FieldSolver/ElectrostaticSolvers/CMakeLists.txt new file mode 100644 index 00000000000..db728f6aaba --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/CMakeLists.txt @@ -0,0 +1,11 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + EffectivePotentialES.cpp + ElectrostaticSolver.cpp + LabFrameExplicitES.cpp + PoissonBoundaryHandler.cpp + RelativisticExplicitES.cpp + ) +endforeach() diff --git a/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.H b/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.H new file mode 100644 index 00000000000..2ade923211c --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.H @@ -0,0 +1,71 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_EFFECTIVEPOTENTIALES_H_ +#define WARPX_EFFECTIVEPOTENTIALES_H_ + +#include "ElectrostaticSolver.H" + +#include +#include + +class EffectivePotentialES final : public ElectrostaticSolver +{ +public: + + EffectivePotentialES (int nlevs_max) : ElectrostaticSolver (nlevs_max) { + ReadParameters(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (nlevs_max == 1), + "Effective potential electrostatic solver only supports one level at present" + ); + } + + void InitData () override; + + void ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) override; + + void ComputeSigma ( amrex::MultiFab& sigma ) const; + + /** + * Compute the potential `phi` by solving the semi-implicit Poisson equation using the Effective Potential method + * with `rho` as the source. + * More specifically, this solves the equation + * \f[ + * \vec{\nabla}\cdot(\sigma\vec{\nabla}) \phi = -\frac{\rho}{\epsilon_0} + * \f] + * \param[out] phi The potential to be computed by this function + * \param[in] rho The total charge density + * \param[in] sigma Represents the modified dielectric + * \param[in] required_precision The relative convergence threshold for the MLMG solver + * \param[in] absolute_tolerance The absolute convergence threshold for the MLMG solver + * \param[in] max_iters The maximum number of iterations allowed for the MLMG solver + * \param[in] verbosity The verbosity setting for the MLMG solver + */ + void computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi + ) const; + void computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + amrex::MultiFab const& sigma, + amrex::Real required_precision, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity + ) const; + +}; + +#endif // WARPX_EFFECTIVEPOTENTIALES_H_ diff --git a/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.cpp b/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.cpp new file mode 100644 index 00000000000..b2f93f7e2b3 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/EffectivePotentialES.cpp @@ -0,0 +1,260 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "EffectivePotentialES.H" +#include "Fluids/MultiFluidContainer_fwd.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" +#include "Particles/MultiParticleContainer_fwd.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +using namespace amrex; + +void EffectivePotentialES::InitData() { + auto & warpx = WarpX::GetInstance(); + m_poisson_boundary_handler->DefinePhiBCs(warpx.Geom(0)); +} + +void EffectivePotentialES::ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + [[maybe_unused]] MultiFluidContainer* mfl, + int max_level) +{ + WARPX_PROFILE("EffectivePotentialES::ComputeSpaceChargeField"); + + using ablastr::fields::MultiLevelScalarField; + using ablastr::fields::MultiLevelVectorField; + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + + // grab the simulation fields + const MultiLevelScalarField rho_fp = fields.get_mr_levels(FieldType::rho_fp, max_level); + const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level, skip_lev0_coarse_patch); + const MultiLevelScalarField phi_fp = fields.get_mr_levels(FieldType::phi_fp, max_level); + const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + + mpc.DepositCharge(rho_fp, 0.0_rt); + if (mfl) { + const int lev = 0; + mfl->DepositCharge(fields, *rho_fp[lev], lev); + } + + // Apply filter, perform MPI exchange, interpolate across levels + const Vector > rho_buf(num_levels); + auto & warpx = WarpX::GetInstance(); + warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); + +#ifndef WARPX_DIM_RZ + for (int lev = 0; lev < num_levels; lev++) { + // Reflect density over PEC boundaries, if needed. + warpx.ApplyRhofieldBoundary(lev, rho_fp[lev], PatchType::fine); + } +#endif + + // set the boundary potentials appropriately + setPhiBC(phi_fp, warpx.gett_new(0)); + + // perform phi calculation + computePhi(rho_fp, phi_fp); + + // Compute the electric field. Note that if an EB is used the electric + // field will be calculated in the computePhi call. + const std::array beta = {0._rt}; + if (!EB::enabled()) { computeE( Efield_fp, phi_fp, beta ); } +} + +void EffectivePotentialES::computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi) const +{ + // Calculate the mass enhancement factor - see Appendix A of + // Barnes, Journal of Comp. Phys., 424 (2021), 109852. + // The "sigma" multifab stores the dressing of the Poisson equation. It + // is a cell-centered multifab. + auto const& ba = convert(rho[0]->boxArray(), IntVect(AMREX_D_DECL(0,0,0))); + MultiFab sigma(ba, rho[0]->DistributionMap(), 1, rho[0]->nGrowVect()); + ComputeSigma(sigma); + + // Use the AMREX MLMG solver + computePhi(rho, phi, sigma, self_fields_required_precision, + self_fields_absolute_tolerance, self_fields_max_iters, + self_fields_verbosity); +} + +void EffectivePotentialES::ComputeSigma (MultiFab& sigma) const +{ + // Reset sigma to 1 + sigma.setVal(1.0_rt); + + // Get the user set value for C_SI (defaults to 4) + amrex::Real C_SI = 4.0; + const ParmParse pp_warpx("warpx"); + utils::parser::queryWithParser(pp_warpx, "effective_potential_factor", C_SI); + + int const lev = 0; + + // sigma is a cell-centered array + amrex::GpuArray const cell_centered = {0, 0, 0}; + // The "coarsening is just 1 i.e. no coarsening" + amrex::GpuArray const coarsen = {1, 1, 1}; + + // GetChargeDensity returns a nodal multifab + // Below we set all the unused dimensions to have cell-centered values for + // rho since these values will be interpolated onto a cell-centered grid + // - if this is not done the Interp function returns nonsense values. +#if defined(WARPX_DIM_3D) + amrex::GpuArray const nodal = {1, 1, 1}; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::GpuArray const nodal = {1, 1, 0}; +#elif defined(WARPX_DIM_1D_Z) + amrex::GpuArray const nodal = {1, 0, 0}; +#endif + + auto& warpx = WarpX::GetInstance(); + auto& mypc = warpx.GetPartContainer(); + + // The effective potential dielectric function is given by + // \varepsilon_{SI} = \varepsilon * (1 + \sum_{i in species} C_{SI}*(w_pi * dt)^2/4) + // Note the use of the plasma frequency in rad/s (not Hz) and the factor of 1/4, + // these choices make it so that C_SI = 1 is the marginal stability threshold. + auto mult_factor = ( + C_SI * warpx.getdt(lev) * warpx.getdt(lev) / (4._rt * PhysConst::ep0) + ); + + // Loop over each species to calculate the Poisson equation dressing + for (auto const& pc : mypc) { + // grab the charge density for this species + auto rho = pc->GetChargeDensity(lev, false); + + // Handle the parallel transfer of guard cells and apply filtering + warpx.ApplyFilterandSumBoundaryRho(lev, lev, *rho, 0, rho->nComp()); + + // get multiplication factor for this species + auto const mult_factor_pc = mult_factor * pc->getCharge() / pc->getMass(); + + // update sigma +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(sigma, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + Array4 const& sigma_arr = sigma.array(mfi); + Array4 const& rho_arr = rho->const_array(mfi); + + // Loop over the cells and update the sigma field + amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Interpolate rho to cell-centered value + auto const rho_cc = ablastr::coarsen::sample::Interp( + rho_arr, nodal, cell_centered, coarsen, i, j, k, 0 + ); + // add species term to sigma: + // C_SI * w_p^2 * dt^2 / 4 = C_SI / 4 * q*rho/(m*eps0) * dt^2 + sigma_arr(i, j, k, 0) += mult_factor_pc * rho_cc; + }); + + } + } +} + + +void EffectivePotentialES::computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + amrex::MultiFab const& sigma, + amrex::Real required_precision, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity +) const +{ + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + // create a vector to our fields, sorted by level + amrex::Vector sorted_rho; + amrex::Vector sorted_phi; + for (int lev = 0; lev < num_levels; ++lev) { + sorted_rho.emplace_back(rho[lev]); + sorted_phi.emplace_back(phi[lev]); + } + + std::optional post_phi_calculation; +#ifdef AMREX_USE_EB + // TODO: double check no overhead occurs on "m_eb_enabled == false" + std::optional > eb_farray_box_factory; +#else + std::optional > const eb_farray_box_factory; +#endif + auto & warpx = WarpX::GetInstance(); + if (EB::enabled()) + { + // EB: use AMReX to directly calculate the electric field since with EB's the + // simple finite difference scheme in WarpX::computeE sometimes fails + + // TODO: maybe make this a helper function or pass Efield_fp directly + amrex::Vector< + amrex::Array + > e_field; + for (int lev = 0; lev < num_levels; ++lev) { + e_field.push_back( +#if defined(WARPX_DIM_1D_Z) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_3D) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#endif + ); + } + post_phi_calculation = EBCalcEfromPhiPerLevel(e_field); + +#ifdef AMREX_USE_EB + amrex::Vector< + amrex::EBFArrayBoxFactory const * + > factories; + for (int lev = 0; lev < num_levels; ++lev) { + factories.push_back(&warpx.fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; +#endif + } + + ablastr::fields::computeEffectivePotentialPhi( + sorted_rho, + sorted_phi, + sigma, + required_precision, + absolute_tolerance, + max_iters, + verbosity, + warpx.Geom(), + warpx.DistributionMap(), + warpx.boxArray(), + WarpX::grid_type, + false, + EB::enabled(), + WarpX::do_single_precision_comms, + warpx.refRatio(), + post_phi_calculation, + *m_poisson_boundary_handler, + warpx.gett_new(0), + eb_farray_box_factory + ); +} diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H new file mode 100755 index 00000000000..0a0b0e2be48 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H @@ -0,0 +1,164 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_ELECTROSTATICSOLVER_H_ +#define WARPX_ELECTROSTATICSOLVER_H_ + +#include "PoissonBoundaryHandler.H" +#include "Fluids/MultiFluidContainer.H" +#include "Particles/MultiParticleContainer.H" +#include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" + +#include + + +/** + * \brief Base class for Electrostatic Solver + * + */ +class ElectrostaticSolver +{ +public: + ElectrostaticSolver() = default; + ElectrostaticSolver( int nlevs_max ); + + virtual ~ElectrostaticSolver(); + + // Prohibit Move and Copy operations + ElectrostaticSolver(const ElectrostaticSolver&) = delete; + ElectrostaticSolver& operator=(const ElectrostaticSolver&) = delete; + ElectrostaticSolver(ElectrostaticSolver&&) = delete; + ElectrostaticSolver& operator=(ElectrostaticSolver&&) = delete; + + void ReadParameters (); + + virtual void InitData () {} + + /** + * \brief Computes charge density, rho, and solves Poisson's equation + * to obtain the associated electrostatic potential, phi. + * Using the electrostatic potential, the electric field is computed + * in lab frame, and if relativistic, then the electric and magnetic + * fields are computed using potential, phi, and + * velocity of source for potential, beta. + * This function must be defined in the derived classes. + */ + virtual void ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) = 0; + + /** + * \brief Set Dirichlet boundary conditions for the electrostatic solver. + * The given potential's values are fixed on the boundaries of the given + * dimension according to the desired values from the simulation input file, + * boundary.potential_lo and boundary.potential_hi. + * \param[inout] phi The electrostatic potential + * \param[in] idim The dimension for which the Dirichlet boundary condition is set + */ + void setPhiBC ( + ablastr::fields::MultiLevelScalarField const& phi, + amrex::Real t + ) const; + + /** + * Compute the potential `phi` by solving the Poisson equation with `rho` as + * a source, assuming that the source moves at a constant speed \f$\vec{\beta}\f$. + * This uses the amrex solver. + * More specifically, this solves the equation + * \f[ + * \vec{\nabla}^2 r \phi - (\vec{\beta}\cdot\vec{\nabla})^2 r \phi = -\frac{r \rho}{\epsilon_0} + * \f] + * \param[in] rho The charge density for a given species (relativistic solver) + * or total charge density (labframe solver) + * \param[out] phi The potential to be computed by this function + * \param[in] beta Represents the velocity of the source of `phi` + * \param[in] required_precision The relative convergence threshold for the MLMG solver + * \param[in] absolute_tolerance The absolute convergence threshold for the MLMG solver + * \param[in] max_iters The maximum number of iterations allowed for the MLMG solver + * \param[in] verbosity The verbosity setting for the MLMG solver + * \param[in] is_igf_2d_slices boolean to select between fully 3D Poisson solver and quasi-3D, i.e. one 2D Poisson solve on every z slice (default: false) + */ + void computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta, + amrex::Real required_precision, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity, + bool is_igf_2d_slices + ) const; + + /** + * \brief Compute the electric field that corresponds to `phi`, and + * add it to the set of MultiFab `E`. + * The electric field is calculated by assuming that the source that + * produces the `phi` potential is moving with a constant speed \f$\vec{\beta}\f$: + * \f[ + * \vec{E} = -\vec{\nabla}\phi + \vec{\beta}(\vec{\beta} \cdot \vec{\nabla}\phi) + * \f] + * (where the second term represent the term \f$\partial_t \vec{A}\f$, in + * the case of a moving source) + * + * \param[inout] E Electric field on the grid + * \param[in] phi The potential from which to compute the electric field + * \param[in] beta Represents the velocity of the source of `phi` + */ + void computeE ( + ablastr::fields::MultiLevelVectorField const& E, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta + ) const; + + /** + * \brief Compute the magnetic field that corresponds to `phi`, and + * add it to the set of MultiFab `B`. + *The magnetic field is calculated by assuming that the source that + *produces the `phi` potential is moving with a constant speed \f$\vec{\beta}\f$: + *\f[ + * \vec{B} = -\frac{1}{c}\vec{\beta}\times\vec{\nabla}\phi + *\f] + *(this represents the term \f$\vec{\nabla} \times \vec{A}\f$, in the case of a moving source) + * + *\param[inout] B Electric field on the grid + *\param[in] phi The potential from which to compute the electric field + *\param[in] beta Represents the velocity of the source of `phi` + */ + void computeB ( + ablastr::fields::MultiLevelVectorField const& B, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta + ) const; + + /** Maximum levels for the electrostatic solver grid */ + int num_levels; + + /** Boundary handler object to set potential for EB and on the domain boundary */ + std::unique_ptr m_poisson_boundary_handler; + + /** Parameters for MLMG Poisson solve */ + amrex::Real self_fields_required_precision = 1e-11; + amrex::Real self_fields_absolute_tolerance = 0.0; + /** Limit on number of MLMG iterations */ + int self_fields_max_iters = 200; + /** Verbosity for the MLMG solver. + * 0 : no verbosity + * 1 : timing and convergence at the end of MLMG + * 2 : convergence progress at every MLMG iteration + */ + int self_fields_verbosity = 2; + + /** Parameters for FFT Poisson solver aka IGF */ + // 0: full 3D, 1: many 2D z-slices (quasi-3D) + bool is_igf_2d_slices = false; +}; + +#endif // WARPX_ELECTROSTATICSOLVER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp new file mode 100755 index 00000000000..429e007b4d0 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp @@ -0,0 +1,551 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ + +#include "ElectrostaticSolver.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" + +#include + + +using namespace amrex; +using warpx::fields::FieldType; + +ElectrostaticSolver::ElectrostaticSolver (int nlevs_max) : num_levels{nlevs_max} +{ + // Create an instance of the boundary handler to properly set boundary + // conditions + m_poisson_boundary_handler = std::make_unique(); +} + +ElectrostaticSolver::~ElectrostaticSolver () = default; + +void ElectrostaticSolver::ReadParameters () { + + ParmParse const pp_warpx("warpx"); + + // Note that with the relativistic version, these parameters would be + // input for each species. + utils::parser::queryWithParser( + pp_warpx, "self_fields_required_precision", self_fields_required_precision); + utils::parser::queryWithParser( + pp_warpx, "self_fields_absolute_tolerance", self_fields_absolute_tolerance); + utils::parser::queryWithParser( + pp_warpx, "self_fields_max_iters", self_fields_max_iters); + utils::parser::queryWithParser( + pp_warpx, "self_fields_verbosity", self_fields_verbosity); + + // FFT solver flags + utils::parser::queryWithParser( + pp_warpx, "use_2d_slices_fft_solver", is_igf_2d_slices); +} + +void +ElectrostaticSolver::setPhiBC ( + ablastr::fields::MultiLevelScalarField const& phi, + amrex::Real t +) const +{ + // check if any dimension has non-periodic boundary conditions + if (!m_poisson_boundary_handler->has_non_periodic) { return; } + + // get the boundary potentials at the current time + amrex::Array phi_bc_values_lo; + amrex::Array phi_bc_values_hi; + phi_bc_values_lo[WARPX_ZINDEX] = m_poisson_boundary_handler->potential_zlo(t); + phi_bc_values_hi[WARPX_ZINDEX] = m_poisson_boundary_handler->potential_zhi(t); +#ifndef WARPX_DIM_1D_Z + phi_bc_values_lo[0] = m_poisson_boundary_handler->potential_xlo(t); + phi_bc_values_hi[0] = m_poisson_boundary_handler->potential_xhi(t); +#endif +#if defined(WARPX_DIM_3D) + phi_bc_values_lo[1] = m_poisson_boundary_handler->potential_ylo(t); + phi_bc_values_hi[1] = m_poisson_boundary_handler->potential_yhi(t); +#endif + + auto dirichlet_flag = m_poisson_boundary_handler->dirichlet_flag; + + auto & warpx = WarpX::GetInstance(); + + // loop over all mesh refinement levels and set the boundary values + for (int lev=0; lev < num_levels; lev++) { + + amrex::Box domain = warpx.Geom(lev).Domain(); + domain.surroundingNodes(); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + // Extract the potential + auto phi_arr = phi[lev]->array(mfi); + // Extract tileboxes for which to loop + const Box& tb = mfi.tilebox( phi[lev]->ixType().toIntVect() ); + + // loop over dimensions + for (int idim=0; idim const beta, + Real const required_precision, + Real absolute_tolerance, + int const max_iters, + int const verbosity, + bool const is_igf_2d +) const +{ + using ablastr::fields::Direction; + + // create a vector to our fields, sorted by level + amrex::Vector sorted_rho; + amrex::Vector sorted_phi; + for (int lev = 0; lev < num_levels; ++lev) { + sorted_rho.emplace_back(rho[lev]); + sorted_phi.emplace_back(phi[lev]); + } + + std::optional post_phi_calculation; +#ifdef AMREX_USE_EB + // TODO: double check no overhead occurs on "m_eb_enabled == false" + std::optional > eb_farray_box_factory; +#else + std::optional > const eb_farray_box_factory; +#endif + auto & warpx = WarpX::GetInstance(); + if (EB::enabled()) + { + if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || + WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + { + // EB: use AMReX to directly calculate the electric field since with EB's the + // simple finite difference scheme in WarpX::computeE sometimes fails + + // TODO: maybe make this a helper function or pass Efield_fp directly + amrex::Vector< + amrex::Array + > e_field; + for (int lev = 0; lev < num_levels; ++lev) { + e_field.push_back( +#if defined(WARPX_DIM_1D_Z) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#elif defined(WARPX_DIM_3D) + amrex::Array{ + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) + } +#endif + ); + } + post_phi_calculation = EBCalcEfromPhiPerLevel(e_field); + } +#ifdef AMREX_USE_EB + amrex::Vector< + amrex::EBFArrayBoxFactory const * + > factories; + for (int lev = 0; lev < num_levels; ++lev) { + factories.push_back(&warpx.fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; +#endif + } + + bool const is_solver_igf_on_lev0 = + WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; + + ablastr::fields::computePhi( + sorted_rho, + sorted_phi, + beta, + required_precision, + absolute_tolerance, + max_iters, + verbosity, + warpx.Geom(), + warpx.DistributionMap(), + warpx.boxArray(), + WarpX::grid_type, + is_solver_igf_on_lev0, + is_igf_2d, + EB::enabled(), + WarpX::do_single_precision_comms, + warpx.refRatio(), + post_phi_calculation, + *m_poisson_boundary_handler, + warpx.gett_new(0), + eb_farray_box_factory + ); + +} + +void +ElectrostaticSolver::computeE ( + ablastr::fields::MultiLevelVectorField const& E, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta ) const +{ + auto & warpx = WarpX::GetInstance(); + for (int lev = 0; lev < num_levels; lev++) { + + const Real* dx = warpx.Geom(lev).CellSize(); + +#ifdef AMREX_USE_OMP +# pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { +#if defined(WARPX_DIM_3D) + const Real inv_dx = 1._rt/dx[0]; + const Real inv_dy = 1._rt/dx[1]; + const Real inv_dz = 1._rt/dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const Real inv_dx = 1._rt/dx[0]; + const Real inv_dz = 1._rt/dx[1]; +#else + const Real inv_dz = 1._rt/dx[0]; +#endif + const amrex::IntVect ex_type = E[lev][0]->ixType().toIntVect(); + const amrex::IntVect ey_type = E[lev][1]->ixType().toIntVect(); + const amrex::IntVect ez_type = E[lev][2]->ixType().toIntVect(); + + const amrex::Box& tbx = mfi.tilebox(ex_type); + const amrex::Box& tby = mfi.tilebox(ey_type); + const amrex::Box& tbz = mfi.tilebox(ez_type); + + const auto& phi_arr = phi[lev]->array(mfi); + const auto& Ex_arr = (*E[lev][0])[mfi].array(); + const auto& Ey_arr = (*E[lev][1])[mfi].array(); + const auto& Ez_arr = (*E[lev][2])[mfi].array(); + + const Real beta_x = beta[0]; + const Real beta_y = beta[1]; + const Real beta_z = beta[2]; + + // Calculate the electric field + // Use discretized derivative that matches the staggering of the grid. + // Nodal solver + if (ex_type == amrex::IntVect::TheNodeVector() && + ey_type == amrex::IntVect::TheNodeVector() && + ez_type == amrex::IntVect::TheNodeVector()) + { +#if defined(WARPX_DIM_3D) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1._rt)*0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) + + beta_x*beta_y *0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) + + beta_x*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ey_arr(i,j,k) += + + beta_y*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) + +(beta_y*beta_y-1._rt)*0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) + + beta_y*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + + beta_z*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k )) + + beta_z*beta_y *0.5_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k )) + +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1)); + } + ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1._rt)*0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) + + beta_x*beta_z *0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ey_arr(i,j,k) += + +beta_x*beta_y*0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) + +beta_y*beta_z*0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + + beta_z*beta_x *0.5_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) + +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)); + } + ); +#else + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ey_arr(i,j,k) += + +beta_y*beta_z*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + +(beta_z*beta_z-1._rt)*0.5_rt*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k)); + } + ); +#endif + } + else // Staggered solver + { +#if defined(WARPX_DIM_3D) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1._rt) *inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i ,j ,k )) + + beta_x*beta_y*0.25_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k ) + + phi_arr(i+1,j+1,k )-phi_arr(i+1,j-1,k )) + + beta_x*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1) + + phi_arr(i+1,j ,k+1)-phi_arr(i+1,j ,k-1)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ey_arr(i,j,k) += + + beta_y*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k ) + + phi_arr(i+1,j+1,k )-phi_arr(i-1,j+1,k )) + +(beta_y*beta_y-1._rt) *inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j ,k )) + + beta_y*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k-1) + + phi_arr(i ,j+1,k+1)-phi_arr(i ,j+1,k-1)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + + beta_z*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k )-phi_arr(i-1,j ,k ) + + phi_arr(i+1,j ,k+1)-phi_arr(i-1,j ,k+1)) + + beta_z*beta_y*0.25_rt*inv_dy*(phi_arr(i ,j+1,k )-phi_arr(i ,j-1,k ) + + phi_arr(i ,j+1,k+1)-phi_arr(i ,j-1,k+1)) + +(beta_z*beta_z-1._rt) *inv_dz*(phi_arr(i ,j ,k+1)-phi_arr(i ,j ,k )); + } + ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ParallelFor( tbx, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1._rt)*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k)) + +beta_x*beta_z*0.25_rt*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j-1,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + +beta_z*beta_x*0.25_rt*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i-1,j+1,k)) + +(beta_z*beta_z-1._rt)*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k)); + } + ); + ignore_unused(beta_y); +#else + amrex::ParallelFor( tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + +(beta_z*beta_z-1._rt)*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k)); + } + ); + ignore_unused(beta_x,beta_y); +#endif + } + } + } +} + + +void ElectrostaticSolver::computeB ( + ablastr::fields::MultiLevelVectorField const& B, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta) const +{ + // return early if beta is 0 since there will be no B-field + if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } + + auto & warpx = WarpX::GetInstance(); + for (int lev = 0; lev < num_levels; lev++) { + + const Real* dx = warpx.Geom(lev).CellSize(); + +#ifdef AMREX_USE_OMP +# pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { +#if defined(WARPX_DIM_3D) + const Real inv_dx = 1._rt/dx[0]; + const Real inv_dy = 1._rt/dx[1]; + const Real inv_dz = 1._rt/dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const Real inv_dx = 1._rt/dx[0]; + const Real inv_dz = 1._rt/dx[1]; +#else + const Real inv_dz = 1._rt/dx[0]; +#endif + const amrex::IntVect bx_type = B[lev][0]->ixType().toIntVect(); + const amrex::IntVect by_type = B[lev][1]->ixType().toIntVect(); + const amrex::IntVect bz_type = B[lev][2]->ixType().toIntVect(); + + const amrex::Box& tbx = mfi.tilebox(bx_type); + const amrex::Box& tby = mfi.tilebox(by_type); + const amrex::Box& tbz = mfi.tilebox(bz_type); + + const auto& phi_arr = phi[lev]->array(mfi); + const auto& Bx_arr = (*B[lev][0])[mfi].array(); + const auto& By_arr = (*B[lev][1])[mfi].array(); + const auto& Bz_arr = (*B[lev][2])[mfi].array(); + + const Real beta_x = beta[0]; + const Real beta_y = beta[1]; + const Real beta_z = beta[2]; + + constexpr Real inv_c = 1._rt/PhysConst::c; + + // Calculate the magnetic field + // Use discretized derivative that matches the staggering of the grid. + // Nodal solver + if (bx_type == amrex::IntVect::TheNodeVector() && + by_type == amrex::IntVect::TheNodeVector() && + bz_type == amrex::IntVect::TheNodeVector()) + { +#if defined(WARPX_DIM_3D) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*0.5_rt*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k-1)) + +beta_z*inv_dy*0.5_rt*(phi_arr(i,j+1,k )-phi_arr(i,j-1,k ))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j,k )-phi_arr(i-1,j,k )) + +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k-1))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + -beta_x*inv_dy*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k)) + +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k))); + } + ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*0.5_rt*(phi_arr(i,j+1,k)-phi_arr(i,j-1,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k)) + +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j,k)-phi_arr(i-1,j,k))); + } + ); +#else + amrex::ParallelFor( tbx, tby, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + +beta_x*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); + } + ); + ignore_unused(beta_z,tbz,Bz_arr); +#endif + } + else // Staggered solver + { +#if defined(WARPX_DIM_3D) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*0.5_rt*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k ) + + phi_arr(i,j+1,k+1)-phi_arr(i,j+1,k )) + +beta_z*inv_dy*0.5_rt*(phi_arr(i,j+1,k )-phi_arr(i,j ,k ) + + phi_arr(i,j+1,k+1)-phi_arr(i,j ,k+1))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j,k )-phi_arr(i ,j,k ) + + phi_arr(i+1,j,k+1)-phi_arr(i ,j,k+1)) + +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k ) + + phi_arr(i+1,j,k+1)-phi_arr(i+1,j,k ))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + -beta_x*inv_dy*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j ,k)) + +beta_y*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i ,j+1,k))); + } + ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*(phi_arr(i,j+1,k)-phi_arr(i,j,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5_rt*(phi_arr(i+1,j ,k)-phi_arr(i ,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i ,j+1,k)) + +beta_x*inv_dz*0.5_rt*(phi_arr(i ,j+1,k)-phi_arr(i ,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j ,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + +beta_y*inv_dx*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); + } + ); +#else + amrex::ParallelFor( tbx, tby, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + +beta_x*inv_dz*(phi_arr(i+1,j,k)-phi_arr(i,j,k))); + } + ); + ignore_unused(beta_z,tbz,Bz_arr); +#endif + } + } + } +} diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver_fwd.H b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver_fwd.H new file mode 100644 index 00000000000..00cd061e975 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver_fwd.H @@ -0,0 +1,15 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_ELECTROSTATICSOLVER_FWD_H +#define WARPX_ELECTROSTATICSOLVER_FWD_H + +class ElectrostaticSolver; + +#endif /* WARPX_ELECTROSTATICSOLVER_FWD_H */ diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H new file mode 100755 index 00000000000..5606ebef2f1 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H @@ -0,0 +1,37 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_LABFRAMEEXPLICITES_H_ +#define WARPX_LABFRAMEEXPLICITES_H_ + +#include "ElectrostaticSolver.H" + +class LabFrameExplicitES final : public ElectrostaticSolver +{ +public: + + LabFrameExplicitES (int nlevs_max) : ElectrostaticSolver (nlevs_max) { + ReadParameters(); + } + + void InitData () override; + + void ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) override; + + void computePhiTriDiagonal ( + const ablastr::fields::MultiLevelScalarField& rho, + const ablastr::fields::MultiLevelScalarField& phi + ); + +}; + +#endif // WARPX_LABFRAMEEXPLICITES_H_ diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp new file mode 100755 index 00000000000..88a0899a7cb --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -0,0 +1,259 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#include "LabFrameExplicitES.H" +#include "Fluids/MultiFluidContainer_fwd.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" +#include "Particles/MultiParticleContainer_fwd.H" +#include "Python/callbacks.H" +#include "WarpX.H" + +using namespace amrex; + +void LabFrameExplicitES::InitData() { + auto & warpx = WarpX::GetInstance(); + m_poisson_boundary_handler->DefinePhiBCs(warpx.Geom(0)); +} + +void LabFrameExplicitES::ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) +{ + using ablastr::fields::MultiLevelScalarField; + using ablastr::fields::MultiLevelVectorField; + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + + const MultiLevelScalarField rho_fp = fields.get_mr_levels(FieldType::rho_fp, max_level); + const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level, skip_lev0_coarse_patch); + const MultiLevelScalarField phi_fp = fields.get_mr_levels(FieldType::phi_fp, max_level); + const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + + mpc.DepositCharge(rho_fp, 0.0_rt); + if (mfl) { + const int lev = 0; + mfl->DepositCharge(fields, *rho_fp[lev], lev); + } + + // Apply filter, perform MPI exchange, interpolate across levels + const Vector > rho_buf(num_levels); + auto & warpx = WarpX::GetInstance(); + warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); + +#ifndef WARPX_DIM_RZ + for (int lev = 0; lev < num_levels; lev++) { + // Reflect density over PEC boundaries, if needed. + warpx.ApplyRhofieldBoundary(lev, rho_fp[lev], PatchType::fine); + } +#endif + // beta is zero in lab frame + // Todo: use simpler finite difference form with beta=0 + const std::array beta = {0._rt}; + + // set the boundary potentials appropriately + setPhiBC(phi_fp, warpx.gett_new(0)); + + // Compute the potential phi, by solving the Poisson equation + if (IsPythonCallbackInstalled("poissonsolver")) { + + // Use the Python level solver (user specified) + ExecutePythonCallback("poissonsolver"); + + } else { + +#if defined(WARPX_DIM_1D_Z) + // Use the tridiag solver with 1D + computePhiTriDiagonal(rho_fp, phi_fp); +#else + // Use the AMREX MLMG or the FFT (IGF) solver otherwise + computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, + self_fields_absolute_tolerance, self_fields_max_iters, + self_fields_verbosity, is_igf_2d_slices); +#endif + + } + + // Compute the electric field. Note that if an EB is used the electric + // field will be calculated in the computePhi call. + if (!EB::enabled()) { computeE( Efield_fp, phi_fp, beta ); } + else { + if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp, phi_fp, beta); } + } +} + +/* \brief Compute the potential by solving Poisson's equation with + a 1D tridiagonal solve. + + \param[in] rho The charge density a given species + \param[out] phi The potential to be computed by this function +*/ +void LabFrameExplicitES::computePhiTriDiagonal ( + const ablastr::fields::MultiLevelScalarField& rho, + const ablastr::fields::MultiLevelScalarField& phi) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(num_levels == 1, + "The tridiagonal solver cannot be used with mesh refinement"); + + const int lev = 0; + auto & warpx = WarpX::GetInstance(); + + const amrex::Real* dx = warpx.Geom(lev).CellSize(); + const amrex::Real xmin = warpx.Geom(lev).ProbLo(0); + const amrex::Real xmax = warpx.Geom(lev).ProbHi(0); + const int nx_full_domain = static_cast( (xmax - xmin)/dx[0] + 0.5_rt ); + + int nx_solve_min = 1; + int nx_solve_max = nx_full_domain - 1; + + auto field_boundary_lo0 = WarpX::field_boundary_lo[0]; + auto field_boundary_hi0 = WarpX::field_boundary_hi[0]; + if (field_boundary_lo0 == FieldBoundaryType::Neumann || field_boundary_lo0 == FieldBoundaryType::Periodic) { + // Neumann or periodic boundary condition + // Solve for the point on the lower boundary + nx_solve_min = 0; + } + if (field_boundary_hi0 == FieldBoundaryType::Neumann || field_boundary_hi0 == FieldBoundaryType::Periodic) { + // Neumann or periodic boundary condition + // Solve for the point on the upper boundary + nx_solve_max = nx_full_domain; + } + + // Create a 1-D MultiFab that covers all of x. + // The tridiag solve will be done in this MultiFab and then copied out afterwards. + const amrex::IntVect lo_full_domain(AMREX_D_DECL(0,0,0)); + const amrex::IntVect hi_full_domain(AMREX_D_DECL(nx_full_domain,0,0)); + const amrex::Box box_full_domain_node(lo_full_domain, hi_full_domain, amrex::IntVect::TheNodeVector()); + const BoxArray ba_full_domain_node(box_full_domain_node); + const amrex::Vector pmap = {0}; // The data will only be on processor 0 + const amrex::DistributionMapping dm_full_domain(pmap); + + // Put the data in the pinned arena since the tridiag solver will be done on the CPU, but have + // the data readily accessible from the GPU. + auto phi1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); + auto zwork1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); + auto rho1d_mf = MultiFab(ba_full_domain_node, dm_full_domain, 1, 0, MFInfo().SetArena(The_Pinned_Arena())); + + if (field_boundary_lo0 == FieldBoundaryType::PEC || field_boundary_hi0 == FieldBoundaryType::PEC) { + // Copy from phi to get the boundary values + phi1d_mf.ParallelCopy(*phi[lev], 0, 0, 1); + } + rho1d_mf.ParallelCopy(*rho[lev], 0, 0, 1); + + // Multiplier on the charge density + const amrex::Real norm = dx[0]*dx[0]/PhysConst::ep0; + rho1d_mf.mult(norm); + + // Use the MFIter loop since when parallel, only process zero has a FAB. + // This skips the loop on all other processors. + for (MFIter mfi(phi1d_mf); mfi.isValid(); ++mfi) { + + const auto& phi1d_arr = phi1d_mf[mfi].array(); + const auto& zwork1d_arr = zwork1d_mf[mfi].array(); + const auto& rho1d_arr = rho1d_mf[mfi].array(); + + // The loops are always performed on the CPU + + amrex::Real diag = 2._rt; + + // The initial values depend on the boundary condition + if (field_boundary_lo0 == FieldBoundaryType::PEC) { + + phi1d_arr(1,0,0) = (phi1d_arr(0,0,0) + rho1d_arr(1,0,0))/diag; + + } else if (field_boundary_lo0 == FieldBoundaryType::Neumann) { + + // Neumann boundary condition + phi1d_arr(0,0,0) = rho1d_arr(0,0,0)/diag; + + zwork1d_arr(1,0,0) = 2._rt/diag; + diag = 2._rt - zwork1d_arr(1,0,0); + phi1d_arr(1,0,0) = (rho1d_arr(1,0,0) - (-1._rt)*phi1d_arr(1-1,0,0))/diag; + + } else if (field_boundary_lo0 == FieldBoundaryType::Periodic) { + + phi1d_arr(0,0,0) = rho1d_arr(0,0,0)/diag; + + zwork1d_arr(1,0,0) = 1._rt/diag; + diag = 2._rt - zwork1d_arr(1,0,0); + phi1d_arr(1,0,0) = (rho1d_arr(1,0,0) - (-1._rt)*phi1d_arr(1-1,0,0))/diag; + + } + + // Loop upward, calculating the Gaussian elimination multipliers and right hand sides + for (int i_up = 2 ; i_up < nx_solve_max ; i_up++) { + + zwork1d_arr(i_up,0,0) = 1._rt/diag; + diag = 2._rt - zwork1d_arr(i_up,0,0); + phi1d_arr(i_up,0,0) = (rho1d_arr(i_up,0,0) - (-1._rt)*phi1d_arr(i_up-1,0,0))/diag; + + } + + // The last value depend on the boundary condition + amrex::Real zwork_product = 1.; // Needed for parallel boundaries + if (field_boundary_hi0 == FieldBoundaryType::PEC) { + + int const nxm1 = nx_full_domain - 1; + zwork1d_arr(nxm1,0,0) = 1._rt/diag; + diag = 2._rt - zwork1d_arr(nxm1,0,0); + phi1d_arr(nxm1,0,0) = (phi1d_arr(nxm1+1,0,0) + rho1d_arr(nxm1,0,0) - (-1._rt)*phi1d_arr(nxm1-1,0,0))/diag; + + } else if (field_boundary_hi0 == FieldBoundaryType::Neumann) { + + // Neumann boundary condition + zwork1d_arr(nx_full_domain,0,0) = 1._rt/diag; + diag = 2._rt - 2._rt*zwork1d_arr(nx_full_domain,0,0); + if (diag == 0._rt) { + // This happens if the lower boundary is also Neumann. + // It this case, the potential is relative to an arbitrary constant, + // so set the upper boundary to zero to force a value. + phi1d_arr(nx_full_domain,0,0) = 0.; + } else { + phi1d_arr(nx_full_domain,0,0) = (rho1d_arr(nx_full_domain,0,0) - (-1._rt)*phi1d_arr(nx_full_domain-1,0,0))/diag; + } + + } else if (field_boundary_hi0 == FieldBoundaryType::Periodic) { + + zwork1d_arr(nx_full_domain,0,0) = 1._rt/diag; + + for (int i = 1 ; i <= nx_full_domain ; i++) { + zwork_product *= zwork1d_arr(i,0,0); + } + + diag = 2._rt - zwork1d_arr(nx_full_domain,0,0) - zwork_product; + // Note that rho1d_arr(0,0,0) is used to ensure that the same value is used + // on both boundaries. + phi1d_arr(nx_full_domain,0,0) = (rho1d_arr(0,0,0) - (-1._rt)*phi1d_arr(nx_full_domain-1,0,0))/diag; + + } + + // Loop downward to calculate the phi + if (field_boundary_lo0 == FieldBoundaryType::Periodic) { + + // With periodic, the right hand column adds an extra term for all rows + for (int i_down = nx_full_domain-1 ; i_down >= 0 ; i_down--) { + zwork_product /= zwork1d_arr(i_down+1,0,0); + phi1d_arr(i_down,0,0) = phi1d_arr(i_down,0,0) + zwork1d_arr(i_down+1,0,0)*phi1d_arr(i_down+1,0,0) + zwork_product*phi1d_arr(nx_full_domain,0,0); + } + + } else { + + for (int i_down = nx_solve_max-1 ; i_down >= nx_solve_min ; i_down--) { + phi1d_arr(i_down,0,0) = phi1d_arr(i_down,0,0) + zwork1d_arr(i_down+1,0,0)*phi1d_arr(i_down+1,0,0); + } + + } + + } + + // Copy phi1d to phi + phi[lev]->ParallelCopy(phi1d_mf, 0, 0, 1); +} diff --git a/Source/FieldSolver/ElectrostaticSolvers/Make.package b/Source/FieldSolver/ElectrostaticSolvers/Make.package new file mode 100644 index 00000000000..673f1c8aa7a --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/Make.package @@ -0,0 +1,7 @@ +CEXE_sources += PoissonBoundaryHandler.cpp +CEXE_sources += LabFrameExplicitES.cpp +CEXE_sources += RelativisticExplicitES.cpp +CEXE_sources += EffectivePotentialES.cpp +CEXE_sources += ElectrostaticSolver.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/ElectrostaticSolvers diff --git a/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.H b/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.H new file mode 100644 index 00000000000..04e427d7122 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.H @@ -0,0 +1,142 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_BOUNDARYHANDLER_H_ +#define WARPX_BOUNDARYHANDLER_H_ + +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +class PoissonBoundaryHandler +{ +public: + PoissonBoundaryHandler (); // constructor + + /** Read runtime parameters. Called in constructor. */ + void ReadParameters (); + + /** + * \brief Read the input settings and set the boundary conditions used + * on each domain boundary for the Poisson solver. + */ + void DefinePhiBCs (const amrex::Geometry& geom); + + /** + * \brief Initialize amrex::Parser objects to get the boundary potential + * values at specified times. + */ + void BuildParsers (); + void BuildParsersEB (); + + /** + * \brief Sets the EB potential string and updates the function parser. + * + * \param [in] potential The string value of the potential + */ + void setPotentialEB(const std::string& potential) { + potential_eb_str = potential; + BuildParsersEB(); + } + + struct PhiCalculatorEB { + + amrex::Real t; + amrex::ParserExecutor<4> potential_eb; + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real operator()(const amrex::Real x, const amrex::Real z) const noexcept { + using namespace amrex::literals; + return potential_eb(x, 0.0_rt, z, t); + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real operator()(const amrex::Real x, const amrex::Real y, const amrex::Real z) const noexcept { + return potential_eb(x, y, z, t); + } + }; + + [[nodiscard]] PhiCalculatorEB + getPhiEB(amrex::Real t) const noexcept + { + return PhiCalculatorEB{t, potential_eb}; + } + + bool m_boundary_potential_specified = false; + + // set default potentials to zero in order for current tests to pass + // but forcing the user to specify a potential might be better + std::string potential_xlo_str = "0"; + std::string potential_xhi_str = "0"; + std::string potential_ylo_str = "0"; + std::string potential_yhi_str = "0"; + std::string potential_zlo_str = "0"; + std::string potential_zhi_str = "0"; + std::string potential_eb_str = "0"; + + amrex::ParserExecutor<1> potential_xlo; + amrex::ParserExecutor<1> potential_xhi; + amrex::ParserExecutor<1> potential_ylo; + amrex::ParserExecutor<1> potential_yhi; + amrex::ParserExecutor<1> potential_zlo; + amrex::ParserExecutor<1> potential_zhi; + amrex::ParserExecutor<1> potential_eb_t; + amrex::ParserExecutor<4> potential_eb; + + amrex::Array lobc, hibc; + std::array dirichlet_flag; + bool has_non_periodic = false; + bool phi_EB_only_t = true; + +private: + + amrex::Parser potential_xlo_parser; + amrex::Parser potential_xhi_parser; + amrex::Parser potential_ylo_parser; + amrex::Parser potential_yhi_parser; + amrex::Parser potential_zlo_parser; + amrex::Parser potential_zhi_parser; + amrex::Parser potential_eb_parser; +}; + +/** use amrex to directly calculate the electric field since with EB's the + * + * simple finite difference scheme in WarpX::computeE sometimes fails + */ +class EBCalcEfromPhiPerLevel { + private: + amrex::Vector< + amrex::Array + > m_e_field; + + public: + EBCalcEfromPhiPerLevel(amrex::Vector > e_field) + : m_e_field(std::move(e_field)) {} + + void operator()(amrex::MLMG & mlmg, int const lev) { + using namespace amrex::literals; + + mlmg.getGradSolution({m_e_field[lev]}); + for (auto &field: m_e_field[lev]) { + field->mult(-1._rt); + } + } +}; + +#endif // WARPX_BOUNDARYHANDLER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.cpp b/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.cpp new file mode 100644 index 00000000000..8afaf4c0587 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/PoissonBoundaryHandler.cpp @@ -0,0 +1,187 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "PoissonBoundaryHandler.H" + +using namespace amrex; + +PoissonBoundaryHandler::PoissonBoundaryHandler () +{ + ReadParameters(); + BuildParsers(); +} + +void PoissonBoundaryHandler::ReadParameters() +{ + // Parse the input file for domain boundary potentials + const ParmParse pp_boundary("boundary"); + + // Read potentials from input file + m_boundary_potential_specified |= pp_boundary.query("potential_lo_x", potential_xlo_str); + m_boundary_potential_specified |= pp_boundary.query("potential_hi_x", potential_xhi_str); + m_boundary_potential_specified |= pp_boundary.query("potential_lo_y", potential_ylo_str); + m_boundary_potential_specified |= pp_boundary.query("potential_hi_y", potential_yhi_str); + m_boundary_potential_specified |= pp_boundary.query("potential_lo_z", potential_zlo_str); + m_boundary_potential_specified |= pp_boundary.query("potential_hi_z", potential_zhi_str); + + const ParmParse pp_warpx("warpx"); + m_boundary_potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", potential_eb_str); + + if (m_boundary_potential_specified & (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary, but \ + also uses the hybrid PIC solver based on Ohm’s law. When using this solver, the \ + electric potential does not have any impact on the simulation.", + ablastr::warn_manager::WarnPriority::low); + } + else if (m_boundary_potential_specified & (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None)) { + ablastr::warn_manager::WMRecordWarning( + "Algorithms", + "The input script specifies the electric potential (phi) at the boundary so \ + an initial Poisson solve will be performed.", + ablastr::warn_manager::WarnPriority::low); + } +} + +void PoissonBoundaryHandler::DefinePhiBCs (const amrex::Geometry& geom) +{ +#ifdef WARPX_DIM_RZ + if (geom.ProbLo(0) == 0){ + lobc[0] = LinOpBCType::Neumann; + dirichlet_flag[0] = false; + + // handle the r_max boundary explicitly + if (WarpX::field_boundary_hi[0] == FieldBoundaryType::PEC) { + hibc[0] = LinOpBCType::Dirichlet; + dirichlet_flag[1] = true; + } + else if (WarpX::field_boundary_hi[0] == FieldBoundaryType::Neumann) { + hibc[0] = LinOpBCType::Neumann; + dirichlet_flag[1] = false; + } + else { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, + "Field boundary condition at the outer radius must be either PEC or neumann " + "when using the electrostatic solver" + ); + } + } + const int dim_start = 1; +#else + const int dim_start = 0; + amrex::ignore_unused(geom); +#endif + for (int idim=dim_start; idim(); + potential_xhi = potential_xhi_parser.compile<1>(); + potential_ylo = potential_ylo_parser.compile<1>(); + potential_yhi = potential_yhi_parser.compile<1>(); + potential_zlo = potential_zlo_parser.compile<1>(); + potential_zhi = potential_zhi_parser.compile<1>(); + + BuildParsersEB(); +} + +void PoissonBoundaryHandler::BuildParsersEB () +{ + potential_eb_parser = utils::parser::makeParser(potential_eb_str, {"x", "y", "z", "t"}); + + // check if the EB potential is a function of space or only of time + const std::set eb_symbols = potential_eb_parser.symbols(); + if ((eb_symbols.count("x") != 0) || (eb_symbols.count("y") != 0) + || (eb_symbols.count("z") != 0)) { + potential_eb = potential_eb_parser.compile<4>(); + phi_EB_only_t = false; + } + else { + potential_eb_parser = utils::parser::makeParser(potential_eb_str, {"t"}); + potential_eb_t = potential_eb_parser.compile<1>(); + } +} diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H new file mode 100755 index 00000000000..cf831a7ab10 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H @@ -0,0 +1,78 @@ + +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Remi Lehe, Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_RELATIVISTICEXPLICITES_H_ +#define WARPX_RELATIVISTICEXPLICITES_H_ + +#include "ElectrostaticSolver.H" +#include "Particles/WarpXParticleContainer.H" + + +class RelativisticExplicitES final : public ElectrostaticSolver +{ +public: + + RelativisticExplicitES (int nlevs_max) : ElectrostaticSolver (nlevs_max) { + ReadParameters(); + } + + void InitData () override; + + /** + * \brief Computes electrostatic fields for species + * that have initialize self fields turned on. + * A loop over all the species is performed and for each species (with self fields) + * the function ``AddSpaceChargeField`` is called. + * This function computes the electrostatic potential for species charge denisyt as source + * and then the electric and magnetic fields are updated to include the + * corresponding fields from the electrostatic potential. + * Then electric and magnetic fields are updated to include potential variation + * due to boundary conditions using the function ``AddBoundaryField`` + * + * \param[in,unused] rho_fp A temporary multifab is used for species charge density + * \param[in,unused] rho_cp A temporary multifab is used to store species charge density on coarse patch + * \param[in] charge_buf Buffer region to synchronize charge density from fine and coarse patch + * \param[in,unused] phi_fp A temporary multifab is used to compute electrostatic potentail for each species + * \param[in] mpc Pointer to multi particle container to access species data + * \param[in,out] Efield_fp Field contribution from phi computed from each species' charge density is added + * \param[in,out] Bfield Field contribution from phi computed from each species' charge density is added + */ + void ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) override; + + /** + * Compute the charge density of the species paricle container, pc, + * and obtain the corresponding electrostatic potential to + * update the electric and magnetic fields. + * \param[in] charge_buf Multifab that stores buffer region to + * synchronize charge density on fine and coarse patch + * \param[in] pc particle container for the selected species + * \param[in] Efield Efield updated to include potential computed for selected species charge density as source + * \param[in] Bfield Bfield updated to include potential computed for selected species charge density as source + */ + void AddSpaceChargeField ( + WarpXParticleContainer& pc, + ablastr::fields::MultiLevelVectorField& Efield_fp, + ablastr::fields::MultiLevelVectorField& Bfield_fp + ); + + /** Compute the potential `phi` by solving the Poisson equation with the + simulation specific boundary conditions and boundary values, then add the + E field due to that `phi` to `Efield_fp`. + * \param[in] Efield Efield updated to include potential gradient from boundary condition + */ + void AddBoundaryField ( + ablastr::fields::MultiLevelVectorField& Efield + ); +}; + +#endif // WARPX_RELATIVISTICEXPLICITES_H_ diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp new file mode 100755 index 00000000000..0b1bcecd1e5 --- /dev/null +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp @@ -0,0 +1,175 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Remi Lehe, Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#include "RelativisticExplicitES.H" + +#include "Fields.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/WarpXParticleContainer.H" +#include "WarpX.H" + + +using namespace amrex; + +void RelativisticExplicitES::InitData () { + auto & warpx = WarpX::GetInstance(); + bool prepare_field_solve = (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic); + // check if any of the particle containers have initialize_self_fields = True + for (auto const& species : warpx.GetPartContainer()) { + prepare_field_solve |= species->initialize_self_fields; + } + prepare_field_solve |= m_poisson_boundary_handler->m_boundary_potential_specified; + + if (prepare_field_solve) { + m_poisson_boundary_handler->DefinePhiBCs(warpx.Geom(0)); + } +} + +void RelativisticExplicitES::ComputeSpaceChargeField ( + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + [[maybe_unused]] MultiFluidContainer* mfl, + int max_level) +{ + WARPX_PROFILE("RelativisticExplicitES::ComputeSpaceChargeField"); + + using ablastr::fields::MultiLevelVectorField; + using warpx::fields::FieldType; + + const bool always_run_solve = (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic); + + MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + MultiLevelVectorField Bfield_fp = fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); + + // Loop over the species and add their space-charge contribution to E and B. + // Note that the fields calculated here does not include the E field + // due to simulation boundary potentials + for (auto const& species : mpc) { + if (always_run_solve || (species->initialize_self_fields)) { + AddSpaceChargeField(*species, Efield_fp, Bfield_fp); + } + } + + // Add the field due to the boundary potentials + if (always_run_solve || (m_poisson_boundary_handler->m_boundary_potential_specified)) + { + AddBoundaryField(Efield_fp); + } +} + +void RelativisticExplicitES::AddSpaceChargeField ( + WarpXParticleContainer& pc, + ablastr::fields::MultiLevelVectorField& Efield_fp, + ablastr::fields::MultiLevelVectorField& Bfield_fp) +{ + WARPX_PROFILE("RelativisticExplicitES::AddSpaceChargeField"); + + if (pc.getCharge() == 0) { return; } + +#ifdef WARPX_DIM_RZ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::n_rz_azimuthal_modes == 1, + "Error: RZ electrostatic only implemented for a single mode"); +#endif + + auto & warpx = WarpX::GetInstance(); + + // Allocate fields for charge and potential + Vector> rho(num_levels); + Vector> rho_coarse(num_levels); // Used in order to interpolate between levels + Vector> phi(num_levels); + // Use number of guard cells used for local deposition of rho + const amrex::IntVect ng = warpx.get_ng_depos_rho(); + for (int lev = 0; lev < num_levels; lev++) { + BoxArray nba = warpx.boxArray(lev); + nba.surroundingNodes(); + rho[lev] = std::make_unique(nba, warpx.DistributionMap(lev), 1, ng); + rho[lev]->setVal(0.); + phi[lev] = std::make_unique(nba, warpx.DistributionMap(lev), 1, 1); + phi[lev]->setVal(0.); + if (lev > 0) { + // For MR levels: allocated the coarsened version of rho + BoxArray cba = nba; + cba.coarsen(warpx.refRatio(lev-1)); + rho_coarse[lev] = std::make_unique(cba, warpx.DistributionMap(lev), 1, ng); + rho_coarse[lev]->setVal(0.); + } + } + // Deposit particle charge density (source of Poisson solver) + // The options below are identical to those in MultiParticleContainer::DepositCharge + bool const local = true; + bool const reset = false; + bool const apply_boundary_and_scale_volume = true; + bool const interpolate_across_levels = false; + if ( !pc.do_not_deposit) { + pc.DepositCharge(amrex::GetVecOfPtrs(rho), + local, reset, apply_boundary_and_scale_volume, + interpolate_across_levels); + } + + // Apply filter, perform MPI exchange, interpolate across levels + const Vector> rho_buf(num_levels); + warpx.SyncRho( + amrex::GetVecOfPtrs(rho), + amrex::GetVecOfPtrs(rho_coarse), + amrex::GetVecOfPtrs(rho_buf)); + + // Get the particle beta vector + bool const local_average = false; // Average across all MPI ranks + std::array beta_pr = pc.meanParticleVelocity(local_average); + std::array beta; + for (int i=0 ; i < static_cast(beta.size()) ; i++) { + beta[i] = beta_pr[i]/PhysConst::c; // Normalize + } + + // Compute the potential phi, by solving the Poisson equation + computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), + beta, pc.self_fields_required_precision, + pc.self_fields_absolute_tolerance, pc.self_fields_max_iters, + pc.self_fields_verbosity, is_igf_2d_slices); + + // Compute the corresponding electric and magnetic field, from the potential phi + computeE( Efield_fp, amrex::GetVecOfPtrs(phi), beta ); + computeB( Bfield_fp, amrex::GetVecOfPtrs(phi), beta ); + +} + +void RelativisticExplicitES::AddBoundaryField (ablastr::fields::MultiLevelVectorField& Efield_fp) +{ + WARPX_PROFILE("RelativisticExplicitES::AddBoundaryField"); + + auto & warpx = WarpX::GetInstance(); + + // Allocate fields for charge and potential + Vector> rho(num_levels); + Vector> phi(num_levels); + // Use number of guard cells used for local deposition of rho + const amrex::IntVect ng = warpx.get_ng_depos_rho(); + for (int lev = 0; lev < num_levels; lev++) { + BoxArray nba = warpx.boxArray(lev); + nba.surroundingNodes(); + rho[lev] = std::make_unique(nba, warpx.DistributionMap(lev), 1, ng); + rho[lev]->setVal(0.); + phi[lev] = std::make_unique(nba, warpx.DistributionMap(lev), 1, 1); + phi[lev]->setVal(0.); + } + + // Set the boundary potentials appropriately + setPhiBC( amrex::GetVecOfPtrs(phi), warpx.gett_new(0)); + + // beta is zero for boundaries + const std::array beta = {0._rt}; + + // Compute the potential phi, by solving the Poisson equation + computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), + beta, self_fields_required_precision, + self_fields_absolute_tolerance, self_fields_max_iters, + self_fields_verbosity, is_igf_2d_slices); + + // Compute the corresponding electric field, from the potential phi. + computeE( Efield_fp, amrex::GetVecOfPtrs(phi), beta ); +} diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H deleted file mode 100644 index 76ff9a74f7b..00000000000 --- a/Source/FieldSolver/Fields.H +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright 2024 Luca Fedeli - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_FIELDS_H_ -#define WARPX_FIELDS_H_ - -namespace warpx::fields -{ - enum struct FieldType : int - { - Efield_aux, - Bfield_aux, - Efield_fp, - Bfield_fp, - current_fp, - current_fp_nodal, - rho_fp, - F_fp, - G_fp, - phi_fp, - vector_potential_fp, - Efield_cp, - Bfield_cp, - current_cp, - rho_cp, - F_cp, - G_cp, - edge_lengths, - face_areas, - Efield_avg_fp, - Bfield_avg_fp, - Efield_avg_cp, - Bfield_avg_cp - }; -} - -#endif //WARPX_FIELDS_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 5e75698903e..e72260fcf4f 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -35,12 +35,14 @@ using namespace amrex; * \brief Update the B field at the boundary, using the Silver-Mueller condition */ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField& Bfield, amrex::Box domain_box, amrex::Real const dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi) { + amrex::Array field_boundary_lo, + amrex::Array field_boundary_hi) { + + using ablastr::fields::Direction; // Ensure that we are using the Yee solver WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -79,14 +81,14 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // tiling is usually set by TilingIfNotGPU() // but here, we set it to false because of potential race condition, // since we grow the tiles by one guard cell after creating them. - for ( MFIter mfi(*Efield[0], false); mfi.isValid(); ++mfi ) { + for ( MFIter mfi(*Efield[Direction{0}], false); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& Er = Efield[0]->array(mfi); - Array4 const& Et = Efield[1]->array(mfi); - Array4 const& Ez = Efield[2]->array(mfi); - Array4 const& Br = Bfield[0]->array(mfi); - Array4 const& Bt = Bfield[1]->array(mfi); - Array4 const& Bz = Bfield[2]->array(mfi); + Array4 const& Er = Efield[Direction{0}]->array(mfi); + Array4 const& Et = Efield[Direction{1}]->array(mfi); + Array4 const& Ez = Efield[Direction{2}]->array(mfi); + Array4 const& Br = Bfield[Direction{0}]->array(mfi); + Array4 const& Bt = Bfield[Direction{1}]->array(mfi); + Array4 const& Bz = Bfield[Direction{2}]->array(mfi); // Extract tileboxes for which to loop Box tbr = mfi.tilebox(Bfield[0]->ixType().toIntVect()); @@ -203,18 +205,18 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // tiling is usually set by TilingIfNotGPU() // but here, we set it to false because of potential race condition, // since we grow the tiles by one guard cell after creating them. - for ( MFIter mfi(*Efield[0], false); mfi.isValid(); ++mfi ) { + for ( MFIter mfi(*Efield[Direction{0}], false); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& Ex = Efield[0]->array(mfi); - Array4 const& Ey = Efield[1]->array(mfi); + Array4 const& Ex = Efield[Direction{0}]->array(mfi); + Array4 const& Ey = Efield[Direction{1}]->array(mfi); #ifndef WARPX_DIM_1D_Z - Array4 const& Ez = Efield[2]->array(mfi); + Array4 const& Ez = Efield[Direction{2}]->array(mfi); #endif - Array4 const& Bx = Bfield[0]->array(mfi); - Array4 const& By = Bfield[1]->array(mfi); + Array4 const& Bx = Bfield[Direction{0}]->array(mfi); + Array4 const& By = Bfield[Direction{1}]->array(mfi); #ifndef WARPX_DIM_1D_Z - Array4 const& Bz = Bfield[2]->array(mfi); + Array4 const& Bz = Bfield[Direction{2}]->array(mfi); #endif // Extract the tileboxes for which to loop diff --git a/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt b/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt index 19c2092d1a6..7539d706632 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt +++ b/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt @@ -3,6 +3,7 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE ComputeDivE.cpp + ComputeCurlA.cpp EvolveB.cpp EvolveBPML.cpp EvolveE.cpp diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeCurlA.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeCurlA.cpp new file mode 100644 index 00000000000..30cbdb60508 --- /dev/null +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeCurlA.cpp @@ -0,0 +1,306 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: S. Eric Clark (Helion Energy) + * + * License: BSD-3-Clause-LBNL + */ + +#include "FiniteDifferenceSolver.H" + +#include "EmbeddedBoundary/Enabled.H" +#ifdef WARPX_DIM_RZ +# include "FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" +#else +# include "FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" +#endif + +#include "Utils/TextMsg.H" +#include "WarpX.H" + +using namespace amrex; + +void FiniteDifferenceSolver::ComputeCurlA ( + ablastr::fields::VectorField& Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, + int lev ) +{ + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) + if (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC) { +#ifdef WARPX_DIM_RZ + ComputeCurlACylindrical ( + Bfield, Afield, eb_update_B, lev + ); + +#else + ComputeCurlACartesian ( + Bfield, Afield, eb_update_B, lev + ); + +#endif + } else { + amrex::Abort(Utils::TextMsg::Err( + "ComputeCurl: Unknown algorithm choice.")); + } +} + +// /** +// * \brief Calculate B from the curl of A +// * i.e. B = curl(A) output field on B field mesh staggering +// * +// * \param[out] curlField output of curl operation +// * \param[in] field input staggered field, should be on E/J/A mesh staggering +// */ +#ifdef WARPX_DIM_RZ +template +void FiniteDifferenceSolver::ComputeCurlACylindrical ( + ablastr::fields::VectorField& Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, + int lev +) +{ + // for the profiler + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // reset Bfield + Bfield[0]->setVal(0); + Bfield[1]->setVal(0); + Bfield[2]->setVal(0); + + // Loop through the grids, and over the tiles within each grid +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*Afield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + Real wt = static_cast(amrex::second()); + + // Extract field data for this grid/tile + Array4 const& Ar = Afield[0]->const_array(mfi); + Array4 const& At = Afield[1]->const_array(mfi); + Array4 const& Az = Afield[2]->const_array(mfi); + Array4 const& Br = Bfield[0]->array(mfi); + Array4 const& Bt = Bfield[1]->array(mfi); + Array4 const& Bz = Bfield[2]->array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries. + amrex::Array4 update_Br_arr, update_Bt_arr, update_Bz_arr; + if (EB::enabled()) { + update_Br_arr = eb_update_B[0]->array(mfi); + update_Bt_arr = eb_update_B[1]->array(mfi); + update_Bz_arr = eb_update_B[2]->array(mfi); + } + + // Extract stencil coefficients + Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); + int const n_coefs_r = static_cast(m_stencil_coefs_r.size()); + Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); + int const n_coefs_z = static_cast(m_stencil_coefs_z.size()); + + // Extract cylindrical specific parameters + Real const dr = m_dr; + int const nmodes = m_nmodes; + Real const rmin = m_rmin; + + // Extract tileboxes for which to loop over + Box const& tbr = mfi.tilebox(Bfield[0]->ixType().toIntVect()); + Box const& tbt = mfi.tilebox(Bfield[1]->ixType().toIntVect()); + Box const& tbz = mfi.tilebox(Bfield[2]->ixType().toIntVect()); + + // Calculate the B-field from the A-field + amrex::ParallelFor(tbr, tbt, tbz, + + // Br calculation + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ + // Skip field update in the embedded boundaries + if (update_Br_arr && update_Br_arr(i, j, 0) == 0) { return; } + + Real const r = rmin + i*dr; // r on nodal point (Br is nodal in r) + if (r != 0) { // Off-axis, regular Maxwell equations + Br(i, j, 0, 0) = - T_Algo::UpwardDz(At, coefs_z, n_coefs_z, i, j, 0, 0); // Mode m=0 + for (int m=1; m(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } +} + +#else + +template +void FiniteDifferenceSolver::ComputeCurlACartesian ( + ablastr::fields::VectorField & Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, + int lev +) +{ + using ablastr::fields::Direction; + + // for the profiler + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // reset Bfield + Bfield[0]->setVal(0); + Bfield[1]->setVal(0); + Bfield[2]->setVal(0); + + // Loop through the grids, and over the tiles within each grid +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*Afield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + // Extract field data for this grid/tile + Array4 const &Bx = Bfield[0]->array(mfi); + Array4 const &By = Bfield[1]->array(mfi); + Array4 const &Bz = Bfield[2]->array(mfi); + Array4 const &Ax = Afield[0]->const_array(mfi); + Array4 const &Ay = Afield[1]->const_array(mfi); + Array4 const &Az = Afield[2]->const_array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries. + amrex::Array4 update_Bx_arr, update_By_arr, update_Bz_arr; + if (EB::enabled()) { + update_Bx_arr = eb_update_B[0]->array(mfi); + update_By_arr = eb_update_B[1]->array(mfi); + update_Bz_arr = eb_update_B[2]->array(mfi); + } + + // Extract stencil coefficients + Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); + Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); + Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); + + // Extract tileboxes for which to loop + Box const& tbx = mfi.tilebox(Bfield[0]->ixType().toIntVect()); + Box const& tby = mfi.tilebox(Bfield[1]->ixType().toIntVect()); + Box const& tbz = mfi.tilebox(Bfield[2]->ixType().toIntVect()); + + // Calculate the curl of A + amrex::ParallelFor(tbx, tby, tbz, + + // Bx calculation + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_Bx_arr && update_Bx_arr(i, j, k) == 0) { return; } + + Bx(i, j, k) = ( + - T_Algo::UpwardDz(Ay, coefs_z, n_coefs_z, i, j, k) + + T_Algo::UpwardDy(Az, coefs_y, n_coefs_y, i, j, k) + ); + }, + + // By calculation + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_By_arr && update_By_arr(i, j, k) == 0) { return; } + + By(i, j, k) = ( + - T_Algo::UpwardDx(Az, coefs_x, n_coefs_x, i, j, k) + + T_Algo::UpwardDz(Ax, coefs_z, n_coefs_z, i, j, k) + ); + }, + + // Bz calculation + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_Bz_arr && update_Bz_arr(i, j, k) == 0) { return; } + + Bz(i, j, k) = ( + - T_Algo::UpwardDy(Ax, coefs_y, n_coefs_y, i, j, k) + + T_Algo::UpwardDx(Ay, coefs_x, n_coefs_x, i, j, k) + ); + } + ); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } +} +#endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 0702b264874..3f757603845 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -16,6 +16,8 @@ # include "FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include + #include #include #include @@ -40,9 +42,10 @@ using namespace amrex; * \brief Update the F field, over one timestep */ void FiniteDifferenceSolver::ComputeDivE ( - const std::array,3>& Efield, - amrex::MultiFab& divEfield ) { - + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divEfield +) +{ // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ @@ -77,7 +80,7 @@ void FiniteDifferenceSolver::ComputeDivE ( template void FiniteDifferenceSolver::ComputeDivECartesian ( - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divEfield ) { // Loop through the grids, and over the tiles within each grid @@ -123,9 +126,10 @@ void FiniteDifferenceSolver::ComputeDivECartesian ( template void FiniteDifferenceSolver::ComputeDivECylindrical ( - const std::array,3>& Efield, - amrex::MultiFab& divEfield ) { - + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divEfield +) +{ // Loop through the grids, and over the tiles within each grid #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index fbc1397b413..c6a1e206200 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -7,6 +7,7 @@ #include "FiniteDifferenceSolver.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" # include "FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" @@ -48,31 +49,50 @@ using namespace amrex; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveB ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, - std::array< std::unique_ptr, 3 >& flag_info_cell, - std::array< std::unique_ptr >, 3 >& borrowing, - int lev, amrex::Real const dt ) { - -#if defined(WARPX_DIM_RZ) || !defined(AMREX_USE_EB) - amrex::ignore_unused(area_mod, ECTRhofield, Venl, flag_info_cell, borrowing); -#endif + ablastr::fields::MultiFabRegister& fields, + int lev, + PatchType patch_type, + [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, + [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, + [[maybe_unused]] amrex::Real const dt ) +{ + + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + fields.get_alldirs(FieldType::Bfield_fp, lev) : fields.get_alldirs(FieldType::Bfield_cp, lev); + const ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? + fields.get_alldirs(FieldType::Efield_fp, lev) : fields.get_alldirs(FieldType::Efield_cp, lev); // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if ((m_fdtd_algo == ElectromagneticSolverAlgo::Yee)|| (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC)){ - ignore_unused(Gfield, face_areas); EvolveBCylindrical ( Bfield, Efield, lev, dt ); #else - if(m_grid_type == GridType::Collocated || m_fdtd_algo != ElectromagneticSolverAlgo::ECT){ - amrex::ignore_unused(face_areas); + + amrex::MultiFab const * Gfield = nullptr; + if (fields.has(FieldType::G_fp, lev)) { + Gfield = patch_type == PatchType::fine ? + fields.get(FieldType::G_fp, lev) : fields.get(FieldType::G_cp, lev); + } + ablastr::fields::VectorField face_areas; + if (fields.has_vector(FieldType::face_areas, lev)) { + face_areas = fields.get_alldirs(FieldType::face_areas, lev); + } + ablastr::fields::VectorField area_mod; + if (fields.has_vector(FieldType::area_mod, lev)) { + area_mod = fields.get_alldirs(FieldType::area_mod, lev); + } + ablastr::fields::VectorField ECTRhofield; + if (fields.has_vector(FieldType::ECTRhofield, lev)) { + ECTRhofield = fields.get_alldirs(FieldType::ECTRhofield, lev); + } + ablastr::fields::VectorField Venl; + if (fields.has_vector(FieldType::Venl, lev)) { + Venl = fields.get_alldirs(FieldType::Venl, lev); } if (m_grid_type == GridType::Collocated) { @@ -87,12 +107,9 @@ void FiniteDifferenceSolver::EvolveB ( } else if (m_fdtd_algo == ElectromagneticSolverAlgo::CKC) { EvolveBCartesian ( Bfield, Efield, Gfield, lev, dt ); -#ifdef AMREX_USE_EB } else if (m_fdtd_algo == ElectromagneticSolverAlgo::ECT) { - EvolveBCartesianECT(Bfield, face_areas, area_mod, ECTRhofield, Venl, flag_info_cell, borrowing, lev, dt); -#endif #endif } else { WARPX_ABORT_WITH_MESSAGE("EvolveB: Unknown algorithm"); @@ -104,9 +121,9 @@ void FiniteDifferenceSolver::EvolveB ( template void FiniteDifferenceSolver::EvolveBCartesian ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Efield, + amrex::MultiFab const * Gfield, int lev, amrex::Real const dt ) { amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -172,7 +189,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( if (Gfield) { // Extract field data for this grid/tile - const Array4 G = Gfield->array(mfi); + Array4 const G = Gfield->array(mfi); // Loop over cells and update G amrex::ParallelFor(tbx, tby, tbz, @@ -203,11 +220,11 @@ void FiniteDifferenceSolver::EvolveBCartesian ( void FiniteDifferenceSolver::EvolveBCartesianECT ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& area_mod, + ablastr::fields::VectorField const& ECTRhofield, + ablastr::fields::VectorField const& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, const int lev, amrex::Real const dt ) { @@ -245,9 +262,9 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( amrex::Array4 const &S = face_areas[idim]->array(mfi); amrex::Array4 const &S_mod = area_mod[idim]->array(mfi); - auto &borrowing_dim = (*borrowing[idim])[mfi]; - auto borrowing_dim_neigh_faces = borrowing_dim.neigh_faces.data(); - auto borrowing_dim_area = borrowing_dim.area.data(); + auto & borrowing_dim = (*borrowing[idim])[mfi]; + auto * borrowing_dim_neigh_faces = borrowing_dim.neigh_faces.data(); + auto * borrowing_dim_area = borrowing_dim.area.data(); auto const &borrowing_inds = (*borrowing[idim])[mfi].inds.data(); auto const &borrowing_size = (*borrowing[idim])[mfi].size.array(); @@ -259,24 +276,23 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( //Take care of the unstable cells amrex::ParallelFor(tb, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (S(i, j, k) <= 0) return; + if (S(i, j, k) <= 0) { return; } - if (!(flag_info_cell_dim(i, j, k) == 0)) - return; + if (!(flag_info_cell_dim(i, j, k) == 0)) { return; } Venl_dim(i, j, k) = Rho(i, j, k) * S(i, j, k); amrex::Real rho_enl; // First we compute the rho of the enlarged face for (int offset = 0; offset void FiniteDifferenceSolver::EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Efield, int lev, amrex::Real const dt ) { amrex::LayoutData* cost = WarpX::getCosts(lev); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 0ad2c8d6802..e3289d52cfe 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -7,6 +7,7 @@ #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "BoundaryConditions/PMLComponent.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" @@ -41,18 +42,27 @@ using namespace amrex; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveBPML ( - std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, amrex::Real const dt, - const bool dive_cleaning) { + const bool dive_cleaning +) +{ + using warpx::fields::FieldType; // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Bfield, Efield, dt, dive_cleaning); + amrex::ignore_unused(fields, patch_type, level, dt, dive_cleaning); WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + fields.get_alldirs(FieldType::pml_B_fp, level) : fields.get_alldirs(FieldType::pml_B_cp, level); + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + fields.get_alldirs(FieldType::pml_E_fp, level) : fields.get_alldirs(FieldType::pml_E_cp, level); + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { EvolveBPMLCartesian (Bfield, Efield, dt, dive_cleaning); @@ -78,7 +88,7 @@ void FiniteDifferenceSolver::EvolveBPML ( template void FiniteDifferenceSolver::EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt, const bool dive_cleaning) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 5f707fbc927..926f52aa8ee 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -6,6 +6,7 @@ */ #include "FiniteDifferenceSolver.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" @@ -13,11 +14,14 @@ #else # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include "EmbeddedBoundary/Enabled.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -41,45 +45,51 @@ #include using namespace amrex; +using namespace ablastr::fields; /** * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveE ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::unique_ptr const& Ffield, - int lev, amrex::Real const dt ) { - -#ifdef AMREX_USE_EB - if (m_fdtd_algo != ElectromagneticSolverAlgo::ECT) { - amrex::ignore_unused(face_areas, ECTRhofield); + ablastr::fields::MultiFabRegister & fields, + int lev, + PatchType patch_type, + ablastr::fields::VectorField const& Efield, + std::array< std::unique_ptr,3 > const& eb_update_E, + amrex::Real const dt +) +{ + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + fields.get_alldirs(FieldType::Bfield_fp, lev) : fields.get_alldirs(FieldType::Bfield_cp, lev); + const ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? + fields.get_alldirs(FieldType::current_fp, lev) : fields.get_alldirs(FieldType::current_cp, lev); + + amrex::MultiFab* Ffield = nullptr; + if (fields.has(FieldType::F_fp, lev)) { + Ffield = patch_type == PatchType::fine ? + fields.get(FieldType::F_fp, lev) : fields.get(FieldType::F_cp, lev); } -#else - amrex::ignore_unused(face_areas, ECTRhofield); -#endif // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee){ - EvolveECylindrical ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECylindrical ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); #else if (m_grid_type == GridType::Collocated) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); } else if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee || m_fdtd_algo == ElectromagneticSolverAlgo::ECT) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); } else if (m_fdtd_algo == ElectromagneticSolverAlgo::CKC) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); #endif } else { @@ -93,17 +103,13 @@ void FiniteDifferenceSolver::EvolveE ( template void FiniteDifferenceSolver::EvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3> const& eb_update_E, + amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::LayoutData* cost = WarpX::getCosts(lev); Real constexpr c2 = PhysConst::c * PhysConst::c; @@ -129,11 +135,13 @@ void FiniteDifferenceSolver::EvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + // Extract structures indicating whether the E field should be updated + amrex::Array4 update_Ex_arr, update_Ey_arr, update_Ez_arr; + if (EB::enabled()) { + update_Ex_arr = eb_update_E[0]->array(mfi); + update_Ey_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -152,10 +160,10 @@ void FiniteDifferenceSolver::EvolveECartesian ( amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Ex_arr && update_Ex_arr(i, j, k) == 0) { return; } + Ex(i, j, k) += c2 * dt * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) + T_Algo::DownwardDy(Bz, coefs_y, n_coefs_y, i, j, k) @@ -163,16 +171,10 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries -#ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; -#elif defined(WARPX_DIM_XZ) - //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif -#endif + + // Skip field push in the embedded boundaries + if (update_Ey_arr && update_Ey_arr(i, j, k) == 0) { return; } + Ey(i, j, k) += c2 * dt * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) + T_Algo::DownwardDz(Bx, coefs_z, n_coefs_z, i, j, k) @@ -180,10 +182,10 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } + Ez(i, j, k) += c2 * dt * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) + T_Algo::DownwardDx(By, coefs_x, n_coefs_x, i, j, k) @@ -197,7 +199,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (Ffield) { // Extract field data for this grid/tile - const Array4 F = Ffield->array(mfi); + const Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, @@ -230,17 +232,13 @@ void FiniteDifferenceSolver::EvolveECartesian ( template void FiniteDifferenceSolver::EvolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, + amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::LayoutData* cost = WarpX::getCosts(lev); // Loop through the grids, and over the tiles within each grid @@ -265,10 +263,13 @@ void FiniteDifferenceSolver::EvolveECylindrical ( Array4 const& jt = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + // Extract structures indicating whether the E field should be updated + amrex::Array4 update_Er_arr, update_Et_arr, update_Ez_arr; + if (EB::enabled()) { + update_Er_arr = eb_update_E[0]->array(mfi); + update_Et_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -292,10 +293,10 @@ void FiniteDifferenceSolver::EvolveECylindrical ( amrex::ParallelFor(ter, tet, tez, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Er_arr && update_Er_arr(i, j, 0) == 0) { return; } + Real const r = rmin + (i + 0.5_rt)*dr; // r on cell-centered point (Er is cell-centered in r) Er(i, j, 0, 0) += c2 * dt*( - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) @@ -313,11 +314,10 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - // The Et field is at a node, so we need to check if the node is covered - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Et_arr && update_Et_arr(i, j, 0) == 0) { return; } + Real const r = rmin + i*dr; // r on a nodal grid (Et is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations Et(i, j, 0, 0) += c2 * dt*( @@ -359,10 +359,10 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i, j, 0) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, 0) == 0) { return; } + Real const r = rmin + i*dr; // r on a nodal grid (Ez is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations Ez(i, j, 0, 0) += c2 * dt*( @@ -399,7 +399,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( if (Ffield) { // Extract field data for this grid/tile - const Array4 F = Ffield->array(mfi); + const Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(ter, tet, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 95f899c98e1..0740a190bec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -42,15 +42,16 @@ #include using namespace amrex; +using namespace ablastr::fields; /** * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveECTRho ( - std::array< std::unique_ptr, 3 > const& Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& ECTRhofield, const int lev) { #if !defined(WARPX_DIM_RZ) and defined(AMREX_USE_EB) @@ -67,10 +68,10 @@ void FiniteDifferenceSolver::EvolveECTRho ( // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_RZ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( - std::array< std::unique_ptr, 3 > const& Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev ) { + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& ECTRhofield, const int lev ) { #ifdef AMREX_USE_EB #if !(defined(WARPX_DIM_3D) || defined(WARPX_DIM_XZ)) @@ -112,7 +113,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( amrex::ParallelFor(trhox, trhoy, trhoz, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sx(i, j, k) <= 0) return; + if (Sx(i, j, k) <= 0) { return; } // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_XZ @@ -122,7 +123,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( }, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sy(i, j, k) <= 0) return; + if (Sy(i, j, k) <= 0) { return; } #ifdef WARPX_DIM_XZ Rhoy(i, j, k) = (Ez(i, j, k) * lz(i, j, k) - Ez(i + 1, j, k) * lz(i + 1, j, k) + @@ -136,7 +137,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( }, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sz(i, j, k) <= 0) return; + if (Sz(i, j, k) <= 0) { return; } // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_XZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 93352ce9896..7a1a05d560d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -16,6 +16,8 @@ #else # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -44,21 +46,38 @@ using namespace amrex; * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveEPML ( - std::array< amrex::MultiFab*, 3 > Efield, - std::array< amrex::MultiFab*, 3 > const Bfield, - std::array< amrex::MultiFab*, 3 > const Jfield, - std::array< amrex::MultiFab*, 3 > const edge_lengths, - amrex::MultiFab* const Ffield, + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, MultiSigmaBox const& sigba, amrex::Real const dt, bool pml_has_particles ) { // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Efield, Bfield, Jfield, Ffield, sigba, dt, pml_has_particles, edge_lengths); + amrex::ignore_unused(fields, patch_type, level, sigba, dt, pml_has_particles); WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + fields.get_alldirs(FieldType::pml_E_fp, level) : fields.get_alldirs(FieldType::pml_E_cp, level); + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + fields.get_alldirs(FieldType::pml_B_fp, level) : fields.get_alldirs(FieldType::pml_B_cp, level); + const ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? + fields.get_alldirs(FieldType::pml_j_fp, level) : fields.get_alldirs(FieldType::pml_j_cp, level); + ablastr::fields::VectorField edge_lengths; + if (fields.has_vector(FieldType::pml_edge_lengths, level)) { + edge_lengths = fields.get_alldirs(FieldType::pml_edge_lengths, level); + } + amrex::MultiFab * Ffield = nullptr; + if (fields.has(FieldType::pml_F_fp, level)) { + Ffield = (patch_type == PatchType::fine) ? + fields.get(FieldType::pml_F_fp, level) : fields.get(FieldType::pml_F_cp, level); + } + if (m_grid_type == GridType::Collocated) { EvolveEPMLCartesian ( @@ -109,11 +128,12 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( Array4 const& By = Bfield[1]->array(mfi); Array4 const& Bz = Bfield[2]->array(mfi); -#ifdef AMREX_USE_EB - Array4 const& lx = edge_lengths[0]->array(mfi); - Array4 const& ly = edge_lengths[1]->array(mfi); - Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -132,9 +152,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - if(lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } Ex(i, j, k, PMLComp::xz) -= c2 * dt * ( T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k, PMLComp::yx) @@ -145,16 +163,15 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } #endif + Ey(i, j, k, PMLComp::yx) -= c2 * dt * ( T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k, PMLComp::zx) + T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k, PMLComp::zy) ); @@ -164,9 +181,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - if(lz(i, j, k) <= 0) return; -#endif + if (lz && lz(i, j, k) <= 0) { return; } Ez(i, j, k, PMLComp::zy) -= c2 * dt * ( T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k, PMLComp::xy) @@ -244,13 +259,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( } ); } - - } - -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - + } // MFIter } -#endif // corresponds to ifndef WARPX_DIM_RZ +#endif // ifndef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 8ce578bb52a..c7f836e47ec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -44,9 +44,9 @@ using namespace amrex; * \brief Update the F field, over one timestep */ void FiniteDifferenceSolver::EvolveF ( - std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* Ffield, + ablastr::fields::VectorField const& Efield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { @@ -82,9 +82,9 @@ void FiniteDifferenceSolver::EvolveF ( template void FiniteDifferenceSolver::EvolveFCartesian ( - std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* Ffield, + ablastr::fields::VectorField const Efield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { @@ -135,9 +135,9 @@ void FiniteDifferenceSolver::EvolveFCartesian ( template void FiniteDifferenceSolver::EvolveFCylindrical ( - std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* Ffield, + ablastr::fields::VectorField const & Efield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp index 4ef056c937a..f14a42f451b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp @@ -41,7 +41,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::EvolveFPML ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt ) { // Select algorithm (The choice of algorithm is a runtime option, @@ -75,7 +75,7 @@ void FiniteDifferenceSolver::EvolveFPML ( template void FiniteDifferenceSolver::EvolveFPMLCartesian ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt ) { // Loop through the grids, and over the tiles within each grid diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp index b6bc8fdca7f..759644201bc 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp @@ -38,8 +38,8 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveG ( - std::unique_ptr& Gfield, - std::array,3> const& Bfield, + amrex::MultiFab* Gfield, + ablastr::fields::VectorField const& Bfield, amrex::Real const dt) { #ifdef WARPX_DIM_RZ @@ -70,8 +70,8 @@ void FiniteDifferenceSolver::EvolveG ( template void FiniteDifferenceSolver::EvolveGCartesian ( - std::unique_ptr& Gfield, - std::array,3> const& Bfield, + amrex::MultiFab* Gfield, + ablastr::fields::VectorField const& Bfield, amrex::Real const dt) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H index 737146f24a3..cf27898cb86 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H @@ -26,7 +26,7 @@ struct CartesianCKCAlgorithm { static void InitializeStencilCoefficients ( - std::array& cell_size, + std::array& cell_size, amrex::Vector& stencil_coefs_x, amrex::Vector& stencil_coefs_y, amrex::Vector& stencil_coefs_z ) { @@ -129,7 +129,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along x on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -186,7 +186,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along y on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -244,7 +244,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H index b693ed8785f..46940e7e306 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H @@ -69,7 +69,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -89,7 +89,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const n_coefs_x, int const i, int const j, int const k, int const ncomp=0 ) { @@ -109,7 +109,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const /*n_coefs_y*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -129,7 +129,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -143,7 +143,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const /*n_coefs_z*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -164,7 +164,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H index c4978287aec..485698802b6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H @@ -67,7 +67,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along x on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -123,7 +123,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along y on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -189,7 +189,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const /*n_coefs_z*/, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index 436f0a83f31..d20e1ef829b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -74,7 +74,7 @@ struct CylindricalYeeAlgorithm { * The input parameter `r` is given at the cell-centered position */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDrr_over_r ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const r, amrex::Real const dr, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -92,7 +92,7 @@ struct CylindricalYeeAlgorithm { * The input parameter `r` is given at the cell-centered position */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDrr_over_r ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const r, amrex::Real const dr, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -108,7 +108,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along r on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDr ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -123,7 +123,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along r on a nodal grid, from a cell-centered field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDr ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -156,7 +156,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const comp ) { @@ -170,7 +170,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along z on a nodal grid, from a cell-centered field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const comp ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H index 05b1db1fe94..ba94eab0b66 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H @@ -42,9 +42,9 @@ struct FieldAccessorMacroscopic } private: /** Array4 of the source field to be scaled and returned by the operator() */ - amrex::Array4 const m_field; + amrex::Array4 m_field; /** Array4 of the macroscopic parameter used to divide m_field in the operator() */ - amrex::Array4 const m_parameter; + amrex::Array4 m_parameter; }; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 00e87525a75..0d12d104436 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -1,7 +1,10 @@ -/* Copyright 2020 Remi Lehe +/* Copyright 2020-2024 The WarpX Community * * This file is part of WarpX. * + * Authors: Remi Lehe (LBNL) + * S. Eric Clark (Helion Energy) + * * License: BSD-3-Clause-LBNL */ @@ -18,6 +21,7 @@ #include "MacroscopicProperties/MacroscopicProperties_fwd.H" #include +#include #include #include @@ -47,56 +51,52 @@ class FiniteDifferenceSolver * \param grid_type Whether the solver is applied to a collocated or staggered grid */ FiniteDifferenceSolver ( - int fdtd_algo, + ElectromagneticSolverAlgo fdtd_algo, std::array cell_size, ablastr::utils::enums::GridType grid_type ); - void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + void EvolveB ( ablastr::fields::MultiFabRegister& fields, + int lev, + PatchType patch_type, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, - int lev, amrex::Real dt ); - - void EvolveE ( std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::unique_ptr const& Ffield, - int lev, amrex::Real dt ); - - void EvolveF ( std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::Real dt ); + + void EvolveE ( ablastr::fields::MultiFabRegister & fields, + int lev, + PatchType patch_type, + ablastr::fields::VectorField const& Efield, + std::array< std::unique_ptr,3 > const& eb_update_E, + amrex::Real dt ); + + void EvolveF ( amrex::MultiFab* Ffield, + ablastr::fields::VectorField const& Efield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); - void EvolveG (std::unique_ptr& Gfield, - std::array,3> const& Bfield, + void EvolveG (amrex::MultiFab* Gfield, + ablastr::fields::VectorField const& Bfield, amrex::Real dt); - void EvolveECTRho ( std::array< std::unique_ptr, 3 > const& Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + void EvolveECTRho ( ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& ECTRhofield, int lev ); - void ApplySilverMuellerBoundary( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 >& Bfield, + void ApplySilverMuellerBoundary ( + ablastr::fields::VectorField & Efield, + ablastr::fields::VectorField & Bfield, amrex::Box domain_box, amrex::Real dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi); + amrex::Array field_boundary_lo, + amrex::Array field_boundary_hi); - void ComputeDivE ( const std::array,3>& Efield, - amrex::MultiFab& divE ); + void ComputeDivE ( + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ); /** * \brief Macroscopic E-update for non-vacuum medium using the user-selected @@ -106,33 +106,38 @@ class FiniteDifferenceSolver * \param[out] Efield vector of electric field MultiFabs updated at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level * \param[in] Jfield vector of current density MultiFabs at a given level - * \param[in] edge_lengths length of edges along embedded boundaries + * \param[in] eb_update_E indicate in which cell E should be updated (related to embedded boundaries) * \param[in] dt timestep of the simulation * \param[in] macroscopic_properties contains user-defined properties of the medium. */ - void MacroscopicEvolveE ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + void MacroscopicEvolveE ( + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real dt, std::unique_ptr const& macroscopic_properties); - void EvolveBPML ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Efield, - amrex::Real dt, - bool dive_cleaning); + void EvolveBPML ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, + amrex::Real dt, + bool dive_cleaning + ); - void EvolveEPML ( std::array< amrex::MultiFab*, 3 > Efield, - std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Jfield, - std::array< amrex::MultiFab*, 3 > edge_lengths, - amrex::MultiFab* Ffield, - MultiSigmaBox const& sigba, - amrex::Real dt, bool pml_has_particles ); + void EvolveEPML ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, + MultiSigmaBox const& sigba, + amrex::Real dt, + bool pml_has_particles + ); void EvolveFPML ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > Efield, - amrex::Real dt ); + ablastr::fields::VectorField Efield, + amrex::Real dt ); /** * \brief E-update in the hybrid PIC algorithm as described in @@ -140,27 +145,25 @@ class FiniteDifferenceSolver * https://link.springer.com/chapter/10.1007/3-540-36530-3_8 * * \param[out] Efield vector of electric field MultiFabs updated at a given level - * \param[in] Jfield vector of total current MultiFabs at a given level + * \param[in] Jfield vector of total plasma current MultiFabs at a given level * \param[in] Jifield vector of ion current density MultiFabs at a given level - * \param[in] Jextfield vector of external current density MultiFabs at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level * \param[in] rhofield scalar ion charge density Multifab at a given level * \param[in] Pefield scalar electron pressure MultiFab at a given level - * \param[in] edge_lengths length of edges along embedded boundaries + * \param[in] eb_update_E indicate in which cell E should be updated (related to embedded boundaries) * \param[in] lev level number for the calculation * \param[in] hybrid_model instance of the hybrid-PIC model - * \param[in] include_resistivity_term boolean flag for whether to include resistivity + * \param[in] solve_for_Faraday boolean flag for whether the E-field is solved to be used in Faraday's equation */ - void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_model, - bool include_resistivity_term ); + void HybridPICSolveE ( ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField & Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3> const& eb_update_E, + int lev, HybridPICModel const* hybrid_model, + bool solve_for_Faraday ); /** * \brief Calculation of total current using Ampere's law (without @@ -168,18 +171,33 @@ class FiniteDifferenceSolver * * \param[out] Jfield vector of current MultiFabs at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level - * \param[in] edge_lengths length of edges along embedded boundaries + * \param[in] eb_update_E indicate in which cell E should be updated (related to embedded boundaries) * \param[in] lev level number for the calculation */ void CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev ); + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3> const& eb_update_E, + int lev ); + + /** + * \brief Calculation of B field from the vector potential A + * B = (curl x A) / mu0. + * + * \param[out] Bfield vector of current MultiFabs at a given level + * \param[in] Afield vector of magnetic field MultiFabs at a given level + * \param[in] edge_lengths length of edges along embedded boundaries + * \param[in] lev level number for the calculation + */ + void ComputeCurlA ( + ablastr::fields::VectorField& Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, + int lev ); private: - int m_fdtd_algo; + ElectromagneticSolverAlgo m_fdtd_algo; ablastr::utils::enums::GridType m_grid_type; #ifdef WARPX_DIM_RZ @@ -206,98 +224,106 @@ class FiniteDifferenceSolver #ifdef WARPX_DIM_RZ template< typename T_Algo > void EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Efield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, + amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveFCylindrical ( - std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* Ffield, + ablastr::fields::VectorField const & Efield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); template< typename T_Algo > void ComputeDivECylindrical ( - const std::array,3>& Efield, - amrex::MultiFab& divE ); + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ); template void HybridPICSolveECylindrical ( - std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3> const& eb_update_E, int lev, HybridPICModel const* hybrid_model, - bool include_resistivity_term ); + bool solve_for_Faraday ); template void CalculateCurrentAmpereCylindrical ( - std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3> const& eb_update_E, + int lev + ); + + template + void ComputeCurlACylindrical ( + ablastr::fields::VectorField& Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, int lev ); #else template< typename T_Algo > void EvolveBCartesian ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Efield, + amrex::MultiFab const * Gfield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, + amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveFCartesian ( - std::unique_ptr& Ffield, - std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* Ffield, + ablastr::fields::VectorField Efield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); template< typename T_Algo > void EvolveGCartesian ( - std::unique_ptr& Gfield, - std::array,3> const& Bfield, + amrex::MultiFab* Gfield, + ablastr::fields::VectorField const& Bfield, amrex::Real dt); void EvolveRhoCartesianECT ( - std::array< std::unique_ptr, 3 > const& Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& ECTRhofield, int lev); void EvolveBCartesianECT ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& area_mod, + ablastr::fields::VectorField const& ECTRhofield, + ablastr::fields::VectorField const& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, int lev, amrex::Real dt @@ -305,28 +331,28 @@ class FiniteDifferenceSolver template< typename T_Algo > void ComputeDivECartesian ( - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ); template< typename T_Algo, typename T_MacroAlgo > void MacroscopicEvolveECartesian ( - std::array< std::unique_ptr< amrex::MultiFab>, 3>& Efield, - std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Bfield, - std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real dt, std::unique_ptr const& macroscopic_properties); template< typename T_Algo > void EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField Efield, amrex::Real dt, bool dive_cleaning); template< typename T_Algo > void EvolveEPMLCartesian ( - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField Efield, std::array< amrex::MultiFab*, 3 > Bfield, std::array< amrex::MultiFab*, 3 > Jfield, std::array< amrex::MultiFab*, 3 > edge_lengths, @@ -336,27 +362,34 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFPMLCartesian ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > Efield, - amrex::Real dt ); + ablastr::fields::VectorField Efield, + amrex::Real dt ); template void HybridPICSolveECartesian ( - std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3> const& eb_update_E, int lev, HybridPICModel const* hybrid_model, - bool include_resistivity_term ); + bool solve_for_Faraday ); template void CalculateCurrentAmpereCartesian ( - std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3> const& eb_update_E, + int lev + ); + + template + void ComputeCurlACartesian ( + ablastr::fields::VectorField & Bfield, + ablastr::fields::VectorField const& Afield, + std::array< std::unique_ptr,3> const& eb_update_B, int lev ); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 9af610031c0..fdd02c5249b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -28,7 +28,7 @@ /* This function initializes the stencil coefficients for the chosen finite-difference algorithm */ FiniteDifferenceSolver::FiniteDifferenceSolver ( - int const fdtd_algo, + ElectromagneticSolverAlgo const fdtd_algo, std::array cell_size, ablastr::utils::enums::GridType grid_type): // Register the type of finite-difference algorithm diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/CMakeLists.txt b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/CMakeLists.txt index 1367578b0aa..bb29baefcb9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/CMakeLists.txt +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/CMakeLists.txt @@ -3,5 +3,6 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE HybridPICModel.cpp + ExternalVectorPotential.cpp ) endforeach() diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.H new file mode 100644 index 00000000000..632ff2bd785 --- /dev/null +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.H @@ -0,0 +1,101 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: S. Eric Clark (Helion Energy) + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_EXTERNAL_VECTOR_POTENTIAL_H_ +#define WARPX_EXTERNAL_VECTOR_POTENTIAL_H_ + +#include "Fields.H" + +#include "Utils/WarpXAlgorithmSelection.H" + +#include "EmbeddedBoundary/Enabled.H" +#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include + +#include +#include +#include +#include +#include + +#include + +/** + * \brief This class contains the parameters needed to evaluate a + * time varying external vector potential, leading to external E/B + * fields to be applied in Hybrid Solver. This class is used to break up + * the passed in fields into a spatial and time dependent solution. + * + * Eventually this can be used in a list to control independent external + * fields with different time profiles. + * + */ +class ExternalVectorPotential +{ +protected: + int m_nFields; + + std::vector m_field_names; + + std::vector m_Ax_ext_grid_function; + std::vector m_Ay_ext_grid_function; + std::vector m_Az_ext_grid_function; + std::vector, 3>> m_A_external_parser; + std::vector, 3>> m_A_external; + + std::vector m_A_ext_time_function; + std::vector> m_A_external_time_parser; + std::vector> m_A_time_scale; + + std::vector m_read_A_from_file; + std::vector m_external_file_path; + +public: + + // Default Constructor + ExternalVectorPotential (); + + void ReadParameters (); + + void AllocateLevelMFs ( + ablastr::fields::MultiFabRegister & fields, + int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + int ncomps, + const amrex::IntVect& ngEB, + const amrex::IntVect& Ex_nodal_flag, + const amrex::IntVect& Ey_nodal_flag, + const amrex::IntVect& Ez_nodal_flag, + const amrex::IntVect& Bx_nodal_flag, + const amrex::IntVect& By_nodal_flag, + const amrex::IntVect& Bz_nodal_flag + ); + + void InitData (); + + void CalculateExternalCurlA (); + void CalculateExternalCurlA (std::string& coil_name); + + AMREX_FORCE_INLINE + void PopulateExternalFieldFromVectorPotential ( + ablastr::fields::VectorField const& dstField, + amrex::Real scale_factor, + ablastr::fields::VectorField const& srcField, + std::array< std::unique_ptr,3> const& eb_update); + + void UpdateHybridExternalFields ( + amrex::Real t, + amrex::Real dt + ); +}; + +#endif //WARPX_TIME_DEPENDENT_VECTOR_POTENTIAL_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.cpp new file mode 100644 index 00000000000..50a62335b57 --- /dev/null +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/ExternalVectorPotential.cpp @@ -0,0 +1,376 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: S. Eric Clark (Helion Energy) + * + * License: BSD-3-Clause-LBNL + */ + +#include "ExternalVectorPotential.H" +#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" +#include "Fields.H" +#include "WarpX.H" + +#include + +using namespace amrex; +using namespace warpx::fields; + +ExternalVectorPotential::ExternalVectorPotential () +{ + ReadParameters(); +} + +void +ExternalVectorPotential::ReadParameters () +{ + const ParmParse pp_ext_A("external_vector_potential"); + + pp_ext_A.queryarr("fields", m_field_names); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!m_field_names.empty(), + "No external field names defined in external_vector_potential.fields"); + + m_nFields = static_cast(m_field_names.size()); + + // Resize vectors and set defaults + m_Ax_ext_grid_function.resize(m_nFields); + m_Ay_ext_grid_function.resize(m_nFields); + m_Az_ext_grid_function.resize(m_nFields); + for (std::string & field : m_Ax_ext_grid_function) { field = "0.0"; } + for (std::string & field : m_Ay_ext_grid_function) { field = "0.0"; } + for (std::string & field : m_Az_ext_grid_function) { field = "0.0"; } + + m_A_external_parser.resize(m_nFields); + m_A_external.resize(m_nFields); + + m_A_ext_time_function.resize(m_nFields); + for (std::string & field_time : m_A_ext_time_function) {field_time = "1.0"; } + + m_A_external_time_parser.resize(m_nFields); + m_A_time_scale.resize(m_nFields); + + m_read_A_from_file.resize(m_nFields); + m_external_file_path.resize(m_nFields); + for (std::string & file_name : m_external_file_path) { file_name = ""; } + + for (int i = 0; i < m_nFields; ++i) { + bool read_from_file = false; + utils::parser::queryWithParser(pp_ext_A, + (m_field_names[i]+".read_from_file").c_str(), read_from_file); + m_read_A_from_file[i] = read_from_file; + + if (m_read_A_from_file[i]) { + pp_ext_A.query((m_field_names[i]+".path").c_str(), m_external_file_path[i]); + } else { + pp_ext_A.query((m_field_names[i]+".Ax_external_grid_function(x,y,z)").c_str(), + m_Ax_ext_grid_function[i]); + pp_ext_A.query((m_field_names[i]+".Ay_external_grid_function(x,y,z)").c_str(), + m_Ay_ext_grid_function[i]); + pp_ext_A.query((m_field_names[i]+".Az_external_grid_function(x,y,z)").c_str(), + m_Az_ext_grid_function[i]); + } + + pp_ext_A.query((m_field_names[i]+".A_time_external_function(t)").c_str(), + m_A_ext_time_function[i]); + } +} + +void +ExternalVectorPotential::AllocateLevelMFs ( + ablastr::fields::MultiFabRegister & fields, + int lev, const BoxArray& ba, const DistributionMapping& dm, + const int ncomps, + const IntVect& ngEB, + const IntVect& Ex_nodal_flag, + const IntVect& Ey_nodal_flag, + const IntVect& Ez_nodal_flag, + const IntVect& Bx_nodal_flag, + const IntVect& By_nodal_flag, + const IntVect& Bz_nodal_flag) +{ + using ablastr::fields::Direction; + for (std::string const & field_name : m_field_names) { + const std::string Aext_field = field_name + std::string{"_Aext"}; + fields.alloc_init(Aext_field, Direction{0}, + lev, amrex::convert(ba, Ex_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(Aext_field, Direction{1}, + lev, amrex::convert(ba, Ey_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(Aext_field, Direction{2}, + lev, amrex::convert(ba, Ez_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + + const std::string curlAext_field = field_name + std::string{"_curlAext"}; + fields.alloc_init(curlAext_field, Direction{0}, + lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(curlAext_field, Direction{1}, + lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(curlAext_field, Direction{2}, + lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + } + fields.alloc_init(FieldType::hybrid_E_fp_external, Direction{0}, + lev, amrex::convert(ba, Ex_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(FieldType::hybrid_E_fp_external, Direction{1}, + lev, amrex::convert(ba, Ey_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(FieldType::hybrid_E_fp_external, Direction{2}, + lev, amrex::convert(ba, Ez_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(FieldType::hybrid_B_fp_external, Direction{0}, + lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(FieldType::hybrid_B_fp_external, Direction{1}, + lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); + fields.alloc_init(FieldType::hybrid_B_fp_external, Direction{2}, + lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, ngEB, 0.0_rt); +} + +void +ExternalVectorPotential::InitData () +{ + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); + + int A_time_dep_count = 0; + + for (int i = 0; i < m_nFields; ++i) { + + const std::string Aext_field = m_field_names[i] + std::string{"_Aext"}; + + if (m_read_A_from_file[i]) { + // Read A fields from file + for (auto lev = 0; lev <= warpx.finestLevel(); ++lev) { +#if defined(WARPX_DIM_RZ) + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{0}, lev), + "A", "r"); + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{1}, lev), + "A", "t"); + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{2}, lev), + "A", "z"); +#else + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{0}, lev), + "A", "x"); + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{1}, lev), + "A", "y"); + warpx.ReadExternalFieldFromFile(m_external_file_path[i], + warpx.m_fields.get(Aext_field, Direction{2}, lev), + "A", "z"); +#endif + } + } else { + // Initialize the A fields from expression + m_A_external_parser[i][0] = std::make_unique( + utils::parser::makeParser(m_Ax_ext_grid_function[i],{"x","y","z","t"})); + m_A_external_parser[i][1] = std::make_unique( + utils::parser::makeParser(m_Ay_ext_grid_function[i],{"x","y","z","t"})); + m_A_external_parser[i][2] = std::make_unique( + utils::parser::makeParser(m_Az_ext_grid_function[i],{"x","y","z","t"})); + m_A_external[i][0] = m_A_external_parser[i][0]->compile<4>(); + m_A_external[i][1] = m_A_external_parser[i][1]->compile<4>(); + m_A_external[i][2] = m_A_external_parser[i][2]->compile<4>(); + + // check if the external current parsers depend on time + for (int idim=0; idim<3; idim++) { + const std::set A_ext_symbols = m_A_external_parser[i][idim]->symbols(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(A_ext_symbols.count("t") == 0, + "Externally Applied Vector potential time variation must be set with A_time_external_function(t)"); + } + + // Initialize data onto grid + for (auto lev = 0; lev <= warpx.finestLevel(); ++lev) { + warpx.ComputeExternalFieldOnGridUsingParser( + Aext_field, + m_A_external[i][0], + m_A_external[i][1], + m_A_external[i][2], + lev, PatchType::fine, + warpx.GetEBUpdateEFlag(), + false); + + for (int idir = 0; idir < 3; ++idir) { + warpx.m_fields.get(Aext_field, Direction{idir}, lev)-> + FillBoundary(warpx.Geom(lev).periodicity()); + } + } + } + + amrex::Gpu::streamSynchronize(); + + CalculateExternalCurlA(m_field_names[i]); + + // Generate parser for time function + m_A_external_time_parser[i] = std::make_unique( + utils::parser::makeParser(m_A_ext_time_function[i],{"t",})); + m_A_time_scale[i] = m_A_external_time_parser[i]->compile<1>(); + + const std::set A_time_ext_symbols = m_A_external_time_parser[i]->symbols(); + A_time_dep_count += static_cast(A_time_ext_symbols.count("t")); + } + + if (A_time_dep_count > 0) { + ablastr::warn_manager::WMRecordWarning( + "HybridPIC ExternalVectorPotential", + "Coulomb Gauge is Expected, please be sure to have a divergence free A. Divergence cleaning of A to be implemented soon.", + ablastr::warn_manager::WarnPriority::low + ); + } + + UpdateHybridExternalFields(warpx.gett_new(0), warpx.getdt(0)); +} + + +void +ExternalVectorPotential::CalculateExternalCurlA () +{ + for (auto fname : m_field_names) { + CalculateExternalCurlA(fname); + } +} + +void +ExternalVectorPotential::CalculateExternalCurlA (std::string& coil_name) +{ + using ablastr::fields::Direction; + auto & warpx = WarpX::GetInstance(); + + // Compute the curl of the reference A field (unscaled by time function) + const std::string Aext_field = coil_name + std::string{"_Aext"}; + const std::string curlAext_field = coil_name + std::string{"_curlAext"}; + + ablastr::fields::MultiLevelVectorField A_ext = + warpx.m_fields.get_mr_levels_alldirs(Aext_field, warpx.finestLevel()); + ablastr::fields::MultiLevelVectorField curlA_ext = + warpx.m_fields.get_mr_levels_alldirs(curlAext_field, warpx.finestLevel()); + + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { + warpx.get_pointer_fdtd_solver_fp(lev)->ComputeCurlA( + curlA_ext[lev], + A_ext[lev], + warpx.GetEBUpdateBFlag()[lev], + lev); + + for (int idir = 0; idir < 3; ++idir) { + warpx.m_fields.get(curlAext_field, Direction{idir}, lev)-> + FillBoundary(warpx.Geom(lev).periodicity()); + } + } +} + +AMREX_FORCE_INLINE +void +ExternalVectorPotential::PopulateExternalFieldFromVectorPotential ( + ablastr::fields::VectorField const& dstField, + amrex::Real scale_factor, + ablastr::fields::VectorField const& srcField, + std::array< std::unique_ptr,3> const& eb_update) +{ + // Loop through the grids, and over the tiles within each grid +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*dstField[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + // Extract field data for this grid/tile + Array4 const& Fx = dstField[0]->array(mfi); + Array4 const& Fy = dstField[1]->array(mfi); + Array4 const& Fz = dstField[2]->array(mfi); + + Array4 const& Sx = srcField[0]->const_array(mfi); + Array4 const& Sy = srcField[1]->const_array(mfi); + Array4 const& Sz = srcField[2]->const_array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries. + amrex::Array4 update_Fx_arr, update_Fy_arr, update_Fz_arr; + if (EB::enabled()) { + update_Fx_arr = eb_update[0]->array(mfi); + update_Fy_arr = eb_update[1]->array(mfi); + update_Fz_arr = eb_update[2]->array(mfi); + } + + // Extract tileboxes for which to loop + Box const& tbx = mfi.tilebox(dstField[0]->ixType().toIntVect()); + Box const& tby = mfi.tilebox(dstField[1]->ixType().toIntVect()); + Box const& tbz = mfi.tilebox(dstField[2]->ixType().toIntVect()); + + // Loop over the cells and update the fields + amrex::ParallelFor(tbx, tby, tbz, + + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_Fx_arr && update_Fx_arr(i, j, k) == 0) { return; } + + Fx(i,j,k) = scale_factor * Sx(i,j,k); + }, + + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_Fy_arr && update_Fy_arr(i, j, k) == 0) { return; } + + Fy(i,j,k) = scale_factor * Sy(i,j,k); + }, + + [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Skip field update in the embedded boundaries + if (update_Fz_arr && update_Fz_arr(i, j, k) == 0) { return; } + + Fz(i,j,k) = scale_factor * Sz(i,j,k); + } + ); + } +} + +void +ExternalVectorPotential::UpdateHybridExternalFields (const amrex::Real t, const amrex::Real dt) +{ + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); + + + ablastr::fields::MultiLevelVectorField B_ext = + warpx.m_fields.get_mr_levels_alldirs(FieldType::hybrid_B_fp_external, warpx.finestLevel()); + ablastr::fields::MultiLevelVectorField E_ext = + warpx.m_fields.get_mr_levels_alldirs(FieldType::hybrid_E_fp_external, warpx.finestLevel()); + + for (int i = 0; i < m_nFields; ++i) { + const std::string Aext_field = m_field_names[i] + std::string{"_Aext"}; + const std::string curlAext_field = m_field_names[i] + std::string{"_curlAext"}; + + // Get B-field Scaling Factor + const amrex::Real scale_factor_B = m_A_time_scale[i](t); + + // Get dA/dt scaling factor based on time centered FD around t + const amrex::Real sf_l = m_A_time_scale[i](t-0.5_rt*dt); + const amrex::Real sf_r = m_A_time_scale[i](t+0.5_rt*dt); + const amrex::Real scale_factor_E = -(sf_r - sf_l)/dt; + + ablastr::fields::MultiLevelVectorField A_ext = + warpx.m_fields.get_mr_levels_alldirs(Aext_field, warpx.finestLevel()); + ablastr::fields::MultiLevelVectorField curlA_ext = + warpx.m_fields.get_mr_levels_alldirs(curlAext_field, warpx.finestLevel()); + + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { + PopulateExternalFieldFromVectorPotential(E_ext[lev], scale_factor_E, A_ext[lev], warpx.GetEBUpdateEFlag()[lev]); + PopulateExternalFieldFromVectorPotential(B_ext[lev], scale_factor_B, curlA_ext[lev], warpx.GetEBUpdateBFlag()[lev]); + + for (int idir = 0; idir < 3; ++idir) { + E_ext[lev][Direction{idir}]->FillBoundary(warpx.Geom(lev).periodicity()); + B_ext[lev][Direction{idir}]->FillBoundary(warpx.Geom(lev).periodicity()); + } + } + } + amrex::Gpu::streamSynchronize(); +} diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 15208727559..2a489e1c806 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -1,8 +1,9 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * * Authors: Roelof Groenewald (TAE Technologies) + * S. Eric Clark (Helion Energy) * * License: BSD-3-Clause-LBNL */ @@ -12,6 +13,9 @@ #include "HybridPICModel_fwd.H" +#include "Fields.H" + +#include "ExternalVectorPotential.H" #include "Utils/WarpXAlgorithmSelection.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" @@ -19,8 +23,13 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include #include +#include +#include +#include #include @@ -31,20 +40,32 @@ class HybridPICModel { public: - HybridPICModel (int nlevs_max); // constructor + HybridPICModel (); /** Read user-defined model parameters. Called in constructor. */ void ReadParameters (); /** Allocate hybrid-PIC specific multifabs. Called in constructor. */ - void AllocateMFs (int nlevs_max); - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, - int ncomps, const amrex::IntVect& ngJ, const amrex::IntVect& ngRho, - const amrex::IntVect& jx_nodal_flag, const amrex::IntVect& jy_nodal_flag, - const amrex::IntVect& jz_nodal_flag, const amrex::IntVect& rho_nodal_flag); - - /** Helper function to clear values from hybrid-PIC specific multifabs. */ - void ClearLevel (int lev); + void AllocateLevelMFs ( + ablastr::fields::MultiFabRegister & fields, + int lev, + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + int ncomps, + const amrex::IntVect& ngJ, + const amrex::IntVect& ngRho, + const amrex::IntVect& ngEB, + const amrex::IntVect& jx_nodal_flag, + const amrex::IntVect& jy_nodal_flag, + const amrex::IntVect& jz_nodal_flag, + const amrex::IntVect& rho_nodal_flag, + const amrex::IntVect& Ex_nodal_flag, + const amrex::IntVect& Ey_nodal_flag, + const amrex::IntVect& Ez_nodal_flag, + const amrex::IntVect& Bx_nodal_flag, + const amrex::IntVect& By_nodal_flag, + const amrex::IntVect& Bz_nodal_flag + ) const; void InitData (); @@ -54,30 +75,24 @@ public: * external current multifab. Note the external current can be a function * of time and therefore this should be re-evaluated at every step. */ - void GetCurrentExternal ( - amrex::Vector, 3>> const& edge_lengths - ); - void GetCurrentExternal ( - std::array< std::unique_ptr, 3> const& edge_lengths, - int lev - ); + void GetCurrentExternal (); /** * \brief - * Function to calculate the total current based on Ampere's law while - * neglecting displacement current (J = curl x B). Used in the Ohm's law - * solver (kinetic-fluid hybrid model). + * Function to calculate the total plasma current based on Ampere's law while + * neglecting displacement current (J = curl x B). Any external current is + * subtracted as well. Used in the Ohm's law solver (kinetic-fluid hybrid model). * * \param[in] Bfield Magnetic field from which the current is calculated. - * \param[in] edge_lengths Length of cell edges taking embedded boundaries into account + * \param[in] eb_update_E Indicate in which cell J should be calculated (related to embedded boundaries). */ - void CalculateCurrentAmpere ( - amrex::Vector, 3>> const& Bfield, - amrex::Vector, 3>> const& edge_lengths + void CalculatePlasmaCurrent ( + ablastr::fields::MultiLevelVectorField const& Bfield, + amrex::Vector,3 > >& eb_update_E ); - void CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3> const& edge_lengths, + void CalculatePlasmaCurrent ( + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3 >& eb_update_E, int lev ); @@ -86,81 +101,82 @@ public: * Function to update the E-field using Ohm's law (hybrid-PIC model). */ void HybridPICSolveE ( - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector, 3>> const& Bfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, - bool include_resistivity_term); + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, + bool solve_for_Faraday) const; void HybridPICSolveE ( - std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, - int lev, bool include_resistivity_term); + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + std::array< std::unique_ptr,3 >& eb_update_E, + int lev, bool solve_for_Faraday) const; void HybridPICSolveE ( - std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, - int lev, PatchType patch_type, bool include_resistivity_term); + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + std::array< std::unique_ptr,3 >& eb_update_E, + int lev, PatchType patch_type, bool solve_for_Faraday) const; void BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, DtType a_dt_type, amrex::IntVect ng, std::optional nodal_sync); void BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, int lev, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); void FieldPush ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); /** * \brief - * Function to calculate the electron pressure at a given timestep type - * using the simulation charge density. Used in the Ohm's law solver - * (kinetic-fluid hybrid model). + * Function to calculate the electron pressure using the simulation charge + * density. Used in the Ohm's law solver (kinetic-fluid hybrid model). */ - void CalculateElectronPressure ( DtType a_dt_type); - void CalculateElectronPressure (int lev, DtType a_dt_type); + void CalculateElectronPressure () const; + void CalculateElectronPressure (int lev) const; /** * \brief Fill the electron pressure multifab given the kinetic particle * charge density (and assumption of quasi-neutrality) using the user * specified electron equation of state. * - * \param[out] Pe_filed scalar electron pressure MultiFab at a given level + * \param[out] Pe_field scalar electron pressure MultiFab at a given level * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) const; + amrex::MultiFab& Pe_field, + amrex::MultiFab const& rho_field ) const; // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ int m_substeps = 10; + bool m_holmstrom_vacuum_region = false; + /** Electron temperature in eV */ amrex::Real m_elec_temp; /** Reference electron density */ @@ -186,33 +202,11 @@ public: std::string m_Jz_ext_grid_function = "0.0"; std::array< std::unique_ptr, 3> m_J_external_parser; std::array< amrex::ParserExecutor<4>, 3> m_J_external; - bool m_external_field_has_time_dependence = false; - - // Declare multifabs specifically needed for the hybrid-PIC model - amrex::Vector< std::unique_ptr > rho_fp_temp; - amrex::Vector, 3 > > current_fp_temp; - amrex::Vector, 3 > > current_fp_ampere; - amrex::Vector, 3 > > current_fp_external; - amrex::Vector< std::unique_ptr > electron_pressure_fp; - - // Helper functions to retrieve hybrid-PIC multifabs - [[nodiscard]] amrex::MultiFab* - get_pointer_current_fp_ampere (int lev, int direction) const - { - return current_fp_ampere[lev][direction].get(); - } + bool m_external_current_has_time_dependence = false; - [[nodiscard]] amrex::MultiFab* - get_pointer_current_fp_external (int lev, int direction) const - { - return current_fp_external[lev][direction].get(); - } - - [[nodiscard]] amrex::MultiFab* - get_pointer_electron_pressure_fp (int lev) const - { - return electron_pressure_fp[lev].get(); - } + /** External E/B fields */ + bool m_add_external_fields = false; + std::unique_ptr m_external_vector_potential; /** Gpu Vector with index type of the Jx multifab */ amrex::GpuArray Jx_IndexType; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 6a72bb3569c..3e5c04e9794 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -1,24 +1,28 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * * Authors: Roelof Groenewald (TAE Technologies) + * S. Eric Clark (Helion Energy) * * License: BSD-3-Clause-LBNL */ #include "HybridPICModel.H" -#include "FieldSolver/Fields.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Python/callbacks.H" +#include "Fields.H" +#include "Particles/MultiParticleContainer.H" +#include "ExternalVectorPotential.H" #include "WarpX.H" using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; -HybridPICModel::HybridPICModel ( int nlevs_max ) +HybridPICModel::HybridPICModel () { ReadParameters(); - AllocateMFs(nlevs_max); } void HybridPICModel::ReadParameters () @@ -29,6 +33,8 @@ void HybridPICModel::ReadParameters () // of sub steps can be specified by the user (defaults to 50). utils::parser::queryWithParser(pp_hybrid, "substeps", m_substeps); + utils::parser::queryWithParser(pp_hybrid, "holmstrom_vacuum_region", m_holmstrom_vacuum_region); + // The hybrid model requires an electron temperature, reference density // and exponent to be given. These values will be used to calculate the // electron pressure according to p = n0 * Te * (n/n0)^gamma @@ -53,61 +59,91 @@ void HybridPICModel::ReadParameters () pp_hybrid.query("Jx_external_grid_function(x,y,z,t)", m_Jx_ext_grid_function); pp_hybrid.query("Jy_external_grid_function(x,y,z,t)", m_Jy_ext_grid_function); pp_hybrid.query("Jz_external_grid_function(x,y,z,t)", m_Jz_ext_grid_function); -} -void HybridPICModel::AllocateMFs (int nlevs_max) -{ - electron_pressure_fp.resize(nlevs_max); - rho_fp_temp.resize(nlevs_max); - current_fp_temp.resize(nlevs_max); - current_fp_ampere.resize(nlevs_max); - current_fp_external.resize(nlevs_max); + // external fields + pp_hybrid.query("add_external_fields", m_add_external_fields); + + if (m_add_external_fields) { + m_external_vector_potential = std::make_unique(); + } } -void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, - const int ncomps, const IntVect& ngJ, const IntVect& ngRho, - const IntVect& jx_nodal_flag, - const IntVect& jy_nodal_flag, - const IntVect& jz_nodal_flag, - const IntVect& rho_nodal_flag) +void HybridPICModel::AllocateLevelMFs ( + ablastr::fields::MultiFabRegister & fields, + int lev, const BoxArray& ba, const DistributionMapping& dm, + const int ncomps, + const IntVect& ngJ, const IntVect& ngRho, + const IntVect& ngEB, + const IntVect& jx_nodal_flag, + const IntVect& jy_nodal_flag, + const IntVect& jz_nodal_flag, + const IntVect& rho_nodal_flag, + const IntVect& Ex_nodal_flag, + const IntVect& Ey_nodal_flag, + const IntVect& Ez_nodal_flag, + const IntVect& Bx_nodal_flag, + const IntVect& By_nodal_flag, + const IntVect& Bz_nodal_flag) const { - // The "electron_pressure_fp" multifab stores the electron pressure calculated + using ablastr::fields::Direction; + + // The "hybrid_electron_pressure_fp" multifab stores the electron pressure calculated // from the specified equation of state. - // The "rho_fp_temp" multifab is used to store the ion charge density + fields.alloc_init(FieldType::hybrid_electron_pressure_fp, + lev, amrex::convert(ba, rho_nodal_flag), + dm, ncomps, ngRho, 0.0_rt); + + // The "hybrid_rho_fp_temp" multifab is used to store the ion charge density // interpolated or extrapolated to appropriate timesteps. - // The "current_fp_temp" multifab is used to store the ion current density + fields.alloc_init(FieldType::hybrid_rho_fp_temp, + lev, amrex::convert(ba, rho_nodal_flag), + dm, ncomps, ngRho, 0.0_rt); + + // The "hybrid_current_fp_temp" multifab is used to store the ion current density // interpolated or extrapolated to appropriate timesteps. - // The "current_fp_ampere" multifab stores the total current calculated as - // the curl of B. - WarpX::AllocInitMultiFab(electron_pressure_fp[lev], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "electron_pressure_fp", 0.0_rt); - - WarpX::AllocInitMultiFab(rho_fp_temp[lev], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "rho_fp_temp", 0.0_rt); - - WarpX::AllocInitMultiFab(current_fp_temp[lev][0], amrex::convert(ba, jx_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_temp[lev][1], amrex::convert(ba, jy_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_temp[lev][2], amrex::convert(ba, jz_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[z]", 0.0_rt); - - WarpX::AllocInitMultiFab(current_fp_ampere[lev][0], amrex::convert(ba, jx_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_ampere[lev][1], amrex::convert(ba, jy_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_ampere[lev][2], amrex::convert(ba, jz_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[z]", 0.0_rt); - - // the external current density multifab is made nodal to avoid needing to interpolate - // to a nodal grid as has to be done for the ion and total current density multifabs - // this also allows the external current multifab to not have any ghost cells - WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[z]", 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{0}, + lev, amrex::convert(ba, jx_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{1}, + lev, amrex::convert(ba, jy_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{2}, + lev, amrex::convert(ba, jz_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + + // The "hybrid_current_fp_plasma" multifab stores the total plasma current calculated + // as the curl of B minus any external current. + fields.alloc_init(FieldType::hybrid_current_fp_plasma, Direction{0}, + lev, amrex::convert(ba, jx_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_plasma, Direction{1}, + lev, amrex::convert(ba, jy_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_plasma, Direction{2}, + lev, amrex::convert(ba, jz_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + + // the external current density multifab matches the current staggering and + // one ghost cell is used since we interpolate the current to a nodal grid + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{0}, + lev, amrex::convert(ba, jx_nodal_flag), + dm, ncomps, IntVect(1), 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{1}, + lev, amrex::convert(ba, jy_nodal_flag), + dm, ncomps, IntVect(1), 0.0_rt); + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{2}, + lev, amrex::convert(ba, jz_nodal_flag), + dm, ncomps, IntVect(1), 0.0_rt); + + if (m_add_external_fields) { + m_external_vector_potential->AllocateLevelMFs( + fields, + lev, ba, dm, + ncomps, ngEB, + Ex_nodal_flag, Ey_nodal_flag, Ez_nodal_flag, + Bx_nodal_flag, By_nodal_flag, Bz_nodal_flag + ); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -116,17 +152,6 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri #endif } -void HybridPICModel::ClearLevel (int lev) -{ - electron_pressure_fp[lev].reset(); - rho_fp_temp[lev].reset(); - for (int i = 0; i < 3; ++i) { - current_fp_temp[lev][i].reset(); - current_fp_ampere[lev][i].reset(); - current_fp_external[lev][i].reset(); - } -} - void HybridPICModel::InitData () { m_resistivity_parser = std::make_unique( @@ -148,21 +173,22 @@ void HybridPICModel::InitData () // check if the external current parsers depend on time for (int i=0; i<3; i++) { const std::set J_ext_symbols = m_J_external_parser[i]->symbols(); - m_external_field_has_time_dependence += J_ext_symbols.count("t"); + m_external_current_has_time_dependence += J_ext_symbols.count("t"); } auto & warpx = WarpX::GetInstance(); + using ablastr::fields::Direction; // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.m_fields.get(FieldType::current_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.m_fields.get(FieldType::current_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.m_fields.get(FieldType::current_fp, Direction{2}, 0)->ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect By_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, 0)->ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, 0)->ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( @@ -226,237 +252,111 @@ void HybridPICModel::InitData () // Initialize external current - note that this approach skips the check // if the current is time dependent which is what needs to be done to // write time independent fields on the first step. - for (int lev = 0; lev <= warpx.finestLevel(); ++lev) - { -#ifdef AMREX_USE_EB - auto& edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); - auto& edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); - auto& edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); - - const auto edge_lengths = std::array< std::unique_ptr, 3 >{ - std::make_unique( - edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp()), - std::make_unique( - edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp()), - std::make_unique( - edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp()) - }; -#else - const auto edge_lengths = std::array< std::unique_ptr, 3 >(); -#endif - GetCurrentExternal(edge_lengths, lev); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { + warpx.ComputeExternalFieldOnGridUsingParser( + FieldType::hybrid_current_fp_external, + m_J_external[0], + m_J_external[1], + m_J_external[2], + lev, PatchType::fine, + warpx.GetEBUpdateEFlag()); } -} - -void HybridPICModel::GetCurrentExternal ( - amrex::Vector, 3>> const& edge_lengths) -{ - if (!m_external_field_has_time_dependence) { return; } - auto& warpx = WarpX::GetInstance(); - for (int lev = 0; lev <= warpx.finestLevel(); ++lev) - { - GetCurrentExternal(edge_lengths[lev], lev); + if (m_add_external_fields) { + m_external_vector_potential->InitData(); } } - -void HybridPICModel::GetCurrentExternal ( - std::array< std::unique_ptr, 3> const& edge_lengths, - int lev) +void HybridPICModel::GetCurrentExternal () { - // This logic matches closely to WarpX::InitializeExternalFieldsOnGridUsingParser - // except that the parsers include time dependence. - auto & warpx = WarpX::GetInstance(); - - auto t = warpx.gett_new(lev); - - auto dx_lev = warpx.Geom(lev).CellSizeArray(); - const RealBox& real_box = warpx.Geom(lev).ProbDomain(); - - auto& mfx = current_fp_external[lev][0]; - auto& mfy = current_fp_external[lev][1]; - auto& mfz = current_fp_external[lev][2]; - - const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); - const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); - const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); - - // avoid implicit lambda capture - auto Jx_external = m_J_external[0]; - auto Jy_external = m_J_external[1]; - auto Jz_external = m_J_external[2]; + if (!m_external_current_has_time_dependence) { return; } - for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { - const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); - const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); - const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); - - auto const& mfxfab = mfx->array(mfi); - auto const& mfyfab = mfy->array(mfi); - auto const& mfzfab = mfz->array(mfi); - -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(ly); -#endif -#else - amrex::ignore_unused(edge_lengths); -#endif - - amrex::ParallelFor (tbx, tby, tbz, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (lx(i, j, k) <= 0) return; -#endif - // Shift required in the x-, y-, or z- position - // depending on the index type of the multifab -#if defined(WARPX_DIM_1D_Z) - const amrex::Real x = 0._rt; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; -#else - const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real fac_y = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; - const amrex::Real fac_z = (1._rt - x_nodal_flag[2]) * dx_lev[2] * 0.5_rt; - const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; -#endif - // Initialize the x-component of the field. - mfxfab(i,j,k) = Jx_external(x,y,z,t); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (ly(i, j, k) <= 0) return; -#endif -#if defined(WARPX_DIM_1D_Z) - const amrex::Real x = 0._rt; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; -#elif defined(WARPX_DIM_3D) - const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real fac_y = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; - const amrex::Real fac_z = (1._rt - y_nodal_flag[2]) * dx_lev[2] * 0.5_rt; - const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; -#endif - // Initialize the y-component of the field. - mfyfab(i,j,k) = Jy_external(x,y,z,t); - }, - [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (lz(i, j, k) <= 0) return; -#endif -#if defined(WARPX_DIM_1D_Z) - const amrex::Real x = 0._rt; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real y = 0._rt; - const amrex::Real fac_z = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; -#elif defined(WARPX_DIM_3D) - const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; - const amrex::Real fac_y = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; - const amrex::Real fac_z = (1._rt - z_nodal_flag[2]) * dx_lev[2] * 0.5_rt; - const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; -#endif - // Initialize the z-component of the field. - mfzfab(i,j,k) = Jz_external(x,y,z,t); - } - ); + warpx.ComputeExternalFieldOnGridUsingParser( + FieldType::hybrid_current_fp_external, + m_J_external[0], + m_J_external[1], + m_J_external[2], + lev, PatchType::fine, + warpx.GetEBUpdateEFlag()); } } -void HybridPICModel::CalculateCurrentAmpere ( - amrex::Vector, 3>> const& Bfield, - amrex::Vector, 3>> const& edge_lengths) +void HybridPICModel::CalculatePlasmaCurrent ( + ablastr::fields::MultiLevelVectorField const& Bfield, + amrex::Vector,3 > >& eb_update_E) { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { - CalculateCurrentAmpere(Bfield[lev], edge_lengths[lev], lev); + CalculatePlasmaCurrent(Bfield[lev], eb_update_E[lev], lev); } } -void HybridPICModel::CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3> const& edge_lengths, +void HybridPICModel::CalculatePlasmaCurrent ( + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3 >& eb_update_E, const int lev) { - WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); + WARPX_PROFILE("HybridPICModel::CalculatePlasmaCurrent()"); auto& warpx = WarpX::GetInstance(); + ablastr::fields::VectorField current_fp_plasma = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_plasma, lev); warpx.get_pointer_fdtd_solver_fp(lev)->CalculateCurrentAmpere( - current_fp_ampere[lev], Bfield, edge_lengths, lev + current_fp_plasma, Bfield, eb_update_E, lev ); // we shouldn't apply the boundary condition to J since J = J_i - J_e but // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } + for (int i=0; i<3; i++) { current_fp_plasma[i]->FillBoundary(warpx.Geom(lev).periodicity()); } + + // Subtract external current from "Ampere" current calculated above. Note + // we need to include 1 ghost cell since later we will interpolate the + // plasma current to a nodal grid. + ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_external, lev); + for (int i=0; i<3; i++) { + current_fp_plasma[i]->minus(*current_fp_external[i], 0, 1, 1); + } + } void HybridPICModel::HybridPICSolveE ( - amrex::Vector, 3>> & Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector, 3>> const& Bfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, - const bool include_resistivity_term) + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, + const bool solve_for_Faraday) const { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { HybridPICSolveE( - Efield[lev], Jfield[lev], Bfield[lev], rhofield[lev], - edge_lengths[lev], lev, include_resistivity_term + Efield[lev], Jfield[lev], Bfield[lev], *rhofield[lev], + eb_update_E[lev], lev, solve_for_Faraday ); } + // Allow execution of Python callback after E-field push + ExecutePythonCallback("afterEpush"); } void HybridPICModel::HybridPICSolveE ( - std::array< std::unique_ptr, 3> & Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, - const int lev, const bool include_resistivity_term) + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + std::array< std::unique_ptr,3 >& eb_update_E, + const int lev, const bool solve_for_Faraday) const { WARPX_PROFILE("WarpX::HybridPICSolveE()"); HybridPICSolveE( - Efield, Jfield, Bfield, rhofield, edge_lengths, lev, - PatchType::fine, include_resistivity_term + Efield, Jfield, Bfield, rhofield, eb_update_E, lev, + PatchType::fine, solve_for_Faraday ); if (lev > 0) { @@ -466,58 +366,58 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - std::array< std::unique_ptr, 3> & Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + std::array< std::unique_ptr,3 >& eb_update_E, const int lev, PatchType patch_type, - const bool include_resistivity_term) + const bool solve_for_Faraday) const { auto& warpx = WarpX::GetInstance(); + ablastr::fields::VectorField current_fp_plasma = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_plasma, lev); + auto* const electron_pressure_fp = warpx.m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); + // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( - Efield, current_fp_ampere[lev], Jfield, current_fp_external[lev], - Bfield, rhofield, - electron_pressure_fp[lev], - edge_lengths, lev, this, include_resistivity_term + Efield, current_fp_plasma, Jfield, Bfield, rhofield, + *electron_pressure_fp, eb_update_E, lev, this, solve_for_Faraday ); - warpx.ApplyEfieldBoundary(lev, patch_type); + amrex::Real const time = warpx.gett_old(0) + warpx.getdt(0); + warpx.ApplyEfieldBoundary(lev, patch_type, time); } -void HybridPICModel::CalculateElectronPressure(DtType a_dt_type) +void HybridPICModel::CalculateElectronPressure() const { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { - CalculateElectronPressure(lev, a_dt_type); + CalculateElectronPressure(lev); } } -void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) +void HybridPICModel::CalculateElectronPressure(const int lev) const { WARPX_PROFILE("WarpX::CalculateElectronPressure()"); auto& warpx = WarpX::GetInstance(); - // The full step uses rho^{n+1}, otherwise use the old or averaged - // charge density. - if (a_dt_type == DtType::Full) { - FillElectronPressureMF( - electron_pressure_fp[lev], warpx.getFieldPointer(FieldType::rho_fp, lev) - ); - } else { - FillElectronPressureMF( - electron_pressure_fp[lev], rho_fp_temp[lev].get() - ); - } + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); + ablastr::fields::ScalarField rho_fp = warpx.m_fields.get(FieldType::rho_fp, lev); + + // Calculate the electron pressure using rho^{n+1}. + FillElectronPressureMF( + *electron_pressure_fp, + *rho_fp + ); warpx.ApplyElectronPressureBoundary(lev, PatchType::fine); - electron_pressure_fp[lev]->FillBoundary(warpx.Geom(lev).periodicity()); + electron_pressure_fp->FillBoundary(warpx.Geom(lev).periodicity()); } void HybridPICModel::FillElectronPressureMF ( - std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) const + amrex::MultiFab& Pe_field, + amrex::MultiFab const& rho_field +) const { const auto n0_ref = m_n0_ref; const auto elec_temp = m_elec_temp; @@ -527,11 +427,11 @@ void HybridPICModel::FillElectronPressureMF ( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for ( MFIter mfi(*Pe_field, TilingIfNotGPU()); mfi.isValid(); ++mfi ) + for ( MFIter mfi(Pe_field, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& rho = rho_field->const_array(mfi); - Array4 const& Pe = Pe_field->array(mfi); + Array4 const& rho = rho_field.const_array(mfi); + Array4 const& Pe = Pe_field.array(mfi); // Extract tileboxes for which to loop const Box& tilebox = mfi.tilebox(); @@ -545,11 +445,11 @@ void HybridPICModel::FillElectronPressureMF ( } void HybridPICModel::BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) { @@ -557,18 +457,18 @@ void HybridPICModel::BfieldEvolveRK ( for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { BfieldEvolveRK( - Bfield, Efield, Jfield, rhofield, edge_lengths, dt, lev, dt_type, + Bfield, Efield, Jfield, rhofield, eb_update_E, dt, lev, dt_type, ng, nodal_sync ); } } void HybridPICModel::BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, int lev, DtType dt_type, IntVect ng, std::optional nodal_sync ) { @@ -595,7 +495,7 @@ void HybridPICModel::BfieldEvolveRK ( // The Runge-Kutta scheme begins here. // Step 1: FieldPush( - Bfield, Efield, Jfield, rhofield, edge_lengths, + Bfield, Efield, Jfield, rhofield, eb_update_E, 0.5_rt*dt, dt_type, ng, nodal_sync ); @@ -611,7 +511,7 @@ void HybridPICModel::BfieldEvolveRK ( // Step 2: FieldPush( - Bfield, Efield, Jfield, rhofield, edge_lengths, + Bfield, Efield, Jfield, rhofield, eb_update_E, 0.5_rt*dt, dt_type, ng, nodal_sync ); @@ -631,7 +531,7 @@ void HybridPICModel::BfieldEvolveRK ( // Step 3: FieldPush( - Bfield, Efield, Jfield, rhofield, edge_lengths, + Bfield, Efield, Jfield, rhofield, eb_update_E, dt, dt_type, ng, nodal_sync ); @@ -647,7 +547,7 @@ void HybridPICModel::BfieldEvolveRK ( // Step 4: FieldPush( - Bfield, Efield, Jfield, rhofield, edge_lengths, + Bfield, Efield, Jfield, rhofield, eb_update_E, 0.5_rt*dt, dt_type, ng, nodal_sync ); @@ -676,23 +576,27 @@ void HybridPICModel::BfieldEvolveRK ( } } + void HybridPICModel::FieldPush ( - amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField const& Jfield, + ablastr::fields::MultiLevelScalarField const& rhofield, + amrex::Vector,3 > >& eb_update_E, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) { auto& warpx = WarpX::GetInstance(); - // Calculate J = curl x B / mu0 - CalculateCurrentAmpere(Bfield, edge_lengths); + amrex::Real const t_old = warpx.gett_old(0); + + // Calculate J = curl x B / mu0 - J_ext + CalculatePlasmaCurrent(Bfield, eb_update_E); // Calculate the E-field from Ohm's law - HybridPICSolveE(Efield, Jfield, Bfield, rhofield, edge_lengths, true); + HybridPICSolveE(Efield, Jfield, Bfield, rhofield, eb_update_E, true); warpx.FillBoundaryE(ng, nodal_sync); + // Push forward the B-field using Faraday's law - warpx.EvolveB(dt, dt_type); + warpx.EvolveB(dt, dt_type, t_old); warpx.FillBoundaryB(ng, nodal_sync); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/Make.package b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/Make.package index 8145cfcef2f..d4fa9bfc390 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/Make.package +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/Make.package @@ -1,3 +1,4 @@ CEXE_sources += HybridPICModel.cpp +CEXE_sources += ExternalVectorPotential.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 456c542a534..f46b2f73e41 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -1,14 +1,16 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * * Authors: Roelof Groenewald (TAE Technologies) + * S. Eric Clark (Helion Energy) * * License: BSD-3-Clause-LBNL */ #include "FiniteDifferenceSolver.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef WARPX_DIM_RZ # include "FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #else @@ -21,11 +23,12 @@ #include using namespace amrex; +using warpx::fields::FieldType; void FiniteDifferenceSolver::CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField & Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev ) { // Select algorithm (The choice of algorithm is a runtime option, @@ -33,12 +36,12 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( if (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC) { #ifdef WARPX_DIM_RZ CalculateCurrentAmpereCylindrical ( - Jfield, Bfield, edge_lengths, lev + Jfield, Bfield, eb_update_E, lev ); #else CalculateCurrentAmpereCartesian ( - Jfield, Bfield, edge_lengths, lev + Jfield, Bfield, eb_update_E, lev ); #endif @@ -58,19 +61,15 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( #ifdef WARPX_DIM_RZ template void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( - std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev ) { // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // reset Jfield Jfield[0]->setVal(0); Jfield[1]->setVal(0); @@ -95,11 +94,17 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( Array4 const& Bt = Bfield[1]->array(mfi); Array4 const& Bz = Bfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lt = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries. + // The plasma current is stored at the same locations as the E-field, + // therefore the `eb_update_E` multifab also appropriately specifies + // where the plasma current should be calculated. + amrex::Array4 update_Jr_arr, update_Jt_arr, update_Jz_arr; + if (EB::enabled()) { + update_Jr_arr = eb_update_E[0]->array(mfi); + update_Jt_arr = eb_update_E[1]->array(mfi); + update_Jz_arr = eb_update_E[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -125,10 +130,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jr calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Jr_arr && update_Jr_arr(i, j, 0) == 0) { return; } + // Mode m=0 Jr(i, j, 0, 0) = one_over_mu0 * ( - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) @@ -151,11 +156,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jt calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // In RZ Jt is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(lt); - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Jt_arr && update_Jt_arr(i, j, 0) == 0) { return; } + // r on a nodal point (Jt is nodal in r) Real const r = rmin + i*dr; // Off-axis, regular curl @@ -199,10 +203,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jz calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lz(i, j, 0) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Jz_arr && update_Jz_arr(i, j, 0) == 0) { return; } + // r on a nodal point (Jz is nodal in r) Real const r = rmin + i*dr; // Off-axis, regular curl @@ -249,19 +253,15 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( template void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( - std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Bfield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev ) { // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // reset Jfield Jfield[0]->setVal(0); Jfield[1]->setVal(0); @@ -272,25 +272,30 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for ( MFIter mfi(*Jfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); } auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile - Array4 const& Jx = Jfield[0]->array(mfi); - Array4 const& Jy = Jfield[1]->array(mfi); - Array4 const& Jz = Jfield[2]->array(mfi); - Array4 const& Bx = Bfield[0]->const_array(mfi); - Array4 const& By = Bfield[1]->const_array(mfi); - Array4 const& Bz = Bfield[2]->const_array(mfi); - -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + Array4 const &Jx = Jfield[0]->array(mfi); + Array4 const &Jy = Jfield[1]->array(mfi); + Array4 const &Jz = Jfield[2]->array(mfi); + Array4 const &Bx = Bfield[0]->const_array(mfi); + Array4 const &By = Bfield[1]->const_array(mfi); + Array4 const &Bz = Bfield[2]->const_array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries. + // The plasma current is stored at the same locations as the E-field, + // therefore the `eb_update_E` multifab also appropriately specifies + // where the plasma current should be calculated. + amrex::Array4 update_Jx_arr, update_Jy_arr, update_Jz_arr; + if (EB::enabled()) { + update_Jx_arr = eb_update_E[0]->array(mfi); + update_Jy_arr = eb_update_E[1]->array(mfi); + update_Jz_arr = eb_update_E[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -313,10 +318,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jx calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Jx_arr && update_Jx_arr(i, j, k) == 0) { return; } + Jx(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) + T_Algo::DownwardDy(Bz, coefs_y, n_coefs_y, i, j, k) @@ -325,16 +330,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jy calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries -#ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; -#elif defined(WARPX_DIM_XZ) - // In XZ Jy is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif -#endif + + // Skip field update in the embedded boundaries + if (update_Jy_arr && update_Jy_arr(i, j, k) == 0) { return; } + Jy(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) + T_Algo::DownwardDz(Bx, coefs_z, n_coefs_z, i, j, k) @@ -343,10 +342,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jz calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Jz_arr && update_Jz_arr(i, j, k) == 0) { return; } + Jz(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) + T_Algo::DownwardDx(By, coefs_x, n_coefs_x, i, j, k) @@ -366,16 +365,15 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( void FiniteDifferenceSolver::HybridPICSolveE ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField& Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev, HybridPICModel const* hybrid_model, - const bool include_resistivity_term) + const bool solve_for_Faraday) { // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -383,15 +381,15 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ HybridPICSolveECylindrical ( - Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, - edge_lengths, lev, hybrid_model, include_resistivity_term + Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + eb_update_E, lev, hybrid_model, solve_for_Faraday ); #else HybridPICSolveECartesian ( - Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, - edge_lengths, lev, hybrid_model, include_resistivity_term + Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + eb_update_E, lev, hybrid_model, solve_for_Faraday ); #endif @@ -404,21 +402,16 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ template void FiniteDifferenceSolver::HybridPICSolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev, HybridPICModel const* hybrid_model, - const bool include_resistivity_term ) + const bool solve_for_Faraday ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // Both steps below do not currently support m > 0 and should be // modified if such support wants to be added WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -436,7 +429,18 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; - const bool include_hyper_resistivity_term = (eta_h > 0.0) && include_resistivity_term; + const bool include_hyper_resistivity_term = (eta_h > 0.0) && solve_for_Faraday; + + const bool include_external_fields = hybrid_model->m_add_external_fields; + + const bool holmstrom_vacuum_region = hybrid_model->m_holmstrom_vacuum_region; + + auto & warpx = WarpX::GetInstance(); + ablastr::fields::VectorField Bfield_external, Efield_external; + if (include_external_fields) { + Bfield_external = warpx.m_fields.get_alldirs(FieldType::hybrid_B_fp_external, 0); // lev=0 + Efield_external = warpx.m_fields.get_alldirs(FieldType::hybrid_E_fp_external, 0); // lev=0 + } // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -468,8 +472,8 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Also note that enE_nodal_mf does not need to have any guard cells since // these values will be interpolated to the Yee mesh which is contained // by the nodal mesh. - auto const& ba = convert(rhofield->boxArray(), IntVect::TheNodeVector()); - MultiFab enE_nodal_mf(ba, rhofield->DistributionMap(), 3, IntVect::TheZeroVector()); + auto const& ba = convert(rhofield.boxArray(), IntVect::TheNodeVector()); + MultiFab enE_nodal_mf(ba, rhofield.DistributionMap(), 3, IntVect::TheZeroVector()); // Loop through the grids, and over the tiles within each grid for the // initial, nodal calculation of E @@ -490,13 +494,17 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& Jir = Jifield[0]->const_array(mfi); Array4 const& Jit = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); - Array4 const& Jextr = Jextfield[0]->const_array(mfi); - Array4 const& Jextt = Jextfield[1]->const_array(mfi); - Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Br = Bfield[0]->const_array(mfi); Array4 const& Bt = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); + Array4 Br_ext, Bt_ext, Bz_ext; + if (include_external_fields) { + Br_ext = Bfield_external[0]->array(mfi); + Bt_ext = Bfield_external[1]->array(mfi); + Bz_ext = Bfield_external[2]->array(mfi); + } + // Loop over the cells and update the nodal E field amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ @@ -511,22 +519,28 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( auto const jiz_interp = Interp(Jiz, Jz_stag, nodal, coarsen, i, j, 0, 0); // interpolate the B field to a nodal grid - auto const Br_interp = Interp(Br, Br_stag, nodal, coarsen, i, j, 0, 0); - auto const Bt_interp = Interp(Bt, Bt_stag, nodal, coarsen, i, j, 0, 0); - auto const Bz_interp = Interp(Bz, Bz_stag, nodal, coarsen, i, j, 0, 0); + auto Br_interp = Interp(Br, Br_stag, nodal, coarsen, i, j, 0, 0); + auto Bt_interp = Interp(Bt, Bt_stag, nodal, coarsen, i, j, 0, 0); + auto Bz_interp = Interp(Bz, Bz_stag, nodal, coarsen, i, j, 0, 0); + + if (include_external_fields) { + Br_interp += Interp(Br_ext, Br_stag, nodal, coarsen, i, j, 0, 0); + Bt_interp += Interp(Bt_ext, Bt_stag, nodal, coarsen, i, j, 0, 0); + Bz_interp += Interp(Bz_ext, Bz_stag, nodal, coarsen, i, j, 0, 0); + } // calculate enE = (J - Ji) x B enE_nodal(i, j, 0, 0) = ( - (jt_interp - jit_interp - Jextt(i, j, 0)) * Bz_interp - - (jz_interp - jiz_interp - Jextz(i, j, 0)) * Bt_interp + (jt_interp - jit_interp) * Bz_interp + - (jz_interp - jiz_interp) * Bt_interp ); enE_nodal(i, j, 0, 1) = ( - (jz_interp - jiz_interp - Jextz(i, j, 0)) * Br_interp - - (jr_interp - jir_interp - Jextr(i, j, 0)) * Bz_interp + (jz_interp - jiz_interp) * Br_interp + - (jr_interp - jir_interp) * Bz_interp ); enE_nodal(i, j, 0, 2) = ( - (jr_interp - jir_interp - Jextr(i, j, 0)) * Bt_interp - - (jt_interp - jit_interp - Jextt(i, j, 0)) * Br_interp + (jr_interp - jir_interp) * Bt_interp + - (jt_interp - jit_interp) * Br_interp ); }); @@ -558,14 +572,24 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& Jt = Jfield[1]->const_array(mfi); Array4 const& Jz = Jfield[2]->const_array(mfi); Array4 const& enE = enE_nodal_mf.const_array(mfi); - Array4 const& rho = rhofield->const_array(mfi); - Array4 const& Pe = Pefield->array(mfi); + Array4 const& rho = rhofield.const_array(mfi); + Array4 const& Pe = Pefield.const_array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries + amrex::Array4 update_Er_arr, update_Et_arr, update_Ez_arr; + if (EB::enabled()) { + update_Er_arr = eb_update_E[0]->array(mfi); + update_Et_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); + } -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lt = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + Array4 Er_ext, Et_ext, Ez_ext; + if (include_external_fields) { + Er_ext = Efield_external[0]->array(mfi); + Et_ext = Efield_external[1]->array(mfi); + Ez_ext = Efield_external[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -586,52 +610,64 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Er calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Er_arr && update_Er_arr(i, j, 0) == 0) { return; } + // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jr_val = Interp(Jr, Jr_stag, Er_stag, coarsen, i, j, 0, 0); - const Real jt_val = Interp(Jt, Jt_stag, Er_stag, coarsen, i, j, 0, 0); - const Real jz_val = Interp(Jz, Jz_stag, Er_stag, coarsen, i, j, 0, 0); - jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Er(i, j, 0) = 0._rt; + } else { + // Get the gradient of the electron pressure if the longitudinal part of + // the E-field should be included, otherwise ignore it since curl x (grad Pe) = 0 + const Real grad_Pe = (!solve_for_Faraday) ? + T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0) + : 0._rt; - // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0); + // interpolate the nodal neE values to the Yee grid + const auto enE_r = Interp(enE, nodal, Er_stag, coarsen, i, j, 0, 0); - // interpolate the nodal neE values to the Yee grid - auto enE_r = Interp(enE, nodal, Er_stag, coarsen, i, j, 0, 0); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; + Er(i, j, 0) = (enE_r - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if (resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jr_val = Jr(i, j, 0); + const Real jt_val = Interp(Jt, Jt_stag, Er_stag, coarsen, i, j, 0, 0); + const Real jz_val = Interp(Jz, Jz_stag, Er_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } - if (include_hyper_resistivity_term) { - // r on cell-centered point (Jr is cell-centered in r) - Real const r = rmin + (i + 0.5_rt)*dr; + Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); - auto nabla2Jr = T_Algo::Dr_rDr_over_r(Jr, r, dr, coefs_r, n_coefs_r, i, j, 0, 0); - Er(i, j, 0) -= eta_h * nabla2Jr; + if (include_hyper_resistivity_term) { + // r on cell-centered point (Jr is cell-centered in r) + const Real r = rmin + (i + 0.5_rt)*dr; + auto nabla2Jr = T_Algo::Dr_rDr_over_r(Jr, r, dr, coefs_r, n_coefs_r, i, j, 0, 0) + + T_Algo::Dzz(Jr, coefs_z, n_coefs_z, i, j, 0, 0) - Jr(i, j, 0)/(r*r); + Er(i, j, 0) -= eta_h * nabla2Jr; + } + } + + if (include_external_fields && (rho_val >= rho_floor)) { + Er(i, j, 0) -= Er_ext(i, j, 0); } }, // Et calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // In RZ Et is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(lt); - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Et_arr && update_Et_arr(i, j, 0) == 0) { return; } + // r on a nodal grid (Et is nodal in r) Real const r = rmin + i*dr; // Mode m=0: // Ensure that Et remains 0 on axis @@ -641,70 +677,103 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( } // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jr_val = Interp(Jr, Jr_stag, Et_stag, coarsen, i, j, 0, 0); - const Real jt_val = Interp(Jt, Jt_stag, Et_stag, coarsen, i, j, 0, 0); - const Real jz_val = Interp(Jz, Jz_stag, Et_stag, coarsen, i, j, 0, 0); - jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Et_stag, coarsen, i, j, 0, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Et(i, j, 0) = 0._rt; + } else { + // Get the gradient of the electron pressure + // -> d/dt = 0 for m = 0 + const auto grad_Pe = 0.0_rt; - // Get the gradient of the electron pressure - // -> d/dt = 0 for m = 0 - auto grad_Pe = 0.0_rt; + // interpolate the nodal neE values to the Yee grid + const auto enE_t = Interp(enE, nodal, Et_stag, coarsen, i, j, 0, 1); - // interpolate the nodal neE values to the Yee grid - auto enE_t = Interp(enE, nodal, Et_stag, coarsen, i, j, 0, 1); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; + Et(i, j, 0) = (enE_t - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if(resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jr_val = Interp(Jr, Jr_stag, Et_stag, coarsen, i, j, 0, 0); + const Real jt_val = Jt(i, j, 0); + const Real jz_val = Interp(Jz, Jz_stag, Et_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } - // Note: Hyper-resisitivity should be revisited here when modal decomposition is implemented + Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); + + if (include_hyper_resistivity_term) { + auto nabla2Jt = T_Algo::Dr_rDr_over_r(Jt, r, dr, coefs_r, n_coefs_r, i, j, 0, 0) + + T_Algo::Dzz(Jt, coefs_z, n_coefs_z, i, j, 0, 0) - Jt(i, j, 0)/(r*r); + Et(i, j, 0) -= eta_h * nabla2Jt; + } + } + + if (include_external_fields && (rho_val >= rho_floor)) { + Et(i, j, 0) -= Et_ext(i, j, 0); + } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB - // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,0) <= 0) { return; } -#endif + + // Skip field update in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, 0) == 0) { return; } + // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, 0, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jr_val = Interp(Jr, Jr_stag, Ez_stag, coarsen, i, j, 0, 0); - const Real jt_val = Interp(Jt, Jt_stag, Ez_stag, coarsen, i, j, 0, 0); - const Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, 0, 0); - jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, 0, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Ez(i, j, 0) = 0._rt; + } else { + // Get the gradient of the electron pressure if the longitudinal part of + // the E-field should be included, otherwise ignore it since curl x (grad Pe) = 0 + const Real grad_Pe = (!solve_for_Faraday) ? + T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, 0, 0) + : 0._rt; - // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, 0, 0); + // interpolate the nodal neE values to the Yee grid + const auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, 0, 2); - // interpolate the nodal neE values to the Yee grid - auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, 0, 2); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Ez(i, j, 0) = (enE_z - grad_Pe) / rho_val; + Ez(i, j, 0) = (enE_z - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, 0) += eta(rho_val, jtot_val) * Jz(i, j, 0); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if (resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jr_val = Interp(Jr, Jr_stag, Ez_stag, coarsen, i, j, 0, 0); + const Real jt_val = Interp(Jt, Jt_stag, Ez_stag, coarsen, i, j, 0, 0); + const Real jz_val = Jz(i, j, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + + Ez(i, j, 0) += eta(rho_val, jtot_val) * Jz(i, j, 0); + + if (include_hyper_resistivity_term) { + // r on nodal point (Jz is nodal in r) + const Real r = rmin + i*dr; + + auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, 0, 0); + if (r > 0.5_rt*dr) { + nabla2Jz += T_Algo::Dr_rDr_over_r(Jz, r, dr, coefs_r, n_coefs_r, i, j, 0, 0); + } + Ez(i, j, 0) -= eta_h * nabla2Jz; + } + } - if (include_hyper_resistivity_term) { - auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, 0, 0); - Ez(i, j, 0) -= eta_h * nabla2Jz; + if (include_external_fields && (rho_val >= rho_floor)) { + Ez(i, j, 0) -= Ez_ext(i, j, 0); } } ); @@ -722,21 +791,16 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( template void FiniteDifferenceSolver::HybridPICSolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, - std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Jfield, + ablastr::fields::VectorField const& Jifield, + ablastr::fields::VectorField const& Bfield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, + std::array< std::unique_ptr,3 > const& eb_update_E, int lev, HybridPICModel const* hybrid_model, - const bool include_resistivity_term ) + const bool solve_for_Faraday ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -748,7 +812,18 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; - const bool include_hyper_resistivity_term = (eta_h > 0.) && include_resistivity_term; + const bool include_hyper_resistivity_term = (eta_h > 0.) && solve_for_Faraday; + + const bool include_external_fields = hybrid_model->m_add_external_fields; + + const bool holmstrom_vacuum_region = hybrid_model->m_holmstrom_vacuum_region; + + auto & warpx = WarpX::GetInstance(); + ablastr::fields::VectorField Bfield_external, Efield_external; + if (include_external_fields) { + Bfield_external = warpx.m_fields.get_alldirs(FieldType::hybrid_B_fp_external, 0); // lev=0 + Efield_external = warpx.m_fields.get_alldirs(FieldType::hybrid_E_fp_external, 0); // lev=0 + } // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -780,8 +855,8 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Also note that enE_nodal_mf does not need to have any guard cells since // these values will be interpolated to the Yee mesh which is contained // by the nodal mesh. - auto const& ba = convert(rhofield->boxArray(), IntVect::TheNodeVector()); - MultiFab enE_nodal_mf(ba, rhofield->DistributionMap(), 3, IntVect::TheZeroVector()); + auto const& ba = convert(rhofield.boxArray(), IntVect::TheNodeVector()); + MultiFab enE_nodal_mf(ba, rhofield.DistributionMap(), 3, IntVect::TheZeroVector()); // Loop through the grids, and over the tiles within each grid for the // initial, nodal calculation of E @@ -802,17 +877,21 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jix = Jifield[0]->const_array(mfi); Array4 const& Jiy = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); - Array4 const& Jextx = Jextfield[0]->const_array(mfi); - Array4 const& Jexty = Jextfield[1]->const_array(mfi); - Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Bx = Bfield[0]->const_array(mfi); Array4 const& By = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); + Array4 Bx_ext, By_ext, Bz_ext; + if (include_external_fields) { + Bx_ext = Bfield_external[0]->array(mfi); + By_ext = Bfield_external[1]->array(mfi); + Bz_ext = Bfield_external[2]->array(mfi); + } + // Loop over the cells and update the nodal E field amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // interpolate the total current to a nodal grid + // interpolate the total plasma current to a nodal grid auto const jx_interp = Interp(Jx, Jx_stag, nodal, coarsen, i, j, k, 0); auto const jy_interp = Interp(Jy, Jy_stag, nodal, coarsen, i, j, k, 0); auto const jz_interp = Interp(Jz, Jz_stag, nodal, coarsen, i, j, k, 0); @@ -823,22 +902,28 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( auto const jiz_interp = Interp(Jiz, Jz_stag, nodal, coarsen, i, j, k, 0); // interpolate the B field to a nodal grid - auto const Bx_interp = Interp(Bx, Bx_stag, nodal, coarsen, i, j, k, 0); - auto const By_interp = Interp(By, By_stag, nodal, coarsen, i, j, k, 0); - auto const Bz_interp = Interp(Bz, Bz_stag, nodal, coarsen, i, j, k, 0); + auto Bx_interp = Interp(Bx, Bx_stag, nodal, coarsen, i, j, k, 0); + auto By_interp = Interp(By, By_stag, nodal, coarsen, i, j, k, 0); + auto Bz_interp = Interp(Bz, Bz_stag, nodal, coarsen, i, j, k, 0); + + if (include_external_fields) { + Bx_interp += Interp(Bx_ext, Bx_stag, nodal, coarsen, i, j, k, 0); + By_interp += Interp(By_ext, By_stag, nodal, coarsen, i, j, k, 0); + Bz_interp += Interp(Bz_ext, Bz_stag, nodal, coarsen, i, j, k, 0); + } // calculate enE = (J - Ji) x B enE_nodal(i, j, k, 0) = ( - (jy_interp - jiy_interp - Jexty(i, j, k)) * Bz_interp - - (jz_interp - jiz_interp - Jextz(i, j, k)) * By_interp + (jy_interp - jiy_interp) * Bz_interp + - (jz_interp - jiz_interp) * By_interp ); enE_nodal(i, j, k, 1) = ( - (jz_interp - jiz_interp - Jextz(i, j, k)) * Bx_interp - - (jx_interp - jix_interp - Jextx(i, j, k)) * Bz_interp + (jz_interp - jiz_interp) * Bx_interp + - (jx_interp - jix_interp) * Bz_interp ); enE_nodal(i, j, k, 2) = ( - (jx_interp - jix_interp - Jextx(i, j, k)) * By_interp - - (jy_interp - jiy_interp - Jexty(i, j, k)) * Bx_interp + (jx_interp - jix_interp) * By_interp + - (jy_interp - jiy_interp) * Bx_interp ); }); @@ -870,14 +955,24 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jy = Jfield[1]->const_array(mfi); Array4 const& Jz = Jfield[2]->const_array(mfi); Array4 const& enE = enE_nodal_mf.const_array(mfi); - Array4 const& rho = rhofield->const_array(mfi); - Array4 const& Pe = Pefield->array(mfi); + Array4 const& rho = rhofield.const_array(mfi); + Array4 const& Pe = Pefield.array(mfi); + + // Extract structures indicating where the fields + // should be updated, given the position of the embedded boundaries + amrex::Array4 update_Ex_arr, update_Ey_arr, update_Ez_arr; + if (EB::enabled()) { + update_Ex_arr = eb_update_E[0]->array(mfi); + update_Ey_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); + } -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + Array4 Ex_ext, Ey_ext, Ez_ext; + if (include_external_fields) { + Ex_ext = Efield_external[0]->array(mfi); + Ey_ext = Efield_external[1]->array(mfi); + Ez_ext = Efield_external[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -896,121 +991,160 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Ex calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + + // Skip field update in the embedded boundaries + if (update_Ex_arr && update_Ex_arr(i, j, k) == 0) { return; } + // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jx_val = Interp(Jx, Jx_stag, Ex_stag, coarsen, i, j, k, 0); - const Real jy_val = Interp(Jy, Jy_stag, Ex_stag, coarsen, i, j, k, 0); - const Real jz_val = Interp(Jz, Jz_stag, Ex_stag, coarsen, i, j, k, 0); - jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Ex(i, j, k) = 0._rt; + } else { + // Get the gradient of the electron pressure if the longitudinal part of + // the E-field should be included, otherwise ignore it since curl x (grad Pe) = 0 + const Real grad_Pe = (!solve_for_Faraday) ? + T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k) + : 0._rt; - // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k); + // interpolate the nodal neE values to the Yee grid + const auto enE_x = Interp(enE, nodal, Ex_stag, coarsen, i, j, k, 0); - // interpolate the nodal neE values to the Yee grid - auto enE_x = Interp(enE, nodal, Ex_stag, coarsen, i, j, k, 0); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; + Ex(i, j, k) = (enE_x - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if (resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jx_val = Jx(i, j, k); + const Real jy_val = Interp(Jy, Jy_stag, Ex_stag, coarsen, i, j, k, 0); + const Real jz_val = Interp(Jz, Jz_stag, Ex_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + + Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); + + if (include_hyper_resistivity_term) { + auto nabla2Jx = T_Algo::Dxx(Jx, coefs_x, n_coefs_x, i, j, k) + + T_Algo::Dyy(Jx, coefs_y, n_coefs_y, i, j, k) + + T_Algo::Dzz(Jx, coefs_z, n_coefs_z, i, j, k); + Ex(i, j, k) -= eta_h * nabla2Jx; + } + } - if (include_hyper_resistivity_term) { - auto nabla2Jx = T_Algo::Dxx(Jx, coefs_x, n_coefs_x, i, j, k); - Ex(i, j, k) -= eta_h * nabla2Jx; + if (include_external_fields && (rho_val >= rho_floor)) { + Ex(i, j, k) -= Ex_ext(i, j, k); } }, // Ey calculation - [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field solve if this cell is fully covered by embedded boundaries -#ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) { return; } -#elif defined(WARPX_DIM_XZ) - //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } -#endif -#endif + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + + // Skip field update in the embedded boundaries + if (update_Ey_arr && update_Ey_arr(i, j, k) == 0) { return; } + // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jx_val = Interp(Jx, Jx_stag, Ey_stag, coarsen, i, j, k, 0); - const Real jy_val = Interp(Jy, Jy_stag, Ey_stag, coarsen, i, j, k, 0); - const Real jz_val = Interp(Jz, Jz_stag, Ey_stag, coarsen, i, j, k, 0); - jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Ey(i, j, k) = 0._rt; + } else { + // Get the gradient of the electron pressure if the longitudinal part of + // the E-field should be included, otherwise ignore it since curl x (grad Pe) = 0 + const Real grad_Pe = (!solve_for_Faraday) ? + T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k) + : 0._rt; - // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k); + // interpolate the nodal neE values to the Yee grid + const auto enE_y = Interp(enE, nodal, Ey_stag, coarsen, i, j, k, 1); - // interpolate the nodal neE values to the Yee grid - auto enE_y = Interp(enE, nodal, Ey_stag, coarsen, i, j, k, 1); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; + Ey(i, j, k) = (enE_y - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if (resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jx_val = Interp(Jx, Jx_stag, Ey_stag, coarsen, i, j, k, 0); + const Real jy_val = Jy(i, j, k); + const Real jz_val = Interp(Jz, Jz_stag, Ey_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + + Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); + + if (include_hyper_resistivity_term) { + auto nabla2Jy = T_Algo::Dxx(Jy, coefs_x, n_coefs_x, i, j, k) + + T_Algo::Dyy(Jy, coefs_y, n_coefs_y, i, j, k) + + T_Algo::Dzz(Jy, coefs_z, n_coefs_z, i, j, k); + Ey(i, j, k) -= eta_h * nabla2Jy; + } + } - if (include_hyper_resistivity_term) { - auto nabla2Jy = T_Algo::Dyy(Jy, coefs_y, n_coefs_y, i, j, k); - Ey(i, j, k) -= eta_h * nabla2Jy; + if (include_external_fields && (rho_val >= rho_floor)) { + Ey(i, j, k) -= Ey_ext(i, j, k); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) { return; } -#endif + + // Skip field update in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } + // Interpolate to get the appropriate charge density in space - Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); - - // Interpolate current to appropriate staggering to match E field - Real jtot_val = 0._rt; - if (include_resistivity_term && resistivity_has_J_dependence) { - const Real jx_val = Interp(Jx, Jx_stag, Ez_stag, coarsen, i, j, k, 0); - const Real jy_val = Interp(Jy, Jy_stag, Ez_stag, coarsen, i, j, k, 0); - const Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, k, 0); - jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); - } + const Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); - // safety condition since we divide by rho_val later - if (rho_val < rho_floor) { rho_val = rho_floor; } + if (rho_val < rho_floor && holmstrom_vacuum_region) { + Ez(i, j, k) = 0._rt; + } else { + // Get the gradient of the electron pressure if the longitudinal part of + // the E-field should be included, otherwise ignore it since curl x (grad Pe) = 0 + const Real grad_Pe = (!solve_for_Faraday) ? + T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k) + : 0._rt; - // Get the gradient of the electron pressure - auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k); + // interpolate the nodal neE values to the Yee grid + const auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, k, 2); - // interpolate the nodal neE values to the Yee grid - auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, k, 2); + // safety condition since we divide by rho + const auto rho_val_limited = std::max(rho_val, rho_floor); - Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; + Ez(i, j, k) = (enE_z - grad_Pe) / rho_val_limited; + } // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } + if (solve_for_Faraday) { + Real jtot_val = 0._rt; + if (resistivity_has_J_dependence) { + // Interpolate current to appropriate staggering to match E field + const Real jx_val = Interp(Jx, Jx_stag, Ez_stag, coarsen, i, j, k, 0); + const Real jy_val = Interp(Jy, Jy_stag, Ez_stag, coarsen, i, j, k, 0); + const Real jz_val = Jz(i, j, k); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + + Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); + + if (include_hyper_resistivity_term) { + auto nabla2Jz = T_Algo::Dxx(Jz, coefs_x, n_coefs_x, i, j, k) + + T_Algo::Dyy(Jz, coefs_y, n_coefs_y, i, j, k) + + T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, k); + Ez(i, j, k) -= eta_h * nabla2Jz; + } + } - if (include_hyper_resistivity_term) { - auto nabla2Jz = T_Algo::Dzz(Jz, coefs_z, n_coefs_z, i, j, k); - Ez(i, j, k) -= eta_h * nabla2Jz; + if (include_external_fields && (rho_val >= rho_floor)) { + Ez(i, j, k) -= Ez_ext(i, j, k); } } ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 3aee7697073..33d368925f7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -7,6 +7,7 @@ # include "FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" # include "FiniteDifferenceAlgorithms/FieldAccessorFunctors.H" #endif +#include "EmbeddedBoundary/Enabled.H" #include "MacroscopicProperties/MacroscopicProperties.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -33,12 +34,13 @@ #include using namespace amrex; +using namespace ablastr::fields; void FiniteDifferenceSolver::MacroscopicEvolveE ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { @@ -46,7 +48,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + amrex::ignore_unused(Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); WARPX_ABORT_WITH_MESSAGE("currently macro E-push does not work for RZ"); #else @@ -59,13 +61,13 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } @@ -76,12 +78,12 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } else if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } @@ -98,17 +100,13 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( template void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, + ablastr::fields::VectorField const& Jfield, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::MultiFab& sigma_mf = macroscopic_properties->getsigma_mf(); amrex::MultiFab& epsilon_mf = macroscopic_properties->getepsilon_mf(); amrex::MultiFab& mu_mf = macroscopic_properties->getmu_mf(); @@ -139,11 +137,12 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 update_Ex_arr, update_Ey_arr, update_Ez_arr; + if (EB::enabled()) { + update_Ex_arr = eb_update_E[0]->array(mfi); + update_Ey_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); + } // material prop // amrex::Array4 const& sigma_arr = sigma_mf.array(mfi); @@ -174,10 +173,10 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Ex_arr && update_Ex_arr(i, j, k) == 0) { return; } + // Interpolate conductivity, sigma, to Ex position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ex_stag, macro_cr, i, j, k, scomp); @@ -193,15 +192,10 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; -#elif defined(WARPX_DIM_XZ) - //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j, k)<=0 || lz(i, j-1, k)<=0) return; -#endif -#endif + + // Skip field push in the embedded boundaries + if (update_Ey_arr && update_Ey_arr(i, j, k) == 0) { return; } + // Interpolate conductivity, sigma, to Ey position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ey_stag, macro_cr, i, j, k, scomp); @@ -218,10 +212,10 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } + // Interpolate conductivity, sigma, to Ez position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ez_stag, macro_cr, i, j, k, scomp); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index a6a389fe056..18c010d9385 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -1,6 +1,6 @@ #include "MacroscopicProperties.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" @@ -23,7 +23,6 @@ #include using namespace amrex; -using namespace warpx::fields; MacroscopicProperties::MacroscopicProperties () { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/Make.package b/Source/FieldSolver/FiniteDifferenceSolver/Make.package index b3708c411fa..bc71b9b51a2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/Make.package +++ b/Source/FieldSolver/FiniteDifferenceSolver/Make.package @@ -5,6 +5,7 @@ CEXE_sources += EvolveF.cpp CEXE_sources += EvolveG.cpp CEXE_sources += EvolveECTRho.cpp CEXE_sources += ComputeDivE.cpp +CEXE_sources += ComputeCurlA.cpp CEXE_sources += MacroscopicEvolveE.cpp CEXE_sources += EvolveBPML.cpp CEXE_sources += EvolveEPML.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt index 6e16f19084c..529336c4d7c 100644 --- a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -2,8 +2,10 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + ImplicitSolver.cpp SemiImplicitEM.cpp ThetaImplicitEM.cpp + StrangImplicitSpectralEM.cpp WarpXImplicitOps.cpp WarpXSolverVec.cpp ) diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H index 88ad6a058fd..0d2083793ac 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -1,4 +1,4 @@ -/* Copyright 2024 Justin Angus +/* Copyright 2024 Justin Angus, Debojyoti Ghosh * * This file is part of WarpX. * @@ -9,9 +9,11 @@ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "NonlinearSolvers/NonlinearSolverLibrary.H" +#include "Utils/WarpXAlgorithmSelection.H" #include #include +#include /** * \brief Base class for implicit time solvers. The base functions are those @@ -55,6 +57,8 @@ public: a_particle_tol = m_particle_tolerance; } + void CreateParticleAttributes () const; + /** * \brief Advance fields and particles by one time step using the specified implicit algorithm */ @@ -81,10 +85,17 @@ public: virtual void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) = 0; + [[nodiscard]] int numAMRLevels () const { return m_num_amr_levels; } + + [[nodiscard]] const amrex::Geometry& GetGeometry (int) const; + [[nodiscard]] const amrex::Array& GetFieldBoundaryLo () const; + [[nodiscard]] const amrex::Array& GetFieldBoundaryHi () const; + [[nodiscard]] amrex::Array GetLinOpBCLo () const; + [[nodiscard]] amrex::Array GetLinOpBCHi () const; + protected: /** @@ -94,6 +105,16 @@ protected: bool m_is_defined = false; + /** + * \brief Number of AMR levels + */ + int m_num_amr_levels = 1; + + /** + * \brief Time step + */ + mutable amrex::Real m_dt = 0.0; + /** * \brief Nonlinear solver type and object */ @@ -140,6 +161,11 @@ protected: } + /** + * \brief Convert from WarpX FieldBoundaryType to amrex::LinOpBCType + */ + [[nodiscard]] amrex::Array convertFieldBCToLinOpBC ( const amrex::Array& ) const; + }; #endif diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp new file mode 100644 index 00000000000..0a934693710 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp @@ -0,0 +1,86 @@ +#include "ImplicitSolver.H" +#include "WarpX.H" +#include "Particles/MultiParticleContainer.H" + +using namespace amrex; + +void ImplicitSolver::CreateParticleAttributes () const +{ + // Set comm to false to that the attributes are not communicated + // nor written to the checkpoint files + int const comm = 0; + + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : m_WarpX->GetPartContainer()) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n", comm); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n", comm); +#endif + pc->AddRealComp("z_n", comm); + pc->AddRealComp("ux_n", comm); + pc->AddRealComp("uy_n", comm); + pc->AddRealComp("uz_n", comm); + } +} + +const Geometry& ImplicitSolver::GetGeometry (const int a_lvl) const +{ + AMREX_ASSERT((a_lvl >= 0) && (a_lvl < m_num_amr_levels)); + return m_WarpX->Geom(a_lvl); +} + +const Array& ImplicitSolver::GetFieldBoundaryLo () const +{ + return m_WarpX->GetFieldBoundaryLo(); +} + +const Array& ImplicitSolver::GetFieldBoundaryHi () const +{ + return m_WarpX->GetFieldBoundaryHi(); +} + +Array ImplicitSolver::GetLinOpBCLo () const +{ + return convertFieldBCToLinOpBC(m_WarpX->GetFieldBoundaryLo()); +} + +Array ImplicitSolver::GetLinOpBCHi () const +{ + return convertFieldBCToLinOpBC(m_WarpX->GetFieldBoundaryHi()); +} + +Array ImplicitSolver::convertFieldBCToLinOpBC (const Array& a_fbc) const +{ + Array lbc; + for (auto& bc : lbc) { bc = LinOpBCType::interior; } + for (int i = 0; i < AMREX_SPACEDIM; i++) { + if (a_fbc[i] == FieldBoundaryType::PML) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Periodic) { + lbc[i] = LinOpBCType::Periodic; + } else if (a_fbc[i] == FieldBoundaryType::PEC) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Damped) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Absorbing_SilverMueller) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Neumann) { + // Also for FieldBoundaryType::PMC + lbc[i] = LinOpBCType::symmetry; + } else if (a_fbc[i] == FieldBoundaryType::PECInsulator) { + ablastr::warn_manager::WMRecordWarning("Implicit solver", + "With PECInsulator, in the Curl-Curl preconditioner Neumann boundary will be used since the full boundary is not yet implemented.", + ablastr::warn_manager::WarnPriority::medium); + lbc[i] = LinOpBCType::symmetry; + } else if (a_fbc[i] == FieldBoundaryType::None) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Open) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else { + WARPX_ABORT_WITH_MESSAGE("Invalid value for FieldBoundaryType"); + } + } + return lbc; +} diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H index 423957ef061..586c7163742 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -9,5 +9,6 @@ #include "SemiImplicitEM.H" // IWYU pragma: export #include "ThetaImplicitEM.H" // IWYU pragma: export +#include "StrangImplicitSpectralEM.H" // IWYU pragma: export #endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package index a4543f94dd3..8f39824d875 100644 --- a/Source/FieldSolver/ImplicitSolvers/Make.package +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -1,5 +1,7 @@ +CEXE_sources += ImplicitSolver.cpp CEXE_sources += SemiImplicitEM.cpp CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += StrangImplicitSpectralEM.cpp CEXE_sources += WarpXImplicitOps.cpp CEXE_sources += WarpXSolverVec.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H index 6e3e5db2c74..62401d7b48f 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -57,14 +57,13 @@ public: void PrintParameters () const override; - void OneStep ( amrex::Real a_time, + void OneStep ( amrex::Real start_time, amrex::Real a_dt, int a_step ) override; void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt, + amrex::Real half_time, int a_nl_iter, bool a_from_jacobian ) override; diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index 897bd5c07f3..bf8441e1992 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -5,9 +5,10 @@ * License: BSD-3-Clause-LBNL */ #include "SemiImplicitEM.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "WarpX.H" -using namespace warpx::fields; +using warpx::fields::FieldType; using namespace amrex::literals; void SemiImplicitEM::Define ( WarpX* a_WarpX ) @@ -20,13 +21,8 @@ void SemiImplicitEM::Define ( WarpX* a_WarpX ) m_WarpX = a_WarpX; // Define E and Eold vectors - m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - - // Need to define the WarpXSolverVec owned dot_mask to do dot - // product correctly for linear and nonlinear solvers - const amrex::Vector& Geom = m_WarpX->Geom(); - m_E.SetDotMask(Geom); + m_E.Define( m_WarpX, "Efield_fp" ); + m_Eold.Define( m_E ); // Parse implicit solver parameters const amrex::ParmParse pp("implicit_evolve"); @@ -41,77 +37,84 @@ void SemiImplicitEM::Define ( WarpX* a_WarpX ) void SemiImplicitEM::PrintParameters () const { if (!m_WarpX->Verbose()) { return; } - amrex::Print() << std::endl; - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << "----------- SEMI IMPLICIT EM SOLVER PARAMETERS ------------" << std::endl; - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; - amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + amrex::Print() << "\n"; + amrex::Print() << "-----------------------------------------------------------\n"; + amrex::Print() << "----------- SEMI IMPLICIT EM SOLVER PARAMETERS ------------\n"; + amrex::Print() << "-----------------------------------------------------------\n"; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << "\n"; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << "\n"; if (m_nlsolver_type==NonlinearSolverType::Picard) { - amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + amrex::Print() << "Nonlinear solver type: Picard\n"; } else if (m_nlsolver_type==NonlinearSolverType::Newton) { - amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + amrex::Print() << "Nonlinear solver type: Newton\n"; } m_nlsolver->PrintParams(); - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------\n\n"; } -void SemiImplicitEM::OneStep ( amrex::Real a_time, +void SemiImplicitEM::OneStep ( amrex::Real start_time, amrex::Real a_dt, int a_step ) { amrex::ignore_unused(a_step); - // Fields have E^{n}, B^{n-1/2} - // Particles have p^{n} and x^{n}. + // Set the member time step + m_dt = a_dt; + + // Fields have Eg^{n}, Bg^{n} + // Particles have up^{n} and xp^{n}. - // Save the values at the start of the time step, + // Save up and xp at the start of the time step m_WarpX->SaveParticlesAtImplicitStepStart ( ); - // Save the fields at the start of the step - m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - m_E.Copy(m_Eold); // initial guess for E + // Save Eg at the start of the time step + m_Eold.Copy( FieldType::Efield_fp ); - // Compute Bfield at time n+1/2 - m_WarpX->EvolveB(a_dt, DtType::Full); - m_WarpX->ApplyMagneticFieldBCs(); + // Advance WarpX owned Bfield_fp from t_{n} to t_{n+1/2} + m_WarpX->EvolveB(0.5_rt*m_dt, DtType::FirstHalf, start_time); + m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true); - const amrex::Real half_time = a_time + 0.5_rt*a_dt; + const amrex::Real half_time = start_time + 0.5_rt*m_dt; - // Solve nonlinear system for E at t_{n+1/2} + // Solve nonlinear system for Eg at t_{n+1/2} // Particles will be advanced to t_{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + m_E.Copy(m_Eold); // initial guess for Eg^{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); // Update WarpX owned Efield_fp to t_{n+1/2} - m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E, half_time ); + m_WarpX->reduced_diags->ComputeDiagsMidStep(a_step); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); - // Advance E fields from time n+1/2 to time n+1 - // Eg^{n+1} = 2.0*E_g^{n+1/2} - E_g^n + // Advance Eg from time n+1/2 to time n+1 + // Eg^{n+1} = 2.0*Eg^{n+1/2} - Eg^n m_E.linComb( 2._rt, m_E, -1._rt, m_Eold ); - m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + const amrex::Real new_time = start_time + m_dt; + m_WarpX->SetElectricFieldAndApplyBCs( m_E, new_time ); + + // Advance WarpX owned Bfield_fp from t_{n+1/2} to t_{n+1} + m_WarpX->EvolveB(0.5_rt*m_dt, DtType::SecondHalf, half_time); + m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true); } void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt, + amrex::Real half_time, int a_nl_iter, bool a_from_jacobian ) { - // update WarpX-owned Efield_fp using current state of E from + // Update WarpX-owned Efield_fp using current state of Eg from // the nonlinear solver at time n+theta - m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + m_WarpX->SetElectricFieldAndApplyBCs( a_E, half_time ); - // Self consistently update particle positions and velocities using the - // current state of the fields E and B. Deposit current density at time n+1/2. - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + // Update particle positions and velocities using the current state + // of Eg and Bg. Deposit current density at time n+1/2 + m_WarpX->ImplicitPreRHSOp( half_time, m_dt, a_nl_iter, a_from_jacobian ); - // RHS = cvac^2*0.5*dt*( curl(B^{n+1/2}) - mu0*J^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); + // RHS = cvac^2*0.5*dt*( curl(Bg^{n+1/2}) - mu0*Jg^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(0.5_rt*m_dt, a_RHS); } diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H new file mode 100644 index 00000000000..fcfcb9821a7 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H @@ -0,0 +1,105 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef STRANG_IMPLICIT_SPECTRALEM_H_ +#define STRANG_IMPLICIT_SPECTRALEM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Implicit spectral electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Advance (Eg^n, Bg^n) -> (Eg^{n+1/2}, Bg^{n+1/2}) source free // E transverse + * Iterate: + * Eg^{n+1} = Eg^n + c^2*dt*( - mu0*Jg^{n+1/2} ) // E longitudinal + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * Advance (Eg^n+1/2, Bg^n+1/2) -> (Eg^{n+1}, Bg^{n+1}) source free // E transverse + * + * The algorithm is exactly energy conserving only with a single box, periodic fft (psatd.periodic_single_box_fft = 1). + * With multiple boxes, energy is not conserved since the ffts in each box assumes periodic in the box which + * is not consistent with the current. + * The algorithm is numerially stable for any time step. + * I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + */ + +class StrangImplicitSpectralEM : public ImplicitSolver +{ +public: + + StrangImplicitSpectralEM() = default; + + ~StrangImplicitSpectralEM() override = default; + + // Prohibit Move and Copy operations + StrangImplicitSpectralEM(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM& operator=(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM(StrangImplicitSpectralEM&&) = delete; + StrangImplicitSpectralEM& operator=(StrangImplicitSpectralEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real start_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real half_time, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( WarpXSolverVec const& a_E, + amrex::Real half_time ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp new file mode 100644 index 00000000000..b8be6b93c63 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp @@ -0,0 +1,141 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "Fields.H" +#include "StrangImplicitSpectralEM.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void StrangImplicitSpectralEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "StrangImplicitSpectralEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX, "Efield_fp" ); + m_Eold.Define( m_E ); + + + // Parse nonlinear solver parameters + const amrex::ParmParse pp_implicit_evolve("implicit_evolve"); + parseNonlinearSolverParams( pp_implicit_evolve ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void StrangImplicitSpectralEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "----------- STRANG SPLIT IMPLICIT SPECTRAL EM SOLVER PARAMETERS --------" << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << "\n"; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << "\n"; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard\n"; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton\n"; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------\n\n"; +} + +void StrangImplicitSpectralEM::OneStep ( amrex::Real start_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Set the member time step + m_dt = a_dt; + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart(); + + // Advance the fields to time n+1/2 source free + m_WarpX->SpectralSourceFreeFieldAdvance(start_time); + + // Save the fields at the start of the step + m_Eold.Copy( FieldType::Efield_fp ); + m_E.Copy(m_Eold); // initial guess for E + + amrex::Real const half_time = start_time + 0.5_rt*m_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2} + UpdateWarpXFields( m_E, half_time ); + m_WarpX->reduced_diags->ComputeDiagsMidStep(a_step); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+1/2 to time n+1 + amrex::Real const new_time = start_time + m_dt; + FinishFieldUpdate( new_time ); + + // Advance the fields to time n+1 source free + m_WarpX->SpectralSourceFreeFieldAdvance(half_time); + +} + +void StrangImplicitSpectralEM::ComputeRHS ( WarpXSolverVec& a_RHS, + WarpXSolverVec const & a_E, + amrex::Real half_time, + int a_nl_iter, + bool a_from_jacobian ) +{ + // Update WarpX-owned Efield_fp and Bfield_fp using current state of + // E from the nonlinear solver at time n+1/2 + UpdateWarpXFields( a_E, half_time ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( half_time, m_dt, a_nl_iter, a_from_jacobian ); + + // For Strang split implicit PSATD, the RHS = -dt*mu*c**2*J + bool const allow_type_mismatch = true; + a_RHS.Copy(FieldType::current_fp, warpx::fields::FieldType::None, allow_type_mismatch); + amrex::Real constexpr coeff = PhysConst::c * PhysConst::c * PhysConst::mu0; + a_RHS.scale(-coeff * 0.5_rt*m_dt); + +} + +void StrangImplicitSpectralEM::UpdateWarpXFields (WarpXSolverVec const & a_E, + amrex::Real half_time ) +{ + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E, half_time ); + +} + +void StrangImplicitSpectralEM::FinishFieldUpdate ( amrex::Real a_new_time ) +{ + // Eg^{n+1} = 2*E_g^{n+1/2} - E_g^n + amrex::Real const c0 = 1._rt/0.5_rt; + amrex::Real const c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E, a_new_time ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H index 009c2c7e546..c9de6b82a00 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -8,13 +8,12 @@ #define THETA_IMPLICIT_EM_H_ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "ImplicitSolver.H" #include #include #include -#include "ImplicitSolver.H" - /** @file * Theta-implicit electromagnetic time solver class. This is a fully implicit * algorithm where both the fields and particles are treated implicitly. @@ -68,19 +67,23 @@ public: void PrintParameters () const override; - void OneStep ( amrex::Real a_time, + /** + * \brief Advances the simulation one time step + * + * \param[in] start_time The time at the start of the time step + * \param[in] a_dt The time step size + * \param[in] a_step The time step number + */ + void OneStep ( amrex::Real start_time, amrex::Real a_dt, int a_step ) override; void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt, + amrex::Real start_time, int a_nl_iter, bool a_from_jacobian ) override; - [[nodiscard]] amrex::Real theta () const { return m_theta; } - private: /** @@ -98,26 +101,17 @@ private: */ WarpXSolverVec m_E, m_Eold; - /** - * \brief B is a derived variable from E. Need to save Bold to update B during - * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. - * It is not used directly by the nonlinear solver, nor is it the same size as the - * solver vector (size E), and so it should not be WarpXSolverVec type. - */ - amrex::Vector, 3 > > m_Bold; - /** * \brief Update the E and B fields owned by WarpX */ void UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ); + amrex::Real start_time ); /** * \brief Nonlinear solver is for the time-centered values of E. After * the solver, need to use m_E and m_Eold to compute E^{n+1} */ - void FinishFieldUpdate ( amrex::Real a_new_time ); + void FinishFieldUpdate ( amrex::Real end_time ); }; diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 026c509c3ba..1e6596f5eaa 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -4,11 +4,12 @@ * * License: BSD-3-Clause-LBNL */ -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "ThetaImplicitEM.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "WarpX.H" -using namespace warpx::fields; +using warpx::fields::FieldType; using namespace amrex::literals; void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) @@ -19,27 +20,23 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) // Retain a pointer back to main WarpX class m_WarpX = a_WarpX; + m_num_amr_levels = 1; // Define E and Eold vectors - m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - - // Need to define the WarpXSolverVec owned dot_mask to do dot - // product correctly for linear and nonlinear solvers - const amrex::Vector& Geom = m_WarpX->Geom(); - m_E.SetDotMask(Geom); - - // Define Bold MultiFab - const int num_levels = 1; - m_Bold.resize(num_levels); // size is number of levels - for (int lev = 0; lev < num_levels; ++lev) { - for (int n=0; n<3; n++) { - const amrex::MultiFab& Bfp = m_WarpX->getField( FieldType::Bfield_fp,lev,n); - m_Bold[lev][n] = std::make_unique( Bfp.boxArray(), - Bfp.DistributionMap(), - Bfp.nComp(), - Bfp.nGrowVect() ); - } + m_E.Define( m_WarpX, "Efield_fp" ); + m_Eold.Define( m_E ); + + // Define B_old MultiFabs + using ablastr::fields::Direction; + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + const auto& ba_Bx = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->boxArray(); + const auto& ba_By = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->boxArray(); + const auto& ba_Bz = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->boxArray(); + const auto& dm = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->DistributionMap(); + const amrex::IntVect ngb = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->nGrowVect(); + m_WarpX->m_fields.alloc_init(FieldType::B_old, Direction{0}, lev, ba_Bx, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init(FieldType::B_old, Direction{1}, lev, ba_By, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init(FieldType::B_old, Direction{2}, lev, ba_Bz, dm, 1, ngb, 0.0_rt); } // Parse theta-implicit solver specific parameters @@ -61,111 +58,113 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) void ThetaImplicitEM::PrintParameters () const { if (!m_WarpX->Verbose()) { return; } - amrex::Print() << std::endl; - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << "----------- THETA IMPLICIT EM SOLVER PARAMETERS -----------" << std::endl; - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << "Time-bias parameter theta: " << m_theta << std::endl; - amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; - amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + amrex::Print() << "\n"; + amrex::Print() << "-----------------------------------------------------------\n"; + amrex::Print() << "----------- THETA IMPLICIT EM SOLVER PARAMETERS -----------\n"; + amrex::Print() << "-----------------------------------------------------------\n"; + amrex::Print() << "Time-bias parameter theta: " << m_theta << "\n"; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << "\n"; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << "\n"; if (m_nlsolver_type==NonlinearSolverType::Picard) { - amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + amrex::Print() << "Nonlinear solver type: Picard\n"; } else if (m_nlsolver_type==NonlinearSolverType::Newton) { - amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + amrex::Print() << "Nonlinear solver type: Newton\n"; } m_nlsolver->PrintParams(); - amrex::Print() << "-----------------------------------------------------------" << std::endl; - amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------\n\n"; } -void ThetaImplicitEM::OneStep ( const amrex::Real a_time, +void ThetaImplicitEM::OneStep ( const amrex::Real start_time, const amrex::Real a_dt, const int a_step ) { amrex::ignore_unused(a_step); - // Fields have E^{n} and B^{n} - // Particles have p^{n} and x^{n}. + // Fields have Eg^{n} and Bg^{n} + // Particles have up^{n} and xp^{n}. + + // Set the member time step + m_dt = a_dt; - // Save the values at the start of the time step, + // Save up and xp at the start of the time step m_WarpX->SaveParticlesAtImplicitStepStart ( ); - // Save the fields at the start of the step - m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); - m_E.Copy(m_Eold); // initial guess for E + // Save Eg at the start of the time step + m_Eold.Copy( FieldType::Efield_fp ); - const int num_levels = static_cast(m_Bold.size()); + const int num_levels = 1; for (int lev = 0; lev < num_levels; ++lev) { - for (int n=0; n<3; n++) { - const amrex::MultiFab& Bfp = m_WarpX->getField(FieldType::Bfield_fp,lev,n); - amrex::MultiFab& Bold = *m_Bold[lev][n]; - amrex::MultiFab::Copy(Bold, Bfp, 0, 0, 1, Bold.nGrowVect()); + const ablastr::fields::VectorField Bfp = m_WarpX->m_fields.get_alldirs(FieldType::Bfield_fp, lev); + ablastr::fields::VectorField B_old = m_WarpX->m_fields.get_alldirs(FieldType::B_old, lev); + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy(*B_old[n], *Bfp[n], 0, 0, B_old[n]->nComp(), + B_old[n]->nGrowVect() ); } } - const amrex::Real theta_time = a_time + m_theta*a_dt; - - // Solve nonlinear system for E at t_{n+theta} + // Solve nonlinear system for Eg at t_{n+theta} // Particles will be advanced to t_{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + m_E.Copy(m_Eold); // initial guess for Eg^{n+theta} + m_nlsolver->Solve( m_E, m_Eold, start_time, m_theta*m_dt ); // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} - UpdateWarpXFields( m_E, theta_time, a_dt ); + UpdateWarpXFields( m_E, start_time ); + m_WarpX->reduced_diags->ComputeDiagsMidStep(a_step); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); - // Advance E and B fields from time n+theta to time n+1 - const amrex::Real new_time = a_time + a_dt; - FinishFieldUpdate( new_time ); + // Advance Eg and Bg from time n+theta to time n+1 + const amrex::Real end_time = start_time + m_dt; + FinishFieldUpdate( end_time ); } void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt, + amrex::Real start_time, int a_nl_iter, bool a_from_jacobian ) { - // update WarpX-owned Efield_fp and Bfield_fp using current state of E from - // the nonlinear solver at time n+theta - UpdateWarpXFields( a_E, a_time, a_dt ); + // Update WarpX-owned Efield_fp and Bfield_fp using current state of + // Eg from the nonlinear solver at time n+theta + UpdateWarpXFields( a_E, start_time ); - // Self consistently update particle positions and velocities using the - // current state of the fields E and B. Deposit current density at time n+1/2. - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + // Update particle positions and velocities using the current state + // of Eg and Bg. Deposit current density at time n+1/2 + const amrex::Real theta_time = start_time + m_theta*m_dt; + m_WarpX->ImplicitPreRHSOp( theta_time, m_dt, a_nl_iter, a_from_jacobian ); - // RHS = cvac^2*m_theta*dt*( curl(B^{n+theta}) - mu0*J^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); + // RHS = cvac^2*m_theta*dt*( curl(Bg^{n+theta}) - mu0*Jg^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE( m_theta*m_dt, a_RHS); } void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ) + amrex::Real start_time ) { - amrex::ignore_unused(a_time); // Update Efield_fp owned by WarpX - m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + const amrex::Real theta_time = start_time + m_theta*m_dt; + m_WarpX->SetElectricFieldAndApplyBCs( a_E, theta_time ); // Update Bfield_fp owned by WarpX - m_WarpX->UpdateMagneticFieldAndApplyBCs( m_Bold, m_theta*a_dt ); + ablastr::fields::MultiLevelVectorField const& B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); + m_WarpX->UpdateMagneticFieldAndApplyBCs( B_old, m_theta*m_dt, start_time ); } -void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) +void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real end_time ) { - amrex::ignore_unused(a_new_time); - // Eg^{n+1} = (1/theta)*E_g^{n+theta} + (1-1/theta)*E_g^n - // Bg^{n+1} = (1/theta)*B_g^{n+theta} + (1-1/theta)*B_g^n + // Eg^{n+1} = (1/theta)*Eg^{n+theta} + (1-1/theta)*Eg^n + // Bg^{n+1} = (1/theta)*Bg^{n+theta} + (1-1/theta)*Bg^n const amrex::Real c0 = 1._rt/m_theta; const amrex::Real c1 = 1._rt - c0; m_E.linComb( c0, m_E, c1, m_Eold ); - m_WarpX->SetElectricFieldAndApplyBCs( m_E ); - m_WarpX->FinishMagneticFieldAndApplyBCs( m_Bold, m_theta ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E, end_time ); + ablastr::fields::MultiLevelVectorField const & B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); + m_WarpX->FinishMagneticFieldAndApplyBCs( B_old, m_theta, end_time ); } diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 7b95abade3e..06e1820854c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -11,14 +11,8 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" -#ifdef WARPX_USE_PSATD -# ifdef WARPX_DIM_RZ -# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" -# else -# include "FieldSolver/SpectralSolver/SpectralSolver.H" -# endif -#endif #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" @@ -58,8 +52,11 @@ WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, bool a_from_jacobian ) { using namespace amrex::literals; + using warpx::fields::FieldType; amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + if (use_filter) { ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), 0); } + // Advance the particle positions by 1/2 dt, // particle velocities by dt, then take average of old and new v, // deposit currents, giving J at n+1/2 @@ -73,40 +70,87 @@ WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, } void -WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) +WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E, amrex::Real a_time ) { - const amrex::Vector, 3 > >& Evec = a_E.getVec(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_E.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "WarpX::SetElectricFieldAndApplyBCs() must be called with Efield_fp type"); + + using warpx::fields::FieldType; + + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + const ablastr::fields::MultiLevelVectorField& Evec = a_E.getArrayVec(); amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); amrex::MultiFab::Copy(*Efield_fp[0][2], *Evec[0][2], 0, 0, ncomps, Evec[0][2]->nGrowVect()); FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyEfieldBoundary(0, PatchType::fine); + ApplyEfieldBoundary(0, PatchType::fine, a_time); } void -WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_thetadt ) +WarpX::UpdateMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_thetadt, amrex::Real start_time ) { - amrex::MultiFab::Copy(*Bfield_fp[0][0], *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][1], *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][2], *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); - EvolveB(a_thetadt, DtType::Full); - ApplyMagneticFieldBCs(); + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + for (int lev = 0; lev <= finest_level; ++lev) { + ablastr::fields::VectorField Bfp = m_fields.get_alldirs(FieldType::Bfield_fp, lev); + amrex::MultiFab::Copy(*Bfp[0], *a_Bn[lev][0], 0, 0, ncomps, a_Bn[lev][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfp[1], *a_Bn[lev][1], 0, 0, ncomps, a_Bn[lev][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfp[2], *a_Bn[lev][2], 0, 0, ncomps, a_Bn[lev][2]->nGrowVect()); + } + EvolveB(a_thetadt, DtType::Full, start_time); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); } void -WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_theta ) +WarpX::FinishMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_theta, amrex::Real a_time ) { - FinishImplicitField(Bfield_fp, a_Bn, a_theta); - ApplyMagneticFieldBCs(); + using warpx::fields::FieldType; + + FinishImplicitField(m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, 0), a_Bn, a_theta); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full, a_time); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); } void -WarpX::ApplyMagneticFieldBCs() +WarpX::SpectralSourceFreeFieldAdvance (amrex::Real start_time) { + using namespace amrex::literals; + using warpx::fields::FieldType; + // Do the first piece of the Strang splitting, source free advance of E and B + // It would be more efficient to write a specialized PSATD advance that does not use J, + // but this works for now. + + // Create temporary MultiFabs to hold J + int const lev = 0; + ablastr::fields::VectorField current_fp = m_fields.get_alldirs(FieldType::current_fp, lev); + amrex::MultiFab* rho_fp = m_fields.get(FieldType::rho_fp, lev); + amrex::MultiFab j0(current_fp[0]->boxArray(), current_fp[0]->DistributionMap(), + current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab j1(current_fp[1]->boxArray(), current_fp[1]->DistributionMap(), + current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab j2(current_fp[2]->boxArray(), current_fp[2]->DistributionMap(), + current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + amrex::MultiFab::Copy(j0, *(current_fp[0]), 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(j1, *(current_fp[1]), 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(j2, *(current_fp[2]), 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + + current_fp[0]->setVal(0._rt); + current_fp[1]->setVal(0._rt); + current_fp[2]->setVal(0._rt); + if (rho_fp) { rho_fp->setVal(0._rt); } + PushPSATD(start_time); // Note that this does dt/2 + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + + // Restore the current_fp MultiFab. Note that this is only needed for diagnostics when + // J is being written out (since current_fp is not otherwise used). + amrex::MultiFab::Copy(*(current_fp[0]), j0, 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[1]), j1, 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[2]), j2, 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); } void @@ -125,7 +169,7 @@ WarpX::SaveParticlesAtImplicitStepStart ( ) #endif { - auto particle_comps = pc->getParticleComps(); + auto particle_comps = pc->GetRealSoANames(); for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { @@ -137,15 +181,15 @@ WarpX::SaveParticlesAtImplicitStepStart ( ) amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); #if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + amrex::ParticleReal* x_n = pti.GetAttribs("x_n").dataPtr(); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + amrex::ParticleReal* y_n = pti.GetAttribs("y_n").dataPtr(); #endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + amrex::ParticleReal* z_n = pti.GetAttribs("z_n").dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs("ux_n").dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs("uy_n").dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs("uz_n").dataPtr(); const long np = pti.numParticles(); @@ -195,7 +239,7 @@ WarpX::FinishImplicitParticleUpdate () #endif { - auto particle_comps = pc->getParticleComps(); + auto particle_comps = pc->GetRealSoANames(); for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { @@ -208,15 +252,15 @@ WarpX::FinishImplicitParticleUpdate () amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); #if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + amrex::ParticleReal* x_n = pti.GetAttribs("x_n").dataPtr(); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + amrex::ParticleReal* y_n = pti.GetAttribs("y_n").dataPtr(); #endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + amrex::ParticleReal* z_n = pti.GetAttribs("z_n").dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs("ux_n").dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs("uy_n").dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs("uz_n").dataPtr(); const long np = pti.numParticles(); @@ -250,9 +294,9 @@ WarpX::FinishImplicitParticleUpdate () } void -WarpX::FinishImplicitField( amrex::Vector, 3 > >& Field_fp, - const amrex::Vector, 3 > >& Field_n, - amrex::Real theta ) +WarpX::FinishImplicitField( ablastr::fields::MultiLevelVectorField const& Field_fp, + ablastr::fields::MultiLevelVectorField const& Field_n, + amrex::Real theta ) { using namespace amrex::literals; @@ -301,7 +345,7 @@ WarpX::FinishImplicitField( amrex::VectorsetVal(0.0); - a_Erhs_vec.getVec()[lev][1]->setVal(0.0); - a_Erhs_vec.getVec()[lev][2]->setVal(0.0); + a_Erhs_vec.getArrayVec()[lev][0]->setVal(0.0); + a_Erhs_vec.getArrayVec()[lev][1]->setVal(0.0); + a_Erhs_vec.getArrayVec()[lev][2]->setVal(0.0); // Compute Efield_rhs in regular cells by calling EvolveE. Because // a_Erhs_vec is set to zero above, calling EvolveE below results in // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], - F_fp[lev], lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE( m_fields, + lev, + patch_type, + a_Erhs_vec.getArrayVec()[lev], + m_eb_update_E[lev], + a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], - F_cp[lev], lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( m_fields, + lev, + patch_type, + a_Erhs_vec.getArrayVec()[lev], + m_eb_update_E[lev], + a_dt ); } // Compute Efield_rhs in PML cells by calling EvolveEPML diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index 89a0b82b700..a4bbbe99f75 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -8,14 +8,15 @@ #define WarpXSolverVec_H_ #include "Utils/TextMsg.H" +#include "Fields.H" #include #include +#include #include #include #include -#include #include #include #include @@ -32,18 +33,25 @@ #include #include + +// forward declaration +class WarpX; + /** - * \brief This is a wrapper class around a Vector of array of pointers to MultiFabs that + * \brief + * This is a wrapper class around a Vector of pointers to MultiFabs that * contains basic math operators and functionality needed to interact with nonlinear - * solvers in WarpX and linear solvers in AMReX, such as GMRES. + * solvers in WarpX and linear solvers in AMReX, such as GMRES. The size of the + * Vector is the number of amr levels. Hardcoded for 1 right now. * - * The size of the Vector is the number of amr levels. Hardcoded for 1 right now. The - * size of the array is the number of MultiFabs. It is hardcoded for 3 right now as it - * is only used for the electric field in the implicit electromagnetic time solvers. In - * the future, the array size can be made a template parameter so that this class can - * be used for other solver vectors, such as electrostatic (array size 1) or Darwin (array size 4). + * A WarpXSolverVec can consist of an array size 3 of MultiFabs (for vector fields + * such as E, B, and A) or of a single MultiFab for scalar fields. Both the array + * size 3 and scalar fields must be of type warpx::fields::FieldType. + * Additionally, a WarpXSolverVec can in general contain both an array size 3 field and a + * scalar field. For example, the array size 3 field can be used for the vector potential A + * and the scalar field can be used for the scalar potential phi, which is the full state of + * unknowns for a Darwin electromagnetic model. */ - class WarpXSolverVec { public: @@ -52,95 +60,103 @@ public: WarpXSolverVec(const WarpXSolverVec&) = delete; - ~WarpXSolverVec() = default; + ~WarpXSolverVec(); using value_type = amrex::Real; using RT = value_type; [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } - inline - void Define (const WarpXSolverVec& a_vec) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - a_vec.IsDefined(), - "WarpXSolverVec::Define(a_vec) called with undefined a_vec"); - Define( a_vec.getVec() ); - } + void Define ( WarpX* a_WarpX, + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name = "none" ); inline - void Define ( const amrex::Vector, 3 > >& a_solver_vec ) + void Define ( const WarpXSolverVec& a_solver_vec ) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !IsDefined(), - "WarpXSolverVec::Define() called on undefined WarpXSolverVec"); - m_field_vec.resize(m_num_amr_levels); - const int lev = 0; - for (int n=0; n<3; n++) { - const amrex::MultiFab& mf_model = *a_solver_vec[lev][n]; - m_field_vec[lev][n] = std::make_unique( mf_model.boxArray(), mf_model.DistributionMap(), - mf_model.nComp(), amrex::IntVect::TheZeroVector() ); - } - m_is_defined = true; + assertIsDefined( a_solver_vec ); + m_num_amr_levels = a_solver_vec.m_num_amr_levels; + Define( WarpXSolverVec::m_WarpX, + a_solver_vec.getVectorType(), + a_solver_vec.getScalarType() ); } - void SetDotMask( const amrex::Vector& a_Geom ); [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; + void Copy ( warpx::fields::FieldType a_array_type, + warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None, + bool allow_type_mismatch = false); + inline - void Copy ( const amrex::Vector, 3 > >& a_solver_vec ) + void Copy ( const WarpXSolverVec& a_solver_vec ) { - AMREX_ASSERT_WITH_MESSAGE( - IsDefined(), - "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); + assertIsDefined( a_solver_vec ); + if (IsDefined()) { assertSameType( a_solver_vec ); } + else { Define(a_solver_vec); } + for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n = 0; n < 3; ++n) { - amrex::MultiFab::Copy(*m_field_vec[lev][n], *a_solver_vec[lev][n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); + if (m_array_type != warpx::fields::FieldType::None) { + for (int n = 0; n < 3; ++n) { + const amrex::MultiFab* this_field = a_solver_vec.getArrayVec()[lev][n]; + amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_field, 0, 0, m_ncomp, + amrex::IntVect::TheZeroVector() ); + } + } + if (m_scalar_type != warpx::fields::FieldType::None) { + const amrex::MultiFab* this_scalar = a_solver_vec.getScalarVec()[lev]; + amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_scalar, 0, 0, m_ncomp, + amrex::IntVect::TheZeroVector() ); } } } - inline - void Copy ( const WarpXSolverVec& a_vec ) - { - AMREX_ASSERT_WITH_MESSAGE( - a_vec.IsDefined(), - "WarpXSolverVec::Copy(a_vec) called with undefined a_vec"); - if (!IsDefined()) { Define(a_vec); } - const amrex::Vector, 3 > >& field_vec = a_vec.getVec(); - Copy(field_vec); - } - // Prohibit Copy assignment operator - WarpXSolverVec& operator= ( const WarpXSolverVec& a_vec ) = delete; + WarpXSolverVec& operator= ( const WarpXSolverVec& a_solver_vec ) = delete; // Move assignment operator WarpXSolverVec(WarpXSolverVec&&) noexcept = default; - WarpXSolverVec& operator= ( WarpXSolverVec&& a_vec ) noexcept + WarpXSolverVec& operator= ( WarpXSolverVec&& a_solver_vec ) noexcept { - if (this != &a_vec) { - m_field_vec = std::move(a_vec.m_field_vec); + if (this != &a_solver_vec) { + m_array_vec = std::move(a_solver_vec.m_array_vec); + m_scalar_vec = std::move(a_solver_vec.m_scalar_vec); + m_array_type = a_solver_vec.m_array_type; + m_scalar_type = a_solver_vec.m_scalar_type; m_is_defined = true; } return *this; } inline - void operator+= ( const WarpXSolverVec& a_vec ) + void operator+= ( const WarpXSolverVec& a_solver_vec ) { + assertIsDefined( a_solver_vec ); + assertSameType( a_solver_vec ); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - m_field_vec[lev][n]->plus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + if (m_array_type != warpx::fields::FieldType::None) { + m_array_vec[lev][0]->plus(*(a_solver_vec.getArrayVec()[lev][0]), 0, 1, 0); + m_array_vec[lev][1]->plus(*(a_solver_vec.getArrayVec()[lev][1]), 0, 1, 0); + m_array_vec[lev][2]->plus(*(a_solver_vec.getArrayVec()[lev][2]), 0, 1, 0); + } + if (m_scalar_type != warpx::fields::FieldType::None) { + m_scalar_vec[lev]->plus(*(a_solver_vec.getScalarVec()[lev]), 0, 1, 0); } } } inline - void operator-= (const WarpXSolverVec& a_vec) + void operator-= (const WarpXSolverVec& a_solver_vec) { + assertIsDefined( a_solver_vec ); + assertSameType( a_solver_vec ); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - m_field_vec[lev][n]->minus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + if (m_array_type != warpx::fields::FieldType::None) { + m_array_vec[lev][0]->minus(*(a_solver_vec.getArrayVec()[lev][0]), 0, 1, 0); + m_array_vec[lev][1]->minus(*(a_solver_vec.getArrayVec()[lev][1]), 0, 1, 0); + m_array_vec[lev][2]->minus(*(a_solver_vec.getArrayVec()[lev][2]), 0, 1, 0); + } + if (m_scalar_type != warpx::fields::FieldType::None) { + m_scalar_vec[lev]->minus(*(a_solver_vec.getScalarVec()[lev]), 0, 1, 0); } } } @@ -151,11 +167,22 @@ public: inline void linComb (const RT a, const WarpXSolverVec& X, const RT b, const WarpXSolverVec& Y) { + assertIsDefined( X ); + assertIsDefined( Y ); + assertSameType( X ); + assertSameType( Y ); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - amrex::MultiFab::LinComb(*m_field_vec[lev][n], a, *X.getVec()[lev][n], 0, - b, *Y.getVec()[lev][n], 0, - 0, 1, 0); + if (m_array_type != warpx::fields::FieldType::None) { + for (int n = 0; n < 3; n++) { + amrex::MultiFab::LinComb(*m_array_vec[lev][n], a, *X.getArrayVec()[lev][n], 0, + b, *Y.getArrayVec()[lev][n], 0, + 0, 1, 0); + } + } + if (m_scalar_type != warpx::fields::FieldType::None) { + amrex::MultiFab::LinComb(*m_scalar_vec[lev], a, *X.getScalarVec()[lev], 0, + b, *Y.getScalarVec()[lev], 0, + 0, 1, 0); } } } @@ -165,9 +192,17 @@ public: */ void increment (const WarpXSolverVec& X, const RT a) { + assertIsDefined( X ); + assertSameType( X ); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - amrex::MultiFab::Saxpy( *m_field_vec[lev][n], a, *X.getVec()[lev][n], + if (m_array_type != warpx::fields::FieldType::None) { + for (int n = 0; n < 3; n++) { + amrex::MultiFab::Saxpy( *m_array_vec[lev][n], a, *X.getArrayVec()[lev][n], + 0, 0, 1, amrex::IntVect::TheZeroVector() ); + } + } + if (m_scalar_type != warpx::fields::FieldType::None) { + amrex::MultiFab::Saxpy( *m_scalar_vec[lev], a, *X.getScalarVec()[lev], 0, 0, 1, amrex::IntVect::TheZeroVector() ); } } @@ -179,9 +214,17 @@ public: inline void scale (RT a_a) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::scale() called on undefined WarpXSolverVec"); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - m_field_vec[lev][n]->mult(a_a, 0, 1); + if (m_array_type != warpx::fields::FieldType::None) { + m_array_vec[lev][0]->mult(a_a, 0, 1); + m_array_vec[lev][1]->mult(a_a, 0, 1); + m_array_vec[lev][2]->mult(a_a, 0, 1); + } + if (m_scalar_type != warpx::fields::FieldType::None) { + m_scalar_vec[lev]->mult(a_a, 0, 1); } } } @@ -192,41 +235,77 @@ public: inline void setVal ( const RT a_val ) { - AMREX_ASSERT_WITH_MESSAGE( + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( IsDefined(), - "WarpXSolverVec::ones() called on undefined WarpXSolverVec"); + "WarpXSolverVec::setVal() called on undefined WarpXSolverVec"); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - for (int n=0; n<3; n++) { - m_field_vec[lev][n]->setVal(a_val); + if (m_array_type != warpx::fields::FieldType::None) { + m_array_vec[lev][0]->setVal(a_val); + m_array_vec[lev][1]->setVal(a_val); + m_array_vec[lev][2]->setVal(a_val); + } + if (m_scalar_type != warpx::fields::FieldType::None) { + m_scalar_vec[lev]->setVal(a_val); } } } + inline + void assertIsDefined( const WarpXSolverVec& a_solver_vec ) const + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_solver_vec.IsDefined(), + "WarpXSolverVec::function(X) called with undefined WarpXSolverVec X"); + } + + inline + void assertSameType( const WarpXSolverVec& a_solver_vec ) const + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_solver_vec.getArrayVecType()==m_array_type && + a_solver_vec.getScalarVecType()==m_scalar_type, + "WarpXSolverVec::function(X) called with WarpXSolverVec X of different type"); + } + [[nodiscard]] inline RT norm2 () const { auto const norm = dotProduct(*this); return std::sqrt(norm); } - [[nodiscard]] const amrex::Vector, 3 > >& getVec() const {return m_field_vec;} - amrex::Vector, 3 > >& getVec() {return m_field_vec;} + [[nodiscard]] const ablastr::fields::MultiLevelVectorField& getArrayVec() const {return m_array_vec;} + ablastr::fields::MultiLevelVectorField& getArrayVec() {return m_array_vec;} + + [[nodiscard]] const ablastr::fields::MultiLevelScalarField& getScalarVec() const {return m_scalar_vec;} + ablastr::fields::MultiLevelScalarField& getScalarVec() {return m_scalar_vec;} + + // solver vector types are type warpx::fields::FieldType + [[nodiscard]] warpx::fields::FieldType getArrayVecType () const { return m_array_type; } + [[nodiscard]] warpx::fields::FieldType getScalarVecType () const { return m_scalar_type; } + + // solver vector type names + [[nodiscard]] std::string getVectorType () const { return m_vector_type_name; } + [[nodiscard]] std::string getScalarType () const { return m_scalar_type_name; } - // clearDotMask() must be called by the highest class that owns WarpXSolverVec() - // after it is done being used ( typically in the destructor ) to avoid the - // following error message after the simulation finishes: - // malloc_consolidate(): unaligned fastbin chunk detected - static void clearDotMask() { m_dotMask.clear(); } private: - bool m_is_defined = false; - amrex::Vector, 3 > > m_field_vec; + bool m_is_defined = false; + + ablastr::fields::MultiLevelVectorField m_array_vec; + ablastr::fields::MultiLevelScalarField m_scalar_vec; + + warpx::fields::FieldType m_array_type = warpx::fields::FieldType::None; + warpx::fields::FieldType m_scalar_type = warpx::fields::FieldType::None; + + std::string m_vector_type_name = "none"; + std::string m_scalar_type_name = "none"; static constexpr int m_ncomp = 1; - static constexpr int m_num_amr_levels = 1; + int m_num_amr_levels = 1; - inline static bool m_dot_mask_defined = false; - inline static amrex::Vector,3>> m_dotMask; + inline static bool m_warpx_ptr_defined = false; + inline static WarpX* m_WarpX = nullptr; }; diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index b181f038fb5..05b5f1caa0c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -5,46 +5,164 @@ * License: BSD-3-Clause-LBNL */ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "WarpX.H" -void WarpXSolverVec::SetDotMask( const amrex::Vector& a_Geom ) +using warpx::fields::FieldType; + +WarpXSolverVec::~WarpXSolverVec () +{ + for (auto & lvl : m_array_vec) + { + for (int i =0; i<3; ++i) + { + delete lvl[i]; + } + } +} + +void WarpXSolverVec::Define ( WarpX* a_WarpX, + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name ) { - if (m_dot_mask_defined) { return; } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "WarpXSolverVec::Define() called on already defined WarpXSolverVec"); + + // Define static member pointer to WarpX + if (!m_warpx_ptr_defined) { + m_WarpX = a_WarpX; + m_warpx_ptr_defined = true; + } + + m_num_amr_levels = 1; + + m_vector_type_name = a_vector_type_name; + m_scalar_type_name = a_scalar_type_name; + + if (m_vector_type_name=="Efield_fp") { + m_array_type = FieldType::Efield_fp; + } + else if (m_vector_type_name=="Bfield_fp") { + m_array_type = FieldType::Bfield_fp; + } + else if (m_vector_type_name=="vector_potential_fp_nodal") { + m_array_type = FieldType::vector_potential_fp; + } + else if (m_vector_type_name!="none") { + WARPX_ABORT_WITH_MESSAGE(a_vector_type_name + +"is not a valid option for array type used in Definining" + +"a WarpXSolverVec. Valid array types are: Efield_fp, Bfield_fp," + +"and vector_potential_fp_nodal"); + } + + if (m_scalar_type_name=="phi_fp") { + m_scalar_type = FieldType::phi_fp; + } + else if (m_scalar_type_name!="none") { + WARPX_ABORT_WITH_MESSAGE(a_scalar_type_name + +"is not a valid option for scalar type used in Definining" + +"a WarpXSolverVec. Valid scalar types are: phi_fp"); + } + + m_array_vec.resize(m_num_amr_levels); + m_scalar_vec.resize(m_num_amr_levels); + + // Define the 3D vector field data container + if (m_array_type != FieldType::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - IsDefined(), - "WarpXSolverVec::SetDotMask() called from undefined instance "); - - m_dotMask.resize(m_num_amr_levels); - for ( int n = 0; n < 3; n++) { - const amrex::BoxArray& grids = m_field_vec[0][n]->boxArray(); - const amrex::MultiFab tmp( grids, m_field_vec[0][n]->DistributionMap(), - 1, 0, amrex::MFInfo().SetAlloc(false) ); - const amrex::Periodicity& period = a_Geom[0].periodicity(); - m_dotMask[0][n] = tmp.OwnerMask(period); - } - m_dot_mask_defined = true; - - // If the function below is not called, then the following - // error message occurs after the simulation finishes: - // malloc_consolidate(): unaligned fastbin chunk detected - amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask); + isFieldArray(m_array_type), + "WarpXSolverVec::Define() called with array_type not an array field"); + + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name, lev); + for (int n = 0; n < 3; n++) { + m_array_vec[lev][n] = new amrex::MultiFab( this_array[n]->boxArray(), + this_array[n]->DistributionMap(), + this_array[n]->nComp(), + amrex::IntVect::TheZeroVector() ); + } + } + + } + + // Define the scalar data container + if (m_scalar_type != FieldType::None) { + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !isFieldArray(m_scalar_type), + "WarpXSolverVec::Define() called with scalar_type not a scalar field "); + + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(m_scalar_type_name,lev); + m_scalar_vec[lev] = new amrex::MultiFab( this_mf->boxArray(), + this_mf->DistributionMap(), + this_mf->nComp(), + amrex::IntVect::TheZeroVector() ); + } + + } + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_array_type != FieldType::None || + m_scalar_type != FieldType::None, + "WarpXSolverVec cannot be defined with both array and scalar vecs FieldType::None"); + + m_is_defined = true; } -[[nodiscard]] amrex::Real WarpXSolverVec::dotProduct ( const WarpXSolverVec& a_X ) const +void WarpXSolverVec::Copy ( FieldType a_array_type, + FieldType a_scalar_type, + bool allow_type_mismatch) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - m_dot_mask_defined, - "WarpXSolverVec::dotProduct called with m_dotMask not yet defined"); + IsDefined(), + "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - a_X.IsDefined(), - "WarpXSolverVec::dotProduct(a_X) called with undefined a_X"); + (a_array_type==m_array_type && + a_scalar_type==m_scalar_type) || allow_type_mismatch, + "WarpXSolverVec::Copy() called with vecs of different types"); + + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + if (m_array_type != FieldType::None) { + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(a_array_type, lev); + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_array[n], 0, 0, m_ncomp, + amrex::IntVect::TheZeroVector() ); + } + } + if (m_scalar_type != FieldType::None) { + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(a_scalar_type,lev); + amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_mf, 0, 0, m_ncomp, + amrex::IntVect::TheZeroVector() ); + } + } +} + +[[nodiscard]] amrex::Real WarpXSolverVec::dotProduct ( const WarpXSolverVec& a_X ) const +{ + assertIsDefined( a_X ); + assertSameType( a_X ); + amrex::Real result = 0.0; - const int lev = 0; const bool local = true; - for (int n = 0; n < 3; ++n) { - auto rtmp = amrex::MultiFab::Dot( *m_dotMask[lev][n], - *m_field_vec[lev][n], 0, - *a_X.getVec()[lev][n], 0, 1, 0, local); - result += rtmp; + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + if (m_array_type != FieldType::None) { + for (int n = 0; n < 3; ++n) { + const amrex::iMultiFab* dotMask = m_WarpX->getFieldDotMaskPointer(m_array_type, lev, ablastr::fields::Direction{n}); + auto rtmp = amrex::MultiFab::Dot( *dotMask, + *m_array_vec[lev][n], 0, + *a_X.getArrayVec()[lev][n], 0, 1, 0, local); + result += rtmp; + } + } + if (m_scalar_type != FieldType::None) { + const amrex::iMultiFab* dotMask = m_WarpX->getFieldDotMaskPointer(m_scalar_type,lev, ablastr::fields::Direction{0}); + auto rtmp = amrex::MultiFab::Dot( *dotMask, + *m_scalar_vec[lev], 0, + *a_X.getScalarVec()[lev], 0, 1, 0, local); + result += rtmp; + } } amrex::ParallelAllReduce::Sum(result, amrex::ParallelContext::CommunicatorSub()); return result; diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index a8bbc954e29..c07551c165c 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -7,6 +7,8 @@ #ifndef WARPX_MAGNETOSTATICSOLVER_H_ #define WARPX_MAGNETOSTATICSOLVER_H_ +#include + #include #include #include @@ -34,23 +36,23 @@ namespace MagnetostaticSolver { */ class EBCalcBfromVectorPotentialPerLevel { private: - const amrex::Vector, 3>>& m_b_field; - const amrex::Vector, 3>>& m_grad_buf_e_stag; - const amrex::Vector, 3>>& m_grad_buf_b_stag; + ablastr::fields::MultiLevelVectorField m_b_field; + ablastr::fields::MultiLevelVectorField m_grad_buf_e_stag; + ablastr::fields::MultiLevelVectorField m_grad_buf_b_stag; public: - EBCalcBfromVectorPotentialPerLevel(const amrex::Vector, 3>>& b_field, - const amrex::Vector, 3>>& grad_buf_e_stag, - const amrex::Vector, 3>>& grad_buf_b_stag) + EBCalcBfromVectorPotentialPerLevel (ablastr::fields::MultiLevelVectorField const & b_field, + ablastr::fields::MultiLevelVectorField const & grad_buf_e_stag, + ablastr::fields::MultiLevelVectorField const & grad_buf_b_stag) : m_b_field(b_field), m_grad_buf_e_stag(grad_buf_e_stag), m_grad_buf_b_stag(grad_buf_b_stag) {} - void operator()(amrex::Array,3> & mlmg, int lev); + void operator() (amrex::Array, 3> & mlmg, int lev); // Function to perform interpolation from cell edges to cell faces - void doInterp(const std::unique_ptr &src, const std::unique_ptr &dst); + void doInterp (amrex::MultiFab & src, amrex::MultiFab & dst); }; } // namespace MagnetostaticSolver diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 26ac1ac96c8..2a744f3f902 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -6,7 +6,9 @@ */ #include "WarpX.H" +#include "Fields.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" +#include "EmbeddedBoundary/Enabled.H" #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" @@ -18,6 +20,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include "Parallelization/WarpXComm_K.H" +#include #include #include #include @@ -62,7 +65,8 @@ WarpX::ComputeMagnetostaticField() // Fields have been reset in Electrostatic solver for this time step, these fields // are added into the B fields after electrostatic solve - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, "Magnetostatic solver not implemented with mesh refinement."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, + "Magnetostatic solver not implemented with mesh refinement."); AddMagnetostaticFieldLabFrame(); } @@ -70,6 +74,9 @@ WarpX::ComputeMagnetostaticField() void WarpX::AddMagnetostaticFieldLabFrame() { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + WARPX_PROFILE("WarpX::AddMagnetostaticFieldLabFrame"); // Store the boundary conditions for the field solver if they haven't been @@ -86,7 +93,7 @@ WarpX::AddMagnetostaticFieldLabFrame() // reset current_fp before depositing current density for this step for (int lev = 0; lev <= max_level; lev++) { for (int dim=0; dim < 3; dim++) { - current_fp[lev][dim]->setVal(0.); + m_fields.get(FieldType::current_fp, Direction{dim}, lev)->setVal(0.); } } @@ -94,32 +101,48 @@ WarpX::AddMagnetostaticFieldLabFrame() for (int ispecies=0; ispeciesnSpecies(); ispecies++){ WarpXParticleContainer& species = mypc->GetParticleContainer(ispecies); if (!species.do_not_deposit) { - species.DepositCurrent(current_fp, dt[0], 0.); + species.DepositCurrent( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + dt[0], 0.); } } #ifdef WARPX_DIM_RZ for (int lev = 0; lev <= max_level; lev++) { - ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), - current_fp[lev][1].get(), - current_fp[lev][2].get(), lev); + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), + lev ); } #endif - SyncCurrent(current_fp, current_cp, current_buf); // Apply filter, perform MPI exchange, interpolate across levels + SyncCurrent("current_fp"); // set the boundary and current density potentials - setVectorPotentialBC(vector_potential_fp_nodal); + setVectorPotentialBC(m_fields.get_mr_levels_alldirs(FieldType::vector_potential_fp_nodal, finest_level)); // Compute the vector potential A, by solving the Poisson equation WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsPythonCallbackInstalled("poissonsolver"), "Python Level Poisson Solve not supported for Magnetostatic implementation."); - const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; + // const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; + // temporary fix!!! + const amrex::Real absolute_tolerance = 0.0; + amrex::Real required_precision; + if constexpr (std::is_same_v) { + required_precision = 1e-5; + } + else { + required_precision = 1e-11; + } - computeVectorPotential( current_fp, vector_potential_fp_nodal, self_fields_required_precision, - magnetostatic_absolute_tolerance, self_fields_max_iters, - self_fields_verbosity ); + computeVectorPotential( + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_fp_nodal, finest_level), + required_precision, absolute_tolerance, magnetostatic_solver_max_iters, + magnetostatic_solver_verbosity + ); } /* Compute the vector potential `A` by solving the Poisson equation with `J` as @@ -139,37 +162,48 @@ WarpX::AddMagnetostaticFieldLabFrame() \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computeVectorPotential (const amrex::Vector,3> >& curr, - amrex::Vector,3> >& A, - Real const required_precision, - Real absolute_tolerance, - int const max_iters, - int const verbosity) const +WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& curr, + ablastr::fields::MultiLevelVectorField const& A, + Real const required_precision, + Real absolute_tolerance, + int const max_iters, + int const verbosity) // const // This breaks non-const m_fields.get_mr_levels_alldirs { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + // create a vector to our fields, sorted by level amrex::Vector> sorted_curr; amrex::Vector> sorted_A; for (int lev = 0; lev <= finest_level; ++lev) { - sorted_curr.emplace_back(amrex::Array ({curr[lev][0].get(), - curr[lev][1].get(), - curr[lev][2].get()})); - sorted_A.emplace_back(amrex::Array ({A[lev][0].get(), - A[lev][1].get(), - A[lev][2].get()})); + sorted_curr.emplace_back(amrex::Array ({curr[lev][0], + curr[lev][1], + curr[lev][2]})); + sorted_A.emplace_back(amrex::Array ({A[lev][0], + A[lev][1], + A[lev][2]})); } + const ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + const std::optional post_A_calculation( + { + Bfield_fp, + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_grad_buf_e_stag, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_grad_buf_b_stag, finest_level) + }); + #if defined(AMREX_USE_EB) - const std::optional post_A_calculation({Bfield_fp, - vector_potential_grad_buf_e_stag, - vector_potential_grad_buf_b_stag}); + std::optional > eb_farray_box_factory; + auto &warpx = WarpX::GetInstance(); - amrex::Vector factories; - for (int lev = 0; lev <= finest_level; ++lev) { - factories.push_back(&WarpX::fieldEBFactory(lev)); + if (EB::enabled()) { + amrex::Vector factories; + for (int lev = 0; lev <= finest_level; ++lev) { + factories.push_back(&warpx.fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; } - const std::optional > eb_farray_box_factory({factories}); #else - const std::optional post_A_calculation; const std::optional > eb_farray_box_factory; #endif @@ -184,6 +218,7 @@ WarpX::computeVectorPotential (const amrex::Vectordmap, this->grids, this->m_vector_poisson_boundary_handler, + EB::enabled(), WarpX::do_single_precision_comms, this->ref_ratio, post_A_calculation, @@ -203,8 +238,10 @@ WarpX::computeVectorPotential (const amrex::Vector,3>>& A ) const +WarpX::setVectorPotentialBC (ablastr::fields::MultiLevelVectorField const& A) const { + using ablastr::fields::Direction; + // check if any dimension has non-periodic boundary conditions if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } @@ -219,11 +256,11 @@ WarpX::setVectorPotentialBC ( amrex::Vectorarray(mfi); + auto A_arr = A[lev][Direction{adim}]->array(mfi); // Extract tileboxes for which to loop - const Box& tb = mfi.tilebox( A[lev][adim]->ixType().toIntVect()); + const Box& tb = mfi.tilebox( A[lev][Direction{adim}]->ixType().toIntVect()); // loop over dimensions for (int idim=0; idim &src, - const std::unique_ptr &dst) +void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::doInterp (amrex::MultiFab & src, + amrex::MultiFab & dst) { WarpX &warpx = WarpX::GetInstance(); @@ -366,20 +403,20 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::doInterp(const std amrex::Real const * stencil_coeffs_z = warpx.device_field_centering_stencil_coeffs_z.data(); // Synchronize the ghost cells, do halo exchange - ablastr::utils::communication::FillBoundary(*src, - src->nGrowVect(), + ablastr::utils::communication::FillBoundary(src, + src.nGrowVect(), WarpX::do_single_precision_comms); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - IntVect const src_stag = src->ixType().toIntVect(); - IntVect const dst_stag = dst->ixType().toIntVect(); + IntVect const src_stag = src.ixType().toIntVect(); + IntVect const dst_stag = dst.ixType().toIntVect(); - Array4 const& src_arr = src->const_array(mfi); - Array4 const& dst_arr = dst->array(mfi); + Array4 const& src_arr = src.const_array(mfi); + Array4 const& dst_arr = dst.array(mfi); const Box bx = mfi.tilebox(); @@ -401,12 +438,12 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: const amrex::Array buf_ptr = { #if defined(WARPX_DIM_3D) - m_grad_buf_e_stag[lev][0].get(), - m_grad_buf_e_stag[lev][1].get(), - m_grad_buf_e_stag[lev][2].get() + m_grad_buf_e_stag[lev][0], + m_grad_buf_e_stag[lev][1], + m_grad_buf_e_stag[lev][2] #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - m_grad_buf_e_stag[lev][0].get(), - m_grad_buf_e_stag[lev][2].get() + m_grad_buf_e_stag[lev][0], + m_grad_buf_e_stag[lev][2] #endif }; @@ -414,13 +451,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[0]->getGradSolution({buf_ptr}); // Interpolate dAx/dz to By grid buffer, then add to By - this->doInterp(m_grad_buf_e_stag[lev][2], - m_grad_buf_b_stag[lev][1]); + this->doInterp(*m_grad_buf_e_stag[lev][2], + *m_grad_buf_b_stag[lev][1]); MultiFab::Add(*(m_b_field[lev][1]), *(m_grad_buf_b_stag[lev][1]), 0, 0, 1, 0 ); // Interpolate dAx/dy to Bz grid buffer, then subtract from Bz - this->doInterp(m_grad_buf_e_stag[lev][1], - m_grad_buf_b_stag[lev][2]); + this->doInterp(*m_grad_buf_e_stag[lev][1], + *m_grad_buf_b_stag[lev][2]); m_grad_buf_b_stag[lev][2]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][2]), *(m_grad_buf_b_stag[lev][2]), 0, 0, 1, 0 ); @@ -428,13 +465,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[1]->getGradSolution({buf_ptr}); // Interpolate dAy/dx to Bz grid buffer, then add to Bz - this->doInterp(m_grad_buf_e_stag[lev][0], - m_grad_buf_b_stag[lev][2]); + this->doInterp(*m_grad_buf_e_stag[lev][0], + *m_grad_buf_b_stag[lev][2]); MultiFab::Add(*(m_b_field[lev][2]), *(m_grad_buf_b_stag[lev][2]), 0, 0, 1, 0 ); // Interpolate dAy/dz to Bx grid buffer, then subtract from Bx - this->doInterp(m_grad_buf_e_stag[lev][2], - m_grad_buf_b_stag[lev][0]); + this->doInterp(*m_grad_buf_e_stag[lev][2], + *m_grad_buf_b_stag[lev][0]); m_grad_buf_b_stag[lev][0]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][0]), *(m_grad_buf_b_stag[lev][0]), 0, 0, 1, 0 ); @@ -442,13 +479,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[2]->getGradSolution({buf_ptr}); // Interpolate dAz/dy to Bx grid buffer, then add to Bx - this->doInterp(m_grad_buf_e_stag[lev][1], - m_grad_buf_b_stag[lev][0]); + this->doInterp(*m_grad_buf_e_stag[lev][1], + *m_grad_buf_b_stag[lev][0]); MultiFab::Add(*(m_b_field[lev][0]), *(m_grad_buf_b_stag[lev][0]), 0, 0, 1, 0 ); // Interpolate dAz/dx to By grid buffer, then subtract from By - this->doInterp(m_grad_buf_e_stag[lev][0], - m_grad_buf_b_stag[lev][1]); + this->doInterp(*m_grad_buf_e_stag[lev][0], + *m_grad_buf_b_stag[lev][1]); m_grad_buf_b_stag[lev][1]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][1]), *(m_grad_buf_b_stag[lev][1]), 0, 0, 1, 0 ); } diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index a8af4c2de97..4c7c41f6f8e 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -2,9 +2,11 @@ CEXE_sources += WarpXPushFieldsEM.cpp CEXE_sources += WarpXPushFieldsHybridPIC.cpp CEXE_sources += ElectrostaticSolver.cpp CEXE_sources += WarpX_QED_Field_Pushers.cpp +CEXE_sources += WarpXSolveFieldsES.cpp ifeq ($(USE_FFT),TRUE) include $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/Make.package endif +include $(WARPX_HOME)/Source/FieldSolver/ElectrostaticSolvers/Make.package include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package include $(WARPX_HOME)/Source/FieldSolver/MagnetostaticSolver/Make.package include $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers/Make.package diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 23c38f33a85..94a16c13ec1 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -55,8 +55,8 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool div_cleaning, - int J_in_time, - int rho_in_time); + JInTime J_in_time, + RhoInTime rho_in_time); /** * \brief Updates E, B, F, and G fields in spectral space, @@ -93,8 +93,8 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm // Other member variables amrex::Real m_dt; bool m_div_cleaning; - int m_J_in_time; - int m_rho_in_time; + JInTime m_J_in_time; + RhoInTime m_rho_in_time; }; #endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_FIRST_ORDER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 14db3c9af48..c5f60e18d24 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -38,8 +38,8 @@ PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder ( ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool div_cleaning, - const int J_in_time, - const int rho_in_time + const JInTime J_in_time, + const RhoInTime rho_in_time ) // Initializer list : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H index c2efbd79935..2cbb4d7402d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H @@ -27,8 +27,8 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ amrex::Real dt_step, bool update_with_rho, bool time_averaging, - int J_in_time, - int rho_in_time, + JInTime J_in_time, + RhoInTime rho_in_time, bool dive_cleaning, bool divb_cleaning); // Redefine functions from base class @@ -65,7 +65,7 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ amrex::Real m_dt; bool m_update_with_rho; bool m_time_averaging; - int m_J_in_time; + JInTime m_J_in_time; bool m_dive_cleaning; bool m_divb_cleaning; SpectralRealCoefficients C_coef, S_ck_coef, X1_coef, X2_coef, X3_coef; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp index efff5c30a41..66d99e29715 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp @@ -25,8 +25,8 @@ PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::Real const dt, bool const update_with_rho, const bool time_averaging, - const int J_in_time, - const int rho_in_time, + const JInTime J_in_time, + const RhoInTime rho_in_time, const bool dive_cleaning, const bool divb_cleaning): // Initialize members of base class and member variables diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index c72e7db250d..462bce23c23 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -13,6 +13,7 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralFieldData.H" +#include #include #include @@ -74,7 +75,7 @@ class SpectralBaseAlgorithm */ void ComputeSpectralDivE ( int lev, SpectralFieldData& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab& divE ); protected: // Meant to be used in the subclasses diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index b3f18dd6912..069b724f96c 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -9,6 +9,8 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -58,8 +60,9 @@ void SpectralBaseAlgorithm::ComputeSpectralDivE ( const int lev, SpectralFieldData& field_data, - const std::array,3>& Efield, - amrex::MultiFab& divE ) + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE +) { const SpectralFieldIndex& Idx = m_spectral_index; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 8e03a2a2559..9f6b5b09219 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -10,6 +10,7 @@ #include "FieldSolver/SpectralSolver/SpectralKSpaceRZ.H" #include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" +#include #include @@ -66,7 +67,7 @@ class SpectralBaseAlgorithmRZ */ void ComputeSpectralDivE ( int lev, SpectralFieldDataRZ& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ); /** diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp index f8ef0ef4730..3e556363a6f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp @@ -6,6 +6,8 @@ */ #include "SpectralBaseAlgorithmRZ.H" +#include + #include using namespace amrex; @@ -18,7 +20,7 @@ void SpectralBaseAlgorithmRZ::ComputeSpectralDivE ( const int lev, SpectralFieldDataRZ& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ) { using amrex::operator""_rt; diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H index 8ced43ced94..42bf32c4724 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H @@ -11,6 +11,7 @@ #include "SpectralFieldData_fwd.H" #include "SpectralKSpace.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpX_Complex.H" #include @@ -59,8 +60,8 @@ class SpectralFieldIndex */ SpectralFieldIndex (bool update_with_rho, bool time_averaging, - int J_in_time, - int rho_in_time, + JInTime J_in_time, + RhoInTime rho_in_time, bool dive_cleaning, bool divb_cleaning, bool pml, @@ -176,11 +177,10 @@ class SpectralFieldData ablastr::math::anyfft::FFTplans forward_plan, backward_plan; // Correcting "shift" factors when performing FFT from/to // a cell-centered grid in real space, instead of a nodal grid - SpectralShiftFactor xshift_FFTfromCell, xshift_FFTtoCell, - zshift_FFTfromCell, zshift_FFTtoCell; -#if defined(WARPX_DIM_3D) - SpectralShiftFactor yshift_FFTfromCell, yshift_FFTtoCell; -#endif + // (0,1,2) is the dimension number + SpectralShiftFactor shift0_FFTfromCell, shift0_FFTtoCell, + shift1_FFTfromCell, shift1_FFTtoCell, + shift2_FFTfromCell, shift2_FFTtoCell; bool m_periodic_single_box; }; diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 20c97f4b5d4..a20429eea68 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -35,8 +35,8 @@ using namespace amrex; SpectralFieldIndex::SpectralFieldIndex (const bool update_with_rho, const bool time_averaging, - const int J_in_time, - const int rho_in_time, + const JInTime J_in_time, + const RhoInTime rho_in_time, const bool dive_cleaning, const bool divb_cleaning, const bool pml, @@ -142,24 +142,21 @@ SpectralFieldData::SpectralFieldData( const int lev, // By default, we assume the FFT is done from/to a nodal grid in real space // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. - xshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 0, + shift0_FFTfromCell = k_space.getSpectralShiftFactor(dm, 0, ShiftType::TransformFromCellCentered); - xshift_FFTtoCell = k_space.getSpectralShiftFactor(dm, 0, + shift0_FFTtoCell = k_space.getSpectralShiftFactor(dm, 0, ShiftType::TransformToCellCentered); -#if defined(WARPX_DIM_3D) - yshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, +#if AMREX_SPACEDIM > 1 + shift1_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, ShiftType::TransformFromCellCentered); - yshift_FFTtoCell = k_space.getSpectralShiftFactor(dm, 1, + shift1_FFTtoCell = k_space.getSpectralShiftFactor(dm, 1, ShiftType::TransformToCellCentered); - zshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 2, +#if AMREX_SPACEDIM > 2 + shift2_FFTfromCell = k_space.getSpectralShiftFactor(dm, 2, ShiftType::TransformFromCellCentered); - zshift_FFTtoCell = k_space.getSpectralShiftFactor(dm, 2, - ShiftType::TransformToCellCentered); -#else - zshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, - ShiftType::TransformFromCellCentered); - zshift_FFTtoCell = k_space.getSpectralShiftFactor(dm, 1, + shift2_FFTtoCell = k_space.getSpectralShiftFactor(dm, 2, ShiftType::TransformToCellCentered); +#endif #endif // Allocate and initialize the FFT plans @@ -221,16 +218,12 @@ SpectralFieldData::ForwardTransform (const int lev, const bool do_costs = WarpXUtilLoadBalance::doCosts(cost, mf.boxArray(), mf.DistributionMap()); // Check field index type, in order to apply proper shift in spectral space -#if (AMREX_SPACEDIM >= 2) - const bool is_nodal_x = mf.is_nodal(0); + const bool is_nodal_0 = mf.is_nodal(0); +#if AMREX_SPACEDIM > 1 + const bool is_nodal_1 = mf.is_nodal(1); +#if AMREX_SPACEDIM > 2 + const bool is_nodal_2 = mf.is_nodal(2); #endif -#if defined(WARPX_DIM_3D) - const bool is_nodal_y = mf.is_nodal(1); - const bool is_nodal_z = mf.is_nodal(2); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const bool is_nodal_z = mf.is_nodal(1); -#elif defined(WARPX_DIM_1D_Z) - const bool is_nodal_z = mf.is_nodal(0); #endif // Loop over boxes @@ -275,13 +268,14 @@ SpectralFieldData::ForwardTransform (const int lev, { const Array4 fields_arr = SpectralFieldData::fields[mfi].array(); const Array4 tmp_arr = tmpSpectralField[mfi].array(); -#if (AMREX_SPACEDIM >= 2) - const Complex* xshift_arr = xshift_FFTfromCell[mfi].dataPtr(); + + const Complex* shift0_arr = shift0_FFTfromCell[mfi].dataPtr(); +#if AMREX_SPACEDIM > 1 + const Complex* shift1_arr = shift1_FFTfromCell[mfi].dataPtr(); +#if AMREX_SPACEDIM > 2 + const Complex* shift2_arr = shift2_FFTfromCell[mfi].dataPtr(); #endif -#if defined(WARPX_DIM_3D) - const Complex* yshift_arr = yshift_FFTfromCell[mfi].dataPtr(); #endif - const Complex* zshift_arr = zshift_FFTfromCell[mfi].dataPtr(); // Loop over indices within one box const Box spectralspace_bx = tmpSpectralField[mfi].box(); @@ -289,16 +283,12 @@ SpectralFieldData::ForwardTransform (const int lev, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { Complex spectral_field_value = tmp_arr(i,j,k); // Apply proper shift in each dimension -#if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } + if (!is_nodal_0) { spectral_field_value *= shift0_arr[i]; } +#if AMREX_SPACEDIM > 1 + if (!is_nodal_1) { spectral_field_value *= shift1_arr[j]; } +#if AMREX_SPACEDIM > 2 + if (!is_nodal_2) { spectral_field_value *= shift2_arr[k]; } #endif -#if defined(WARPX_DIM_3D) - if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } - if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } -#elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; @@ -328,32 +318,9 @@ SpectralFieldData::BackwardTransform (const int lev, const bool do_costs = WarpXUtilLoadBalance::doCosts(cost, mf.boxArray(), mf.DistributionMap()); // Check field index type, in order to apply proper shift in spectral space -#if (AMREX_SPACEDIM >= 2) - const bool is_nodal_x = mf.is_nodal(0); -#endif -#if defined(WARPX_DIM_3D) - const bool is_nodal_y = mf.is_nodal(1); - const bool is_nodal_z = mf.is_nodal(2); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const bool is_nodal_z = mf.is_nodal(1); -#elif defined(WARPX_DIM_1D_Z) - const bool is_nodal_z = mf.is_nodal(0); -#endif - -#if (AMREX_SPACEDIM >= 2) - const int si = (is_nodal_x) ? 1 : 0; -#endif -#if defined(WARPX_DIM_1D_Z) - const int si = (is_nodal_z) ? 1 : 0; - const int sj = 0; - const int sk = 0; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int sj = (is_nodal_z) ? 1 : 0; - const int sk = 0; -#elif defined(WARPX_DIM_3D) - const int sj = (is_nodal_y) ? 1 : 0; - const int sk = (is_nodal_z) ? 1 : 0; -#endif + const bool is_nodal_0 = mf.is_nodal(0); + const bool is_nodal_1 = (AMREX_SPACEDIM > 1 ? mf.is_nodal(1) : 0); + const bool is_nodal_2 = (AMREX_SPACEDIM > 2 ? mf.is_nodal(2) : 0); // Numbers of guard cells const amrex::IntVect& mf_ng = mf.nGrowVect(); @@ -375,13 +342,13 @@ SpectralFieldData::BackwardTransform (const int lev, { const Array4 field_arr = SpectralFieldData::fields[mfi].array(); const Array4 tmp_arr = tmpSpectralField[mfi].array(); -#if (AMREX_SPACEDIM >= 2) - const Complex* xshift_arr = xshift_FFTtoCell[mfi].dataPtr(); + const Complex* shift0_arr = shift0_FFTtoCell[mfi].dataPtr(); +#if AMREX_SPACEDIM > 1 + const Complex* shift1_arr = shift1_FFTtoCell[mfi].dataPtr(); +#if AMREX_SPACEDIM > 2 + const Complex* shift2_arr = shift2_FFTtoCell[mfi].dataPtr(); #endif -#if defined(WARPX_DIM_3D) - const Complex* yshift_arr = yshift_FFTtoCell[mfi].dataPtr(); #endif - const Complex* zshift_arr = zshift_FFTtoCell[mfi].dataPtr(); // Loop over indices within one box const Box spectralspace_bx = tmpSpectralField[mfi].box(); @@ -389,16 +356,12 @@ SpectralFieldData::BackwardTransform (const int lev, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { Complex spectral_field_value = field_arr(i,j,k,field_index); // Apply proper shift in each dimension -#if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } + if (!is_nodal_0) { spectral_field_value *= shift0_arr[i]; } +#if AMREX_SPACEDIM > 1 + if (!is_nodal_1) { spectral_field_value *= shift1_arr[j]; } +#if AMREX_SPACEDIM > 2 + if (!is_nodal_2) { spectral_field_value *= shift2_arr[k]; } #endif -#if defined(WARPX_DIM_3D) - if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } - if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } -#elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into temporary array tmp_arr(i,j,k) = spectral_field_value; @@ -419,28 +382,18 @@ SpectralFieldData::BackwardTransform (const int lev, // Total number of cells, including ghost cells (nj represents ny in 3D and nz in 2D) const int ni = mf_box.length(0); -#if defined(WARPX_DIM_1D_Z) - constexpr int nj = 1; - constexpr int nk = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int nj = mf_box.length(1); - constexpr int nk = 1; -#elif defined(WARPX_DIM_3D) - const int nj = mf_box.length(1); - const int nk = mf_box.length(2); -#endif + const int nj = (AMREX_SPACEDIM > 1 ? mf_box.length(1) : 1); + const int nk = (AMREX_SPACEDIM > 2 ? mf_box.length(2) : 1); + + const int si = (is_nodal_0) ? 1 : 0; + const int sj = (is_nodal_1) ? 1 : 0; + const int sk = (is_nodal_2) ? 1 : 0; + // Lower bound of the box (lo_j represents lo_y in 3D and lo_z in 2D) const int lo_i = amrex::lbound(mf_box).x; -#if defined(WARPX_DIM_1D_Z) - constexpr int lo_j = 0; - constexpr int lo_k = 0; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int lo_j = amrex::lbound(mf_box).y; - constexpr int lo_k = 0; -#elif defined(WARPX_DIM_3D) - const int lo_j = amrex::lbound(mf_box).y; - const int lo_k = amrex::lbound(mf_box).z; -#endif + const int lo_j = (AMREX_SPACEDIM > 1 ? amrex::lbound(mf_box).y : 0); + const int lo_k = (AMREX_SPACEDIM > 2 ? amrex::lbound(mf_box).z : 0); + // If necessary, do not fill the guard cells // (shrink box by passing negative number of cells) if (!m_periodic_single_box) diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H index 16f93d8292a..fcf1a2ccd02 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -35,9 +36,9 @@ using SpectralShiftFactor = amrex::LayoutData< // Indicate the type of correction "shift" factor to apply // when the FFT is performed from/to a cell-centered grid in real space. -struct ShiftType { - enum{ TransformFromCellCentered=0, TransformToCellCentered=1 }; -}; +AMREX_ENUM(ShiftType, + TransformFromCellCentered, + TransformToCellCentered); /** * \brief Class that represents the spectral space. @@ -69,7 +70,7 @@ class SpectralKSpace SpectralShiftFactor getSpectralShiftFactor( const amrex::DistributionMapping& dm, int i_dim, - int shift_type ) const; + ShiftType shift_type ) const; protected: amrex::Array k_vec; diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 94bd384f265..5313409553f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -7,10 +7,12 @@ */ #include "SpectralKSpace.H" -#include "WarpX.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" +#include +#include + #include #include #include @@ -143,7 +145,7 @@ SpectralKSpace::getKComponent( const DistributionMapping& dm, SpectralShiftFactor SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, const int i_dim, - const int shift_type ) const + const ShiftType shift_type ) const { // Initialize an empty DeviceVector in each box SpectralShiftFactor shift_factor( spectralspace_ba, dm ); @@ -211,7 +213,8 @@ SpectralKSpace::getModifiedKComponent (const DistributionMapping& dm, } else { // Compute real-space stencil coefficients - Vector h_stencil_coef = WarpX::getFornbergStencilCoefficients(n_order, grid_type); + Vector h_stencil_coef = + ablastr::math::getFornbergStencilCoefficients(n_order, grid_type); Gpu::DeviceVector d_stencil_coef(h_stencil_coef.size()); Gpu::copyAsync(Gpu::hostToDevice, h_stencil_coef.begin(), h_stencil_coef.end(), d_stencil_coef.begin()); @@ -237,7 +240,7 @@ SpectralKSpace::getModifiedKComponent (const DistributionMapping& dm, { p_modified_k[i] = 0; for (int n=0; n #include #include @@ -86,9 +88,9 @@ class SpectralSolver bool periodic_single_box, bool update_with_rho, bool fft_do_time_averaging, - int psatd_solution_type, - int J_in_time, - int rho_in_time, + PSATDSolutionType psatd_solution_type, + JInTime J_in_time, + RhoInTime rho_in_time, bool dive_cleaning, bool divb_cleaning); @@ -125,9 +127,12 @@ class SpectralSolver * \brief Public interface to call the member function ComputeSpectralDivE * of the base class SpectralBaseAlgorithm from objects of class SpectralSolver */ - void ComputeSpectralDivE ( int lev, - const std::array,3>& Efield, - amrex::MultiFab& divE ) { + void ComputeSpectralDivE ( + int lev, + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ) + { algorithm->ComputeSpectralDivE( lev, field_data, Efield, divE ); } @@ -193,6 +198,9 @@ class SpectralSolver SpectralFieldIndex m_spectral_index; + // Solve time step size + amrex::Real m_dt; + protected: amrex::IntVect m_fill_guards; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index c6192187064..59f7c2d6d38 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -37,11 +37,12 @@ SpectralSolver::SpectralSolver ( const bool pml, const bool periodic_single_box, const bool update_with_rho, const bool fft_do_time_averaging, - const int psatd_solution_type, - const int J_in_time, - const int rho_in_time, + const PSATDSolutionType psatd_solution_type, + const JInTime J_in_time, + const RhoInTime rho_in_time, const bool dive_cleaning, const bool divb_cleaning) + : m_dt(dt) { // Initialize all structures using the same distribution mapping dm diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H index 2e94fe95da2..7e1a4f970d2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H @@ -12,6 +12,7 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithmRZ.H" #include "SpectralFieldDataRZ.H" +#include #include @@ -41,8 +42,8 @@ class SpectralSolverRZ bool with_pml, bool update_with_rho, bool fft_do_time_averaging, - int J_in_time, - int rho_in_time, + JInTime J_in_time, + RhoInTime rho_in_time, bool dive_cleaning, bool divb_cleaning); @@ -95,7 +96,8 @@ class SpectralSolverRZ * \brief Public interface to call the member function ComputeSpectralDivE * of the base class SpectralBaseAlgorithmRZ from objects of class SpectralSolverRZ */ - void ComputeSpectralDivE (int lev, const std::array,3>& Efield, + void ComputeSpectralDivE (int lev, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE); /** @@ -150,6 +152,9 @@ class SpectralSolverRZ SpectralFieldIndex m_spectral_index; + // Solve time step size + amrex::Real m_dt; + private: SpectralKSpaceRZ k_space; // Save the instance to initialize filtering diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp index 3529247ef56..7672b646b2e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp @@ -36,11 +36,11 @@ SpectralSolverRZ::SpectralSolverRZ (const int lev, bool const with_pml, bool const update_with_rho, const bool fft_do_time_averaging, - const int J_in_time, - const int rho_in_time, + const JInTime J_in_time, + const RhoInTime rho_in_time, const bool dive_cleaning, const bool divb_cleaning) - : k_space(realspace_ba, dm, dx) + : m_dt(dt), k_space(realspace_ba, dm, dx) { // Initialize all structures using the same distribution mapping dm @@ -142,8 +142,10 @@ SpectralSolverRZ::pushSpectralFields (const bool doing_pml) { */ void SpectralSolverRZ::ComputeSpectralDivE (const int lev, - const std::array,3>& Efield, - amrex::MultiFab& divE) { + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE +) +{ algorithm->ComputeSpectralDivE(lev, field_data, Efield, divE); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index bc75e08be00..24a6483c869 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -10,6 +10,7 @@ #include "BoundaryConditions/PML.H" #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #if defined(WARPX_USE_FFT) # include "FieldSolver/SpectralSolver/SpectralFieldData.H" @@ -53,6 +54,7 @@ #include using namespace amrex; +using warpx::fields::FieldType; #ifdef WARPX_USE_FFT namespace { @@ -64,7 +66,7 @@ namespace { #else SpectralSolver& solver, #endif - const std::array,3>& vector_field, + const ablastr::fields::VectorField& vector_field, const int compx, const int compy, const int compz) { #ifdef WARPX_DIM_RZ @@ -84,7 +86,7 @@ namespace { #else SpectralSolver& solver, #endif - const std::array,3>& vector_field, + const ablastr::fields::VectorField& vector_field, const int compx, const int compy, const int compz, const amrex::IntVect& fill_guards) { @@ -100,63 +102,93 @@ namespace { } } -void WarpX::PSATDForwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp) +void WarpX::PSATDForwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has_vector(Efield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz); + } if (spectral_solver_cp[lev]) { - ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has_vector(Efield_cp_string, lev)) { + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has_vector(Bfield_cp_string, lev)) { + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz); + } } } } -void WarpX::PSATDBackwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp) +void WarpX::PSATDBackwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has_vector(Efield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } if (spectral_solver_cp[lev]) { - BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has_vector(Efield_cp_string, lev)) { + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has_vector(Bfield_cp_string, lev)) { + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } } } // Damp the fields in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, E_fp[lev], B_fp[lev]); + if (m_fields.has_vector(Efield_fp_string, lev) && m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + DampFieldsInGuards(lev, E_fp, B_fp); + } } } void WarpX::PSATDBackwardTransformEBavg ( - const amrex::Vector,3>>& E_avg_fp, - const amrex::Vector,3>>& B_avg_fp, - const amrex::Vector,3>>& E_avg_cp, - const amrex::Vector,3>>& B_avg_cp) + ablastr::fields::MultiLevelVectorField const& E_avg_fp, + ablastr::fields::MultiLevelVectorField const& B_avg_fp, + ablastr::fields::MultiLevelVectorField const& E_avg_cp, + ablastr::fields::MultiLevelVectorField const& B_avg_cp) { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; @@ -184,11 +216,15 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } + if (m_fields.has(FieldType::F_fp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F); + } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } + if (m_fields.has(FieldType::F_cp, lev)) { + spectral_solver_cp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F); + } } } } @@ -201,17 +237,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } + if (m_fields.has(FieldType::F_fp, lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F); } #else - if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } + if (m_fields.has(FieldType::F_fp, lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } + if (m_fields.has(FieldType::F_cp, lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F); } #else - if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } + if (m_fields.has(FieldType::F_cp, lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F, m_fill_guards_fields); } #endif } } @@ -219,7 +255,7 @@ WarpX::PSATDBackwardTransformF () // Damp the field in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, F_fp[lev]); + DampFieldsInGuards(lev, m_fields.get(FieldType::F_fp, lev)); } } @@ -230,11 +266,15 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } + if (m_fields.has(FieldType::G_fp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::G_fp, lev), Idx.G); + } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } + if (m_fields.has(FieldType::G_cp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::G_cp, lev), Idx.G); + } } } } @@ -246,34 +286,38 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { + if (m_fields.has(FieldType::G_fp, lev)) { + MultiFab* G_fp = m_fields.get(FieldType::G_fp, lev); #ifdef WARPX_DIM_RZ - if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G); #else - if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G, m_fill_guards_fields); #endif + DampFieldsInGuards(lev, G_fp); + } + if (spectral_solver_cp[lev]) { + if (m_fields.has(FieldType::G_cp, lev)) { + MultiFab* G_cp = m_fields.get(FieldType::G_cp, lev); #ifdef WARPX_DIM_RZ - if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G); #else - if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G, m_fill_guards_fields); #endif + } } } - - // Damp the field in the guard cells - for (int lev = 0; lev <= finest_level; ++lev) - { - DampFieldsInGuards(lev, G_fp[lev]); - } } void WarpX::PSATDForwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, const bool apply_kspace_filter) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -285,7 +329,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField const J_fp = m_fields.get_alldirs(J_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, idx_jx, idx_jy, idx_jz); + } if (spectral_solver_cp[lev]) { @@ -295,7 +342,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField const J_cp = m_fields.get_alldirs(J_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, idx_jx, idx_jy, idx_jz); + } } } @@ -331,9 +381,11 @@ void WarpX::PSATDForwardTransformJ ( } void WarpX::PSATDBackwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp) + std::string const & J_fp_string, + std::string const & J_cp_string) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -347,8 +399,11 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField const J_fp = m_fields.get_alldirs(J_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } if (spectral_solver_cp[lev]) { @@ -360,26 +415,35 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField const J_cp = m_fields.get_alldirs(J_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } } } } void WarpX::PSATDForwardTransformRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) { return; } + if (!m_fields.has(charge_fp_string, 0)) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } + if (m_fields.has(charge_fp_string, lev)) { + amrex::MultiFab const & charge_fp = *m_fields.get(charge_fp_string, lev); + spectral_solver_fp[lev]->ForwardTransform(lev, charge_fp, dcomp, icomp); + } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } + if (m_fields.has(charge_cp_string, lev)) { + amrex::MultiFab const & charge_cp = *m_fields.get(charge_cp_string, lev); + spectral_solver_cp[lev]->ForwardTransform(lev, charge_cp, dcomp, icomp); + } } } @@ -430,6 +494,8 @@ void WarpX::PSATDVayDeposition () void WarpX::PSATDSubtractCurrentPartialSumsAvg () { + using ablastr::fields::Direction; + // Subtraction of cumulative sum for Vay deposition // implemented only in 2D and 3D Cartesian geometry #if !defined (WARPX_DIM_1D_Z) && !defined (WARPX_DIM_RZ) @@ -441,15 +507,15 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () { const std::array& dx = WarpX::CellSize(lev); - amrex::MultiFab const& Dx = *current_fp_vay[lev][0]; - amrex::MultiFab const& Dy = *current_fp_vay[lev][1]; - amrex::MultiFab const& Dz = *current_fp_vay[lev][2]; + amrex::MultiFab const& Dx = *m_fields.get(FieldType::current_fp_vay, Direction{0}, lev); + amrex::MultiFab const& Dy = *m_fields.get(FieldType::current_fp_vay, Direction{1}, lev); + amrex::MultiFab const& Dz = *m_fields.get(FieldType::current_fp_vay, Direction{2}, lev); #if defined (WARPX_DIM_XZ) amrex::ignore_unused(Dy); #endif - amrex::MultiFab& Jx = *current_fp[lev][0]; + amrex::MultiFab& Jx = *m_fields.get(FieldType::current_fp, Direction{0}, lev); #ifdef AMREX_USE_OMP @@ -480,7 +546,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #if defined (WARPX_DIM_3D) // Subtract average of cumulative sum from Jy - amrex::MultiFab& Jy = *current_fp[lev][1]; + amrex::MultiFab& Jy = *m_fields.get(FieldType::current_fp, Direction{1}, lev);; for (amrex::MFIter mfi(Jy); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); @@ -505,7 +571,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #endif // Subtract average of cumulative sum from Jz - amrex::MultiFab& Jz = *current_fp[lev][2]; + amrex::MultiFab& Jz = *m_fields.get(FieldType::current_fp, Direction{2}, lev); for (amrex::MFIter mfi(Jz); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); @@ -648,56 +714,68 @@ WarpX::PSATDScaleAverageFields (const amrex::Real scale_factor) #endif // WARPX_USE_FFT void -WarpX::PushPSATD () +WarpX::PushPSATD (amrex::Real start_time) { #ifndef WARPX_USE_FFT + amrex::ignore_unused(start_time); WARPX_ABORT_WITH_MESSAGE( "PushFieldsEM: PSATD solver selected but not built"); #else + bool const skip_lev0_coarse_patch = true; + const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; + + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level); + std::string current_fp_string = "current_fp"; + std::string const current_cp_string = "current_cp"; + if (fft_periodic_single_box) { if (current_correction) { // FFT of J and rho - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D and rho (if used) // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - PSATDForwardTransformJ(current_fp_vay, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // TODO Cumulative sums need to be fixed with periodic single box PSATDSubtractCurrentPartialSumsAvg(); // FFT of J after subtraction of cumulative sums - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ(current_fp_string, current_cp_string); } else // no current correction, no Vay deposition { // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } } else // no periodic single box @@ -709,35 +787,37 @@ WarpX::PushPSATD () // In RZ geometry, do not apply filtering here, since it is // applied in the subsequent calls to these functions (below) const bool apply_kspace_filter = false; - PSATDForwardTransformJ(current_fp, current_cp, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new, apply_kspace_filter); + PSATDForwardTransformJ(current_fp_string, current_cp_string, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new, apply_kspace_filter); #else - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); #endif // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // Synchronize J and rho - SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncCurrent("current_fp"); + SyncRho(); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D - PSATDForwardTransformJ(current_fp_vay, current_cp); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); PSATDSubtractCurrentPartialSumsAvg(); // Synchronize J and rho (if used). @@ -747,17 +827,17 @@ WarpX::PushPSATD () // TODO This works only without mesh refinement const int lev = 0; SumBoundaryJ(current_fp, lev, Geom(lev).periodicity()); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } // FFT of E and B - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDForwardTransformEB(); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -771,33 +851,39 @@ WarpX::PushPSATD () PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G - PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB(); if (WarpX::fft_do_time_averaging) { + auto Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); + auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); + auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_lev0_coarse_patch); + auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_lev0_coarse_patch); PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); } if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } - // Evolve the fields in the PML boxes for (int lev = 0; lev <= finest_level; ++lev) { + // Evolve the fields in the PML boxes if (pml[lev] && pml[lev]->ok()) { - pml[lev]->PushPSATD(lev); + pml[lev]->PushPSATD(m_fields, lev); } - ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } - ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } + + amrex::Real const new_time = start_time + spectral_solver_fp[lev]->m_dt; + ApplyEfieldBoundary(lev, PatchType::fine, new_time); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse, new_time); } + ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf, new_time); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf, new_time); } } #endif } void -WarpX::EvolveB (amrex::Real a_dt, DtType a_dt_type) +WarpX::EvolveB (amrex::Real a_dt, DtType a_dt_type, amrex::Real start_time) { for (int lev = 0; lev <= finest_level; ++lev) { - EvolveB(lev, a_dt, a_dt_type); + EvolveB(lev, a_dt, a_dt_type, start_time); } // Allow execution of Python callback after B-field push @@ -805,52 +891,54 @@ WarpX::EvolveB (amrex::Real a_dt, DtType a_dt_type) } void -WarpX::EvolveB (int lev, amrex::Real a_dt, DtType a_dt_type) +WarpX::EvolveB (int lev, amrex::Real a_dt, DtType a_dt_type, amrex::Real start_time) { WARPX_PROFILE("WarpX::EvolveB()"); - EvolveB(lev, PatchType::fine, a_dt, a_dt_type); + EvolveB(lev, PatchType::fine, a_dt, a_dt_type, start_time); if (lev > 0) { - EvolveB(lev, PatchType::coarse, a_dt, a_dt_type); + EvolveB(lev, PatchType::coarse, a_dt, a_dt_type, start_time); } } void -WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) +WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type, amrex::Real start_time) { - // Evolve B field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], G_fp[lev], - m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fdtd_solver_fp[lev]->EvolveB( m_fields, + lev, + patch_type, + m_flag_info_face[lev], m_borrowing[lev], a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], G_cp[lev], - m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fdtd_solver_cp[lev]->EvolveB( m_fields, + lev, + patch_type, + m_flag_info_face[lev], m_borrowing[lev], a_dt ); } // Evolve B field in PML cells if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveBPML( - pml[lev]->GetB_fp(), pml[lev]->GetE_fp(), a_dt, WarpX::do_dive_cleaning); + m_fields, patch_type, lev, a_dt, WarpX::do_dive_cleaning); } else { m_fdtd_solver_cp[lev]->EvolveBPML( - pml[lev]->GetB_cp(), pml[lev]->GetE_cp(), a_dt, WarpX::do_dive_cleaning); + m_fields, patch_type, lev, a_dt, WarpX::do_dive_cleaning); } } - ApplyBfieldBoundary(lev, patch_type, a_dt_type); + amrex::Real const new_time = start_time + a_dt; + ApplyBfieldBoundary(lev, patch_type, a_dt_type, new_time); } void -WarpX::EvolveE (amrex::Real a_dt) +WarpX::EvolveE (amrex::Real a_dt, amrex::Real start_time) { for (int lev = 0; lev <= finest_level; ++lev) { - EvolveE(lev, a_dt); + EvolveE(lev, a_dt, start_time); } // Allow execution of Python callback after E-field push @@ -858,63 +946,74 @@ WarpX::EvolveE (amrex::Real a_dt) } void -WarpX::EvolveE (int lev, amrex::Real a_dt) +WarpX::EvolveE (int lev, amrex::Real a_dt, amrex::Real start_time) { WARPX_PROFILE("WarpX::EvolveE()"); - EvolveE(lev, PatchType::fine, a_dt); + EvolveE(lev, PatchType::fine, a_dt, start_time); if (lev > 0) { - EvolveE(lev, PatchType::coarse, a_dt); + EvolveE(lev, PatchType::coarse, a_dt, start_time); } } void -WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) +WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt, amrex::Real start_time) { // Evolve E field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE(Efield_fp[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], - F_fp[lev], lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE( m_fields, + lev, + patch_type, + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_eb_update_E[lev], + a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE(Efield_cp[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], - F_cp[lev], lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( m_fields, + lev, + patch_type, + m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_eb_update_E[lev], + a_dt ); } // Evolve E field in PML cells if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveEPML( - pml[lev]->GetE_fp(), pml[lev]->GetB_fp(), - pml[lev]->Getj_fp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_fp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_fp(), a_dt, pml_has_particles ); } else { m_fdtd_solver_cp[lev]->EvolveEPML( - pml[lev]->GetE_cp(), pml[lev]->GetB_cp(), - pml[lev]->Getj_cp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_cp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_cp(), a_dt, pml_has_particles ); } } - ApplyEfieldBoundary(lev, patch_type); + amrex::Real const new_time = start_time + a_dt; + ApplyEfieldBoundary(lev, patch_type, new_time); // ECTRhofield must be recomputed at the very end of the Efield update to ensure // that ECTRhofield is consistent with Efield #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), + m_fields.get_alldirs(FieldType::ECTRhofield, lev), + lev ); } else { - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), + m_fields.get_alldirs(FieldType::ECTRhofield, lev), + lev); } } #endif @@ -952,21 +1051,27 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( F_fp[lev], Efield_fp[lev], - rho_fp[lev], rhocomp, a_dt ); + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get(FieldType::F_fp, lev), + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get(FieldType::rho_fp,lev), rhocomp, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveF( F_cp[lev], Efield_cp[lev], - rho_cp[lev], rhocomp, a_dt ); + m_fdtd_solver_cp[lev]->EvolveF( m_fields.get(FieldType::F_cp, lev), + m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get(FieldType::rho_cp,lev), rhocomp, a_dt ); } // Evolve F field in PML cells if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveFPML( - pml[lev]->GetF_fp(), pml[lev]->GetE_fp(), a_dt ); + m_fields.get(FieldType::pml_F_fp, lev), + m_fields.get_alldirs(FieldType::pml_E_fp, lev), + a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveFPML( - pml[lev]->GetF_cp(), pml[lev]->GetE_cp(), a_dt ); + m_fields.get(FieldType::pml_F_cp, lev), + m_fields.get_alldirs(FieldType::pml_E_cp, lev), + a_dt ); } } } @@ -1002,29 +1107,37 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t WARPX_PROFILE("WarpX::EvolveG()"); + bool const skip_lev0_coarse_patch = true; + // Evolve G field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveG(G_fp[lev], Bfield_fp[lev], a_dt); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + m_fdtd_solver_fp[lev]->EvolveG( + m_fields.get(FieldType::G_fp, lev), + Bfield_fp[lev], a_dt); } else // coarse patch { - m_fdtd_solver_cp[lev]->EvolveG(G_cp[lev], Bfield_cp[lev], a_dt); + ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level, skip_lev0_coarse_patch); + m_fdtd_solver_cp[lev]->EvolveG( + m_fields.get(FieldType::G_cp, lev), + Bfield_cp_new[lev], a_dt); } // TODO Evolution in PML cells will go here } void -WarpX::MacroscopicEvolveE (amrex::Real a_dt) +WarpX::MacroscopicEvolveE (amrex::Real a_dt, amrex::Real start_time) { for (int lev = 0; lev <= finest_level; ++lev ) { - MacroscopicEvolveE(lev, a_dt); + MacroscopicEvolveE(lev, a_dt, start_time); } } void -WarpX::MacroscopicEvolveE (int lev, amrex::Real a_dt) { +WarpX::MacroscopicEvolveE (int lev, amrex::Real a_dt, amrex::Real start_time) { WARPX_PROFILE("WarpX::MacroscopicEvolveE()"); @@ -1033,11 +1146,11 @@ WarpX::MacroscopicEvolveE (int lev, amrex::Real a_dt) { "Macroscopic EvolveE is not implemented for lev>0, yet." ); - MacroscopicEvolveE(lev, PatchType::fine, a_dt); + MacroscopicEvolveE(lev, PatchType::fine, a_dt, start_time); } void -WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { +WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt, amrex::Real start_time) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( patch_type == PatchType::fine, @@ -1045,35 +1158,38 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { ); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( - Efield_fp[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + m_fields.get_alldirs(FieldType::current_fp, lev), + m_eb_update_E[lev], a_dt, m_macroscopic_properties); if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveEPML( - pml[lev]->GetE_fp(), pml[lev]->GetB_fp(), - pml[lev]->Getj_fp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_fp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_fp(), a_dt, pml_has_particles ); } else { m_fdtd_solver_cp[lev]->EvolveEPML( - pml[lev]->GetE_cp(), pml[lev]->GetB_cp(), - pml[lev]->Getj_cp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_cp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_cp(), a_dt, pml_has_particles ); } } - ApplyEfieldBoundary(lev, patch_type); + amrex::Real const new_time = start_time + a_dt; + ApplyEfieldBoundary(lev, patch_type, new_time); } void WarpX::DampFieldsInGuards(const int lev, - const std::array,3>& Efield, - const std::array,3>& Bfield) { + const ablastr::fields::VectorField& Efield, + const ablastr::fields::VectorField& Bfield) { // Loop over dimensions for (int dampdir = 0 ; dampdir < AMREX_SPACEDIM ; dampdir++) @@ -1169,7 +1285,7 @@ WarpX::DampFieldsInGuards(const int lev, } } -void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& mf) +void WarpX::DampFieldsInGuards(const int lev, amrex::MultiFab* mf) { // Loop over dimensions for (int dampdir = 0; dampdir < AMREX_SPACEDIM; dampdir++) @@ -1219,7 +1335,7 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& // It is faster to apply this on the grid than to do it particle by particle. // It is put here since there isn't another nice place for it. void -WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, MultiFab* Jz, int lev) +WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, MultiFab* Jz, int lev) const { const amrex::IntVect ngJ = Jx->nGrowVect(); const std::array& dx = WarpX::CellSize(lev); @@ -1228,7 +1344,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu constexpr int NODE = amrex::IndexType::NODE; // See Verboncoeur JCP 174, 421-427 (2001) for the modified volume factor - const amrex::Real axis_volume_factor = (verboncoeur_axis_correction ? 1._rt/3._rt : 1._rt/4._rt); + const amrex::Real axis_volume_factor = (m_verboncoeur_axis_correction ? 1._rt/3._rt : 1._rt/4._rt); for ( MFIter mfi(*Jx, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -1392,7 +1508,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu } void -WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) +WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) const { const amrex::IntVect ngRho = Rho->nGrowVect(); const std::array& dx = WarpX::CellSize(lev); @@ -1401,7 +1517,7 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) constexpr int NODE = amrex::IndexType::NODE; // See Verboncoeur JCP 174, 421-427 (2001) for the modified volume factor - const amrex::Real axis_volume_factor = (verboncoeur_axis_correction ? 1._rt/3._rt : 1._rt/4._rt); + const amrex::Real axis_volume_factor = (m_verboncoeur_axis_correction ? 1._rt/3._rt : 1._rt/4._rt); Box tilebox; diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 4dbe10c4e5a..b57def5c4fe 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -1,12 +1,14 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * * Authors: Roelof Groenewald (TAE Technologies) + * S. Eric Clark (Helion Energy) * * License: BSD-3-Clause-LBNL */ #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" @@ -15,10 +17,16 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + + using namespace amrex; void WarpX::HybridPICEvolveFields () { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); // The below deposition is hard coded for a single level simulation @@ -26,17 +34,46 @@ void WarpX::HybridPICEvolveFields () finest_level == 0, "Ohm's law E-solve only works with a single level."); + // Get requested number of substeps to use + const int sub_steps = m_hybrid_pic_model->m_substeps; + + // Get flag to include external fields. + const bool add_external_fields = m_hybrid_pic_model->m_add_external_fields; + + // Handle field splitting for Hybrid field push + if (add_external_fields) { + // Get the external fields + m_hybrid_pic_model->m_external_vector_potential->UpdateHybridExternalFields( + gett_old(0), + 0.5_rt*dt[0]); + + // If using split fields, subtract the external field at the old time + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { + MultiFab::Subtract( + *m_fields.get(FieldType::Bfield_fp, Direction{idim}, lev), + *m_fields.get(FieldType::hybrid_B_fp_external, Direction{idim}, lev), + 0, 0, 1, + m_fields.get(FieldType::Bfield_fp, Direction{idim}, lev)->nGrowVect()); + } + } + } + // The particles have now been pushed to their t_{n+1} positions. // Perform charge deposition in component 0 of rho_fp at t_{n+1}. - mypc->DepositCharge(rho_fp, 0._rt); + mypc->DepositCharge(m_fields.get_mr_levels(FieldType::rho_fp, finest_level), 0._rt); // Perform current deposition at t_{n+1/2}. - mypc->DepositCurrent(current_fp, dt[0], -0.5_rt * dt[0]); + mypc->DepositCurrent(m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), dt[0], -0.5_rt * dt[0]); // Deposit cold-relativistic fluid charge and current if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge(lev, *rho_fp[lev]); - myfl->DepositCurrent(lev, *current_fp[lev][0], *current_fp[lev][1], *current_fp[lev][2]); + myfl->DepositCharge(m_fields, *m_fields.get(FieldType::rho_fp, lev), lev); + myfl->DepositCurrent(m_fields, + *m_fields.get(FieldType::current_fp, Direction{0}, lev), + *m_fields.get(FieldType::current_fp, Direction{1}, lev), + *m_fields.get(FieldType::current_fp, Direction{2}, lev), + lev); } // Synchronize J and rho: @@ -49,19 +86,16 @@ void WarpX::HybridPICEvolveFields () // a nodal grid for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < 3; ++idim) { - current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + m_fields.get(FieldType::current_fp, Direction{idim}, lev)->FillBoundary(Geom(lev).periodicity()); } } - // Get requested number of substeps to use - const int sub_steps = m_hybrid_pic_model->m_substeps; - // Get the external current - m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); + m_hybrid_pic_model->GetCurrentExternal(); // Reference hybrid-PIC multifabs - auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; - auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level); // During the above deposition the charge and current density were updated // so that, at this time, we have rho^{n} in rho_fp_temp, rho{n+1} in the @@ -82,23 +116,23 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], 0.5_rt, *current_fp_temp[lev][idim], 0, - 0.5_rt, *current_fp[lev][idim], 0, + 0.5_rt, *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } } - // Calculate the electron pressure at t=n using rho^n - m_hybrid_pic_model->CalculateElectronPressure(DtType::FirstHalf); - // Push the B field from t=n to t=n+1/2 using the current and density // at t=n, while updating the E field along with B using the electron // momentum equation for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp_temp, rho_fp_temp, - m_edge_lengths, 0.5_rt/sub_steps*dt[0], + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), + current_fp_temp, rho_fp_temp, + m_eb_update_E, + 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points ); @@ -112,19 +146,27 @@ void WarpX::HybridPICEvolveFields () // the result into the 0'th index of `rho_fp_temp[lev]` MultiFab::LinComb( *rho_fp_temp[lev], 0.5_rt, *rho_fp_temp[lev], 0, - 0.5_rt, *rho_fp[lev], 0, 0, 1, rho_fp_temp[lev]->nGrowVect() + 0.5_rt, *m_fields.get(FieldType::rho_fp, lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect() ); } - // Calculate the electron pressure at t=n+1/2 - m_hybrid_pic_model->CalculateElectronPressure(DtType::SecondHalf); + if (add_external_fields) { + // Get the external fields at E^{n+1/2} + m_hybrid_pic_model->m_external_vector_potential->UpdateHybridExternalFields( + gett_old(0) + 0.5_rt*dt[0], + 0.5_rt*dt[0]); + } // Now push the B field from t=n+1/2 to t=n+1 using the n+1/2 quantities for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp, rho_fp_temp, - m_edge_lengths, 0.5_rt/sub_steps*dt[0], + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + rho_fp_temp, + m_eb_update_E, + 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points ); @@ -142,45 +184,91 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], -1._rt, *current_fp_temp[lev][idim], 0, - 2._rt, *current_fp[lev][idim], 0, + 2._rt, *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } } + if (add_external_fields) { + m_hybrid_pic_model->m_external_vector_potential->UpdateHybridExternalFields( + gett_new(0), + 0.5_rt*dt[0]); + } + // Calculate the electron pressure at t=n+1 - m_hybrid_pic_model->CalculateElectronPressure(DtType::Full); + m_hybrid_pic_model->CalculateElectronPressure(); // Update the E field to t=n+1 using the extrapolated J_i^n+1 value - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); + m_hybrid_pic_model->CalculatePlasmaCurrent( + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_eb_update_E); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, false - ); + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), + current_fp_temp, + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_eb_update_E, false); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + // Handle field splitting for Hybrid field push + if (add_external_fields) { + // If using split fields, add the external field at the new time + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { + MultiFab::Add( + *m_fields.get(FieldType::Bfield_fp, Direction{idim}, lev), + *m_fields.get(FieldType::hybrid_B_fp_external, Direction{idim}, lev), + 0, 0, 1, + m_fields.get(FieldType::Bfield_fp, Direction{idim}, lev)->nGrowVect()); + MultiFab::Add( + *m_fields.get(FieldType::Efield_fp, Direction{idim}, lev), + *m_fields.get(FieldType::hybrid_E_fp_external, Direction{idim}, lev), + 0, 0, 1, + m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)->nGrowVect()); + } + } + } + // Copy the rho^{n+1} values to rho_fp_temp and the J_i^{n+1/2} values to // current_fp_temp since at the next step those values will be needed as // rho^{n} and J_i^{n-1/2}. for (int lev = 0; lev <= finest_level; ++lev) { // copy 1 component value starting at index 0 to index 0 - MultiFab::Copy(*rho_fp_temp[lev], *rho_fp[lev], + MultiFab::Copy(*rho_fp_temp[lev], *m_fields.get(FieldType::rho_fp, lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect()); for (int idim = 0; idim < 3; ++idim) { - MultiFab::Copy(*current_fp_temp[lev][idim], *current_fp[lev][idim], + MultiFab::Copy(*current_fp_temp[lev][idim], *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect()); } } + + // Check that the E-field does not have nan or inf values, otherwise print a clear message + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + for (int lev = 0; lev <= finest_level; ++lev) + { + for (int idim = 0; idim < 3; ++idim) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + Efield_fp[lev][idim]->is_finite(), + "Non-finite value detected in E-field; this indicates more substeps should be used in the field solver." + ); + } + } } void WarpX::HybridPICDepositInitialRhoAndJ () { - auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; - auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level); mypc->DepositCharge(rho_fp_temp, 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); - SyncRho(rho_fp_temp, rho_cp, charge_buf); - SyncCurrent(current_fp_temp, current_cp, current_buf); + SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch), m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_lev0_coarse_patch)); + SyncCurrent("hybrid_current_fp_temp"); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to @@ -189,13 +277,25 @@ void WarpX::HybridPICDepositInitialRhoAndJ () current_fp_temp[lev][1]->FillBoundary(Geom(lev).periodicity()); current_fp_temp[lev][2]->FillBoundary(Geom(lev).periodicity()); - ApplyRhofieldBoundary(lev, rho_fp_temp[lev].get(), PatchType::fine); + ApplyRhofieldBoundary(lev, rho_fp_temp[lev], PatchType::fine); // Set current density at PEC boundaries, if needed. ApplyJfieldBoundary( - lev, current_fp_temp[lev][0].get(), - current_fp_temp[lev][1].get(), - current_fp_temp[lev][2].get(), + lev, current_fp_temp[lev][0], + current_fp_temp[lev][1], + current_fp_temp[lev][2], PatchType::fine ); } } + +void +WarpX::CalculateExternalCurlA() { + WARPX_PROFILE("WarpX::CalculateExternalCurlA()"); + + auto & warpx = WarpX::GetInstance(); + + // Get reference to External Field Object + auto* ext_vector = warpx.m_hybrid_pic_model->m_external_vector_potential.get(); + ext_vector->CalculateExternalCurlA(); + +} diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp new file mode 100644 index 00000000000..6194570cd2d --- /dev/null +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -0,0 +1,35 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Remi Lehe, Roelof Groenewald, Arianna Formenti, Revathi Jambunathan + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" + +#include "Fields.H" +#include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" + + +void WarpX::ComputeSpaceChargeField (bool const reset_fields) +{ + WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + if (reset_fields) { + // Reset all E and B fields to 0, before calculating space-charge fields + WARPX_PROFILE("WarpX::ComputeSpaceChargeField::reset_fields"); + for (int lev = 0; lev <= max_level; lev++) { + for (int comp=0; comp<3; comp++) { + m_fields.get(FieldType::Efield_fp, Direction{comp}, lev)->setVal(0); + m_fields.get(FieldType::Bfield_fp, Direction{comp}, lev)->setVal(0); + } + } + } + + m_electrostatic_solver->ComputeSpaceChargeField( + m_fields, *mypc, myfl.get(), max_level ); +} diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 1d3cb04a8a7..1ff1d1f866d 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -6,6 +6,7 @@ */ #include "WarpX.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" @@ -69,36 +70,41 @@ WarpX::Hybrid_QED_Push (int lev, amrex::Real a_dt) void WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + const int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const std::array& dx_vec= WarpX::CellSize(patch_level); const Real dx = dx_vec[0]; const Real dy = dx_vec[1]; const Real dz = dx_vec[2]; + using ablastr::fields::Direction; + MultiFab *Ex, *Ey, *Ez, *Bx, *By, *Bz, *Jx, *Jy, *Jz; if (patch_type == PatchType::fine) { - Ex = Efield_fp[lev][0].get(); - Ey = Efield_fp[lev][1].get(); - Ez = Efield_fp[lev][2].get(); - Bx = Bfield_fp[lev][0].get(); - By = Bfield_fp[lev][1].get(); - Bz = Bfield_fp[lev][2].get(); - Jx = current_fp[lev][0].get(); - Jy = current_fp[lev][1].get(); - Jz = current_fp[lev][2].get(); + Ex = m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + Ey = m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + Ez = m_fields.get(FieldType::Efield_fp, Direction{2}, lev); + Bx = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev); + By = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev); + Bz = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev); + Jx = m_fields.get(FieldType::current_fp, Direction{0}, lev); + Jy = m_fields.get(FieldType::current_fp, Direction{1}, lev); + Jz = m_fields.get(FieldType::current_fp, Direction{2}, lev); } else { - Ex = Efield_cp[lev][0].get(); - Ey = Efield_cp[lev][1].get(); - Ez = Efield_cp[lev][2].get(); - Bx = Bfield_cp[lev][0].get(); - By = Bfield_cp[lev][1].get(); - Bz = Bfield_cp[lev][2].get(); - Jx = current_cp[lev][0].get(); - Jy = current_cp[lev][1].get(); - Jz = current_cp[lev][2].get(); + Ex = m_fields.get(FieldType::Efield_cp, Direction{0}, lev); + Ey = m_fields.get(FieldType::Efield_cp, Direction{1}, lev); + Ez = m_fields.get(FieldType::Efield_cp, Direction{2}, lev); + Bx = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev); + By = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev); + Bz = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev); + Jx = m_fields.get(FieldType::current_cp, Direction{0}, lev); + Jy = m_fields.get(FieldType::current_cp, Direction{1}, lev); + Jz = m_fields.get(FieldType::current_cp, Direction{2}, lev); } amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -169,7 +175,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) ); // Make local copy of xi, to use on device. - const Real xi_c2 = WarpX::quantum_xi_c2; + const Real xi_c2 = m_quantum_xi_c2; // Apply QED correction to electric field, using temporary arrays. amrex::ParallelFor( diff --git a/Source/Fields.H b/Source/Fields.H new file mode 100644 index 00000000000..271d5a835a3 --- /dev/null +++ b/Source/Fields.H @@ -0,0 +1,141 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + * Authors: Luca Fedeli, Justin Angus, Remi Lehe, Axel Huebl + */ +#ifndef WARPX_FIELDS_H_ +#define WARPX_FIELDS_H_ + +#include + +#include + +#include +#include + + +namespace warpx::fields +{ + /** Unique identifiers for WarpX scalar and vector fields. + * + * These are implemented as amrex::MultiFab (one or one per component "direction", + * respectively) and stored in the ablastr::fields::MultiFabRegister . + */ + AMREX_ENUM(FieldType, + None, + Efield_aux, /**< Field that the particles gather from. Obtained from Efield_fp (and Efield_cp when using MR); see UpdateAuxilaryData */ + Bfield_aux, /**< Field that the particles gather from. Obtained from Bfield_fp (and Bfield_cp when using MR); see UpdateAuxilaryData */ + Efield_fp, /**< The field that is updated by the field solver at each timestep */ + Bfield_fp, /**< The field that is updated by the field solver at each timestep */ + Efield_fp_external, /**< Stores grid particle fields provided by the user as through an openPMD file */ + Bfield_fp_external, /**< Stores grid particle fields provided by the user as through an openPMD file */ + current_fp, /**< The current that is used as a source for the field solver */ + current_fp_nodal, /**< Only used when using nodal current deposition */ + current_fp_vay, /**< Only used when using Vay current deposition */ + current_buf, /**< Particles that are close to the edge of the MR patch (i.e. in the deposition buffer) deposit to this field. */ + current_store, /**< Only used when doing subcycling with mesh refinement, for book-keeping of currents */ + rho_buf, /**< Particles that are close to the edge of the MR patch (i.e. in the deposition buffer) deposit to this field. */ + rho_fp, /**< The charge density that is used as a source for the field solver (mostly for labframe electrostatic and PSATD) */ + F_fp, /**< Used for divE cleaning */ + G_fp, /**< Used for divB cleaning */ + phi_fp, /**< Obtained by the Poisson solver, for labframe electrostatic */ + vector_potential_fp, /**< Obtained by the magnetostatic solver */ + vector_potential_fp_nodal, + vector_potential_grad_buf_e_stag, + vector_potential_grad_buf_b_stag, + hybrid_electron_pressure_fp, /**< Used with Ohm's law solver. Stores the electron pressure */ + hybrid_rho_fp_temp, /**< Used with Ohm's law solver. Stores the time interpolated/extrapolated charge density */ + hybrid_current_fp_temp, /**< Used with Ohm's law solver. Stores the time interpolated/extrapolated current density */ + hybrid_current_fp_plasma, /**< Used with Ohm's law solver. Stores plasma current calculated as J_plasma = curl x B / mu0 - J_ext */ + hybrid_current_fp_external, /**< Used with Ohm's law solver. Stores external current */ + hybrid_B_fp_external, /**< Used with Ohm's law solver. Stores external B field */ + hybrid_E_fp_external, /**< Used with Ohm's law solver. Stores external E field */ + Efield_cp, /**< Only used with MR. The field that is updated by the field solver at each timestep, on the coarse patch of each level */ + Bfield_cp, /**< Only used with MR. The field that is updated by the field solver at each timestep, on the coarse patch of each level */ + current_cp, /**< Only used with MR. The current that is used as a source for the field solver, on the coarse patch of each level */ + rho_cp, /**< Only used with MR. The charge density that is used as a source for the field solver, on the coarse patch of each level */ + F_cp, /**< Only used with MR. Used for divE cleaning, on the coarse patch of each level */ + G_cp, /**< Only used with MR. Used for divB cleaning, on the coarse patch of each level */ + Efield_cax, /**< Only used with MR. Particles that are close to the edge of the MR patch (i.e. in the gather buffer) gather from this field */ + Bfield_cax, /**< Only used with MR. Particles that are close to the edge of the MR patch (i.e. in the gather buffer) gather from this field */ + E_external_particle_field, /**< Stores external particle fields provided by the user as through an openPMD file */ + B_external_particle_field, /**< Stores external particle fields provided by the user as through an openPMD file */ + distance_to_eb, /**< Only used with embedded boundaries (EB). Stores the distance to the nearest EB */ + edge_lengths, /**< Only used with the ECT solver. Indicates the length of the cell edge that is covered by the EB, in SI units */ + face_areas, /**< Only used with the ECT solver. Indicates the area of the cell face that is covered by the EB, in SI units */ + area_mod, + pml_E_fp, + pml_B_fp, + pml_j_fp, + pml_F_fp, + pml_G_fp, + pml_E_cp, + pml_B_cp, + pml_j_cp, + pml_F_cp, + pml_G_cp, + pml_edge_lengths, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp, + B_old, /**< Stores the value of B at the beginning of the timestep, for the implicit solver */ + ECTRhofield, + Venl + ); + + /** these are vector fields */ + constexpr FieldType ArrayFieldTypes[] = { + FieldType::Efield_aux, + FieldType::Bfield_aux, + FieldType::Efield_fp, + FieldType::Bfield_fp, + FieldType::current_fp, + FieldType::current_fp_nodal, + FieldType::current_fp_vay, + FieldType::current_buf, + FieldType::current_store, + FieldType::vector_potential_fp, + FieldType::vector_potential_fp_nodal, + FieldType::vector_potential_grad_buf_e_stag, + FieldType::vector_potential_grad_buf_b_stag, + FieldType::hybrid_current_fp_temp, + FieldType::hybrid_current_fp_plasma, + FieldType::hybrid_current_fp_external, + FieldType::hybrid_B_fp_external, + FieldType::hybrid_E_fp_external, + FieldType::Efield_cp, + FieldType::Bfield_cp, + FieldType::current_cp, + FieldType::Efield_cax, + FieldType::Bfield_cax, + FieldType::E_external_particle_field, + FieldType::B_external_particle_field, + FieldType::pml_E_fp, + FieldType::pml_B_fp, + FieldType::pml_j_fp, + FieldType::pml_E_cp, + FieldType::pml_B_cp, + FieldType::pml_j_cp, + FieldType::Efield_avg_fp, + FieldType::Bfield_avg_fp, + FieldType::Efield_avg_cp, + FieldType::Bfield_avg_cp, + FieldType::B_old, + FieldType::ECTRhofield, + FieldType::Venl + }; + + /** Returns true if a FieldType represents a vector field */ + inline bool + isFieldArray (const FieldType field_type) + { + return std::any_of( std::begin(ArrayFieldTypes), std::end(ArrayFieldTypes), + [field_type](const FieldType& f) { return f == field_type; }); + } + +} + +#endif //WARPX_FIELDS_H_ diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index a095bce6ae3..66976045943 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -64,34 +64,25 @@ void BilinearFilter::ComputeStencils(){ WARPX_PROFILE("BilinearFilter::ComputeStencils()"); int i = 0; for (const auto& el : npass_each_dir ) { - stencil_length_each_dir[i++] = static_cast(el); + stencil_length_each_dir[i++] = static_cast(el) + 1; } - stencil_length_each_dir += 1.; + + m_stencil_0.resize( 1u + npass_each_dir[0] ); + compute_stencil(m_stencil_0, npass_each_dir[0]); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) + m_stencil_1.resize( 1u + npass_each_dir[1] ); + compute_stencil(m_stencil_1, npass_each_dir[1]); +#endif #if defined(WARPX_DIM_3D) - // npass_each_dir = npass_x npass_y npass_z - stencil_x.resize( 1u + npass_each_dir[0] ); - stencil_y.resize( 1u + npass_each_dir[1] ); - stencil_z.resize( 1u + npass_each_dir[2] ); - compute_stencil(stencil_x, npass_each_dir[0]); - compute_stencil(stencil_y, npass_each_dir[1]); - compute_stencil(stencil_z, npass_each_dir[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - // npass_each_dir = npass_x npass_z - stencil_x.resize( 1u + npass_each_dir[0] ); - stencil_z.resize( 1u + npass_each_dir[1] ); - compute_stencil(stencil_x, npass_each_dir[0]); - compute_stencil(stencil_z, npass_each_dir[1]); -#elif defined(WARPX_DIM_1D_Z) - // npass_each_dir = npass_z - stencil_z.resize( 1u + npass_each_dir[0] ); - compute_stencil(stencil_z, npass_each_dir[0]); + m_stencil_2.resize( 1u + npass_each_dir[2] ); + compute_stencil(m_stencil_2, npass_each_dir[2]); #endif + slen = stencil_length_each_dir.dim3(); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#if defined(WARPX_DIM_1D_Z) || defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) slen.z = 1; #endif #if defined(WARPX_DIM_1D_Z) slen.y = 1; - slen.z = 1; #endif } diff --git a/Source/Filter/Filter.H b/Source/Filter/Filter.H index 5c648a75c4c..584f6b151d7 100644 --- a/Source/Filter/Filter.H +++ b/Source/Filter/Filter.H @@ -21,9 +21,9 @@ public: // Apply stencil on MultiFab. // Guard cells are handled inside this function - void ApplyStencil(amrex::MultiFab& dstmf, - const amrex::MultiFab& srcmf, int lev, int scomp=0, - int dcomp=0, int ncomp=10000); + void ApplyStencil (amrex::MultiFab& dstmf, + const amrex::MultiFab& srcmf, int lev, int scomp=0, + int dcomp=0, int ncomp=10000); // Apply stencil on a FabArray. void ApplyStencil (amrex::FArrayBox& dstfab, @@ -31,20 +31,18 @@ public: int scomp=0, int dcomp=0, int ncomp=10000); // public for cuda - void DoFilter(const amrex::Box& tbx, - amrex::Array4 const& tmp, - amrex::Array4 const& dst, - int scomp, int dcomp, int ncomp); + void DoFilter (const amrex::Box& tbx, + amrex::Array4 const& tmp, + amrex::Array4 const& dst, + int scomp, int dcomp, int ncomp); - // In 2D, stencil_length_each_dir = {length(stencil_x), length(stencil_z)} + // Length of stencil in each included direction amrex::IntVect stencil_length_each_dir; protected: // Stencil along each direction. - // in 2D, stencil_y is not initialized. - amrex::Gpu::DeviceVector stencil_x, stencil_y, stencil_z; - // Length of each stencil. - // In 2D, slen = {length(stencil_x), length(stencil_z), 1} + amrex::Gpu::DeviceVector m_stencil_0, m_stencil_1, m_stencil_2; + // Length of each stencil, 1 for dimensions not included amrex::Dim3 slen; private: diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp index 6243ce4ebbf..40dfc54f8cb 100644 --- a/Source/Filter/Filter.cpp +++ b/Source/Filter/Filter.cpp @@ -87,23 +87,21 @@ Filter::ApplyStencil (FArrayBox& dstfab, const FArrayBox& srcfab, DoFilter(tbx, src, dst, scomp, dcomp, ncomp); } -/* \brief Apply stencil (2D/3D, CPU/GPU) +/* \brief Apply stencil (CPU/GPU) */ void Filter::DoFilter (const Box& tbx, Array4 const& src, Array4 const& dst, int scomp, int dcomp, int ncomp) { -#if (AMREX_SPACEDIM >= 2) - amrex::Real const* AMREX_RESTRICT sx = stencil_x.data(); -#endif -#if defined(WARPX_DIM_3D) - amrex::Real const* AMREX_RESTRICT sy = stencil_y.data(); -#endif - amrex::Real const* AMREX_RESTRICT sz = stencil_z.data(); + AMREX_D_TERM( + amrex::Real const* AMREX_RESTRICT s0 = m_stencil_0.data();, + amrex::Real const* AMREX_RESTRICT s1 = m_stencil_1.data();, + amrex::Real const* AMREX_RESTRICT s2 = m_stencil_2.data(); + ) Dim3 slen_local = slen; -#if defined(WARPX_DIM_3D) +#if AMREX_SPACEDIM == 3 AMREX_PARALLEL_FOR_4D ( tbx, ncomp, i, j, k, n, { Real d = 0.0; @@ -115,25 +113,25 @@ void Filter::DoFilter (const Box& tbx, return src.contains(jj,kk,ll) ? src(jj,kk,ll,nn) : 0.0_rt; }; - for (int iz=0; iz < slen_local.z; ++iz){ - for (int iy=0; iy < slen_local.y; ++iy){ - for (int ix=0; ix < slen_local.x; ++ix){ - Real sss = sx[ix]*sy[iy]*sz[iz]; - d += sss*( src_zeropad(i-ix,j-iy,k-iz,scomp+n) - +src_zeropad(i+ix,j-iy,k-iz,scomp+n) - +src_zeropad(i-ix,j+iy,k-iz,scomp+n) - +src_zeropad(i+ix,j+iy,k-iz,scomp+n) - +src_zeropad(i-ix,j-iy,k+iz,scomp+n) - +src_zeropad(i+ix,j-iy,k+iz,scomp+n) - +src_zeropad(i-ix,j+iy,k+iz,scomp+n) - +src_zeropad(i+ix,j+iy,k+iz,scomp+n)); + for (int i2=0; i2 < slen_local.z; ++i2){ + for (int i1=0; i1 < slen_local.y; ++i1){ + for (int i0=0; i0 < slen_local.x; ++i0){ + Real sss = s0[i0]*s1[i1]*s2[i2]; + d += sss*( src_zeropad(i-i0,j-i1,k-i2,scomp+n) + +src_zeropad(i+i0,j-i1,k-i2,scomp+n) + +src_zeropad(i-i0,j+i1,k-i2,scomp+n) + +src_zeropad(i+i0,j+i1,k-i2,scomp+n) + +src_zeropad(i-i0,j-i1,k+i2,scomp+n) + +src_zeropad(i+i0,j-i1,k+i2,scomp+n) + +src_zeropad(i-i0,j+i1,k+i2,scomp+n) + +src_zeropad(i+i0,j+i1,k+i2,scomp+n)); } } } dst(i,j,k,dcomp+n) = d; }); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif AMREX_SPACEDIM == 2 AMREX_PARALLEL_FOR_4D ( tbx, ncomp, i, j, k, n, { Real d = 0.0; @@ -145,21 +143,21 @@ void Filter::DoFilter (const Box& tbx, return src.contains(jj,kk,ll) ? src(jj,kk,ll,nn) : 0.0_rt; }; - for (int iz=0; iz < slen_local.z; ++iz){ - for (int iy=0; iy < slen_local.y; ++iy){ - for (int ix=0; ix < slen_local.x; ++ix){ - Real sss = sx[ix]*sz[iy]; - d += sss*( src_zeropad(i-ix,j-iy,k,scomp+n) - +src_zeropad(i+ix,j-iy,k,scomp+n) - +src_zeropad(i-ix,j+iy,k,scomp+n) - +src_zeropad(i+ix,j+iy,k,scomp+n)); + for (int i2=0; i2 < slen_local.z; ++i2){ + for (int i1=0; i1 < slen_local.y; ++i1){ + for (int i0=0; i0 < slen_local.x; ++i0){ + Real sss = s0[i0]*s1[i1]; + d += sss*( src_zeropad(i-i0,j-i1,k,scomp+n) + +src_zeropad(i+i0,j-i1,k,scomp+n) + +src_zeropad(i-i0,j+i1,k,scomp+n) + +src_zeropad(i+i0,j+i1,k,scomp+n)); } } } dst(i,j,k,dcomp+n) = d; }); -#elif defined(WARPX_DIM_1D_Z) +#elif AMREX_SPACEDIM == 1 AMREX_PARALLEL_FOR_4D ( tbx, ncomp, i, j, k, n, { Real d = 0.0; @@ -171,21 +169,18 @@ void Filter::DoFilter (const Box& tbx, return src.contains(jj,kk,ll) ? src(jj,kk,ll,nn) : 0.0_rt; }; - for (int iz=0; iz < slen_local.z; ++iz){ - for (int iy=0; iy < slen_local.y; ++iy){ - for (int ix=0; ix < slen_local.x; ++ix){ - Real sss = sz[iy]; - d += sss*( src_zeropad(i-ix,j,k,scomp+n) - +src_zeropad(i+ix,j,k,scomp+n)); + for (int i2=0; i2 < slen_local.z; ++i2){ + for (int i1=0; i1 < slen_local.y; ++i1){ + for (int i0=0; i0 < slen_local.x; ++i0){ + Real sss = s0[i0]; + d += sss*( src_zeropad(i-i0,j,k,scomp+n) + +src_zeropad(i+i0,j,k,scomp+n)); } } } dst(i,j,k,dcomp+n) = d; }); -#else - WARPX_ABORT_WITH_MESSAGE( - "Filter not implemented for the current geometry!"); #endif } @@ -278,13 +273,11 @@ void Filter::DoFilter (const Box& tbx, const auto lo = amrex::lbound(tbx); const auto hi = amrex::ubound(tbx); // tmp and dst are of type Array4 (Fortran ordering) -#if (AMREX_SPACEDIM >= 2) - amrex::Real const* AMREX_RESTRICT sx = stencil_x.data(); -#endif -#if defined(WARPX_DIM_3D) - amrex::Real const* AMREX_RESTRICT sy = stencil_y.data(); -#endif - amrex::Real const* AMREX_RESTRICT sz = stencil_z.data(); + AMREX_D_TERM( + amrex::Real const* AMREX_RESTRICT s0 = m_stencil_0.data();, + amrex::Real const* AMREX_RESTRICT s1 = m_stencil_1.data();, + amrex::Real const* AMREX_RESTRICT s2 = m_stencil_2.data(); + ) for (int n = 0; n < ncomp; ++n) { // Set dst value to 0. for (int k = lo.z; k <= hi.z; ++k) { @@ -295,41 +288,32 @@ void Filter::DoFilter (const Box& tbx, } } // 3 nested loop on 3D stencil - for (int iz=0; iz < slen.z; ++iz){ - for (int iy=0; iy < slen.y; ++iy){ - for (int ix=0; ix < slen.x; ++ix){ -#if defined(WARPX_DIM_3D) - const Real sss = sx[ix]*sy[iy]*sz[iz]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real sss = sx[ix]*sz[iy]; -#else - const Real sss = sz[ix]; -#endif + for (int i2=0; i2 < slen.z; ++i2){ + for (int i1=0; i1 < slen.y; ++i1){ + for (int i0=0; i0 < slen.x; ++i0){ + const Real sss = AMREX_D_TERM(s0[i0], *s1[i1], *s2[i2]); // 3 nested loop on 3D array for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { -#if defined(WARPX_DIM_3D) - dst(i,j,k,dcomp+n) += sss*(tmp(i-ix,j-iy,k-iz,scomp+n) - +tmp(i+ix,j-iy,k-iz,scomp+n) - +tmp(i-ix,j+iy,k-iz,scomp+n) - +tmp(i+ix,j+iy,k-iz,scomp+n) - +tmp(i-ix,j-iy,k+iz,scomp+n) - +tmp(i+ix,j-iy,k+iz,scomp+n) - +tmp(i-ix,j+iy,k+iz,scomp+n) - +tmp(i+ix,j+iy,k+iz,scomp+n)); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - dst(i,j,k,dcomp+n) += sss*(tmp(i-ix,j-iy,k,scomp+n) - +tmp(i+ix,j-iy,k,scomp+n) - +tmp(i-ix,j+iy,k,scomp+n) - +tmp(i+ix,j+iy,k,scomp+n)); -#elif defined(WARPX_DIM_1D_Z) - dst(i,j,k,dcomp+n) += sss*(tmp(i-ix,j,k,scomp+n) - +tmp(i+ix,j,k,scomp+n)); -#else - WARPX_ABORT_WITH_MESSAGE( - "Filter not implemented for the current geometry!"); +#if AMREX_SPACEDIM == 3 + dst(i,j,k,dcomp+n) += sss*(tmp(i-i0,j-i1,k-i2,scomp+n) + +tmp(i+i0,j-i1,k-i2,scomp+n) + +tmp(i-i0,j+i1,k-i2,scomp+n) + +tmp(i+i0,j+i1,k-i2,scomp+n) + +tmp(i-i0,j-i1,k+i2,scomp+n) + +tmp(i+i0,j-i1,k+i2,scomp+n) + +tmp(i-i0,j+i1,k+i2,scomp+n) + +tmp(i+i0,j+i1,k+i2,scomp+n)); +#elif AMREX_SPACEDIM == 2 + dst(i,j,k,dcomp+n) += sss*(tmp(i-i0,j-i1,k,scomp+n) + +tmp(i+i0,j-i1,k,scomp+n) + +tmp(i-i0,j+i1,k,scomp+n) + +tmp(i+i0,j+i1,k,scomp+n)); +#elif AMREX_SPACEDIM == 1 + dst(i,j,k,dcomp+n) += sss*(tmp(i-i0,j,k,scomp+n) + +tmp(i+i0,j,k,scomp+n)); #endif } } diff --git a/Source/Filter/NCIGodfreyFilter.cpp b/Source/Filter/NCIGodfreyFilter.cpp index 9567bdf1bb2..a73efb0ec64 100644 --- a/Source/Filter/NCIGodfreyFilter.cpp +++ b/Source/Filter/NCIGodfreyFilter.cpp @@ -121,17 +121,18 @@ void NCIGodfreyFilter::ComputeStencils() # endif h_stencil_z[0] /= 2._rt; - stencil_x.resize(h_stencil_x.size()); + m_stencil_0.resize(h_stencil_x.size()); + Gpu::copyAsync(Gpu::hostToDevice,h_stencil_x.begin(),h_stencil_x.end(),m_stencil_0.begin()); # if defined(WARPX_DIM_3D) - stencil_y.resize(h_stencil_y.size()); + m_stencil_1.resize(h_stencil_y.size()); + m_stencil_2.resize(h_stencil_z.size()); + Gpu::copyAsync(Gpu::hostToDevice,h_stencil_y.begin(),h_stencil_y.end(),m_stencil_1.begin()); + Gpu::copyAsync(Gpu::hostToDevice,h_stencil_z.begin(),h_stencil_z.end(),m_stencil_2.begin()); +# elif (AMREX_SPACEDIM == 2) + // In 2D, the filter applies stencil_1 to the 2nd dimension + m_stencil_1.resize(h_stencil_z.size()); + Gpu::copyAsync(Gpu::hostToDevice,h_stencil_z.begin(),h_stencil_z.end(),m_stencil_1.begin()); # endif - stencil_z.resize(h_stencil_z.size()); - - Gpu::copyAsync(Gpu::hostToDevice,h_stencil_x.begin(),h_stencil_x.end(),stencil_x.begin()); -# if defined(WARPX_DIM_3D) - Gpu::copyAsync(Gpu::hostToDevice,h_stencil_y.begin(),h_stencil_y.end(),stencil_y.begin()); -# endif - Gpu::copyAsync(Gpu::hostToDevice,h_stencil_z.begin(),h_stencil_z.end(),stencil_z.begin()); Gpu::synchronize(); } diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 23f0c46590b..c2cdfc3e19f 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -10,6 +10,8 @@ #include "WarpXFluidContainer_fwd.H" +#include + #include #include @@ -34,7 +36,7 @@ class MultiFluidContainer public: - MultiFluidContainer (int nlevs_max); + MultiFluidContainer (); ~MultiFluidContainer() = default; @@ -52,25 +54,26 @@ public: } #endif - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev); - void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + void InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev); /// /// This evolves all the fluids by one PIC time step, including current deposition, the /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. /// - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::Real cur_time, bool skip_deposition=false); + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string const& current_fp_string, + amrex::Real cur_time, + bool skip_deposition=false); [[nodiscard]] int nSpecies() const {return static_cast(species_names.size());} - void DepositCharge (int lev, amrex::MultiFab &rho); - void DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + void DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev); + void DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + int lev); private: diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index 234cefb4f07..b160817d886 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -13,7 +13,7 @@ using namespace amrex; -MultiFluidContainer::MultiFluidContainer (int nlevs_max) +MultiFluidContainer::MultiFluidContainer () { const ParmParse pp_fluids("fluids"); pp_fluids.queryarr("species_names", species_names); @@ -22,52 +22,52 @@ MultiFluidContainer::MultiFluidContainer (int nlevs_max) allcontainers.resize(nspecies); for (int i = 0; i < nspecies; ++i) { - allcontainers[i] = std::make_unique(nlevs_max, i, species_names[i]); + allcontainers[i] = std::make_unique(i, species_names[i]); } } void -MultiFluidContainer::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm) +MultiFluidContainer::AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const BoxArray& ba, const DistributionMapping& dm, int lev) { for (auto& fl : allcontainers) { - fl->AllocateLevelMFs(lev, ba, dm); + fl->AllocateLevelMFs(m_fields, ba, dm, lev); } } void -MultiFluidContainer::InitData (int lev, amrex::Box init_box, amrex::Real cur_time) +MultiFluidContainer::InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev) { for (auto& fl : allcontainers) { - fl->InitData(lev, init_box, cur_time); + fl->InitData(m_fields, init_box, cur_time, lev); } } void -MultiFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho) +MultiFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev) { for (auto& fl : allcontainers) { - fl->DepositCharge(lev,rho); + fl->DepositCharge(m_fields,rho,lev); } } void -MultiFluidContainer::DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz) +MultiFluidContainer::DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, int lev) { for (auto& fl : allcontainers) { - fl->DepositCurrent(lev,jx,jy,jz); + fl->DepositCurrent(m_fields,jx,jy,jz,lev); } } void -MultiFluidContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab* rho, MultiFab& jx, MultiFab& jy, MultiFab& jz, - amrex::Real cur_time, bool skip_deposition) +MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string const& current_fp_string, + amrex::Real cur_time, + bool skip_deposition) { for (auto& fl : allcontainers) { - fl->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, rho, jx, jy, jz, cur_time, skip_deposition); + fl->Evolve(fields, lev, current_fp_string, cur_time, skip_deposition); } } diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 04ec4d9e80d..f3ea2d9a498 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -30,7 +30,7 @@ class WarpXFluidContainer public: friend MultiFluidContainer; - WarpXFluidContainer (int nlevs_max, int ispecies, const std::string& name); + WarpXFluidContainer (int ispecies, const std::string &name); ~WarpXFluidContainer() = default; WarpXFluidContainer (WarpXFluidContainer const &) = delete; @@ -38,20 +38,20 @@ public: WarpXFluidContainer(WarpXFluidContainer&& ) = default; WarpXFluidContainer& operator=(WarpXFluidContainer&& ) = default; - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev) const; - void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + void InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev); void ReadParameters (); /** * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::Real cur_time, bool skip_deposition=false); + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, + amrex::Real cur_time, + bool skip_deposition=false); /** * AdvectivePush_Muscl takes a single timestep (dt) of the cold relativistic fluid equations @@ -61,7 +61,7 @@ public: * * \param[in] lev refinement level */ - void AdvectivePush_Muscl (int lev); + void AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& m_fields, int lev); /** @@ -72,7 +72,7 @@ public: * * \param[in] lev refinement level */ - void ApplyBcFluidsAndComms (int lev); + void ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& m_fields, int lev); #if defined(WARPX_DIM_RZ) /** @@ -83,7 +83,7 @@ public: * * \param[in] lev refinement level */ - void centrifugal_source_rz (int lev); + void centrifugal_source_rz (ablastr::fields::MultiFabRegister& m_fields, int lev); #endif /** @@ -101,10 +101,10 @@ public: * \param[in] Bz Yee magnetic field (z) * \param[in] t Current time */ - void GatherAndPush (int lev, + void GatherAndPush (ablastr::fields::MultiFabRegister& m_fields, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::Real t); + amrex::Real t, int lev); /** * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and @@ -117,8 +117,8 @@ public: * \param[in,out] jy current density MultiFab y comp. * \param[in,out] jz current density MultiFab z comp. */ - void DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + void DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, int lev); /** * DepositCharge interpolates the fluid charge density onto the Yee grid and @@ -129,7 +129,7 @@ public: * \param[in] lev refinement level * \param[in,out] rho charge density MultiFab. */ - void DepositCharge (int lev, amrex::MultiFab &rho, int icomp = 0); + void DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev, int icomp = 0); [[nodiscard]] amrex::Real getCharge () const {return charge;} [[nodiscard]] amrex::Real getMass () const {return mass;} @@ -185,9 +185,9 @@ protected: public: - // MultiFabs that contain the density (N) and momentum density (NU) of this fluid species, for each refinement level - amrex::Vector< std::unique_ptr > N; - amrex::Vector, 3 > > NU; + // Names of Multifabs that will be added to the mfs register + std::string name_mf_N = "fluid_density_"+species_name; + std::string name_mf_NU = "fluid_momentum_density_"+species_name; }; diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 11f25678dc0..0a0ca4b8818 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -4,22 +4,25 @@ * * License: BSD-3-Clause-LBNL */ -#include "ablastr/coarsen/sample.H" +#include "Fields.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Utils/WarpXProfilerWrapper.H" #include "MusclHancockUtils.H" #include "Fluids/WarpXFluidContainer.H" -#include "WarpX.H" -#include #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXUtil.H" #include "Utils/SpeciesUtils.H" +#include "WarpX.H" + +#include +#include using namespace ablastr::utils::communication; using namespace amrex; -WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std::string &name): + +WarpXFluidContainer::WarpXFluidContainer(int ispecies, const std::string &name): species_id{ispecies}, species_name{name} { @@ -50,9 +53,6 @@ WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std: } amrex::Gpu::synchronize(); - // Resize the list of MultiFabs for the right number of levels - N.resize(nlevs_max); - NU.resize(nlevs_max); } void WarpXFluidContainer::ReadParameters() @@ -139,31 +139,33 @@ void WarpXFluidContainer::ReadParameters() } } -void WarpXFluidContainer::AllocateLevelMFs(int lev, const BoxArray &ba, const DistributionMapping &dm) +void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& fields, const BoxArray &ba, const DistributionMapping &dm, int lev) const { + using ablastr::fields::Direction; const int ncomps = 1; const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); - // set human-readable tag for each MultiFab - auto const tag = [lev](std::string tagname) - { - tagname.append("[l=").append(std::to_string(lev)).append("]"); - return tagname; - }; - - WarpX::AllocInitMultiFab(N[lev], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid density"), 0.0_rt); - - WarpX::AllocInitMultiFab(NU[lev][0], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [x]"), 0.0_rt); - WarpX::AllocInitMultiFab(NU[lev][1], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [y]"), 0.0_rt); - WarpX::AllocInitMultiFab(NU[lev][2], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [z]"), 0.0_rt); + fields.alloc_init( + name_mf_N, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + fields.alloc_init( + name_mf_NU, Direction{0}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + fields.alloc_init( + name_mf_NU, Direction{1}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + fields.alloc_init( + name_mf_NU, Direction{2}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + } -void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur_time) +void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& fields, amrex::Box init_box, amrex::Real cur_time, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::InitData"); // Convert initialization box to nodal box @@ -186,14 +188,14 @@ void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Box const tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Return the intersection of all cells and the ones we wish to update amrex::Box const init_box_intersection = init_box & tile_box; @@ -253,54 +255,68 @@ void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur void WarpXFluidContainer::Evolve( + ablastr::fields::MultiFabRegister& fields, int lev, - const amrex::MultiFab &Ex, const amrex::MultiFab &Ey, const amrex::MultiFab &Ez, - const amrex::MultiFab &Bx, const amrex::MultiFab &By, const amrex::MultiFab &Bz, - amrex::MultiFab* rho, amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, - amrex::Real cur_time, bool skip_deposition) + const std::string& current_fp_string, + amrex::Real cur_time, + bool skip_deposition) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("WarpXFluidContainer::Evolve"); - if (rho && ! skip_deposition && ! do_not_deposit) { + if (fields.has(FieldType::rho_fp,lev) && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. - DepositCharge(lev, *rho, 0); + DepositCharge(fields, *fields.get(FieldType::rho_fp,lev), lev, 0); } // Step the Lorentz Term if(!do_not_gather){ - GatherAndPush(lev, Ex, Ey, Ez, Bx, By, Bz, cur_time); + GatherAndPush(fields, + *fields.get(FieldType::Efield_aux, Direction{0}, lev), + *fields.get(FieldType::Efield_aux, Direction{1}, lev), + *fields.get(FieldType::Efield_aux, Direction{2}, lev), + *fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *fields.get(FieldType::Bfield_aux, Direction{2}, lev), + cur_time, lev); } // Cylindrical centrifugal term if(!do_not_push){ #if defined(WARPX_DIM_RZ) - centrifugal_source_rz(lev); + centrifugal_source_rz(fields, lev); #endif // Apply (non-periodic) BC on the fluids (needed for spatial derivative), // and communicate N, NU at boundaries - ApplyBcFluidsAndComms(lev); + ApplyBcFluidsAndComms(fields, lev); // Step the Advective term - AdvectivePush_Muscl(lev); + AdvectivePush_Muscl(fields, lev); } // Deposit rho to the simulation mesh // Deposit charge (end of the step) - if (rho && ! skip_deposition && ! do_not_deposit) { - DepositCharge(lev, *rho, 1); + if (fields.has(FieldType::rho_fp,lev) && ! skip_deposition && ! do_not_deposit) { + DepositCharge(fields, *fields.get(FieldType::rho_fp,lev), lev, 1); } // Deposit J to the simulation mesh if (!skip_deposition && ! do_not_deposit) { - DepositCurrent(lev, jx, jy, jz); + DepositCurrent(fields, + *fields.get(current_fp_string, Direction{0}, lev), + *fields.get(current_fp_string, Direction{1}, lev), + *fields.get(current_fp_string, Direction{2}, lev), + lev); } } // Momentum source due to curvature -void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) +void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::ApplyBcFluidsAndComms"); WarpX &warpx = WarpX::GetInstance(); @@ -315,15 +331,15 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - const amrex::Array4 N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + const amrex::Array4 N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); //Grow the tilebox tile_box.grow(1); @@ -395,15 +411,16 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) } // Fill guard cells - FillBoundary(*N[lev], N[lev]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][0], NU[lev][0]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][1], NU[lev][1]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][2], NU[lev][2]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_N, lev), fields.get(name_mf_N, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{0}, lev), fields.get(name_mf_NU, Direction{0}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{1}, lev), fields.get(name_mf_NU, Direction{1}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{2}, lev), fields.get(name_mf_NU, Direction{2}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); } // Muscl Advection Update -void WarpXFluidContainer::AdvectivePush_Muscl (int lev) +void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::AdvectivePush_Muscl"); // Grab the grid spacing @@ -434,31 +451,31 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) const amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[0]); #endif - const amrex::BoxArray ba = N[lev]->boxArray(); + const amrex::BoxArray ba = fields.get(name_mf_N, lev)->boxArray(); // Temporary Half-step values #if defined(WARPX_DIM_3D) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #else - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #endif // Fill edge values of N and U at the half timestep for MUSCL #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { // Loop over a box with one extra gridpoint in the ghost region to avoid @@ -476,10 +493,10 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) return tt; }(); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Boxes are computed to avoid going out of bounds. // Grow the entire domain @@ -741,13 +758,13 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - const amrex::Array4 N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + const amrex::Box tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + const amrex::Array4 N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); #if defined(WARPX_DIM_3D) amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); @@ -878,8 +895,9 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Momentum source due to curvature #if defined(WARPX_DIM_RZ) -void WarpXFluidContainer::centrifugal_source_rz (int lev) +void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegister& fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::centrifugal_source_rz"); WarpX &warpx = WarpX::GetInstance(); @@ -894,15 +912,15 @@ void WarpXFluidContainer::centrifugal_source_rz (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept @@ -947,11 +965,13 @@ void WarpXFluidContainer::centrifugal_source_rz (int lev) // Momentum source from fields void WarpXFluidContainer::GatherAndPush ( - int lev, + ablastr::fields::MultiFabRegister& fields, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - Real t) + Real t, + int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::GatherAndPush"); WarpX &warpx = WarpX::GetInstance(); @@ -978,7 +998,7 @@ void WarpXFluidContainer::GatherAndPush ( auto Bz_type = amrex::GpuArray{0, 0, 0}; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - Nodal_type[i] = N[lev]->ixType()[i]; + Nodal_type[i] = fields.get(name_mf_N, lev)->ixType()[i]; Ex_type[i] = Ex.ixType()[i]; Ey_type[i] = Ey.ixType()[i]; Ez_type[i] = Ez.ixType()[i]; @@ -990,24 +1010,23 @@ void WarpXFluidContainer::GatherAndPush ( // External field parsers external_e_fields = (m_E_ext_s == "parse_e_ext_function"); external_b_fields = (m_B_ext_s == "parse_b_ext_function"); + amrex::ParserExecutor<4> Exfield_parser; amrex::ParserExecutor<4> Eyfield_parser; amrex::ParserExecutor<4> Ezfield_parser; amrex::ParserExecutor<4> Bxfield_parser; amrex::ParserExecutor<4> Byfield_parser; amrex::ParserExecutor<4> Bzfield_parser; + if (external_e_fields){ - constexpr int num_arguments = 4; //x,y,z,t - Exfield_parser = m_Ex_parser->compile(); - Eyfield_parser = m_Ey_parser->compile(); - Ezfield_parser = m_Ez_parser->compile(); + Exfield_parser = m_Ex_parser->compile<4>(); + Eyfield_parser = m_Ey_parser->compile<4>(); + Ezfield_parser = m_Ez_parser->compile<4>(); } - if (external_b_fields){ - constexpr int num_arguments = 4; //x,y,z,t - Bxfield_parser = m_Bx_parser->compile(); - Byfield_parser = m_By_parser->compile(); - Bzfield_parser = m_Bz_parser->compile(); + Bxfield_parser = m_Bx_parser->compile<4>(); + Byfield_parser = m_By_parser->compile<4>(); + Bzfield_parser = m_Bz_parser->compile<4>(); } @@ -1015,15 +1034,15 @@ void WarpXFluidContainer::GatherAndPush ( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::Array4 const& Ex_arr = Ex.array(mfi); amrex::Array4 const& Ey_arr = Ey.array(mfi); @@ -1035,14 +1054,27 @@ void WarpXFluidContainer::GatherAndPush ( // Here, we do not perform any coarsening. const amrex::GpuArray coarsening_ratio = {1, 1, 1}; - amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + enum exte_flags : int { no_exte, has_exte }; + enum extb_flags : int { no_extb, has_extb }; + enum boost_flags : int { no_gamma_boost, has_gamma_boost }; + const int exte_runtime_flag = external_e_fields ? has_exte : no_exte; + const int extb_runtime_flag = external_b_fields ? has_extb : no_extb; + const int boost_runtime_flag = (gamma_boost > 1._rt) ? has_gamma_boost : no_gamma_boost; + + amrex::ParallelFor(TypeList, + CompileTimeOptions, + CompileTimeOptions>{}, + {exte_runtime_flag, extb_runtime_flag, boost_runtime_flag}, + tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k, + auto exte_control, auto extb_control, auto boost_control) noexcept { // Only run if density is positive if (N_arr(i,j,k)>0.0) { // Interpolate fields from tmp to Nodal points + // NOLINTBEGIN(misc-const-correctness) amrex::Real Ex_Nodal = ablastr::coarsen::sample::Interp(Ex_arr, Ex_type, Nodal_type, coarsening_ratio, i, j, k, 0); amrex::Real Ey_Nodal = ablastr::coarsen::sample::Interp(Ey_arr, @@ -1055,9 +1087,16 @@ void WarpXFluidContainer::GatherAndPush ( By_type, Nodal_type, coarsening_ratio, i, j, k, 0); amrex::Real Bz_Nodal = ablastr::coarsen::sample::Interp(Bz_arr, Bz_type, Nodal_type, coarsening_ratio, i, j, k, 0); + // NOLINTEND(misc-const-correctness) - if (gamma_boost > 1._rt) { // Lorentz transform fields due to moving frame - if ( ( external_b_fields ) || ( external_e_fields ) ){ +#ifdef AMREX_USE_CUDA + amrex::ignore_unused(Exfield_parser, Eyfield_parser, Ezfield_parser, + Bxfield_parser, Byfield_parser, Bzfield_parser, + gamma_boost, problo, dx, t, beta_boost); +#endif + + if constexpr (boost_control == has_gamma_boost) { // Lorentz transform fields due to moving frame + if constexpr (exte_control == has_exte || extb_control == has_extb) { // Lorentz transform z (from boosted to lab frame) amrex::Real Ex_ext_boost, Ey_ext_boost, Ez_ext_boost; @@ -1086,7 +1125,7 @@ void WarpXFluidContainer::GatherAndPush ( const amrex::Real z_lab = gamma_boost*(z + beta_boost*PhysConst::c*t); // Grab the external fields in the lab frame: - if ( external_e_fields ) { + if ( exte_control == has_exte ) { Ex_ext_lab = Exfield_parser(x, y, z_lab, t_lab); Ey_ext_lab = Eyfield_parser(x, y, z_lab, t_lab); Ez_ext_lab = Ezfield_parser(x, y, z_lab, t_lab); @@ -1095,7 +1134,7 @@ void WarpXFluidContainer::GatherAndPush ( Ey_ext_lab = 0.0; Ez_ext_lab = 0.0; } - if ( external_b_fields ) { + if ( extb_control == has_extb ) { Bx_ext_lab = Bxfield_parser(x, y, z_lab, t_lab); By_ext_lab = Byfield_parser(x, y, z_lab, t_lab); Bz_ext_lab = Bzfield_parser(x, y, z_lab, t_lab); @@ -1126,7 +1165,7 @@ void WarpXFluidContainer::GatherAndPush ( } else { // Added external e fields: - if ( external_e_fields ){ + if constexpr ( exte_control == has_exte ){ #if defined(WARPX_DIM_3D) const amrex::Real x = problo[0] + i * dx[0]; const amrex::Real y = problo[1] + j * dx[1]; @@ -1147,7 +1186,7 @@ void WarpXFluidContainer::GatherAndPush ( } // Added external b fields: - if ( external_b_fields ){ + if ( extb_control == has_extb ){ #if defined(WARPX_DIM_3D) const amrex::Real x = problo[0] + i * dx[0]; const amrex::Real y = problo[1] + j * dx[1]; @@ -1198,7 +1237,7 @@ void WarpXFluidContainer::GatherAndPush ( } } -void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icomp) +void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& fields, amrex::MultiFab &rho, int lev, int icomp) { WARPX_PROFILE("WarpXFluidContainer::DepositCharge"); @@ -1215,11 +1254,11 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); const amrex::Array4 rho_arr = rho.array(mfi); const amrex::Array4 owner_mask_rho_arr = owner_mask_rho->array(mfi); @@ -1235,15 +1274,17 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom void WarpXFluidContainer::DepositCurrent( - int lev, - amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz) + ablastr::fields::MultiFabRegister& fields, + amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, + int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::DepositCurrent"); // Temporary nodal currents - amrex::MultiFab tmp_jx_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jy_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jz_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jx_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jy_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jz_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); const amrex::Real inv_clight_sq = 1.0_prt / PhysConst::c / PhysConst::c; const amrex::Real q = getCharge(); @@ -1273,14 +1314,14 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); const amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); const amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); @@ -1308,7 +1349,7 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { amrex::Box const &tile_box_x = mfi.tilebox(jx.ixType().toIntVect()); amrex::Box const &tile_box_y = mfi.tilebox(jy.ixType().toIntVect()); diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index bfb134d6660..4a5e62bb55a 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -16,3 +16,5 @@ foreach(D IN LISTS WarpX_DIMS) WarpXInitData.cpp ) endforeach() + +add_subdirectory(DivCleaner) diff --git a/Source/Initialization/DivCleaner/CMakeLists.txt b/Source/Initialization/DivCleaner/CMakeLists.txt new file mode 100644 index 00000000000..9acedc05fb0 --- /dev/null +++ b/Source/Initialization/DivCleaner/CMakeLists.txt @@ -0,0 +1,7 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + ProjectionDivCleaner.cpp + ) +endforeach() diff --git a/Source/Initialization/DivCleaner/Make.package b/Source/Initialization/DivCleaner/Make.package new file mode 100644 index 00000000000..2ef55d8f4fc --- /dev/null +++ b/Source/Initialization/DivCleaner/Make.package @@ -0,0 +1,3 @@ +CEXE_sources += ProjectionDivCleaner.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization/DivCleaner diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H new file mode 100644 index 00000000000..2fedb83cd36 --- /dev/null +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H @@ -0,0 +1,98 @@ + /* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: S. Eric Clark (Helion Energy, Inc.) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PROJECTION_DIV_CLEANER_H_ +#define WARPX_PROJECTION_DIV_CLEANER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Fields.H" +#include "Utils/Parser/ParserUtils.H" + +namespace warpx::initialization { + +class ProjectionDivCleaner +{ +protected: + int m_levels = 1; // Hard coded to 1 for now, will only clean first level + + int m_ref_ratio = 1; + + // For MLMG solver + int m_verbose = 2; + int m_bottom_verbose = 0; + int m_max_iter = 5000; + int m_max_fmg_iter = 1000; + int m_linop_maxorder = 3; + bool m_agglomeration = false; + bool m_consolidation = false; + bool m_semicoarsening = true; + int m_max_coarsening_level = 10; + int m_max_semicoarsening_level = 10; + amrex::BottomSolver m_bottom_solver = amrex::BottomSolver::bicgstab; + + + amrex::Real m_rtol; + amrex::Real m_atol; + + std::string m_field_name; + +public: + amrex::Vector< std::unique_ptr > m_solution; + amrex::Vector< std::unique_ptr > m_source; + + amrex::Vector m_h_stencil_coefs_x; +#if !defined(WARPX_DIM_RZ) + amrex::Vector m_h_stencil_coefs_y; +#endif + amrex::Vector m_h_stencil_coefs_z; + + amrex::Gpu::DeviceVector m_stencil_coefs_x; +#if !defined(WARPX_DIM_RZ) + amrex::Gpu::DeviceVector m_stencil_coefs_y; +#endif + amrex::Gpu::DeviceVector m_stencil_coefs_z; + + // Default Constructor + ProjectionDivCleaner (std::string const& a_field_name); + + void ReadParameters (); + + void solve (); + void setSourceFromBfield (); + void correctBfield (); + +}; + +} // end namespace warpx::initialization + +#endif // WARPX_PROJECTION_DIV_CLEANER_H_ diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp new file mode 100644 index 00000000000..d7a3bb3ac92 --- /dev/null +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -0,0 +1,361 @@ + /* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: S. Eric Clark (Helion Energy, Inc.) + * + * License: BSD-3-Clause-LBNL + */ + +#include "ProjectionDivCleaner.H" + +#include +#include +#include + +#include +#if defined WARPX_DIM_RZ +#include +#else +#include +#endif +#include "Fields.H" +#include +#include +#include + +#include + +using namespace amrex; + +namespace warpx::initialization { + +ProjectionDivCleaner::ProjectionDivCleaner(std::string const& a_field_name) : + m_field_name(a_field_name) +{ + using ablastr::fields::Direction; + ReadParameters(); + + auto& warpx = WarpX::GetInstance(); + + // Only div clean level 0 + if (warpx.finestLevel() > 0) { + ablastr::warn_manager::WMRecordWarning("Projection Div Cleaner", + "Multiple AMR levels detected, only first level has been cleaned.", + ablastr::warn_manager::WarnPriority::low); + } + + m_solution.resize(m_levels); + m_source.resize(m_levels); + + const int ncomps = WarpX::ncomps; + auto const& ng = warpx.m_fields.get(m_field_name, Direction{0}, 0)->nGrowVect(); + + for (int lev = 0; lev < m_levels; ++lev) + { + // Default BoxArray and DistributionMap for initializing the output MultiFab, m_mf_output. + const amrex::BoxArray& ba = warpx.boxArray(lev); + const amrex::DistributionMapping& dmap = warpx.DistributionMap(lev); + + m_solution[lev].reset(); + m_source[lev].reset(); + + const auto tag1 = amrex::MFInfo().SetTag("div_cleaner_solution"); + m_solution[lev] = std::make_unique(amrex::convert(ba, IntVect::TheCellVector()), + dmap, ncomps, ng, tag1); + const auto tag2 = amrex::MFInfo().SetTag("div_cleaner_source"); + m_source[lev] = std::make_unique(amrex::convert(ba, IntVect::TheCellVector()), + dmap, ncomps, ng, tag2); + + m_solution[lev]->setVal(0.0, ng); + m_source[lev]->setVal(0.0, ng); + } + + auto cell_size = WarpX::CellSize(0); +#if defined(WARPX_DIM_RZ) + CylindricalYeeAlgorithm::InitializeStencilCoefficients( cell_size, + m_h_stencil_coefs_x, m_h_stencil_coefs_z ); +#else + CartesianYeeAlgorithm::InitializeStencilCoefficients( cell_size, + m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z ); +#endif + + m_stencil_coefs_x.resize(m_h_stencil_coefs_x.size()); +#if !defined(WARPX_DIM_RZ) + m_stencil_coefs_y.resize(m_h_stencil_coefs_y.size()); +#endif + m_stencil_coefs_z.resize(m_h_stencil_coefs_z.size()); + + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, + m_h_stencil_coefs_x.begin(), m_h_stencil_coefs_x.end(), + m_stencil_coefs_x.begin()); +#if !defined(WARPX_DIM_RZ) + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, + m_h_stencil_coefs_y.begin(), m_h_stencil_coefs_y.end(), + m_stencil_coefs_y.begin()); +#endif + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, + m_h_stencil_coefs_z.begin(), m_h_stencil_coefs_z.end(), + m_stencil_coefs_z.begin()); + amrex::Gpu::synchronize(); +} + +void +ProjectionDivCleaner::ReadParameters () +{ + // Initialize tolerance based on field precision + if constexpr (std::is_same_v) { + m_rtol = 5e-5; + m_atol = 0.0; + } + else { + m_rtol = 5e-12; + m_atol = 0.0; + } + + const ParmParse pp_divb_cleaner("projection_divb_cleaner"); + + // Defaults to rtol 5e-12 for double fields and 5e-5 for single + utils::parser::queryWithParser(pp_divb_cleaner, "atol", m_atol); + utils::parser::queryWithParser(pp_divb_cleaner, "rtol", m_rtol); +} + +void +ProjectionDivCleaner::solve () +{ + // Get WarpX object + auto & warpx = WarpX::GetInstance(); + + const auto& ba = warpx.boxArray(); + const auto& dmap = warpx.DistributionMap(); + const auto& geom = warpx.Geom(); + + // Pull boundary conditions from WarpX class + // bogus values are overwritten. + amrex::Array lobc({AMREX_D_DECL(LinOpBCType::bogus, + LinOpBCType::bogus, + LinOpBCType::bogus)}); + amrex::Array hibc({AMREX_D_DECL(LinOpBCType::bogus, + LinOpBCType::bogus, + LinOpBCType::bogus)}); + + std::map bcmap{ + {FieldBoundaryType::PEC, LinOpBCType::Dirichlet}, + {FieldBoundaryType::Neumann, LinOpBCType::Neumann}, // Note that PMC is the same as Neumann + {FieldBoundaryType::Periodic, LinOpBCType::Periodic}, + {FieldBoundaryType::None, LinOpBCType::Neumann} + }; + + for (int idim=0; idim 0) { + mlpoisson.setCoarseFineBC(m_solution[ilev-1].get(), m_ref_ratio); + } + + mlpoisson.setLevelBC(ilev, m_solution[ilev].get()); + + MLMG mlmg(mlpoisson); + mlmg.setMaxIter(m_max_iter); + mlmg.setMaxFmgIter(m_max_fmg_iter); + mlmg.setBottomSolver(m_bottom_solver); + mlmg.setVerbose(m_verbose); + mlmg.setBottomVerbose(m_bottom_verbose); + mlmg.setAlwaysUseBNorm(false); + mlmg.solve({m_solution[ilev].get()}, {m_source[ilev].get()}, m_rtol, m_atol); + + // Synchronize the ghost cells, do halo exchange + ablastr::utils::communication::FillBoundary(*m_solution[ilev], + m_solution[ilev]->nGrowVect(), + WarpX::do_single_precision_comms, + geom[ilev].periodicity()); + } +} + +void +ProjectionDivCleaner::setSourceFromBfield () +{ + using ablastr::fields::Direction; + + // Get WarpX object + auto & warpx = WarpX::GetInstance(); + const auto& geom = warpx.Geom(); + + // This function will compute -divB and store it in the source multifab + for (int ilev = 0; ilev < m_levels; ++ilev) + { + WarpX::ComputeDivB( + *m_source[ilev], + 0, + {warpx.m_fields.get(m_field_name, Direction{0}, ilev), + warpx.m_fields.get(m_field_name, Direction{1}, ilev), + warpx.m_fields.get(m_field_name, Direction{2}, ilev)}, + WarpX::CellSize(0) + ); + + m_source[ilev]->mult(-1._rt); + + // Synchronize the ghost cells, do halo exchange + ablastr::utils::communication::FillBoundary(*m_source[ilev], + m_source[ilev]->nGrowVect(), + WarpX::do_single_precision_comms, + geom[ilev].periodicity()); + } +} + +void +ProjectionDivCleaner::correctBfield () +{ + using ablastr::fields::Direction; + + // Get WarpX object + auto & warpx = WarpX::GetInstance(); + const auto& geom = warpx.Geom(); + + // This function computes the gradient of the solution and subtracts out divB component from B + for (int ilev = 0; ilev < m_levels; ++ilev) + { + // Grab B-field multifabs at this level + amrex::MultiFab* Bx = warpx.m_fields.get(m_field_name, Direction{0}, ilev); + amrex::MultiFab* By = warpx.m_fields.get(m_field_name, Direction{1}, ilev); + amrex::MultiFab* Bz = warpx.m_fields.get(m_field_name, Direction{2}, ilev); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*m_solution[ilev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + // Grab references to B field arrays for this grid/tile + amrex::Array4 const& Bx_arr = Bx->array(mfi); +#if !defined(WARPX_DIM_RZ) + amrex::Array4 const& By_arr = By->array(mfi); +#endif + amrex::Array4 const& Bz_arr = Bz->array(mfi); + + // Extract stencil coefficients + Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); +#if !defined(WARPX_DIM_RZ) + Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); +#endif + Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); + + const Box& tbx = mfi.tilebox(Bx->ixType().toIntVect()); +#if !defined(WARPX_DIM_RZ) + const Box& tby = mfi.tilebox(By->ixType().toIntVect()); +#endif + const Box& tbz = mfi.tilebox(Bz->ixType().toIntVect()); + + amrex::Array4 const& sol_arr = m_solution[ilev]->array(mfi); + +#if defined(WARPX_DIM_RZ) + amrex::ParallelFor(tbx, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Bx_arr(i,j,0) += CylindricalYeeAlgorithm::DownwardDr(sol_arr, coefs_x, n_coefs_x, i, j, 0, 0); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Bz_arr(i,j,0) += CylindricalYeeAlgorithm::DownwardDz(sol_arr, coefs_z, n_coefs_z, i, j, 0, 0); + }); +#else + amrex::ParallelFor(tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + Bx_arr(i,j,k) += CartesianYeeAlgorithm::DownwardDx(sol_arr, coefs_x, n_coefs_x, i, j, k); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + By_arr(i,j,k) += CartesianYeeAlgorithm::DownwardDy(sol_arr, coefs_y, n_coefs_y, i, j, k); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + Bz_arr(i,j,k) += CartesianYeeAlgorithm::DownwardDz(sol_arr, coefs_z, n_coefs_z, i, j, k); + }); +#endif + } + // Synchronize the ghost cells, do halo exchange + ablastr::utils::communication::FillBoundary(*Bx, + Bx->nGrowVect(), + WarpX::do_single_precision_comms, + geom[ilev].periodicity()); + ablastr::utils::communication::FillBoundary(*By, + By->nGrowVect(), + WarpX::do_single_precision_comms, + geom[ilev].periodicity()); + ablastr::utils::communication::FillBoundary(*Bz, + Bz->nGrowVect(), + WarpX::do_single_precision_comms, + geom[ilev].periodicity()); + amrex::Gpu::synchronize(); + } +} + +} // namespace warpx::initialization + +void +WarpX::ProjectionCleanDivB() { + WARPX_PROFILE("WarpX::ProjectionDivCleanB()"); + + if (grid_type == GridType::Collocated) { + ablastr::warn_manager::WMRecordWarning("Projection Div Cleaner", + "Grid Type is collocated, so divB not cleaned. Interpolation may lead to non-zero B field divergence.", + ablastr::warn_manager::WarnPriority::low); + } else if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee + || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC + || ( (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame + || WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + && WarpX::poisson_solver_id == PoissonSolverAlgo::Multigrid)) { + amrex::Print() << Utils::TextMsg::Info( "Starting Projection B-Field divergence cleaner."); + + if constexpr (!std::is_same_v) { + ablastr::warn_manager::WMRecordWarning("Projection Div Cleaner", + "WarpX is running with a field precision of SINGLE." + "Convergence of projection based div cleaner is not optimal and may fail.", + ablastr::warn_manager::WarnPriority::low); + } + + warpx::initialization::ProjectionDivCleaner dc("Bfield_fp_external"); + + + dc.setSourceFromBfield(); + dc.solve(); + dc.correctBfield(); + + amrex::Print() << Utils::TextMsg::Info( "Finished Projection B-Field divergence cleaner."); + } else { + ablastr::warn_manager::WMRecordWarning("Projection Div Cleaner", + "Only Yee, HybridPIC, and MLMG based static Labframe solvers are currently supported, so divB not cleaned. " + "Interpolation may lead to non-zero B field divergence.", + ablastr::warn_manager::WarnPriority::low); + } +} diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H index b32e361bff4..4cd4c41ccf2 100644 --- a/Source/Initialization/ExternalField.H +++ b/Source/Initialization/ExternalField.H @@ -23,7 +23,8 @@ enum class ExternalFieldType default_zero, constant, parse_ext_grid_function, - read_from_file + read_from_file, + load_from_python }; /** diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index 69120c0ae3c..504fb1ce7a5 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -47,6 +47,9 @@ namespace else if ( s == "read_from_file"){ return ExternalFieldType::read_from_file; } + else if ( s == "load_from_python"){ + return ExternalFieldType::load_from_python; + } else{ WARPX_ABORT_WITH_MESSAGE( "'" + s + "' is an unknown external field type!"); @@ -124,11 +127,11 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) str_Bz_ext_grid_function); Bxfield_parser = std::make_unique( - utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z","t"})); Byfield_parser = std::make_unique( - utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z","t"})); Bzfield_parser = std::make_unique( - utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z","t"})); } //___________________________________________________________________________ @@ -160,11 +163,11 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) str_Ez_ext_grid_function); Exfield_parser = std::make_unique( - utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z","t"})); Eyfield_parser = std::make_unique( - utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z","t"})); Ezfield_parser = std::make_unique( - utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z","t"})); } //___________________________________________________________________________ diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index e643e61dc2c..5593758c407 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -11,4 +11,6 @@ CEXE_sources += WarpXAMReXInit.cpp CEXE_sources += WarpXInit.cpp CEXE_sources += WarpXInitData.cpp +include $(WARPX_HOME)/Source/Initialization/DivCleaner/Make.package + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index b9fe2323290..f14720d271c 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -131,6 +131,8 @@ public: int flux_normal_axis; int flux_direction; // -1 for left, +1 for right + bool m_inject_from_eb = false; // whether to inject from the embedded boundary + bool radially_weighted = true; std::string str_flux_function; diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 2e338b20c09..468d9e7e336 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -9,6 +9,7 @@ */ #include "PlasmaInjector.H" +#include "EmbeddedBoundary/Enabled.H" #include "Initialization/GetTemperature.H" #include "Initialization/GetVelocity.H" #include "Initialization/InjectorDensity.H" @@ -51,11 +52,11 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, { #ifdef AMREX_USE_GPU - static_assert(std::is_trivially_copyable::value, + static_assert(std::is_trivially_copyable_v, "InjectorPosition must be trivially copyable"); - static_assert(std::is_trivially_copyable::value, + static_assert(std::is_trivially_copyable_v, "InjectorDensity must be trivially copyable"); - static_assert(std::is_trivially_copyable::value, + static_assert(std::is_trivially_copyable_v, "InjectorMomentum must be trivially copyable"); #endif @@ -303,50 +304,66 @@ void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) "(Please visit PR#765 for more information.)"); } #endif - utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); - std::string flux_normal_axis_string; - utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); - flux_normal_axis = -1; + + // Check whether injection from the embedded boundary is requested + utils::parser::queryWithParser(pp_species, source_name, "inject_from_embedded_boundary", m_inject_from_eb); + if (m_inject_from_eb) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( EB::enabled(), + "Error: Embedded boundary injection is only available when " + "embedded boundaries are enabled."); + flux_normal_axis = 2; // Interpret z as the normal direction to the EB + flux_direction = 1; + } else { + // Injection is through a plane in this case. + // Parse the parameters of the plane (position, normal direction, etc.) + + utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); + std::string flux_normal_axis_string; + utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); + flux_normal_axis = -1; #ifdef WARPX_DIM_RZ - if (flux_normal_axis_string == "r" || flux_normal_axis_string == "R") { - flux_normal_axis = 0; - } - if (flux_normal_axis_string == "t" || flux_normal_axis_string == "T") { - flux_normal_axis = 1; - } + if (flux_normal_axis_string == "r" || flux_normal_axis_string == "R") { + flux_normal_axis = 0; + } + if (flux_normal_axis_string == "t" || flux_normal_axis_string == "T") { + flux_normal_axis = 1; + } #else # ifndef WARPX_DIM_1D_Z - if (flux_normal_axis_string == "x" || flux_normal_axis_string == "X") { - flux_normal_axis = 0; - } + if (flux_normal_axis_string == "x" || flux_normal_axis_string == "X") { + flux_normal_axis = 0; + } # endif #endif #ifdef WARPX_DIM_3D - if (flux_normal_axis_string == "y" || flux_normal_axis_string == "Y") { - flux_normal_axis = 1; - } + if (flux_normal_axis_string == "y" || flux_normal_axis_string == "Y") { + flux_normal_axis = 1; + } #endif - if (flux_normal_axis_string == "z" || flux_normal_axis_string == "Z") { - flux_normal_axis = 2; - } + if (flux_normal_axis_string == "z" || flux_normal_axis_string == "Z") { + flux_normal_axis = 2; + } #ifdef WARPX_DIM_3D - const std::string flux_normal_axis_help = "'x', 'y', or 'z'."; + const std::string flux_normal_axis_help = "'x', 'y', or 'z'."; #else # ifdef WARPX_DIM_RZ - const std::string flux_normal_axis_help = "'r' or 'z'."; + const std::string flux_normal_axis_help = "'r' or 'z'."; # elif WARPX_DIM_XZ - const std::string flux_normal_axis_help = "'x' or 'z'."; + const std::string flux_normal_axis_help = "'x' or 'z'."; # else - const std::string flux_normal_axis_help = "'z'."; + const std::string flux_normal_axis_help = "'z'."; # endif -#endif - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_normal_axis >= 0, - "Error: Invalid value for flux_normal_axis. It must be " + flux_normal_axis_help); - utils::parser::getWithParser(pp_species, source_name, "flux_direction", flux_direction); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_direction == +1 || flux_direction == -1, - "Error: flux_direction must be -1 or +1."); + #endif + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_normal_axis >= 0, + "Error: Invalid value for flux_normal_axis. It must be " + flux_normal_axis_help); + utils::parser::getWithParser(pp_species, source_name, "flux_direction", flux_direction); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_direction == +1 || flux_direction == -1, + "Error: flux_direction must be -1 or +1."); + } + // Construct InjectorPosition with InjectorPositionRandom. h_flux_pos = std::make_unique( (InjectorPositionRandomPlane*)nullptr, diff --git a/Source/Initialization/ReconnectionPerturbation.H b/Source/Initialization/ReconnectionPerturbation.H index 50e2388c019..6de121cde9a 100644 --- a/Source/Initialization/ReconnectionPerturbation.H +++ b/Source/Initialization/ReconnectionPerturbation.H @@ -50,9 +50,8 @@ public: Reconnection_Perturbation (){} #ifndef WARPX_DIM_RZ - static void AddBfieldPerturbation (amrex::MultiFab *Bx, - amrex::MultiFab *By, - amrex::MultiFab *Bz, + static void AddBfieldPerturbation ( + warpx::fields::FieldType field, amrex::ParserExecutor<3> const& xfield_parser, amrex::ParserExecutor<3> const& yfield_parser, amrex::ParserExecutor<3> const& zfield_parser, const int lev, diff --git a/Source/Initialization/ReconnectionPerturbation.cpp b/Source/Initialization/ReconnectionPerturbation.cpp index 32fb717dcdc..f0c946af595 100644 --- a/Source/Initialization/ReconnectionPerturbation.cpp +++ b/Source/Initialization/ReconnectionPerturbation.cpp @@ -48,9 +48,8 @@ using namespace amrex; #ifndef WARPX_DIM_RZ void -Reconnection_Perturbation::AddBfieldPerturbation (amrex::MultiFab *Bx, - amrex::MultiFab *By, - amrex::MultiFab *Bz, +Reconnection_Perturbation::AddBfieldPerturbation ( + warpx::fields::FieldType field, amrex::ParserExecutor<3> const& xfield_parser, amrex::ParserExecutor<3> const& yfield_parser, amrex::ParserExecutor<3> const& zfield_parser, const int lev, @@ -65,6 +64,12 @@ Reconnection_Perturbation::AddBfieldPerturbation (amrex::MultiFab *Bx, } } const RealBox& real_box = warpx.Geom(lev).ProbDomain(); + + using ablastr::fields::Direction; + amrex::MultiFab* Bx = warpx.m_fields.get(field, Direction{0}, lev); + amrex::MultiFab* By = warpx.m_fields.get(field, Direction{1}, lev); + amrex::MultiFab* Bz = warpx.m_fields.get(field, Direction{2}, lev); + amrex::IntVect x_nodal_flag = Bx->ixType().toIntVect(); amrex::IntVect z_nodal_flag = Bz->ixType().toIntVect(); amrex::ignore_unused(xfield_parser, yfield_parser, By); diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 900dbb53e06..2c43168f5ef 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -7,76 +7,167 @@ #include "Initialization/WarpXAMReXInit.H" +#include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include +#include #include +#include #include -#include +#include +#include #include namespace { - /** Overwrite defaults in AMReX Inputs - * - * This overwrites defaults in amrex::ParmParse for inputs. - */ - void - overwrite_amrex_parser_defaults () - { - amrex::ParmParse pp_amrex("amrex"); +#ifdef AMREX_USE_GPU + constexpr auto amrex_use_gpu = true; +#else + constexpr auto amrex_use_gpu = false; +#endif + + void override_default_abort_on_out_of_gpu_memory () + { // https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters - bool abort_on_out_of_gpu_memory = true; // AMReX' default: false + auto pp_amrex = amrex::ParmParse{"amrex"}; + bool abort_on_out_of_gpu_memory = true; // AMReX's default: false pp_amrex.queryAdd("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory); + } - bool the_arena_is_managed = false; // AMReX' default: true + void override_default_the_arena_is_managed () + { + auto pp_amrex = amrex::ParmParse{"amrex"}; + bool the_arena_is_managed = false; // AMReX's default: true pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed); + } + void override_default_omp_threads () + { // https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html - std::string omp_threads = "nosmt"; // AMReX' default: system + auto pp_amrex = amrex::ParmParse{"amrex"}; + std::string omp_threads = "nosmt"; // AMReX's default: system pp_amrex.queryAdd("omp_threads", omp_threads); + } + void set_device_synchronization () + { + //See https://github.com/AMReX-Codes/amrex/pull/3763 + auto warpx_do_device_synchronize = amrex_use_gpu; + + auto pp_warpx = amrex::ParmParse{"warpx"}; + pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize); + bool do_device_synchronize = warpx_do_device_synchronize; + + auto pp_tiny_profiler = amrex::ParmParse{"tiny_profiler"}; + if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + do_device_synchronize == warpx_do_device_synchronize, + "tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize."); + } + + } + + void apply_workaround_for_warpx_numprocs () + { // Work-around: // If warpx.numprocs is used for the domain decomposition, we will not use blocking factor // to generate grids. Nonetheless, AMReX has asserts in place that validate that the // number of cells is a multiple of blocking factor. We set the blocking factor to 1 so those // AMReX asserts will always pass. - const amrex::ParmParse pp_warpx("warpx"); + const auto pp_warpx = amrex::ParmParse{"warpx"}; if (pp_warpx.contains("numprocs")) { amrex::ParmParse pp_amr("amr"); pp_amr.add("blocking_factor", 1); } + } - //See https://github.com/AMReX-Codes/amrex/pull/3763 -#ifdef AMREX_USE_GPU - bool warpx_do_device_synchronize = true; -#else - bool warpx_do_device_synchronize = false; -#endif - pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize); - bool do_device_synchronize = warpx_do_device_synchronize; - amrex::ParmParse pp_tiny_profiler("tiny_profiler"); - if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - do_device_synchronize == warpx_do_device_synchronize, - "tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize."); - } - + void override_default_tiling_option_for_particles () + { // Here we override the default tiling option for particles, which is always // "false" in AMReX, to "false" if compiling for GPU execution and "true" // if compiling for CPU. - { - amrex::ParmParse pp_particles("particles"); -#ifdef AMREX_USE_GPU - bool do_tiling = false; // By default, tiling is off on GPU -#else - bool do_tiling = true; -#endif - pp_particles.queryAdd("do_tiling", do_tiling); - } + auto pp_particles = amrex::ParmParse{"particles"}; + auto do_tiling = !amrex_use_gpu; // By default, tiling is off on GPU + pp_particles.queryAdd("do_tiling", do_tiling); + } + + /** Overwrite defaults in AMReX Inputs + * + * This overwrites defaults in amrex::ParmParse for inputs. + */ + void + overwrite_amrex_parser_defaults () + { + override_default_abort_on_out_of_gpu_memory(); + override_default_the_arena_is_managed(); + override_default_omp_threads(); + apply_workaround_for_warpx_numprocs(); + set_device_synchronization(); + override_default_tiling_option_for_particles(); + } + + /** Parse prob_lo and hi + * + * Parse prob_lo and hi evaluating any expressions since geometry + * does not parse its input. Note that this operation has to be + * performed after having initialized AMReX + */ + void parse_geometry_input () + { + auto pp_geometry = amrex::ParmParse {"geometry"}; + + auto prob_lo = amrex::Vector(AMREX_SPACEDIM); + auto prob_hi = amrex::Vector(AMREX_SPACEDIM); + + utils::parser::getArrWithParser( + pp_geometry, "prob_lo", prob_lo, 0, AMREX_SPACEDIM); + utils::parser::getArrWithParser( + pp_geometry, "prob_hi", prob_hi, 0, AMREX_SPACEDIM); + + AMREX_ALWAYS_ASSERT(prob_lo.size() == AMREX_SPACEDIM); + AMREX_ALWAYS_ASSERT(prob_hi.size() == AMREX_SPACEDIM); + + pp_geometry.addarr("prob_lo", prob_lo); + pp_geometry.addarr("prob_hi", prob_hi); + + // Parse amr input, evaluating any expressions since amr does not parse its input + auto pp_amr = amrex::ParmParse{"amr"}; + + // Note that n_cell is replaced so that only the parsed version is written out to the + // warpx_job_info file. This must be done since yt expects to be able to parse + // the value of n_cell from that file. For the rest, this doesn't matter. + auto preparse_amrex_input_int_array = + [&pp_amr](const std::string& input_str, const bool replace = false) + { + const auto *const c_input_str = input_str.c_str(); + if (pp_amr.contains(c_input_str)) { + amrex::Vector input_array; + utils::parser::getArrWithParser(pp_amr,c_input_str, input_array); + if (replace) { + pp_amr.remove(c_input_str); + } + pp_amr.addarr(c_input_str, input_array); + } + }; + + preparse_amrex_input_int_array("n_cell", true); + + const auto params_to_parse = std::vector{ + "max_grid_size", "max_grid_size_x", "max_grid_size_y", "max_grid_size_z", + "blocking_factor", "blocking_factor_x", "blocking_factor_y", "blocking_factor_z"}; + std::for_each(params_to_parse.begin(), params_to_parse.end(), preparse_amrex_input_int_array); + } + + /** This method groups calls to functions related to the initialization of AMReX + * that can run only after having called amrex::Initialize + */ + void amrex_post_initialize () + { + parse_geometry_input(); } } @@ -86,13 +177,18 @@ namespace warpx::initialization amrex::AMReX* amrex_init (int& argc, char**& argv, bool build_parm_parse) { - return amrex::Initialize( - argc, - argv, - build_parm_parse, - MPI_COMM_WORLD, - ::overwrite_amrex_parser_defaults - ); + amrex::AMReX* amrex = + amrex::Initialize( + argc, + argv, + build_parm_parse, + MPI_COMM_WORLD, + ::overwrite_amrex_parser_defaults + ); + + ::amrex_post_initialize(); + + return amrex; } } diff --git a/Source/Initialization/WarpXInit.H b/Source/Initialization/WarpXInit.H index ce179e2e997..85e3b8d068e 100644 --- a/Source/Initialization/WarpXInit.H +++ b/Source/Initialization/WarpXInit.H @@ -17,14 +17,23 @@ namespace warpx::initialization * @param[in] argc number of arguments from main() * @param[in] argv argument strings from main() */ - void initialize_external_libraries(int argc, char* argv[]); + void initialize_external_libraries (int argc, char* argv[]); /** Initializes, in the following order: * - the FFT library through the anyfft::cleanup() function in ablastr * - the AMReX library * - the MPI library through the mpi_finalize helper function in ablastr */ - void finalize_external_libraries(); + void finalize_external_libraries (); + + /** + * Initializes the Warning manager in ablastr + */ + void initialize_warning_manager (); + + /** Check that warpx.dims matches the binary name + */ + void check_dims (); } #endif //WARPX_INIT_H_ diff --git a/Source/Initialization/WarpXInit.cpp b/Source/Initialization/WarpXInit.cpp index 7e00760bf30..555bea52a7f 100644 --- a/Source/Initialization/WarpXInit.cpp +++ b/Source/Initialization/WarpXInit.cpp @@ -8,11 +8,17 @@ #include "WarpXInit.H" #include "Initialization/WarpXAMReXInit.H" +#include "Utils/TextMsg.H" #include +#include #include #include +#include + +#include +#include void warpx::initialization::initialize_external_libraries(int argc, char* argv[]) { @@ -21,9 +27,68 @@ void warpx::initialization::initialize_external_libraries(int argc, char* argv[] ablastr::math::anyfft::setup(); } -void warpx::initialization::finalize_external_libraries() +void warpx::initialization::finalize_external_libraries () { ablastr::math::anyfft::cleanup(); amrex::Finalize(); ablastr::parallelization::mpi_finalize(); } + +void warpx::initialization::initialize_warning_manager () +{ + const auto pp_warpx = amrex::ParmParse{"warpx"}; + + //"Synthetic" warning messages may be injected in the Warning Manager via + // inputfile for debug&testing purposes. + ablastr::warn_manager::GetWMInstance().debug_read_warnings_from_input(pp_warpx); + + // Set the flag to control if WarpX has to emit a warning message as soon as a warning is recorded + bool always_warn_immediately = false; + pp_warpx.query("always_warn_immediately", always_warn_immediately); + ablastr::warn_manager::GetWMInstance().SetAlwaysWarnImmediately(always_warn_immediately); + + // Set the WarnPriority threshold to decide if WarpX has to abort when a warning is recorded + if(std::string str_abort_on_warning_threshold; + pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ + std::optional abort_on_warning_threshold = std::nullopt; + if (str_abort_on_warning_threshold == "high") { + abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; + } else if (str_abort_on_warning_threshold == "medium" ) { + abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; + } else if (str_abort_on_warning_threshold == "low") { + abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; + } else { + WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold + +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); + } + ablastr::warn_manager::GetWMInstance().SetAbortThreshold(abort_on_warning_threshold); + } +} + +void warpx::initialization::check_dims() +{ + // Ensure that geometry.dims is set properly. +#if defined(WARPX_DIM_3D) + std::string const dims_compiled = "3"; +#elif defined(WARPX_DIM_XZ) + std::string const dims_compiled = "2"; +#elif defined(WARPX_DIM_1D_Z) + std::string const dims_compiled = "1"; +#elif defined(WARPX_DIM_RZ) + std::string const dims_compiled = "RZ"; +#endif + const amrex::ParmParse pp_geometry("geometry"); + std::string dims; + std::string dims_error = "The selected WarpX executable was built as '"; + dims_error.append(dims_compiled).append("'-dimensional, but the "); + if (pp_geometry.contains("dims")) { + pp_geometry.get("dims", dims); + dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); + dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + } else { + dims = "Not specified"; + dims_error.append("inputs file does not declare 'geometry.dims'. Please add 'geometry.dims = "); + dims_error.append(dims_compiled).append("' to inputs file."); + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(dims == dims_compiled, dims_error); +} diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 715c25d2e8b..5fa92cf1c2f 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -16,12 +16,18 @@ #endif #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "EmbeddedBoundary/Enabled.H" +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/EmbeddedBoundaryInit.H" +#endif +#include "Fields.H" +#include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" #include "Filter/NCIGodfreyFilter.H" #include "Initialization/ExternalField.H" +#include "Initialization/DivCleaner/ProjectionDivCleaner.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/LinearInterpolation.H" #include "Utils/Logo/GetLogo.H" @@ -34,6 +40,7 @@ #include "Initialization/ReconnectionPerturbation.H" #include "Python/callbacks.H" +#include #include #include #include @@ -90,12 +97,40 @@ using namespace amrex; namespace { + + /** Print dt and dx,dy,dz */ + void PrintDtDxDyDz ( + int max_level, const amrex::Vector& geom, const amrex::Vector& dt) + { + for (int lev=0; lev <= max_level; lev++) { + const amrex::Real* dx_lev = geom[lev].CellSize(); + amrex::Print() << "Level " << lev << ": dt = " << dt[lev] + #if defined(WARPX_DIM_1D_Z) + << " ; dz = " << dx_lev[0] << '\n'; + #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + << " ; dx = " << dx_lev[0] + << " ; dz = " << dx_lev[1] << '\n'; + #elif defined(WARPX_DIM_3D) + << " ; dx = " << dx_lev[0] + << " ; dy = " << dx_lev[1] + << " ; dz = " << dx_lev[2] << '\n'; + #endif + } + } + /** * \brief Check that the number of guard cells is smaller than the number of valid cells, * for a given MultiFab, and abort otherwise. */ - void CheckGuardCells(amrex::MultiFab const& mf) + void CheckGuardCells ( + ablastr::fields::MultiFabRegister& fields, + const std::string& mf_name, + int lev + ) { + if (!fields.has(mf_name, lev)) { return; } + auto & mf = *fields.get(mf_name, lev); + for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { const amrex::IntVect vc = mfi.validbox().enclosedCells().size(); @@ -110,6 +145,71 @@ namespace WARPX_ALWAYS_ASSERT_WITH_MESSAGE(vc.allGT(gc), ss_msg.str()); } } + + /** + * \brief Check the requested resources and write performance hints + * + * @param[in] total_nboxes total number of boxes in the simulation + * @param[in] nprocs number of MPI processes + */ + void PerformanceHints (const amrex::Long total_nboxes, const amrex::Long nprocs) + { + // Check: are there more MPI ranks than Boxes? + if (nprocs > total_nboxes) { + std::stringstream warnMsg; + warnMsg << "Too many resources / too little work!\n" + << " It looks like you requested more compute resources than " + << "there are total number of boxes of cells available (" + << total_nboxes << "). " + << "You started with (" << nprocs + << ") MPI ranks, so (" << nprocs - total_nboxes + << ") rank(s) will have no work.\n" + #ifdef AMREX_USE_GPU + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + #endif + << "Consider decreasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + + #ifdef AMREX_USE_GPU + // Check: Are there more than 12 boxes per GPU? + if (total_nboxes > nprocs * 12) { + std::stringstream warnMsg; + warnMsg << "Too many boxes per GPU!\n" + << " It looks like you split your simulation domain " + << "in too many boxes (" << total_nboxes << "), which " + << "results in an average number of (" + << amrex::Long(total_nboxes/nprocs) << ") per GPU. " + << "This causes severe overhead in the communication of " + << "border/guard regions.\n" + << " On GPUs, consider using 1-8 boxes per GPU that together fill " + << "each GPU's memory sufficiently. If you do not rely on dynamic " + << "load-balancing, then one large box per GPU is ideal.\n" + << "Consider increasing the amr.blocking_factor and " + << "amr.max_grid_size parameters and/or using more MPI ranks.\n" + << " More information:\n" + << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; + + ablastr::warn_manager::WMRecordWarning( + "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + } + #endif + + // TODO: warn if some ranks have disproportionally more work than all others + // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down + // all other ranks; we need to measure this with our load-balancing + // routines and issue a warning only of some ranks stall all other ranks + // TODO: check MPI-rank to GPU ratio (should be 1:1) + // TODO: check memory per MPI rank, especially if GPUs are underutilized + // TODO: CPU tiling hints with OpenMP + } } void @@ -206,20 +306,24 @@ WarpX::PrintMainPICparameters () amrex::Print() << "Operation mode: | Electrostatic" << "\n"; amrex::Print() << " | - laboratory frame, electrostatic + magnetostatic" << "\n"; } + else if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameEffectivePotential){ + amrex::Print() << "Operation mode: | Electrostatic" << "\n"; + amrex::Print() << " | - laboratory frame, effective potential scheme" << "\n"; + } else{ amrex::Print() << "Operation mode: | Electromagnetic" << "\n"; } - if (em_solver_medium == MediumForEM::Vacuum ){ + if (m_em_solver_medium == MediumForEM::Vacuum ){ amrex::Print() << " | - vacuum" << "\n"; } - else if (em_solver_medium == MediumForEM::Macroscopic ){ + else if (m_em_solver_medium == MediumForEM::Macroscopic ){ amrex::Print() << " | - macroscopic" << "\n"; } - if ( (em_solver_medium == MediumForEM::Macroscopic) && + if ( (m_em_solver_medium == MediumForEM::Macroscopic) && (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff)){ amrex::Print() << " | - Lax-Wendroff algorithm\n"; } - else if ((em_solver_medium == MediumForEM::Macroscopic) && + else if ((m_em_solver_medium == MediumForEM::Macroscopic) && (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler)){ amrex::Print() << " | - Backward Euler algorithm\n"; } @@ -431,6 +535,10 @@ void WarpX::InitData () { WARPX_PROFILE("WarpX::InitData()"); + + using ablastr::fields::Direction; + using warpx::fields::FieldType; + ablastr::parallelization::check_mpi_thread_level(); #ifdef WARPX_QED @@ -450,20 +558,20 @@ WarpX::InitData () // WarpX::computeMaxStepBoostAccelerator // needs to start from the initial zmin_domain_boost, // even if restarting from a checkpoint file - if (do_compute_max_step_from_zmax) { + if (m_zmax_plasma_to_compute_max_step.has_value()) { zmin_domain_boost_step_0 = geom[0].ProbLo(WARPX_ZINDEX); } if (restart_chkfile.empty()) { ComputeDt(); - WarpX::PrintDtDxDyDz(); + ::PrintDtDxDyDz(max_level, geom, dt); InitFromScratch(); InitDiagnostics(); } else { InitFromCheckpoint(); - WarpX::PrintDtDxDyDz(); + ::PrintDtDxDyDz(max_level, geom, dt); PostRestart(); reduced_diags->InitData(); } @@ -478,16 +586,18 @@ WarpX::InitData () BuildBufferMasks(); - if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + if (m_em_solver_medium == MediumForEM::Macroscopic) { const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + m_fields.get(FieldType::Efield_fp, Direction{0}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev_zero)->ixType().toIntVect() ); } + m_electrostatic_solver->InitData(); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { m_hybrid_pic_model->InitData(); } @@ -507,6 +617,11 @@ WarpX::InitData () } WriteUsedInputsFile(); + // Run div cleaner here on loaded external fields + if (m_do_divb_cleaning_external) { + WarpX::ProjectionCleanDivB(); + } + if (restart_chkfile.empty()) { // Loop through species and calculate their space-charge field @@ -517,6 +632,14 @@ WarpX::InitData () if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); } + // Add external fields to the fine patch fields. This makes it so that the + // net fields are the sum of the field solutions and any external fields. + for (int lev = 0; lev <= max_level; ++lev) { + AddExternalFields(lev); + } + } + else { + ExecutePythonCallback("afterInitatRestart"); } if (restart_chkfile.empty() || write_diagnostics_on_restart) { @@ -531,36 +654,49 @@ WarpX::InitData () } } - PerformanceHints(); + // Computes available boxes on all levels. + amrex::Long total_nboxes = 0; + for (int ilev = 0; ilev <= finestLevel(); ++ilev) { + total_nboxes += boxArray(ilev).size(); + } + auto const nprocs = ParallelDescriptor::NProcs(); + + ::PerformanceHints(total_nboxes, nprocs); CheckKnownIssues(); } void -WarpX::AddExternalFields (int const lev) { +WarpX::AddExternalFields (int const lev) +{ + using ablastr::fields::Direction; + using warpx::fields::FieldType; + // FIXME: RZ multimode has more than one component for all these if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][0], *m_fields.get(FieldType::Efield_fp_external, Direction{0}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][1], *m_fields.get(FieldType::Efield_fp_external, Direction{1}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][2], *m_fields.get(FieldType::Efield_fp_external, Direction{2}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get(FieldType::Bfield_fp_external, Direction{0}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][1], *m_fields.get(FieldType::Bfield_fp_external, Direction{1}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][2], *m_fields.get(FieldType::Bfield_fp_external, Direction{2}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } } @@ -583,21 +719,7 @@ WarpX::InitFromScratch () m_implicit_solver->Define(this); m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme, particle_tol_in_implicit_scheme ); - - // Add space to save the positions and velocities at the start of the time steps - for (auto const& pc : *mypc) { -#if (AMREX_SPACEDIM >= 2) - pc->AddRealComp("x_n"); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - pc->AddRealComp("y_n"); -#endif - pc->AddRealComp("z_n"); - pc->AddRealComp("ux_n"); - pc->AddRealComp("uy_n"); - pc->AddRealComp("uz_n"); - } - + m_implicit_solver->CreateParticleAttributes(); } mypc->AllocData(); @@ -620,9 +742,10 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) { do_pml = 1; } + if (max_level > 0) { do_pml = 1; } if (do_pml) { + bool const eb_enabled = EB::enabled(); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) do_pml_Lo[0][0] = 0; // no PML at r=0, in cylindrical geometry pml_rz[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), pml_ncell, do_pml_in_domain); @@ -635,16 +758,17 @@ WarpX::InitPML () pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), dt[0], nox_fft, noy_fft, noz_fft, grid_type, do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, + m_psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, amrex::IntVect(0), amrex::IntVect(0), + eb_enabled, guard_cells.ng_FieldSolver.max(), v_particle_pml, do_cubic_sigma_pml, pml_damping_strength, do_pml_Lo[0], do_pml_Hi[0]); #endif - for (int lev = 1; lev <= finest_level; ++lev) + for (int lev = 1; lev <= max_level; ++lev) { do_pml_Lo[lev] = amrex::IntVect::TheUnitVector(); do_pml_Hi[lev] = amrex::IntVect::TheUnitVector(); @@ -676,8 +800,9 @@ WarpX::InitPML () pml_ncell, pml_delta, refRatio(lev-1), dt[lev], nox_fft, noy_fft, noz_fft, grid_type, do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, + m_psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, amrex::IntVect(0), amrex::IntVect(0), + eb_enabled, guard_cells.ng_FieldSolver.max(), v_particle_pml, do_cubic_sigma_pml, pml_damping_strength, @@ -691,7 +816,7 @@ WarpX::ComputePMLFactors () { if (do_pml) { - for (int lev = 0; lev <= finest_level; ++lev) + for (int lev = 0; lev <= max_level; ++lev) { if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); @@ -703,7 +828,7 @@ WarpX::ComputePMLFactors () void WarpX::ComputeMaxStep () { - if (do_compute_max_step_from_zmax) { + if (m_zmax_plasma_to_compute_max_step.has_value()) { computeMaxStepBoostAccelerator(); } } @@ -736,7 +861,7 @@ WarpX::computeMaxStepBoostAccelerator() { // End of the plasma: Transform input argument // zmax_plasma_to_compute_max_step to boosted frame. - const Real len_plasma_boost = zmax_plasma_to_compute_max_step/gamma_boost; + const Real len_plasma_boost = m_zmax_plasma_to_compute_max_step.value()/gamma_boost; // Plasma velocity const Real v_plasma_boost = -beta_boost * PhysConst::c; // Get time at which the lower end of the simulation domain passes the @@ -744,12 +869,12 @@ WarpX::computeMaxStepBoostAccelerator() { const Real interaction_time_boost = (len_plasma_boost-zmin_domain_boost_step_0)/ (moving_window_v-v_plasma_boost); // Divide by dt, and update value of max_step. - const auto computed_max_step = (do_subcycling)? + const auto computed_max_step = (m_do_subcycling)? static_cast(interaction_time_boost/dt[0]): static_cast(interaction_time_boost/dt[maxLevel()]); max_step = computed_max_step; Print()<<"max_step computed in computeMaxStepBoostAccelerator: " - <setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_avg_fp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } if (lev > 0) { - Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_aux, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { - Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_avg_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } } } @@ -843,21 +971,21 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { if (fft_do_time_averaging) { - Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_avg_fp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } - if (lev > 0) { - Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_aux, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { - Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_avg_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } } } } #ifdef AMREX_USE_EB - InitializeEBGridData(lev); + bool const eb_enabled = EB::enabled(); + if (eb_enabled) { InitializeEBGridData(lev); } #endif // if the input string for the B-field is "parse_b_ext_grid_function", @@ -868,42 +996,29 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev <= maxlevel_extEMfield_init)) { - InitializeExternalFieldsOnGridUsingParser( - Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); + ComputeExternalFieldOnGridUsingParser( + FieldType::Bfield_fp, + m_p_ext_field_params->Bxfield_parser->compile<4>(), + m_p_ext_field_params->Byfield_parser->compile<4>(), + m_p_ext_field_params->Bzfield_parser->compile<4>(), + lev, PatchType::fine, 'f', + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev)); if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser( - Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); + ComputeExternalFieldOnGridUsingParser( + FieldType::Bfield_aux, + m_p_ext_field_params->Bxfield_parser->compile<4>(), + m_p_ext_field_params->Byfield_parser->compile<4>(), + m_p_ext_field_params->Bzfield_parser->compile<4>(), + lev, PatchType::fine, m_eb_update_B); + + ComputeExternalFieldOnGridUsingParser( + FieldType::Bfield_cp, + m_p_ext_field_params->Bxfield_parser->compile<4>(), + m_p_ext_field_params->Byfield_parser->compile<4>(), + m_p_ext_field_params->Bzfield_parser->compile<4>(), + lev, PatchType::coarse, m_eb_update_B); } } ParmParse pp_warpx("warpx"); @@ -919,29 +1034,12 @@ WarpX::InitLevelData (int lev, Real /*time*/) "For reconnection simulations with perturbation, managed memory should be used with amrex.the_arena_is_managed=1\n"); #endif - Reconnection_Perturbation::AddBfieldPerturbation (Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::fine); - - if (lev > 0) { - Reconnection_Perturbation::AddBfieldPerturbation (Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), + Reconnection_Perturbation::AddBfieldPerturbation ( + FieldType::Bfield_fp, m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::fine); - - Reconnection_Perturbation::AddBfieldPerturbation (Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::coarse); - } - + #endif } @@ -966,44 +1064,44 @@ WarpX::InitLevelData (int lev, Real /*time*/) lev, PatchType::fine); #ifdef AMREX_USE_EB - // We initialize ECTRhofield consistently with the Efield - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_fdtd_solver_fp[lev]->EvolveECTRho( - Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + if (eb_enabled) { + // We initialize ECTRhofield consistently with the Efield + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + m_fdtd_solver_fp[lev]->EvolveECTRho( + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev], + m_fields.get_alldirs(FieldType::ECTRhofield, lev), + lev); + } } #endif if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser( - Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get(), - m_p_ext_field_params->Exfield_parser->compile<3>(), - m_p_ext_field_params->Eyfield_parser->compile<3>(), - m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser( - Efield_cp[lev][0].get(), - Efield_cp[lev][1].get(), - Efield_cp[lev][2].get(), - m_p_ext_field_params->Exfield_parser->compile<3>(), - m_p_ext_field_params->Eyfield_parser->compile<3>(), - m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::coarse); + ComputeExternalFieldOnGridUsingParser( + FieldType::Efield_aux, + m_p_ext_field_params->Exfield_parser->compile<4>(), + m_p_ext_field_params->Eyfield_parser->compile<4>(), + m_p_ext_field_params->Ezfield_parser->compile<4>(), + lev, PatchType::fine, m_eb_update_E); + + ComputeExternalFieldOnGridUsingParser( + FieldType::Efield_cp, + m_p_ext_field_params->Exfield_parser->compile<4>(), + m_p_ext_field_params->Eyfield_parser->compile<4>(), + m_p_ext_field_params->Ezfield_parser->compile<4>(), + lev, PatchType::coarse, m_eb_update_E); #ifdef AMREX_USE_EB - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - // We initialize ECTRhofield consistently with the Efield - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); - + if (eb_enabled) { + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + // We initialize ECTRhofield consistently with the Efield + m_fdtd_solver_cp[lev]->EvolveECTRho( + m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev], + m_fields.get_alldirs(FieldType::ECTRhofield, lev), + lev); + } } #endif } @@ -1011,8 +1109,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) // load external grid fields into E/Bfield_fp_external multifabs LoadExternalFields(lev); - // add the external fields to the fine patch fields as initial conditions for the fields - AddExternalFields(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1023,31 +1119,41 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } -void -WarpX::InitializeExternalFieldsOnGridUsingParser ( - MultiFab *mfx, MultiFab *mfy, MultiFab *mfz, - ParserExecutor<3> const& xfield_parser, ParserExecutor<3> const& yfield_parser, - ParserExecutor<3> const& zfield_parser, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - const char field, - const int lev, PatchType patch_type) +template +void ComputeExternalFieldOnGridUsingParser_template ( + T field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field, + bool use_eb_flags) { + auto &warpx = WarpX::GetInstance(); + auto const &geom = warpx.Geom(lev); + + auto t = warpx.gett_new(lev); + + auto dx_lev = geom.CellSizeArray(); + const RealBox& real_box = geom.ProbDomain(); - auto dx_lev = geom[lev].CellSizeArray(); amrex::IntVect refratio = (lev > 0 ) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1); if (patch_type == PatchType::coarse) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { dx_lev[idim] = dx_lev[idim] * refratio[idim]; } } - const RealBox& real_box = geom[lev].ProbDomain(); + + using ablastr::fields::Direction; + amrex::MultiFab* mfx = warpx.m_fields.get(field, Direction{0}, lev); + amrex::MultiFab* mfy = warpx.m_fields.get(field, Direction{1}, lev); + amrex::MultiFab* mfz = warpx.m_fields.get(field, Direction{2}, lev); + const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); - for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) - { + for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); @@ -1056,41 +1162,19 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( auto const& mfyfab = mfy->array(mfi); auto const& mfzfab = mfz->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); - amrex::Array4 const& Sx = face_areas[0]->array(mfi); - amrex::Array4 const& Sy = face_areas[1]->array(mfi); - amrex::Array4 const& Sz = face_areas[2]->array(mfi); - -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Dim3 lx_lo = amrex::lbound(lx); - const amrex::Dim3 lx_hi = amrex::ubound(lx); - const amrex::Dim3 lz_lo = amrex::lbound(lz); - const amrex::Dim3 lz_hi = amrex::ubound(lz); -#endif - -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(ly, Sx, Sz); -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(lx, ly, lz, Sx, Sy, Sz); -#endif - -#else - amrex::ignore_unused(edge_lengths, face_areas, field); -#endif + amrex::Array4 update_fx_arr, update_fy_arr, update_fz_arr; + if (use_eb_flags && EB::enabled()) { + update_fx_arr = eb_update_field[lev][0]->array(mfi); + update_fy_arr = eb_update_field[lev][1]->array(mfi); + update_fz_arr = eb_update_field[lev][2]->array(mfi); + } amrex::ParallelFor (tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if((field=='E' and lx(i, j, k)<=0) or (field=='B' and Sx(i, j, k)<=0)) return; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ex is associated with a x-edge, while Bx is associated with a z-edge - if((field=='E' and lx(i, j, k)<=0) or (field=='B' and lz(i, j, k)<=0)) return; -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fx_arr && update_fx_arr(i,j,k) == 0) { return; } + // Shift required in the x-, y-, or z- position // depending on the index type of the multifab #if defined(WARPX_DIM_1D_Z) @@ -1113,21 +1197,13 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; #endif // Initialize the x-component of the field. - mfxfab(i,j,k) = xfield_parser(x,y,z); + mfxfab(i,j,k) = fx_parser(x,y,z,t); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if((field=='E' and ly(i, j, k)<=0) or (field=='B' and Sy(i, j, k)<=0)) return; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - if((field=='E' and (lx(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)<=0 - || lx(std::max(i-1, lx_lo.x), std::min(j , lx_hi.y), k)<=0 - || lz(std::min(i , lz_hi.x), std::min(j , lz_hi.y), k)<=0 - || lz(std::min(i , lz_hi.x), std::max(j-1, lz_lo.y), k)<=0)) or - (field=='B' and Sy(i,j,k)<=0)) return; -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fy_arr && update_fy_arr(i,j,k) == 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; @@ -1148,17 +1224,13 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; #endif // Initialize the y-component of the field. - mfyfab(i,j,k) = yfield_parser(x,y,z); + mfyfab(i,j,k) = fy_parser(x,y,z,t); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if((field=='E' and lz(i, j, k)<=0) or (field=='B' and Sz(i, j, k)<=0)) return; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ez is associated with a z-edge, while Bz is associated with a x-edge - if((field=='E' and lz(i, j, k)<=0) or (field=='B' and lx(i, j, k)<=0)) return; -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fz_arr && update_fz_arr(i,j,k) == 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; @@ -1179,141 +1251,114 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; #endif // Initialize the z-component of the field. - mfzfab(i,j,k) = zfield_parser(x,y,z); + mfzfab(i,j,k) = fz_parser(x,y,z,t); } ); } } -void -WarpX::PerformanceHints () +void WarpX::ComputeExternalFieldOnGridUsingParser ( + warpx::fields::FieldType field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field, + bool use_eb_flags) { - // Check requested MPI ranks and available boxes - amrex::Long total_nboxes = 0; // on all MPI ranks - for (int ilev = 0; ilev <= finestLevel(); ++ilev) { - total_nboxes += boxArray(ilev).size(); - } - auto const nprocs = ParallelDescriptor::NProcs(); - - // Check: are there more MPI ranks than Boxes? - if (nprocs > total_nboxes) { - std::stringstream warnMsg; - warnMsg << "Too many resources / too little work!\n" - << " It looks like you requested more compute resources than " - << "there are total number of boxes of cells available (" - << total_nboxes << "). " - << "You started with (" << nprocs - << ") MPI ranks, so (" << nprocs - total_nboxes - << ") rank(s) will have no work.\n" -#ifdef AMREX_USE_GPU - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" -#endif - << "Consider decreasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using fewer MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; - - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } + ComputeExternalFieldOnGridUsingParser_template ( + field, + fx_parser, fy_parser, fz_parser, + lev, patch_type, eb_update_field, + use_eb_flags); +} -#ifdef AMREX_USE_GPU - // Check: Are there more than 12 boxes per GPU? - if (total_nboxes > nprocs * 12) { - std::stringstream warnMsg; - warnMsg << "Too many boxes per GPU!\n" - << " It looks like you split your simulation domain " - << "in too many boxes (" << total_nboxes << "), which " - << "results in an average number of (" - << amrex::Long(total_nboxes/nprocs) << ") per GPU. " - << "This causes severe overhead in the communication of " - << "border/guard regions.\n" - << " On GPUs, consider using 1-8 boxes per GPU that together fill " - << "each GPU's memory sufficiently. If you do not rely on dynamic " - << "load-balancing, then one large box per GPU is ideal.\n" - << "Consider increasing the amr.blocking_factor and " - << "amr.max_grid_size parameters and/or using more MPI ranks.\n" - << " More information:\n" - << " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n"; +void WarpX::ComputeExternalFieldOnGridUsingParser ( + std::string const& field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field, + bool use_eb_flags) +{ + ComputeExternalFieldOnGridUsingParser_template ( + field, + fx_parser, fy_parser, fz_parser, + lev, patch_type, eb_update_field, + use_eb_flags); +} - ablastr::warn_manager::WMRecordWarning( - "Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - } -#endif +void WarpX::ComputeExternalFieldOnGridUsingParser ( + warpx::fields::FieldType field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field) +{ + ComputeExternalFieldOnGridUsingParser_template ( + field, + fx_parser, fy_parser, fz_parser, + lev, patch_type, eb_update_field, + true); +} - // TODO: warn if some ranks have disproportionally more work than all others - // tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down - // all other ranks; we need to measure this with our load-balancing - // routines and issue a warning only of some ranks stall all other ranks - // TODO: check MPI-rank to GPU ratio (should be 1:1) - // TODO: check memory per MPI rank, especially if GPUs are underutilized - // TODO: CPU tiling hints with OpenMP +void WarpX::ComputeExternalFieldOnGridUsingParser ( + std::string const& field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field) +{ + ComputeExternalFieldOnGridUsingParser_template ( + field, + fx_parser, fy_parser, fz_parser, + lev, patch_type, eb_update_field, + true); } void WarpX::CheckGuardCells() { - for (int lev = 0; lev <= finest_level; ++lev) + for (int lev = 0; lev <= max_level; ++lev) { for (int dim = 0; dim < 3; ++dim) { - ::CheckGuardCells(*Efield_fp[lev][dim]); - ::CheckGuardCells(*Bfield_fp[lev][dim]); - ::CheckGuardCells(*current_fp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "current_fp[" + std::to_string(dim) + "]", lev); if (WarpX::fft_do_time_averaging) { - ::CheckGuardCells(*Efield_avg_fp[lev][dim]); - ::CheckGuardCells(*Bfield_avg_fp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_avg_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_avg_fp[" + std::to_string(dim) + "]", lev); } } - if (rho_fp[lev]) - { - ::CheckGuardCells(*rho_fp[lev]); - } - - if (F_fp[lev]) - { - ::CheckGuardCells(*F_fp[lev]); - } - - if (G_fp[lev]) - { - ::CheckGuardCells(*G_fp[lev]); - } + ::CheckGuardCells(m_fields, "rho_fp", lev); + ::CheckGuardCells(m_fields, "F_fp", lev); + ::CheckGuardCells(m_fields, "G_fp", lev); // MultiFabs on coarse patch if (lev > 0) { for (int dim = 0; dim < 3; ++dim) { - ::CheckGuardCells(*Efield_cp[lev][dim]); - ::CheckGuardCells(*Bfield_cp[lev][dim]); - ::CheckGuardCells(*current_cp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "current_cp[" + std::to_string(dim) + "]", lev); if (WarpX::fft_do_time_averaging) { - ::CheckGuardCells(*Efield_avg_cp[lev][dim]); - ::CheckGuardCells(*Bfield_avg_cp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_avg_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_avg_cp[" + std::to_string(dim) + "]", lev); } } - if (rho_cp[lev]) - { - ::CheckGuardCells(*rho_cp[lev]); - } - - if (F_cp[lev]) - { - ::CheckGuardCells(*F_cp[lev]); - } - - if (G_cp[lev]) - { - ::CheckGuardCells(*G_cp[lev]); - } + ::CheckGuardCells(m_fields, "rho_cp", lev); + ::CheckGuardCells(m_fields, "F_cp", lev); + ::CheckGuardCells(m_fields, "G_cp", lev); } } } @@ -1323,32 +1368,53 @@ void WarpX::InitializeEBGridData (int lev) #ifdef AMREX_USE_EB if (lev == maxLevel()) { - // Throw a warning if EB is on and particle_shape > 1 - bool flag_eb_on = not fieldEBFactory(lev).isAllRegular(); + auto const eb_fact = fieldEBFactory(lev); - if ((nox > 1 or noy > 1 or noz > 1) and flag_eb_on) + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { - ablastr::warn_manager::WMRecordWarning("Particles", - "when algo.particle_shape > 1, numerical artifacts will be present when\n" - "particles are close to embedded boundaries"); - } - - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { - - auto const eb_fact = fieldEBFactory(lev); - - ComputeEdgeLengths(m_edge_lengths[lev], eb_fact); - ScaleEdges(m_edge_lengths[lev], CellSize(lev)); - ComputeFaceAreas(m_face_areas[lev], eb_fact); - ScaleAreas(m_face_areas[lev], CellSize(lev)); + using warpx::fields::FieldType; if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - MarkCells(); + + auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); + warpx::embedded_boundary::ComputeEdgeLengths(edge_lengths_lev, eb_fact); + warpx::embedded_boundary::ScaleEdges(edge_lengths_lev, CellSize(lev)); + + auto face_areas_lev = m_fields.get_alldirs(FieldType::face_areas, lev); + warpx::embedded_boundary::ComputeFaceAreas(face_areas_lev, eb_fact); + warpx::embedded_boundary::ScaleAreas(face_areas_lev, CellSize(lev)); + + // Compute additional quantities required for the ECT solver + const auto& area_mod = m_fields.get_alldirs(FieldType::area_mod, maxLevel()); + warpx::embedded_boundary::MarkExtensionCells( + CellSize(maxLevel()), m_flag_info_face[maxLevel()], m_flag_ext_face[maxLevel()], + m_fields.get_alldirs(FieldType::Bfield_fp, maxLevel()), + face_areas_lev, + edge_lengths_lev, area_mod); ComputeFaceExtensions(); + + // Mark on which grid points E should be updated + warpx::embedded_boundary::MarkUpdateECellsECT( m_eb_update_E[lev], edge_lengths_lev ); + // Mark on which grid points B should be updated + warpx::embedded_boundary::MarkUpdateBCellsECT( m_eb_update_B[lev], face_areas_lev, edge_lengths_lev); + + } else { + // Mark on which grid points E should be updated (stair-case approximation) + warpx::embedded_boundary::MarkUpdateCellsStairCase( + m_eb_update_E[lev], + m_fields.get_alldirs(FieldType::Efield_fp, lev), + eb_fact ); + // Mark on which grid points B should be updated (stair-case approximation) + warpx::embedded_boundary::MarkUpdateCellsStairCase( + m_eb_update_B[lev], + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + eb_fact ); } + } ComputeDistanceToEB(); + warpx::embedded_boundary::MarkReducedShapeCells( m_eb_reduce_particle_shape[lev], eb_fact, nox, Geom(0).periodicity()); } #else @@ -1405,6 +1471,9 @@ void WarpX::CheckKnownIssues() void WarpX::LoadExternalFields (int const lev) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external // fields should be updated every time the window moves. @@ -1418,65 +1487,38 @@ WarpX::LoadExternalFields (int const lev) "External fields from file are not compatible with the moving window." ); } - //// External grid fields - //if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { - // // Initialize Bfield_fp_external with external function - // InitializeExternalFieldsOnGridUsingParser( - // Bfield_fp_external[lev][0].get(), - // Bfield_fp_external[lev][1].get(), - // Bfield_fp_external[lev][2].get(), - // m_p_ext_field_params->Bxfield_parser->compile<3>(), - // m_p_ext_field_params->Byfield_parser->compile<3>(), - // m_p_ext_field_params->Bzfield_parser->compile<3>(), - // m_edge_lengths[lev], - // m_face_areas[lev], - // 'B', - // lev, PatchType::fine); - //} - //else if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{0},lev), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{1},lev), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{2},lev), "B", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{0}, lev), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{1}, lev), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{2}, lev), "B", "z"); #endif } - //if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { - // // Initialize Efield_fp_external with external function - // InitializeExternalFieldsOnGridUsingParser( - // Efield_fp_external[lev][0].get(), - // Efield_fp_external[lev][1].get(), - // Efield_fp_external[lev][2].get(), - // m_p_ext_field_params->Exfield_parser->compile<3>(), - // m_p_ext_field_params->Eyfield_parser->compile<3>(), - // m_p_ext_field_params->Ezfield_parser->compile<3>(), - // m_edge_lengths[lev], - // m_face_areas[lev], - // 'E', - // lev, PatchType::fine); - //} - //else if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{0},lev), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{1},lev), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{2},lev), "E", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{0}, lev), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{1}, lev), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{2}, lev), "E", "z"); #endif } + if (lev == finestLevel()) { + // Call Python callback which might write values to external field multifabs + ExecutePythonCallback("loadExternalFields"); + } // External particle fields if (mypc->m_B_ext_particle_s == "read_from_file") { @@ -1486,13 +1528,25 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{0}, lev), + "B", "r"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{1}, lev), + "B", "t"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{2}, lev), + "B", "z"); #else - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{0}, lev), + "B", "x"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{1}, lev), + "B", "y"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::B_external_particle_field, Direction{2}, lev), + "B", "z"); #endif } if (mypc->m_E_ext_particle_s == "read_from_file") { @@ -1502,13 +1556,25 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{0}, lev), + "E", "r"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{1}, lev), + "E", "t"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{2}, lev), + "E", "z"); #else - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{0}, lev), + "E", "x"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{1}, lev), + "E", "y"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get(FieldType::E_external_particle_field, Direction{2}, lev), + "E", "z"); #endif } } diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp index 96d61d920f1..a1162e9a7a5 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -117,8 +117,8 @@ WarpXLaserProfiles::GaussianLaserProfile::fill_amplitude ( // Time stretching due to STCs and phi2 complex envelope // (1 if zeta=0, beta=0, phi2=0) const Complex stretch_factor = 1._rt + 4._rt * - (m_params.zeta+m_params.beta*m_params.focal_distance*inv_tau2) - * (m_params.zeta+m_params.beta*m_params.focal_distance*inv_complex_waist_2) + ((m_params.zeta+m_params.beta*m_params.focal_distance)*inv_tau2) + * ((m_params.zeta+m_params.beta*m_params.focal_distance)*inv_complex_waist_2) + 2._rt*I*(m_params.phi2-m_params.beta*m_params.beta*k0*m_params.focal_distance)*inv_tau2; // Amplitude and monochromatic oscillations diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 18405ca548d..47476de4f10 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -151,7 +151,7 @@ endif ifeq ($(USE_OPENPMD), TRUE) # try pkg-config query - ifeq (0, $(shell pkg-config "openPMD >= 0.15.1"; echo $$?)) + ifeq (0, $(shell pkg-config "openPMD >= 0.16.1"; echo $$?)) CXXFLAGS += $(shell pkg-config --cflags openPMD) LIBRARY_LOCATIONS += $(shell pkg-config --variable=libdir openPMD) libraries += $(shell pkg-config --libs-only-l openPMD) @@ -183,6 +183,8 @@ ifeq ($(USE_FFT),TRUE) INCLUDE_LOCATIONS += $(ROC_PATH)/rocfft/include LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib libraries += -lrocfft + else ifeq ($(USE_SYCL),TRUE) + # nothing else # Running on CPU # Use FFTW ifeq ($(PRECISION),FLOAT) diff --git a/Source/NonlinearSolvers/CurlCurlMLMGPC.H b/Source/NonlinearSolvers/CurlCurlMLMGPC.H new file mode 100644 index 00000000000..b3fcc6fe38f --- /dev/null +++ b/Source/NonlinearSolvers/CurlCurlMLMGPC.H @@ -0,0 +1,356 @@ +/* Copyright 2024 Debojyoti Ghosh + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef CURL_CURL_MLMG_PC_H_ +#define CURL_CURL_MLMG_PC_H_ + +#include "Fields.H" +#include "Utils/WarpXConst.H" +#include "Preconditioner.H" + +#include + +#include +#include +#include +#include +#include +#ifndef WARPX_DIM_1D_Z // currently not implemented in 1D +#include +#include +#include +#include +#endif + +/** + * \brief Curl-curl Preconditioner + * + * Preconditioner that solves the curl-curl equation for the E-field, given + * a RHS. Uses AMReX's curl-curl linear operator and multigrid solver. + * + * The equation solves for Eg in: + * curl ( alpha * curl ( Eg ) ) + beta * Eg = b + * where + * + alpha is a scalar + * + beta can either be a scalar that is constant in space or a MultiFab + * + Eg is the electric field. + * + b is a specified RHS with the same layout as Eg + * + * This class is templated on a solution-type class T and an operator class Ops. + * + * The Ops class must have the following function: + * + Return number of AMR levels + * + Return the amrex::Geometry object given an AMR level + * + Return hi and lo linear operator boundaries + * + Return the time step factor (theta) for the time integration scheme + * + * The T class must have the following functions: + * + Return underlying vector of amrex::MultiFab arrays + */ + +template +class CurlCurlMLMGPC : public Preconditioner +{ + public: + + using RT = typename T::value_type; + + /** + * \brief Default constructor + */ + CurlCurlMLMGPC () = default; + + /** + * \brief Default destructor + */ + ~CurlCurlMLMGPC () override = default; + + // Prohibit move and copy operations + CurlCurlMLMGPC(const CurlCurlMLMGPC&) = delete; + CurlCurlMLMGPC& operator=(const CurlCurlMLMGPC&) = delete; + CurlCurlMLMGPC(CurlCurlMLMGPC&&) noexcept = delete; + CurlCurlMLMGPC& operator=(CurlCurlMLMGPC&&) noexcept = delete; + + /** + * \brief Define the preconditioner + */ + void Define (const T&, Ops*) override; + + /** + * \brief Update the preconditioner + */ + void Update (const T&) override; + + /** + * \brief Apply (solve) the preconditioner given a RHS + * + * Given a right-hand-side b, solve: + * A x = b + * where A is the linear operator, in this case, the curl-curl operator: + * A x = curl (alpha * curl (x) ) + beta * x + */ + void Apply (T&, const T&) override; + + /** + * \brief Print parameters + */ + void printParameters() const override; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] inline bool IsDefined () const override { return m_is_defined; } + + protected: + + using MFArr = amrex::Array; + + bool m_is_defined = false; + + bool m_verbose = true; + bool m_bottom_verbose = false; + bool m_agglomeration = true; + bool m_consolidation = true; + bool m_use_gmres = false; + bool m_use_gmres_pc = true; + + int m_max_iter = 10; + int m_max_coarsening_level = 30; + + RT m_atol = 1.0e-16; + RT m_rtol = 1.0e-4; + + Ops* m_ops = nullptr; + + int m_num_amr_levels = 0; + amrex::Vector m_geom; + amrex::Vector m_grids; + amrex::Vector m_dmap; + amrex::IntVect m_gv; + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + amrex::Array m_bc_lo; + amrex::Array m_bc_hi; + + std::unique_ptr m_info; + std::unique_ptr m_curl_curl; + std::unique_ptr> m_solver; + std::unique_ptr> m_gmres_solver; +#endif + + /** + * \brief Read parameters + */ + void readParameters(); + + private: + +}; + +template +void CurlCurlMLMGPC::printParameters() const +{ + using namespace amrex; + auto pc_name = getEnumNameString(PreconditionerType::pc_curl_curl_mlmg); + Print() << pc_name << " verbose: " << (m_verbose?"true":"false") << "\n"; + Print() << pc_name << " bottom verbose: " << (m_bottom_verbose?"true":"false") << "\n"; + Print() << pc_name << " max iter: " << m_max_iter << "\n"; + Print() << pc_name << " agglomeration: " << m_agglomeration << "\n"; + Print() << pc_name << " consolidation: " << m_consolidation << "\n"; + Print() << pc_name << " max_coarsening_level: " << m_max_coarsening_level << "\n"; + Print() << pc_name << " absolute tolerance: " << m_atol << "\n"; + Print() << pc_name << " relative tolerance: " << m_rtol << "\n"; + Print() << pc_name << " use GMRES: " << (m_use_gmres?"true":"false") << "\n"; + if (m_use_gmres) { + Print() << pc_name + << " use PC for GMRES: " + << (m_use_gmres_pc?"true":"false") << "\n"; + } +} + +template +void CurlCurlMLMGPC::readParameters() +{ + const amrex::ParmParse pp(amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg)); + pp.query("verbose", m_verbose); + pp.query("bottom_verbose", m_bottom_verbose); + pp.query("max_iter", m_max_iter); + pp.query("agglomeration", m_agglomeration); + pp.query("consolidation", m_consolidation); + pp.query("max_coarsening_level", m_max_coarsening_level); + pp.query("absolute_tolerance", m_atol); + pp.query("relative_tolerance", m_rtol); + pp.query("use_gmres", m_use_gmres); + pp.query("use_gmres_pc", m_use_gmres_pc); +} + +template +void CurlCurlMLMGPC::Define ( const T& a_U, + Ops* const a_ops ) +{ + using namespace amrex; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "CurlCurlMLMGPC::Define() called on defined object" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (a_ops != nullptr), + "CurlCurlMLMGPC::Define(): a_ops is nullptr" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_U.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Define() must be called with Efield_fp type"); + + m_ops = a_ops; + // read preconditioner parameters + readParameters(); + +// currently not implemented in 1D +#ifdef WARPX_DIM_1D_Z + WARPX_ABORT_WITH_MESSAGE("CurlCurlMLMGPC not yet implemented for 1D"); +#else + // create info object for curl-curl op + m_info = std::make_unique(); + m_info->setAgglomeration(m_agglomeration); + m_info->setConsolidation(m_consolidation); + m_info->setMaxCoarseningLevel(m_max_coarsening_level); + + // Get data vectors from a_U + auto& u_mfarrvec = a_U.getArrayVec(); + + // Set number of AMR levels and create geometry, grids, and + // distribution mapping vectors. + m_num_amr_levels = m_ops->numAMRLevels(); + m_geom.resize(m_num_amr_levels); + m_grids.resize(m_num_amr_levels); + m_dmap.resize(m_num_amr_levels); + for (int n = 0; n < m_num_amr_levels; n++) { + m_geom[n] = m_ops->GetGeometry(n); + m_dmap[n] = u_mfarrvec[n][0]->DistributionMap(); + + BoxArray ba = u_mfarrvec[n][0]->boxArray(); + m_grids[n] = ba.enclosedCells(); + } + + // Construct the curl-curl linear operator and set its BCs + m_curl_curl = std::make_unique(m_geom, m_grids, m_dmap, *m_info); + m_curl_curl->setDomainBC(m_ops->GetLinOpBCLo(), m_ops->GetLinOpBCHi()); + + // Dummy value for alpha and beta to avoid abort due to degenerate matrix by MLMG solver + m_curl_curl->setScalars(1.0, 1.0); + + // Construct the MLMG solver + m_solver = std::make_unique>(*m_curl_curl); + m_solver->setMaxIter(m_max_iter); + m_solver->setFixedIter(m_max_iter); + m_solver->setVerbose(static_cast(m_verbose)); + m_solver->setBottomVerbose(static_cast(m_bottom_verbose)); + + // If using GMRES solver, construct it + if (m_use_gmres) { + m_gmres_solver = std::make_unique>(*m_solver); + m_gmres_solver->usePrecond(m_use_gmres_pc); + m_gmres_solver->setPrecondNumIters(m_max_iter); + m_gmres_solver->setVerbose(static_cast(m_verbose)); + } +#endif + + m_is_defined = true; +} + +template +void CurlCurlMLMGPC::Update (const T& a_U) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "CurlCurlMLMGPC::Update() called on undefined object" ); + + // a_U is not needed for a linear operator + amrex::ignore_unused(a_U); + + // set the coefficients alpha and beta for curl-curl op + // (m_dt here is actually theta<=0.5 times simulation dt) + const RT alpha = (this->m_dt*PhysConst::c) * (this->m_dt*PhysConst::c); + const RT beta = RT(1.0); + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + m_curl_curl->setScalars(alpha, beta); +#endif + + if (m_verbose) { + amrex::Print() << "Updating " << amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg) + << ": theta*dt = " << this->m_dt << ", " + << " coefficients: " + << "alpha = " << alpha << ", " + << "beta = " << beta << "\n"; + } +} + +template +void CurlCurlMLMGPC::Apply (T& a_x, const T& a_b) +{ + // Given a right-hand-side b, solve: + // A x = b + // where A is the linear operator, in this case, the curl-curl + // operator: + // A x = curl (alpha * curl (x) ) + beta * x + + using namespace amrex; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "CurlCurlMLMGPC::Apply() called on undefined object" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_x.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Apply() - a_x must be Efield_fp type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_b.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Apply() - a_b must be Efield_fp type"); + + // Get the data vectors + auto& b_mfarrvec = a_b.getArrayVec(); + auto& x_mfarrvec = a_x.getArrayVec(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + ((b_mfarrvec.size() == m_num_amr_levels) && (x_mfarrvec.size() == m_num_amr_levels)), + "Error in CurlCurlMLMGPC::Apply() - mismatch in number of levels." ); + + for (int n = 0; n < m_num_amr_levels; n++) { + + // Copy initial guess to local object +#if defined(WARPX_DIM_1D_Z) + // Missing dimensions is x,y in WarpX and y,z in AMReX + WARPX_ABORT_WITH_MESSAGE("CurlCurlMLMGPC not yet implemented for 1D"); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Missing dimension is y in WarpX and z in AMReX + Array solution { MultiFab(*x_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][2], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][1], make_alias, 0, 1) }; + Array rhs { MultiFab(*b_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][2], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][1], make_alias, 0, 1) }; +#elif defined(WARPX_DIM_3D) + Array solution { MultiFab(*x_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][1], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][2], make_alias, 0, 1) }; + Array rhs { MultiFab(*b_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][1], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][2], make_alias, 0, 1) }; +#endif + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + m_curl_curl->prepareRHS({&rhs}); + if (m_use_gmres) { + m_gmres_solver->solve(solution, rhs, m_rtol, m_atol); + } else { + m_solver->solve({&solution}, {&rhs}, m_rtol, m_atol); + } +#endif + } +} + +#endif diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H index 823523df23c..1a30dde4250 100644 --- a/Source/NonlinearSolvers/JacobianFunctionMF.H +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -7,6 +7,9 @@ #ifndef JacobianFunctionMF_H_ #define JacobianFunctionMF_H_ +#include "CurlCurlMLMGPC.H" +#include + /** * \brief This is a linear function class for computing the action of a * Jacobian on a vector using a matrix-free finite-difference method. @@ -21,8 +24,8 @@ class JacobianFunctionMF using RT = typename T::value_type; - JacobianFunctionMF() = default; - ~JacobianFunctionMF() = default; + JacobianFunctionMF() = default; + ~JacobianFunctionMF() = default; // Default move and copy operations JacobianFunctionMF(const JacobianFunctionMF&) = default; @@ -35,14 +38,18 @@ class JacobianFunctionMF inline void precond ( T& a_U, const T& a_X ) { - if (m_usePreCond) { a_U.zero(); } - else { a_U.Copy(a_X); } + if (m_usePreCond) { + a_U.zero(); + m_preCond->Apply(a_U, a_X); + } else { + a_U.Copy(a_X); + } } inline void updatePreCondMat ( const T& a_X ) { - amrex::ignore_unused(a_X); + if (m_usePreCond) { m_preCond->Update(a_X); } } inline @@ -133,15 +140,25 @@ class JacobianFunctionMF void curTime ( RT a_time ) { m_cur_time = a_time; + if (m_usePreCond) { m_preCond->CurTime(a_time); } } inline void curTimeStep ( RT a_dt ) { m_dt = a_dt; + if (m_usePreCond) { m_preCond->CurTimeStep(a_dt); } + } + + inline + void printParams () const + { + if (m_pc_type != PreconditionerType::none) { + m_preCond->printParameters(); + } } - void define( const T&, Ops* ); + void define( const T&, Ops*, const PreconditionerType& ); private: @@ -151,16 +168,18 @@ class JacobianFunctionMF RT m_epsJFNK = RT(1.0e-6); RT m_normY0; RT m_cur_time, m_dt; - std::string m_pc_type; - T m_Z, m_Y0, m_R0, m_R; - Ops* m_ops; + PreconditionerType m_pc_type = PreconditionerType::none; + T m_Z, m_Y0, m_R0, m_R; + Ops* m_ops = nullptr; + std::unique_ptr> m_preCond = nullptr; }; template -void JacobianFunctionMF::define ( const T& a_U, - Ops* a_ops ) +void JacobianFunctionMF::define ( const T& a_U, + Ops* a_ops, + const PreconditionerType& a_pc_type ) { m_Z.Define(a_U); m_Y0.Define(a_U); @@ -169,6 +188,20 @@ void JacobianFunctionMF::define ( const T& a_U, m_ops = a_ops; + m_usePreCond = (a_pc_type != PreconditionerType::none); + if (m_usePreCond) { + m_pc_type = a_pc_type; + if (m_pc_type == PreconditionerType::pc_curl_curl_mlmg) { + m_preCond = std::make_unique>(); + } else { + std::stringstream convergenceMsg; + convergenceMsg << "JacobianFunctionMF::define(): " << amrex::getEnumNameString(m_pc_type) + << " is not a valid preconditioner type."; + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } + m_preCond->Define(a_U, a_ops); + } + m_is_defined = true; } @@ -220,7 +253,7 @@ void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) const RT eps_inv = 1.0_rt/eps; m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU - m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, -1, true ); // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU // = dU - (R(Z)-R(Y0))/eps diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H index 93ad432208a..f92687d6b34 100644 --- a/Source/NonlinearSolvers/NewtonSolver.H +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -9,10 +9,11 @@ #include "NonlinearSolver.H" #include "JacobianFunctionMF.H" +#include "Preconditioner.H" +#include "Utils/TextMsg.H" #include #include -#include "Utils/TextMsg.H" #include @@ -28,9 +29,9 @@ class NewtonSolver : public NonlinearSolver { public: - NewtonSolver() = default; + NewtonSolver() = default; - ~NewtonSolver() override = default; + ~NewtonSolver() override = default; // Prohibit Move and Copy operations NewtonSolver(const NewtonSolver&) = delete; @@ -69,16 +70,19 @@ public: void PrintParams () const override { - amrex::Print() << "Newton verbose: " << (this->m_verbose?"true":"false") << std::endl; - amrex::Print() << "Newton max iterations: " << m_maxits << std::endl; - amrex::Print() << "Newton relative tolerance: " << m_rtol << std::endl; - amrex::Print() << "Newton absolute tolerance: " << m_atol << std::endl; - amrex::Print() << "Newton require convergence: " << (m_require_convergence?"true":"false") << std::endl; - amrex::Print() << "GMRES verbose: " << m_gmres_verbose_int << std::endl; - amrex::Print() << "GMRES restart length: " << m_gmres_restart_length << std::endl; - amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << std::endl; - amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << std::endl; - amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << std::endl; + amrex::Print() << "Newton verbose: " << (this->m_verbose?"true":"false") << "\n"; + amrex::Print() << "Newton max iterations: " << m_maxits << "\n"; + amrex::Print() << "Newton relative tolerance: " << m_rtol << "\n"; + amrex::Print() << "Newton absolute tolerance: " << m_atol << "\n"; + amrex::Print() << "Newton require convergence: " << (m_require_convergence?"true":"false") << "\n"; + amrex::Print() << "GMRES verbose: " << m_gmres_verbose_int << "\n"; + amrex::Print() << "GMRES restart length: " << m_gmres_restart_length << "\n"; + amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << "\n"; + amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << "\n"; + amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << "\n"; + amrex::Print() << "Preconditioner type: " << amrex::getEnumNameString(m_pc_type) << "\n"; + + m_linear_function->printParams(); } private: @@ -138,9 +142,12 @@ private: */ int m_gmres_restart_length = 30; + /** + * \brief Preconditioner type + */ + PreconditionerType m_pc_type = PreconditionerType::none; + mutable amrex::Real m_cur_time, m_dt; - mutable bool m_update_pc = false; - mutable bool m_update_pc_init = false; /** * \brief The linear function used by GMRES to compute A*v. @@ -162,7 +169,6 @@ private: const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const; }; @@ -184,7 +190,7 @@ void NewtonSolver::Define ( const Vec& a_U, m_ops = a_ops; m_linear_function = std::make_unique>(); - m_linear_function->define(m_F, m_ops); + m_linear_function->define(m_F, m_ops, m_pc_type); m_linear_solver = std::make_unique>>(); m_linear_solver->define(*m_linear_function); @@ -212,6 +218,9 @@ void NewtonSolver::ParseParameters () pp_gmres.query("absolute_tolerance", m_gmres_atol); pp_gmres.query("relative_tolerance", m_gmres_rtol); pp_gmres.query("max_iterations", m_gmres_maxits); + + const amrex::ParmParse pp_jac("jacobian"); + pp_jac.query("pc_type", m_pc_type); } template @@ -242,7 +251,7 @@ void NewtonSolver::Solve ( Vec& a_U, for (iter = 0; iter < m_maxits;) { // Compute residual: F(U) = U - b - R(U) - EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + EvalResidual(m_F, a_U, a_b, a_time, iter); // Compute norm of the residual norm_abs = m_F.norm2(); @@ -261,19 +270,19 @@ void NewtonSolver::Solve ( Vec& a_U, if (norm_abs < m_rtol) { amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter - << ". Satisfied absolute tolerance " << m_atol << std::endl; + << ". Satisfied absolute tolerance " << m_atol << "\n"; break; } if (norm_rel < m_rtol) { amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter - << ". Satisfied relative tolerance " << m_rtol << std::endl; + << ". Satisfied relative tolerance " << m_rtol << "\n"; break; } if (norm_abs > 100._rt*norm0) { amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter - << ". SOLVER DIVERGED! relative tolerance = " << m_rtol << std::endl; + << ". SOLVER DIVERGED! relative tolerance = " << m_rtol << "\n"; std::stringstream convergenceMsg; convergenceMsg << "Newton: exiting at iteration " << std::setw(3) << iter << ". SOLVER DIVERGED! absolute norm = " << norm_abs << @@ -291,7 +300,7 @@ void NewtonSolver::Solve ( Vec& a_U, iter++; if (iter >= m_maxits) { amrex::Print() << "Newton: exiting at iter = " << std::setw(3) << iter - << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + << ". Maximum iteration reached: iter = " << m_maxits << "\n"; break; } @@ -304,7 +313,7 @@ void NewtonSolver::Solve ( Vec& a_U, " and the relative tolerance is " << m_rtol << ". Absolute norm is " << norm_abs << " and the absolute tolerance is " << m_atol; - if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << "\n"; } if (m_require_convergence) { WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); } else { @@ -319,21 +328,17 @@ void NewtonSolver::EvalResidual ( Vec& a_F, const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const { - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, a_iter, false ); // set base U and R(U) for matrix-free Jacobian action calculation m_linear_function->setBaseSolution(a_U); m_linear_function->setBaseRHS(m_R); // update preconditioner - if (m_update_pc || m_update_pc_init) { - m_linear_function->updatePreCondMat(a_U); - } - m_update_pc_init = false; + m_linear_function->updatePreCondMat(a_U); // Compute residual: F(U) = U - b - R(U) a_F.Copy(a_U); diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H index 5587826474c..9daa3489f11 100644 --- a/Source/NonlinearSolvers/NonlinearSolver.H +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -16,7 +16,7 @@ * This class is templated on a vector class Vec, and an operator class Ops. * * The Ops class must have the following function: - * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * ComputeRHS( R_vec, U_vec, time, nl_iter, from_jacobian ), * where U_vec and R_vec are of type Vec. * * The Vec class must have basic math operators, such as Copy, +=, -=, @@ -28,9 +28,9 @@ class NonlinearSolver { public: - NonlinearSolver() = default; + NonlinearSolver() = default; - virtual ~NonlinearSolver() = default; + virtual ~NonlinearSolver() = default; // Prohibit Move and Copy operations NonlinearSolver(const NonlinearSolver&) = delete; diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H index f05b9a106e6..62323b64a23 100644 --- a/Source/NonlinearSolvers/PicardSolver.H +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -26,9 +26,9 @@ class PicardSolver : public NonlinearSolver { public: - PicardSolver() = default; + PicardSolver() = default; - ~PicardSolver() override = default; + ~PicardSolver() override = default; // Prohibit Move and Copy operations PicardSolver(const PicardSolver&) = delete; @@ -55,10 +55,10 @@ public: void PrintParams () const override { - amrex::Print() << "Picard max iterations: " << m_maxits << std::endl; - amrex::Print() << "Picard relative tolerance: " << m_rtol << std::endl; - amrex::Print() << "Picard absolute tolerance: " << m_atol << std::endl; - amrex::Print() << "Picard require convergence: " << (m_require_convergence?"true":"false") << std::endl; + amrex::Print() << "Picard max iterations: " << m_maxits << "\n"; + amrex::Print() << "Picard relative tolerance: " << m_rtol << "\n"; + amrex::Print() << "Picard absolute tolerance: " << m_atol << "\n"; + amrex::Print() << "Picard require convergence: " << (m_require_convergence?"true":"false") << "\n"; } private: @@ -138,6 +138,7 @@ void PicardSolver::Solve ( Vec& a_U, WARPX_ALWAYS_ASSERT_WITH_MESSAGE( this->m_is_defined, "PicardSolver::Solve() called on undefined object"); + amrex::ignore_unused(a_dt); using namespace amrex::literals; // @@ -156,7 +157,7 @@ void PicardSolver::Solve ( Vec& a_U, m_Usave.Copy(a_U); // Update the solver state (a_U = a_b + m_R) - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, iter, false ); a_U.Copy(a_b); a_U += m_R; @@ -179,19 +180,19 @@ void PicardSolver::Solve ( Vec& a_U, if (norm_abs < m_atol) { amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter - << ". Satisfied absolute tolerance " << m_atol << std::endl; + << ". Satisfied absolute tolerance " << m_atol << "\n"; break; } if (norm_rel < m_rtol) { amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter - << ". Satisfied relative tolerance " << m_rtol << std::endl; + << ". Satisfied relative tolerance " << m_rtol << "\n"; break; } if (iter >= m_maxits) { amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter - << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + << ". Maximum iteration reached: iter = " << m_maxits << "\n"; break; } @@ -204,7 +205,7 @@ void PicardSolver::Solve ( Vec& a_U, " and the relative tolerance is " << m_rtol << ". Absolute norm is " << norm_abs << " and the absolute tolerance is " << m_atol; - if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << "\n"; } if (m_require_convergence) { WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); } else { diff --git a/Source/NonlinearSolvers/Preconditioner.H b/Source/NonlinearSolvers/Preconditioner.H new file mode 100644 index 00000000000..191a48d00bc --- /dev/null +++ b/Source/NonlinearSolvers/Preconditioner.H @@ -0,0 +1,100 @@ +/* Copyright 2024 Debojyoti Ghosh + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PRECONDITIONER_H_ +#define WARPX_PRECONDITIONER_H_ + +#include + +/** + * \brief Types for preconditioners for field solvers + */ +AMREX_ENUM(PreconditionerType, pc_curl_curl_mlmg, none); + +/** + * \brief Base class for preconditioners + * + * This class is templated on a solution-type class T and an operator class Ops. + * + * The Ops class must have the following function: + * (this will depend on the specific preconditioners inheriting from this class) + * + * The T class must have the following functions: + * (this will depend on the specific preconditioners inheriting from this class) + */ + +template +class Preconditioner +{ + public: + + using RT = typename T::value_type; + + /** + * \brief Default constructor + */ + Preconditioner () = default; + + /** + * \brief Default destructor + */ + virtual ~Preconditioner () = default; + + // Default move and copy operations + Preconditioner(const Preconditioner&) = default; + Preconditioner& operator=(const Preconditioner&) = default; + Preconditioner(Preconditioner&&) noexcept = default; + Preconditioner& operator=(Preconditioner&&) noexcept = default; + + /** + * \brief Define the preconditioner + */ + virtual void Define (const T&, Ops*) = 0; + + /** + * \brief Update the preconditioner + */ + virtual void Update ( const T& ) = 0; + + /** + * \brief Apply (solve) the preconditioner given a RHS + * + * Given a right-hand-side b, solve: + * A x = b + * where A is a linear operator. + */ + virtual void Apply (T& a_x, const T& a_b) = 0; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] virtual bool IsDefined () const = 0; + + /** + * \brief Print parameters + */ + virtual void printParameters() const { } + + /** + * \brief Set the current time. + */ + inline void CurTime (const RT a_time) { m_time = a_time; } + + /** + * \brief Set the current time step size. + */ + inline void CurTimeStep (const RT a_dt) { m_dt = a_dt; } + + protected: + + RT m_time = 0.0; + RT m_dt = 0.0; + + private: + +}; + +#endif diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index c70bd6d3a35..c9de3d8b250 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,6 +7,7 @@ #ifndef WARPX_GUARDCELLMANAGER_H_ #define WARPX_GUARDCELLMANAGER_H_ +#include #include #include @@ -63,7 +64,7 @@ public: int nox, int nox_fft, int noy_fft, int noz_fft, int nci_corr_stencil, - int electromagnetic_solver_id, + ElectromagneticSolverAlgo electromagnetic_solver_id, int max_level, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 4c2f978484d..a36e39d9497 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -42,7 +42,7 @@ guardCellManager::Init ( const int nox, const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, - const int electromagnetic_solver_id, + const ElectromagneticSolverAlgo electromagnetic_solver_id, const int max_level, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, @@ -101,9 +101,9 @@ guardCellManager::Init ( // the fine grid by a number of cells equal to the ref_ratio in the moving // window direction. if (do_moving_window) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(ref_ratios.size() <= 1, + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1, "The number of grow cells for the moving window currently assumes 2 levels max."); - const auto nlevs = static_cast(ref_ratios.size()+1); + const auto nlevs = static_cast(max_level+1); const int max_r = (nlevs > 1) ? ref_ratios[0][moving_window_dir] : 2; ngx = std::max(ngx,max_r); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 6c44df061fd..3adf4389a46 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -12,6 +12,7 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif +#include "Fields.H" #include "Filter/BilinearFilter.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -20,6 +21,7 @@ #include "WarpXSumGuardCells.H" #include "Particles/MultiParticleContainer.H" +#include #include #include @@ -49,13 +51,20 @@ #include using namespace amrex; +using warpx::fields::FieldType; void WarpX::UpdateAuxilaryData () { WARPX_PROFILE("WarpX::UpdateAuxilaryData()"); - if (Bfield_aux[0][0]->ixType() == Bfield_fp[0][0]->ixType()) { + using ablastr::fields::Direction; + + amrex::MultiFab *Bfield_aux_lvl0_0 = m_fields.get(FieldType::Bfield_aux, Direction{0}, 0); + + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + + if (Bfield_aux_lvl0_0->ixType() == Bfield_fp[0][0]->ixType()) { UpdateAuxilaryDataSameType(); } else { UpdateAuxilaryDataStagToNodal(); @@ -64,14 +73,18 @@ WarpX::UpdateAuxilaryData () // When loading particle fields from file: add the external fields: for (int lev = 0; lev <= finest_level; ++lev) { if (mypc->m_E_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Efield_aux[lev][0], *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Efield_aux[lev][1], *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Efield_aux[lev][2], *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + ablastr::fields::VectorField Efield_aux = m_fields.get_alldirs(FieldType::Efield_aux, lev); + const auto& E_ext_lev = m_fields.get_alldirs(FieldType::E_external_particle_field, lev); + amrex::MultiFab::Add(*Efield_aux[0], *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[1], *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[2], *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Bfield_aux[lev][0], *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux[lev][1], *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux[lev][2], *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + ablastr::fields::VectorField Bfield_aux = m_fields.get_alldirs(FieldType::Bfield_aux, lev); + const auto& B_ext_lev = m_fields.get_alldirs(FieldType::B_external_particle_field, lev); + amrex::MultiFab::Add(*Bfield_aux[0], *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[1], *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[2], *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } } @@ -87,11 +100,21 @@ WarpX::UpdateAuxilaryDataStagToNodal () "WarpX build with spectral solver support."); } #endif - - amrex::Vector,3>> const & Bmf = WarpX::fft_do_time_averaging ? - Bfield_avg_fp : Bfield_fp; - amrex::Vector,3>> const & Emf = WarpX::fft_do_time_averaging ? - Efield_avg_fp : Efield_fp; + using ablastr::fields::Direction; + + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); + + ablastr::fields::MultiLevelVectorField const & Bmf = + WarpX::fft_do_time_averaging ? + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level) : + Bfield_fp; + ablastr::fields::MultiLevelVectorField const & Emf = + WarpX::fft_do_time_averaging ? + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level) : + Efield_fp; const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); const amrex::IntVect& By_stag = Bmf[0][1]->ixType().toIntVect(); @@ -173,10 +196,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Btmp; - if (Bfield_cax[lev][0]) { + if (m_fields.has_vector(FieldType::Bfield_cax, lev)) { for (int i = 0; i < 3; ++i) { Btmp[i] = std::make_unique( - *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + *m_fields.get(FieldType::Bfield_cax, Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); @@ -200,13 +223,13 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -216,12 +239,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); - Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); - Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); + Array4 const& bx_fp = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& by_fp = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& bz_fp = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->const_array(mfi); + Array4 const& bx_cp = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->const_array(mfi); + Array4 const& by_cp = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->const_array(mfi); + Array4 const& bz_cp = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->const_array(mfi); Array4 const& bx_c = Btmp[0]->const_array(mfi); Array4 const& by_c = Btmp[1]->const_array(mfi); Array4 const& bz_c = Btmp[2]->const_array(mfi); @@ -267,10 +290,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Etmp; - if (Efield_cax[lev][0]) { + if (m_fields.has_vector(FieldType::Efield_cax, lev)) { for (int i = 0; i < 3; ++i) { Etmp[i] = std::make_unique( - *Efield_cax[lev][i], amrex::make_alias, 0, 1); + *m_fields.get(FieldType::Efield_cax, Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); @@ -295,13 +318,13 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType().toIntVect(); - const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -311,12 +334,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); - Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); - Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); + Array4 const& ex_cp = m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_cp = m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_cp = m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->const_array(mfi); Array4 const& ex_c = Etmp[0]->const_array(mfi); Array4 const& ey_c = Etmp[1]->const_array(mfi); Array4 const& ez_c = Etmp[2]->const_array(mfi); @@ -332,9 +355,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () } } else { // electrostatic - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -343,9 +366,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); const Box& bx = mfi.growntilebox(); amrex::ParallelFor(bx, @@ -367,17 +390,23 @@ WarpX::UpdateAuxilaryDataSameType () // Update aux field, including guard cells, up to ng_FieldGather const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); + // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. if (WarpX::fft_do_time_averaging) { - MultiFab::Copy(*Efield_aux[0][0], *Efield_avg_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][1], *Efield_avg_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][2], *Efield_avg_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][0], *Bfield_avg_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][1], *Bfield_avg_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][2], *Bfield_avg_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][0], *m_fields.get(FieldType::Efield_avg_fp, Direction{0}, 0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *m_fields.get(FieldType::Efield_avg_fp, Direction{1}, 0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *m_fields.get(FieldType::Efield_avg_fp, Direction{2}, 0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, 0), 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, 0), 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, 0), 0, 0, Bfield_aux[0][2]->nComp(), ng_src); } else { @@ -391,16 +420,19 @@ WarpX::UpdateAuxilaryDataSameType () for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); - const IntVect& ng = Bfield_cp[lev][0]->nGrowVect(); - const DistributionMapping& dm = Bfield_cp[lev][0]->DistributionMap(); + const IntVect& ng = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nGrowVect(); + const DistributionMapping& dm = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->DistributionMap(); // B field { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); - MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); - MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); + MultiFab dBx(m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab dBy(m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab dBz(m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->nComp(), ng); dBx.setVal(0.0); dBy.setVal(0.0); dBz.setVal(0.0); @@ -418,15 +450,18 @@ WarpX::UpdateAuxilaryDataSameType () Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, crse_period); - if (Bfield_cax[lev][0]) + if (m_fields.has_vector(FieldType::Bfield_cax, lev)) { - MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{0}, lev), dBx, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{1}, lev), dBy, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{2}, lev), dBz, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{2}, lev)->nComp(), ng); } - MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); + MultiFab::Subtract(dBx, *m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab::Subtract(dBy, *m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab::Subtract(dBz, *m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -475,9 +510,12 @@ WarpX::UpdateAuxilaryDataSameType () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); - MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); - MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); + MultiFab dEx(m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab dEy(m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab dEz(m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->nComp(), ng); dEx.setVal(0.0); dEy.setVal(0.0); dEz.setVal(0.0); @@ -497,15 +535,18 @@ WarpX::UpdateAuxilaryDataSameType () WarpX::do_single_precision_comms, crse_period); - if (Efield_cax[lev][0]) + if (m_fields.has_vector(FieldType::Efield_cax, lev)) { - MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{0}, lev), dEx, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{1}, lev), dEy, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{2}, lev), dEz, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{2}, lev)->nComp(), ng); } - MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); + MultiFab::Subtract(dEx, *m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab::Subtract(dEy, *m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab::Subtract(dEz, *m_fields.get(FieldType::Efield_cp, Direction{2}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -521,9 +562,9 @@ WarpX::UpdateAuxilaryDataSameType () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); Array4 const& ex_c = dEx.const_array(mfi); Array4 const& ey_c = dEy.const_array(mfi); Array4 const& ez_c = dEz.const_array(mfi); @@ -545,9 +586,9 @@ WarpX::UpdateAuxilaryDataSameType () } else // electrostatic { - MultiFab::Copy(*Efield_aux[lev][0], *Efield_fp[lev][0], 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][1], *Efield_fp[lev][1], 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][2], *Efield_fp[lev][2], 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][0], *m_fields.get(FieldType::Efield_fp, Direction{0}, lev), 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *m_fields.get(FieldType::Efield_fp, Direction{1}, lev), 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *m_fields.get(FieldType::Efield_fp, Direction{2}, lev), 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } @@ -668,14 +709,20 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In std::array mf; amrex::Periodicity period; + using ablastr::fields::Direction; + if (patch_type == PatchType::fine) { - mf = {Efield_fp[lev][0].get(), Efield_fp[lev][1].get(), Efield_fp[lev][2].get()}; + mf = {m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev)}; period = Geom(lev).periodicity(); } else // coarse patch { - mf = {Efield_cp[lev][0].get(), Efield_cp[lev][1].get(), Efield_cp[lev][2].get()}; + mf = {m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + m_fields.get(FieldType::Efield_cp, Direction{2}, lev)}; period = Geom(lev-1).periodicity(); } @@ -686,16 +733,18 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In if (pml[lev] && pml[lev]->ok()) { const std::array mf_pml = - (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); + (patch_type == PatchType::fine) ? + m_fields.get_alldirs(FieldType::pml_E_fp, lev) : + m_fields.get_alldirs(FieldType::pml_E_cp, lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryE(patch_type, nodal_sync); + pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); + pml_rz[lev]->FillBoundaryE(m_fields, patch_type, nodal_sync); } #endif } @@ -707,7 +756,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); - const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; + const amrex::IntVect nghost = (m_safe_guard_cells) ? mf[i]->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*mf[i], nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -725,14 +774,20 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In std::array mf; amrex::Periodicity period; + using ablastr::fields::Direction; + if (patch_type == PatchType::fine) { - mf = {Bfield_fp[lev][0].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get()}; + mf = {m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)}; period = Geom(lev).periodicity(); } else // coarse patch { - mf = {Bfield_cp[lev][0].get(), Bfield_cp[lev][1].get(), Bfield_cp[lev][2].get()}; + mf = {m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)}; period = Geom(lev-1).periodicity(); } @@ -743,16 +798,18 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In if (pml[lev] && pml[lev]->ok()) { const std::array mf_pml = - (patch_type == PatchType::fine) ? pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); + (patch_type == PatchType::fine) ? + m_fields.get_alldirs(FieldType::pml_B_fp, lev) : + m_fields.get_alldirs(FieldType::pml_B_cp, lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryB(patch_type, nodal_sync); + pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); + pml_rz[lev]->FillBoundaryB(m_fields, patch_type, nodal_sync); } #endif } @@ -764,7 +821,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); - const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; + const amrex::IntVect nghost = (m_safe_guard_cells) ? mf[i]->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*mf[i], nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -779,6 +836,8 @@ WarpX::FillBoundaryE_avg(int lev, IntVect ng) void WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) { + bool const skip_lev0_coarse_patch = true; + if (patch_type == PatchType::fine) { if (do_pml && pml[lev]->ok()) @@ -786,9 +845,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); - if ( safe_guard_cells ){ - const Vector mf{Efield_avg_fp[lev][0].get(),Efield_avg_fp[lev][1].get(),Efield_avg_fp[lev][2].get()}; + if ( m_safe_guard_cells ){ + const Vector mf{Efield_avg_fp[lev][0],Efield_avg_fp[lev][1],Efield_avg_fp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -806,9 +867,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_lev0_coarse_patch); + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); - if ( safe_guard_cells ) { - const Vector mf{Efield_avg_cp[lev][0].get(),Efield_avg_cp[lev][1].get(),Efield_avg_cp[lev][2].get()}; + if ( m_safe_guard_cells ) { + const Vector mf{Efield_avg_cp[lev][0],Efield_avg_cp[lev][1],Efield_avg_cp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { @@ -833,19 +896,26 @@ WarpX::FillBoundaryB_avg (int lev, IntVect ng) void WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) { + using ablastr::fields::Direction; + + bool const skip_lev0_coarse_patch = true; + if (patch_type == PatchType::fine) { if (do_pml && pml[lev]->ok()) { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); - if ( safe_guard_cells ) { - const Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; + if ( m_safe_guard_cells ) { + const Vector mf{Bfield_avg_fp[lev][0],Bfield_avg_fp[lev][1],Bfield_avg_fp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng.allLE(Bfield_fp[lev][0]->nGrowVect()), + ng.allLE(m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -859,9 +929,11 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_lev0_coarse_patch); + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); - if ( safe_guard_cells ){ - const Vector mf{Bfield_avg_cp[lev][0].get(),Bfield_avg_cp[lev][1].get(),Bfield_avg_cp[lev][2].get()}; + if ( m_safe_guard_cells ){ + const Vector mf{Bfield_avg_cp[lev][0],Bfield_avg_cp[lev][1],Bfield_avg_cp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -888,30 +960,38 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } - pml[lev]->FillBoundaryF(patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_F_fp, lev) && m_fields.has(FieldType::F_fp, lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_F_fp, lev), m_fields.get(FieldType::F_fp, lev), patch_type, do_pml_in_domain); + } + if (m_fields.has(FieldType::pml_F_fp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_F_fp, lev), patch_type, nodal_sync); + } } - if (F_fp[lev]) + if (m_fields.has(FieldType::F_fp, lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? F_fp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*F_fp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (m_safe_guard_cells) ? m_fields.get(FieldType::F_fp, lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get(FieldType::F_fp, lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } - pml[lev]->FillBoundaryF(patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_F_cp, lev) && m_fields.has(FieldType::F_cp, lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_F_cp, lev), m_fields.get(FieldType::F_cp, lev), patch_type, do_pml_in_domain); + } + if (m_fields.has(FieldType::pml_F_cp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_F_cp, lev), patch_type, nodal_sync); + } } - if (F_cp[lev]) + if (m_fields.has(FieldType::F_cp, lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? F_cp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*F_cp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (m_safe_guard_cells) ? m_fields.get(FieldType::F_cp, lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get(FieldType::F_cp, lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } } @@ -932,30 +1012,40 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } - pml[lev]->FillBoundaryG(patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_G_fp,lev) && m_fields.has(FieldType::G_fp,lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_G_fp, lev), m_fields.get(FieldType::G_fp, lev), patch_type, do_pml_in_domain); + } + if (m_fields.has(FieldType::pml_G_fp,lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_G_fp, lev), patch_type, nodal_sync); + } } - if (G_fp[lev]) + if (m_fields.has(FieldType::G_fp,lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*G_fp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + MultiFab* G_fp = m_fields.get(FieldType::G_fp,lev); + const amrex::IntVect& nghost = (m_safe_guard_cells) ? G_fp->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*G_fp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } - pml[lev]->FillBoundaryG(patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_G_cp,lev) && m_fields.has(FieldType::G_cp,lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_G_cp, lev), m_fields.get(FieldType::G_cp, lev), patch_type, do_pml_in_domain); + } + if (m_fields.has(FieldType::pml_G_cp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_G_cp, lev), patch_type, nodal_sync); + } } - if (G_cp[lev]) + if (m_fields.has(FieldType::G_cp,lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*G_cp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + MultiFab* G_cp = m_fields.get(FieldType::G_cp,lev); + const amrex::IntVect& nghost = (m_safe_guard_cells) ? G_cp->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*G_cp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } } @@ -972,6 +1062,9 @@ WarpX::FillBoundaryAux (IntVect ng) void WarpX::FillBoundaryAux (int lev, IntVect ng) { + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); ablastr::utils::communication::FillBoundary(*Efield_aux[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Efield_aux[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -982,23 +1075,28 @@ WarpX::FillBoundaryAux (int lev, IntVect ng) } void -WarpX::SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer) +WarpX::SyncCurrent (const std::string& current_fp_string) { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::SyncCurrent()"); + bool const skip_lev0_coarse_patch = true; + + ablastr::fields::MultiLevelVectorField const& J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level); + // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { + ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs(FieldType::current_fp_nodal, finest_level); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); for (int lev = 0; lev <= finest_level; lev++) { - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][0], *current_fp_nodal[lev][0]); - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][1], *current_fp_nodal[lev][1]); - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][2], *current_fp_nodal[lev][2]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{0}], *J_fp_nodal[lev][Direction{0}]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{1}], *J_fp_nodal[lev][Direction{1}]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{2}], *J_fp_nodal[lev][Direction{2}]); } } @@ -1073,7 +1171,7 @@ WarpX::SyncCurrent ( { for (int lev = finest_level; lev >= 0; --lev) { - const int ncomp = J_fp[lev][idim]->nComp(); + const int ncomp = J_fp[lev][Direction{idim}]->nComp(); auto const& period = Geom(lev).periodicity(); if (lev < finest_level) @@ -1081,8 +1179,8 @@ WarpX::SyncCurrent ( // On a coarse level, the data in mf_comm comes from the // coarse patch of the fine level. They are unfiltered and uncommunicated. // We need to add it to the fine patch of the current level. - MultiFab fine_lev_cp(J_fp[lev][idim]->boxArray(), - J_fp[lev][idim]->DistributionMap(), + MultiFab fine_lev_cp(J_fp[lev][Direction{idim}]->boxArray(), + J_fp[lev][Direction{idim}]->DistributionMap(), ncomp, 0); fine_lev_cp.setVal(0.0); fine_lev_cp.ParallelAdd(*mf_comm, 0, 0, ncomp, mf_comm->nGrowVect(), @@ -1091,7 +1189,7 @@ WarpX::SyncCurrent ( auto owner_mask = amrex::OwnerMask(fine_lev_cp, period); auto const& mma = owner_mask->const_arrays(); auto const& sma = fine_lev_cp.const_arrays(); - auto const& dma = J_fp[lev][idim]->arrays(); + auto const& dma = J_fp[lev][Direction{idim}]->arrays(); amrex::ParallelFor(fine_lev_cp, IntVect(0), ncomp, [=] AMREX_GPU_DEVICE (int bno, int i, int j, int k, int n) { @@ -1100,9 +1198,10 @@ WarpX::SyncCurrent ( } }); // Now it's safe to apply filter and sumboundary on J_cp + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch); if (use_filter) { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); } SumBoundaryJ(J_cp, lev+1, idim, period); } @@ -1114,29 +1213,32 @@ WarpX::SyncCurrent ( // filtering depends on the level. This is also done before any // same-level communication because it's easier this way to // avoid double counting. - J_cp[lev][idim]->setVal(0.0); - ablastr::coarsen::average::Coarsen(*J_cp[lev][idim], - *J_fp[lev][idim], + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_lev0_coarse_patch); + J_cp[lev][Direction{idim}]->setVal(0.0); + ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}], + *J_fp[lev][Direction{idim}], refRatio(lev-1)); - if (J_buffer[lev][idim]) + if (m_fields.has(FieldType::current_buf, Direction{idim}, lev)) { - IntVect const& ng = J_cp[lev][idim]->nGrowVect(); - AMREX_ASSERT(ng.allLE(J_buffer[lev][idim]->nGrowVect())); - MultiFab::Add(*J_buffer[lev][idim], *J_cp[lev][idim], + ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_lev0_coarse_patch); + + IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect(); + AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect())); + MultiFab::Add(*J_buffer[lev][Direction{idim}], *J_cp[lev][Direction{idim}], 0, 0, ncomp, ng); mf_comm = std::make_unique - (*J_buffer[lev][idim], amrex::make_alias, 0, ncomp); + (*J_buffer[lev][Direction{idim}], amrex::make_alias, 0, ncomp); } else { mf_comm = std::make_unique - (*J_cp[lev][idim], amrex::make_alias, 0, ncomp); + (*J_cp[lev][Direction{idim}], amrex::make_alias, 0, ncomp); } } if (use_filter) { - ApplyFilterJ(J_fp, lev, idim); + ApplyFilterMF(J_fp, lev, idim); } SumBoundaryJ(J_fp, lev, idim, period); } @@ -1145,14 +1247,25 @@ WarpX::SyncCurrent ( void WarpX::SyncRho () { - SyncRho(rho_fp, rho_cp, charge_buf); + bool const skip_lev0_coarse_patch = true; + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has(FieldType::rho_fp, 0) ? + m_fields.get_mr_levels(FieldType::rho_fp, finest_level) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has(FieldType::rho_cp, 1) ? + m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_lev0_coarse_patch) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; + const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has(FieldType::rho_buf, 1) ? + m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_lev0_coarse_patch) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; + + SyncRho(rho_fp, rho_cp, rho_buf); } void WarpX::SyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const amrex::Vector>& charge_buffer) + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_buffer) { WARPX_PROFILE("WarpX::SyncRho()"); @@ -1227,8 +1340,8 @@ WarpX::SyncRho ( * averaging the values of the current of the fine patch (on the same level). */ void WarpX::RestrictCurrentFromFineToCoarsePatch ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, const int lev) { J_cp[lev][0]->setVal(0.0); @@ -1237,51 +1350,55 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( const IntVect& refinement_ratio = refRatio(lev-1); - std::array fine { J_fp[lev][0].get(), - J_fp[lev][1].get(), - J_fp[lev][2].get() }; - std::array< MultiFab*,3> crse { J_cp[lev][0].get(), - J_cp[lev][1].get(), - J_cp[lev][2].get() }; + std::array fine { J_fp[lev][0], + J_fp[lev][1], + J_fp[lev][2] }; + std::array< MultiFab*,3> crse { J_cp[lev][0], + J_cp[lev][1], + J_cp[lev][2] }; ablastr::coarsen::average::Coarsen(*crse[0], *fine[0], refinement_ratio ); ablastr::coarsen::average::Coarsen(*crse[1], *fine[1], refinement_ratio ); ablastr::coarsen::average::Coarsen(*crse[2], *fine[2], refinement_ratio ); } -void WarpX::ApplyFilterJ ( - const amrex::Vector,3>>& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev, const int idim) { - amrex::MultiFab& J = *current[lev][idim]; + using ablastr::fields::Direction; - const int ncomp = J.nComp(); - const amrex::IntVect ngrow = J.nGrowVect(); - amrex::MultiFab Jf(J.boxArray(), J.DistributionMap(), ncomp, ngrow); - bilinear_filter.ApplyStencil(Jf, J, lev); + amrex::MultiFab& mf = *mfvec[lev][Direction{idim}]; + + const int ncomp = mf.nComp(); + const amrex::IntVect ngrow = mf.nGrowVect(); + amrex::MultiFab mf_filtered(mf.boxArray(), mf.DistributionMap(), ncomp, ngrow); + bilinear_filter.ApplyStencil(mf_filtered, mf, lev); const int srccomp = 0; const int dstcomp = 0; - amrex::MultiFab::Copy(J, Jf, srccomp, dstcomp, ncomp, ngrow); + amrex::MultiFab::Copy(mf, mf_filtered, srccomp, dstcomp, ncomp, ngrow); } -void WarpX::ApplyFilterJ ( - const amrex::Vector,3>>& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev) { for (int idim=0; idim<3; ++idim) { - ApplyFilterJ(current, lev, idim); + ApplyFilterMF(mfvec, lev, idim); } } void WarpX::SumBoundaryJ ( - const amrex::Vector,3>>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev, const int idim, const amrex::Periodicity& period) { - amrex::MultiFab& J = *current[lev][idim]; + using ablastr::fields::Direction; + + amrex::MultiFab& J = *current[lev][Direction{idim}]; const amrex::IntVect ng = J.nGrowVect(); amrex::IntVect ng_depos_J = get_ng_depos_J(); @@ -1314,7 +1431,7 @@ void WarpX::SumBoundaryJ ( } void WarpX::SumBoundaryJ ( - const amrex::Vector,3>>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev, const amrex::Periodicity& period) { @@ -1338,16 +1455,16 @@ void WarpX::SumBoundaryJ ( * patch (and buffer region) of `lev+1` */ void WarpX::AddCurrentFromFineLevelandSumBoundary ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer, const int lev) { const amrex::Periodicity& period = Geom(lev).periodicity(); if (use_filter) { - ApplyFilterJ(J_fp, lev); + ApplyFilterMF(J_fp, lev); } SumBoundaryJ(J_fp, lev, period); @@ -1366,8 +1483,8 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter && J_buffer[lev+1][idim]) { - ApplyFilterJ(J_cp, lev+1, idim); - ApplyFilterJ(J_buffer, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); + ApplyFilterMF(J_buffer, lev+1, idim); MultiFab::Add( *J_buffer[lev+1][idim], *J_cp[lev+1][idim], @@ -1381,7 +1498,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } else if (use_filter) // but no buffer { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); ablastr::utils::communication::ParallelAdd( mf, *J_cp[lev+1][idim], 0, 0, @@ -1415,28 +1532,25 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } } -void WarpX::RestrictRhoFromFineToCoarsePatch ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const int lev) +void WarpX::RestrictRhoFromFineToCoarsePatch ( const int lev ) { - if (charge_fp[lev]) { - charge_cp[lev]->setVal(0.0); + if (m_fields.has(FieldType::rho_fp, lev)) { + m_fields.get(FieldType::rho_cp, lev)->setVal(0.0); const IntVect& refinement_ratio = refRatio(lev-1); - ablastr::coarsen::average::Coarsen(*charge_cp[lev], *charge_fp[lev], refinement_ratio ); + ablastr::coarsen::average::Coarsen(*m_fields.get(FieldType::rho_cp, lev), *m_fields.get(FieldType::rho_fp, lev), refinement_ratio ); } } void WarpX::ApplyFilterandSumBoundaryRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const int lev, PatchType patch_type, const int icomp, const int ncomp) { const int glev = (patch_type == PatchType::fine) ? lev : lev-1; - const std::unique_ptr& rho = (patch_type == PatchType::fine) ? + amrex::MultiFab* rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); @@ -1474,9 +1588,9 @@ void WarpX::ApplyFilterandSumBoundaryRho (int /*lev*/, int glev, amrex::MultiFab * patch (and buffer region) of `lev+1` */ void WarpX::AddRhoFromFineLevelandSumBoundary ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const amrex::Vector>& charge_buffer, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_buffer, const int lev, const int icomp, const int ncomp) @@ -1553,51 +1667,3 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( MultiFab::Add(*charge_fp[lev], mf, 0, icomp, ncomp, 0); } } - -void WarpX::NodalSyncJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const int lev, - PatchType patch_type) -{ - if (!override_sync_intervals.contains(istep[0])) { return; } - - if (patch_type == PatchType::fine) - { - const amrex::Periodicity& period = Geom(lev).periodicity(); - ablastr::utils::communication::OverrideSync(*J_fp[lev][0], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*J_fp[lev][1], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*J_fp[lev][2], WarpX::do_single_precision_comms, period); - } - else if (patch_type == PatchType::coarse) - { - const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); - ablastr::utils::communication::OverrideSync(*J_cp[lev][0], WarpX::do_single_precision_comms, cperiod); - ablastr::utils::communication::OverrideSync(*J_cp[lev][1], WarpX::do_single_precision_comms, cperiod); - ablastr::utils::communication::OverrideSync(*J_cp[lev][2], WarpX::do_single_precision_comms, cperiod); - } -} - -void WarpX::NodalSyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const int lev, - PatchType patch_type, - const int icomp, - const int ncomp) -{ - if (!override_sync_intervals.contains(istep[0])) { return; } - - if (patch_type == PatchType::fine && charge_fp[lev]) - { - const amrex::Periodicity& period = Geom(lev).periodicity(); - MultiFab rhof(*charge_fp[lev], amrex::make_alias, icomp, ncomp); - ablastr::utils::communication::OverrideSync(rhof, WarpX::do_single_precision_comms, period); - } - else if (patch_type == PatchType::coarse && charge_cp[lev]) - { - const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); - MultiFab rhoc(*charge_cp[lev], amrex::make_alias, icomp, ncomp); - ablastr::utils::communication::OverrideSync(rhoc, WarpX::do_single_precision_comms, cperiod); - } -} diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index c3362087ad9..79f2b34fba0 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -44,18 +44,19 @@ void warpx_interp (int j, int k, int l, // Refinement ratio const int rj = rr[0]; - const int rk = (AMREX_SPACEDIM == 1) ? 1 : rr[1]; - const int rl = (AMREX_SPACEDIM <= 2) ? 1 : rr[2]; + const int rk = (AMREX_SPACEDIM > 1) ? rr[1] : 1; + const int rl = (AMREX_SPACEDIM > 2) ? rr[2] : 1; // Staggering (0: cell-centered; 1: nodal) + // Unused dimensions are considered nodal. const int sj = arr_stag[0]; - const int sk = (AMREX_SPACEDIM == 1) ? 0 : arr_stag[1]; - const int sl = (AMREX_SPACEDIM <= 2) ? 0 : arr_stag[2]; + const int sk = (AMREX_SPACEDIM > 1) ? arr_stag[1] : 1; + const int sl = (AMREX_SPACEDIM > 2) ? arr_stag[2] : 1; // Number of points used for interpolation from coarse grid to fine grid const int nj = 2; - const int nk = 2; - const int nl = 2; + const int nk = (AMREX_SPACEDIM > 1) ? 2 : 1; + const int nl = (AMREX_SPACEDIM > 2) ? 2 : 1; const int jc = (sj == 0) ? amrex::coarsen(j - rj/2, rj) : amrex::coarsen(j, rj); const int kc = (sk == 0) ? amrex::coarsen(k - rk/2, rk) : amrex::coarsen(k, rk); @@ -133,38 +134,20 @@ void warpx_interp (int j, int k, int l, // Refinement ratio const int rj = rr[0]; -#if defined(WARPX_DIM_1D_Z) - constexpr int rk = 1; - constexpr int rl = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int rk = rr[1]; - constexpr int rl = 1; -#else - const int rk = rr[1]; - const int rl = rr[2]; -#endif + const int rk = (AMREX_SPACEDIM > 1) ? rr[1] : 1; + const int rl = (AMREX_SPACEDIM > 2) ? rr[2] : 1; // Staggering of fine array (0: cell-centered; 1: nodal) + // Unused dimensions are considered nodal. const int sj_fp = arr_fine_stag[0]; -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int sk_fp = arr_fine_stag[1]; -#elif defined(WARPX_DIM_3D) - const int sk_fp = arr_fine_stag[1]; - const int sl_fp = arr_fine_stag[2]; -#endif + const int sk_fp = (AMREX_SPACEDIM > 1) ? arr_fine_stag[1] : 1; + const int sl_fp = (AMREX_SPACEDIM > 2) ? arr_fine_stag[2] : 1; // Staggering of coarse array (0: cell-centered; 1: nodal) + // Unused dimensions are considered nodal. const int sj_cp = arr_coarse_stag[0]; -#if defined(WARPX_DIM_1D_Z) - constexpr int sk_cp = 0; - constexpr int sl_cp = 0; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int sk_cp = arr_coarse_stag[1]; - constexpr int sl_cp = 0; -#else - const int sk_cp = arr_coarse_stag[1]; - const int sl_cp = arr_coarse_stag[2]; -#endif + const int sk_cp = (AMREX_SPACEDIM > 1) ? arr_coarse_stag[1] : 1; + const int sl_cp = (AMREX_SPACEDIM > 2) ? arr_coarse_stag[2] : 1; // Number of points used for interpolation from coarse grid to fine grid int nj; @@ -182,27 +165,19 @@ void warpx_interp (int j, int k, int l, // 1) Interpolation from coarse nodal to fine nodal nj = 2; -#if defined(WARPX_DIM_1D_Z) - nk = 1; - nl = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - nk = 2; - nl = 1; -#else - nk = 2; - nl = 2; -#endif + nk = (AMREX_SPACEDIM > 1) ? 2 : 1; + nl = (AMREX_SPACEDIM > 2) ? 2 : 1; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { auto c = arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); c *= (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); -#if (AMREX_SPACEDIM >= 2) +#if (AMREX_SPACEDIM > 1) c *= (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); -#endif -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM > 2) c *= (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); +#endif #endif tmp += c; } @@ -212,16 +187,8 @@ void warpx_interp (int j, int k, int l, // 2) Interpolation from coarse staggered to fine nodal nj = 2; -#if defined(WARPX_DIM_1D_Z) - nk = 1; - nl = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - nk = 2; - nl = 1; -#else - nk = 2; - nl = 2; -#endif + nk = (AMREX_SPACEDIM > 1) ? 2 : 1; + nl = (AMREX_SPACEDIM > 2) ? 2 : 1; const int jn = (sj_cp == 1) ? j : j - rj / 2; const int kn = (sk_cp == 1) ? k : k - rk / 2; @@ -236,11 +203,11 @@ void warpx_interp (int j, int k, int l, for (int ll = 0; ll < nl; ll++) { auto c = arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); c *= (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); -#if (AMREX_SPACEDIM >= 2) +#if (AMREX_SPACEDIM > 1) c *= (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); -#endif -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM > 2) c *= (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); +#endif #endif coarse += c; } @@ -250,28 +217,12 @@ void warpx_interp (int j, int k, int l, // 3) Interpolation from fine staggered to fine nodal nj = (sj_fp == 0) ? 2 : 1; -#if defined(WARPX_DIM_1D_Z) - nk = 1; - nl = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - nk = (sk_fp == 0) ? 2 : 1; - nl = 1; -#else nk = (sk_fp == 0) ? 2 : 1; nl = (sl_fp == 0) ? 2 : 1; -#endif const int jm = (sj_fp == 0) ? j-1 : j; -#if defined(WARPX_DIM_1D_Z) - const int km = k; - const int lm = l; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int km = (sk_fp == 0) ? k-1 : k; - const int lm = l; -#else const int km = (sk_fp == 0) ? k-1 : k; const int lm = (sl_fp == 0) ? l-1 : l; -#endif for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { @@ -285,6 +236,7 @@ void warpx_interp (int j, int k, int l, // Final result arr_aux(j,k,l) = tmp + (fine - coarse); } + /** * \brief Interpolation function called within WarpX::UpdateAuxilaryDataStagToNodal * to interpolate data from the coarse and fine grids to the fine aux grid, @@ -320,13 +272,10 @@ void warpx_interp (int j, int k, int l, // - (x,y,z) in 3D // Staggering of fine array (0: cell-centered; 1: nodal) + // Unused dimensions are considered nodal. const int sj_fp = arr_fine_stag[0]; -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int sk_fp = arr_fine_stag[1]; -#elif defined(WARPX_DIM_3D) - const int sk_fp = arr_fine_stag[1]; - const int sl_fp = arr_fine_stag[2]; -#endif + const int sk_fp = (AMREX_SPACEDIM > 1) ? arr_fine_stag[1] : 1; + const int sl_fp = (AMREX_SPACEDIM > 2) ? arr_fine_stag[2] : 1; // Number of points used for interpolation from coarse grid to fine grid int nj; @@ -338,28 +287,12 @@ void warpx_interp (int j, int k, int l, // 3) Interpolation from fine staggered to fine nodal nj = (sj_fp == 0) ? 2 : 1; -#if defined(WARPX_DIM_1D_Z) - nk = 1; - nl = 1; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - nk = (sk_fp == 0) ? 2 : 1; - nl = 1; -#else nk = (sk_fp == 0) ? 2 : 1; nl = (sl_fp == 0) ? 2 : 1; -#endif - const int jm = (sj_fp == 0) ? j-1 : j; -#if defined(WARPX_DIM_1D_Z) - const int km = k; - const int lm = l; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int km = (sk_fp == 0) ? k-1 : k; - const int lm = l; -#else - const int km = (sk_fp == 0) ? k-1 : k; - const int lm = (sl_fp == 0) ? l-1 : l; -#endif + int const jm = (sj_fp == 0) ? j-1 : j; + int const km = (sk_fp == 0) ? k-1 : k; + int const lm = (sl_fp == 0) ? l-1 : l; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { @@ -418,11 +351,7 @@ void warpx_interp (const int j, }; // Avoid compiler warnings -#if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(nox, noy, stencil_coeffs_x, stencil_coeffs_y); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(noy, stencil_coeffs_y); -#endif // If dst_nodal = true , we are centering from a staggered grid to a nodal grid // If dst_nodal = false, we are centering from a nodal grid to a staggered grid @@ -432,70 +361,32 @@ void warpx_interp (const int j, const int shift = (dst_nodal) ? 0 : 1; // Staggering (s = 0 if cell-centered, s = 1 if nodal) + // Unused dimensions are considered nodal. const int sj = (dst_nodal) ? src_stag[0] : dst_stag[0]; -#if (AMREX_SPACEDIM >= 2) - const int sk = (dst_nodal) ? src_stag[1] : dst_stag[1]; -#endif -#if defined(WARPX_DIM_3D) - const int sl = (dst_nodal) ? src_stag[2] : dst_stag[2]; -#endif + const int sk = (AMREX_SPACEDIM > 1) ? ((dst_nodal) ? src_stag[1] : dst_stag[1]) : 1; + const int sl = (AMREX_SPACEDIM > 2) ? ((dst_nodal) ? src_stag[2] : dst_stag[2]) : 1; // Interpolate along j,k,l only if source MultiFab is staggered along j,k,l const bool interp_j = (sj == 0); -#if (AMREX_SPACEDIM >= 2) const bool interp_k = (sk == 0); -#endif -#if defined(WARPX_DIM_3D) const bool interp_l = (sl == 0); -#endif -#if defined(WARPX_DIM_1D_Z) - const int noj = noz; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const int noj = nox; - const int nok = noz; -#elif defined(WARPX_DIM_3D) - const int noj = nox; - const int nok = noy; - const int nol = noz; -#endif + const int noj = AMREX_D_PICK(noz, nox, nox); + const int nok = AMREX_D_PICK(0 , noz, noy); + const int nol = AMREX_D_PICK(0 , 0 , noz); // Additional normalization factor const amrex::Real wj = (interp_j) ? 0.5_rt : 1.0_rt; -#if defined(WARPX_DIM_1D_Z) - constexpr amrex::Real wk = 1.0_rt; - constexpr amrex::Real wl = 1.0_rt; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real wk = (interp_k) ? 0.5_rt : 1.0_rt; - constexpr amrex::Real wl = 1.0_rt; -#elif defined(WARPX_DIM_3D) const amrex::Real wk = (interp_k) ? 0.5_rt : 1.0_rt; const amrex::Real wl = (interp_l) ? 0.5_rt : 1.0_rt; -#endif - // Min and max for interpolation loop along j + // Min and max for interpolation loop const int jmin = (interp_j) ? j - noj/2 + shift : j; const int jmax = (interp_j) ? j + noj/2 + shift - 1 : j; - - // Min and max for interpolation loop along k -#if defined(WARPX_DIM_1D_Z) - // k = 0 always - const int kmin = k; - const int kmax = k; -#else const int kmin = (interp_k) ? k - nok/2 + shift : k; const int kmax = (interp_k) ? k + nok/2 + shift - 1 : k; -#endif - - // Min and max for interpolation loop along l -#if (AMREX_SPACEDIM <= 2) - // l = 0 always - const int lmin = l; - const int lmax = l; -#elif defined(WARPX_DIM_3D) const int lmin = (interp_l) ? l - nol/2 + shift : l; const int lmax = (interp_l) ? l + nol/2 + shift - 1 : l; -#endif // Number of interpolation points const int nj = jmax - jmin; @@ -543,31 +434,16 @@ void warpx_interp (const int j, amrex::Real res = 0.0_rt; -#if defined(WARPX_DIM_1D_Z) - amrex::Real const* scj = stencil_coeffs_z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Real const* scj = stencil_coeffs_x; - amrex::Real const* sck = stencil_coeffs_z; -#elif defined(WARPX_DIM_3D) - amrex::Real const* scj = stencil_coeffs_x; - amrex::Real const* sck = stencil_coeffs_y; - amrex::Real const* scl = stencil_coeffs_z; -#endif + amrex::Real const* scj = AMREX_D_PICK(stencil_coeffs_z, stencil_coeffs_x, stencil_coeffs_x); + amrex::Real const* sck = AMREX_D_PICK(nullptr , stencil_coeffs_z, stencil_coeffs_y); + amrex::Real const* scl = AMREX_D_PICK(nullptr , nullptr , stencil_coeffs_z); for (int ll = 0; ll <= nl; ll++) { -#if defined(WARPX_DIM_3D) - const amrex::Real cl = (interp_l)? scl[ll] : 1.0_rt; -#else - const amrex::Real cl = 1.0_rt; -#endif + const amrex::Real cl = (interp_l)? scl[ll] : 1.0_rt; for (int kk = 0; kk <= nk; kk++) { -#if (AMREX_SPACEDIM >= 2) const amrex::Real ck = (interp_k)? sck[kk] : 1.0_rt; -#else - const amrex::Real ck = 1.0_rt; -#endif for (int jj = 0; jj <= nj; jj++) { const amrex::Real cj = (interp_j)? scj[jj] : 1.0_rt; diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 594875d55b0..9063f8b4241 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -10,7 +10,9 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "EmbeddedBoundary/Enabled.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" @@ -21,6 +23,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include +#include #include #include @@ -278,93 +281,51 @@ WarpX::LoadBalance () void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; - const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ + const auto RemakeMultiFab = [&](auto& mf){ if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); auto pmf = std::remove_reference_t{}; AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } mf = std::move(pmf); }; + bool const eb_enabled = EB::enabled(); if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } + m_fields.remake_level(lev, dm); + // Fine patch + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], true); - RemakeMultiFab(Efield_fp[lev][idim], true); - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], true); - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], true); - } - if (mypc->m_B_ext_particle_s == "read_from_file") { - RemakeMultiFab(B_external_particle_field[lev][idim], true); - } - if (mypc->m_E_ext_particle_s == "read_from_file") { - RemakeMultiFab(E_external_particle_field[lev][idim], true); - } - RemakeMultiFab(current_fp[lev][idim], false); - RemakeMultiFab(current_store[lev][idim], false); - if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], false); - } - if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], false); - } - if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], true); - RemakeMultiFab(Bfield_avg_fp[lev][idim], true); - } - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); - } -#ifdef AMREX_USE_EB - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], false); - RemakeMultiFab(m_face_areas[lev][idim], false); - if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT){ - RemakeMultiFab(Venl[lev][idim], false); - RemakeMultiFab(m_flag_info_face[lev][idim], false); - RemakeMultiFab(m_flag_ext_face[lev][idim], false); - RemakeMultiFab(m_area_mod[lev][idim], false); - RemakeMultiFab(ECTRhofield[lev][idim], false); - m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); + if (eb_enabled) { + RemakeMultiFab( m_eb_reduce_particle_shape[lev] ); + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + RemakeMultiFab( m_eb_update_E[lev][idim] ); + RemakeMultiFab( m_eb_update_B[lev][idim] ); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); + } } } -#endif - } - - RemakeMultiFab(F_fp[lev], true); - RemakeMultiFab(rho_fp[lev], false); - // phi_fp should be redistributed since we use the solution from - // the last step as the initial guess for the next solve - RemakeMultiFab(phi_fp[lev], true); - - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); - RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); } + if (eb_enabled) { #ifdef AMREX_USE_EB - RemakeMultiFab(m_distance_to_eb[lev], false); - - int max_guard = guard_cells.ng_FieldSolver.max(); - m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, - {max_guard, max_guard, max_guard}, - amrex::EBSupport::full); - - InitializeEBGridData(lev); -#else - m_field_factory[lev] = std::make_unique(); + int const max_guard = guard_cells.ng_FieldSolver.max(); + m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, + {max_guard, max_guard, max_guard}, + amrex::EBSupport::full); #endif + InitializeEBGridData(lev); + } else { + m_field_factory[lev] = std::make_unique(); + } #ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -400,35 +361,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } #endif - // Aux patch - if (lev == 0 && Bfield_aux[0][0]->ixType() == Bfield_fp[0][0]->ixType()) - { - for (int idim = 0; idim < 3; ++idim) { - Bfield_aux[lev][idim] = std::make_unique(*Bfield_fp[lev][idim], amrex::make_alias, 0, Bfield_aux[lev][idim]->nComp()); - Efield_aux[lev][idim] = std::make_unique(*Efield_fp[lev][idim], amrex::make_alias, 0, Efield_aux[lev][idim]->nComp()); - } - } else { - for (int idim=0; idim < 3; ++idim) - { - RemakeMultiFab(Bfield_aux[lev][idim], false); - RemakeMultiFab(Efield_aux[lev][idim], false); - } - } - // Coarse patch if (lev > 0) { - for (int idim=0; idim < 3; ++idim) - { - RemakeMultiFab(Bfield_cp[lev][idim], true); - RemakeMultiFab(Efield_cp[lev][idim], true); - RemakeMultiFab(current_cp[lev][idim], false); - if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_cp[lev][idim], true); - RemakeMultiFab(Bfield_avg_cp[lev][idim], true); - } - } - RemakeMultiFab(F_cp[lev], true); - RemakeMultiFab(rho_cp[lev], false); #ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -466,24 +400,13 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } if (lev > 0 && (n_field_gather_buffer > 0 || n_current_deposition_buffer > 0)) { - for (int idim=0; idim < 3; ++idim) - { - RemakeMultiFab(Bfield_cax[lev][idim], false); - RemakeMultiFab(Efield_cax[lev][idim], false); - RemakeMultiFab(current_buf[lev][idim], false); - } - RemakeMultiFab(charge_buf[lev], false); - // we can avoid redistributing these since we immediately re-build the values via BuildBufferMasks() - RemakeMultiFab(current_buffer_masks[lev], false); - RemakeMultiFab(gather_buffer_masks[lev], false); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); } } // Re-initialize the lattice element finder with the new ba and dm. - m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); + m_accelerator_lattice[lev]->InitElementFinder(lev, gamma_boost, ba, dm); if (costs[lev] != nullptr) { @@ -513,9 +436,12 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi void WarpX::ComputeCostsHeuristic (amrex::Vector > >& a_costs) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + for (int lev = 0; lev <= finest_level; ++lev) { - const auto & mypc_ref = GetInstance().GetPartContainer(); + const auto & mypc_ref = GetPartContainer(); const auto nSpecies = mypc_ref.nSpecies(); // Species loop @@ -531,7 +457,7 @@ WarpX::ComputeCostsHeuristic (amrex::Vector +#include +#include +#include +#include +#include + +struct PDim3 { + amrex::ParticleReal x, y, z; + + AMREX_GPU_HOST_DEVICE + PDim3(const amrex::XDim3& a): + x{static_cast(a.x)}, + y{static_cast(a.y)}, + z{static_cast(a.z)} + {} + + AMREX_GPU_HOST_DEVICE + ~PDim3() = default; + + AMREX_GPU_HOST_DEVICE + PDim3(PDim3 const &) = default; + AMREX_GPU_HOST_DEVICE + PDim3& operator=(PDim3 const &) = default; + AMREX_GPU_HOST_DEVICE + PDim3(PDim3&&) = default; + AMREX_GPU_HOST_DEVICE + PDim3& operator=(PDim3&&) = default; +}; + +/* + Finds the overlap region between the given tile_realbox and part_realbox, returning true + if an overlap exists and false if otherwise. This also sets the parameters overlap_realbox, + overlap_box, and shifted to the appropriate values. + */ +bool find_overlap (const amrex::RealBox& tile_realbox, const amrex::RealBox& part_realbox, + const amrex::GpuArray& dx, + const amrex::GpuArray& prob_lo, + amrex::RealBox& overlap_realbox, amrex::Box& overlap_box, amrex::IntVect& shifted); + +/* + Finds the overlap region between the given tile_realbox, part_realbox and the surface used for + flux injection, returning true if an overlap exists and false if otherwise. This also sets the + parameters overlap_realbox, overlap_box, and shifted to the appropriate values. + */ +bool find_overlap_flux (const amrex::RealBox& tile_realbox, const amrex::RealBox& part_realbox, + const amrex::GpuArray& dx, + const amrex::GpuArray& prob_lo, + const PlasmaInjector& plasma_injector, + amrex::RealBox& overlap_realbox, amrex::Box& overlap_box, amrex::IntVect& shifted); + +/* + This computes the scale_fac (used for setting the particle weights) on a volumetric basis. + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real compute_scale_fac_volume (const amrex::GpuArray& dx, + const amrex::Long pcount) { + using namespace amrex::literals; + return (pcount != 0) ? AMREX_D_TERM(dx[0],*dx[1],*dx[2])/pcount : 0.0_rt; +} + +/* + Given a refinement ratio, this computes the total increase in resolution for a plane + defined by the normal_axis. + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +int compute_area_weights (const amrex::IntVect& iv, const int normal_axis) { + int r = AMREX_D_TERM(iv[0],*iv[1],*iv[2]); +#if defined(WARPX_DIM_3D) + r /= iv[normal_axis]; +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (normal_axis == 0) { r /= iv[0]; } + else if (normal_axis == 2) { r /= iv[1]; } +#elif defined(WARPX_DIM_1D_Z) + if (normal_axis == 2) { r /= iv[0]; } +#endif + return r; +} + + +#ifdef AMREX_USE_EB +/* + * \brief This computes the scale_fac (used for setting the particle weights) on a on area basis + * (used for flux injection from the embedded boundary). + * + * \param[in] dx: cell size in each direction + * \param[in] num_ppc_real: number of particles per cell + * \param[in] eb_bnd_normal_arr: array containing the normal to the embedded boundary + * \param[in] i, j, k: indices of the cell + * + * \return scale_fac: the scaling factor to be applied to the weight of the particles + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real compute_scale_fac_area_eb ( + const amrex::GpuArray& dx, + const amrex::Real num_ppc_real, + AMREX_D_DECL(const amrex::Real n0, + const amrex::Real n1, + const amrex::Real n2)) { + using namespace amrex::literals; + // Scale particle weight by the area of the emitting surface, within one cell + // By definition, eb_bnd_area_arr is normalized (unitless). + // Here we undo the normalization (i.e. multiply by the surface used for normalization in amrex: + // see https://amrex-codes.github.io/amrex/docs_html/EB.html#embedded-boundary-data-structures) +#if defined(WARPX_DIM_3D) + amrex::Real scale_fac = std::sqrt(amrex::Math::powi<2>(n0*dx[1]*dx[2]) + + amrex::Math::powi<2>(n1*dx[0]*dx[2]) + + amrex::Math::powi<2>(n2*dx[0]*dx[1])); + +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + amrex::Real scale_fac = std::sqrt(amrex::Math::powi<2>(n0*dx[1]) + + amrex::Math::powi<2>(n1*dx[0])); +#else + amrex::ignore_unused(dx, AMREX_D_DECL(n0,n1,n2)); + amrex::Real scale_fac = 0.0_rt; +#endif + // Do not multiply by eb_bnd_area_arr(i,j,k) here because this + // already taken into account by emitting a number of macroparticles + // that is proportional to the area of eb_bnd_area_arr(i,j,k). + scale_fac /= num_ppc_real; + return scale_fac; +} + +/* \brief Rotate the momentum of the particle so that the z direction + * transforms to the normal of the embedded boundary. + * + * More specifically, before calling this function, `pu.z` has the + * momentum distribution that is meant for the direction normal to the + * embedded boundary, and `pu.x`/`pu.y` have the momentum distribution that + * is meant for the tangentional direction. After calling this function, + * `pu.x`, `pu.y`, `pu.z` will have the correct momentum distribution, + * consistent with the local normal to the embedded boundary. + * + * \param[inout] pu momentum of the particle + * \param[in] eb_bnd_normal_arr: array containing the normal to the embedded boundary + * \param[in] i, j, k: indices of the cell + * */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void rotate_momentum_eb ( + PDim3 & pu, + AMREX_D_DECL(const amrex::Real n0, + const amrex::Real n1, + const amrex::Real n2)) +{ + using namespace amrex::literals; + +#if defined(WARPX_DIM_3D) + // The minus sign below takes into account the fact that eb_bnd_normal_arr + // points towards the covered region, while particles are to be emitted + // *away* from the covered region. + amrex::Real const nx = -n0; + amrex::Real const ny = -n1; + amrex::Real const nz = -n2; + + // Rotate the momentum in theta and phi + amrex::Real const cos_theta = nz; + amrex::Real const sin_theta = std::sqrt(1._rt-nz*nz); + amrex::Real const nperp = std::sqrt(nx*nx + ny*ny); + amrex::Real cos_phi = 1; + amrex::Real sin_phi = 0; + if ( nperp > 0.0 ) { + cos_phi = nx/nperp; + sin_phi = ny/nperp; + } + // Apply rotation matrix + amrex::Real const ux = pu.x*cos_theta*cos_phi - pu.y*sin_phi + pu.z*sin_theta*cos_phi; + amrex::Real const uy = pu.x*cos_theta*sin_phi + pu.y*cos_phi + pu.z*sin_theta*sin_phi; + amrex::Real const uz = -pu.x*sin_theta + pu.z*cos_theta; + pu.x = ux; + pu.y = uy; + pu.z = uz; + +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // The minus sign below takes into account the fact that eb_bnd_normal_arr + // points towards the covered region, while particles are to be emitted + // *away* from the covered region. + amrex::Real const sin_theta = -n0; + amrex::Real const cos_theta = -n1; + amrex::Real const uz = pu.z*cos_theta - pu.x*sin_theta; + amrex::Real const ux = pu.x*cos_theta + pu.z*sin_theta; + pu.x = ux; + pu.z = uz; +#else + amrex::ignore_unused(pu, AMREX_D_DECL(n0,n1,n2)); +#endif +} +#endif //AMREX_USE_EB + +/* + This computes the scale_fac (used for setting the particle weights) on a on area basis + (used for flux injection from a plane). + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real compute_scale_fac_area_plane (const amrex::GpuArray& dx, + const amrex::Real num_ppc_real, const int flux_normal_axis) { + using namespace amrex::literals; + amrex::Real scale_fac = AMREX_D_TERM(dx[0],*dx[1],*dx[2])/num_ppc_real; + // Scale particle weight by the area of the emitting surface, within one cell +#if defined(WARPX_DIM_3D) + scale_fac /= dx[flux_normal_axis]; +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // When emission is in the r direction, the emitting surface is a cylinder. + // The factor 2*pi*r is added later below. + if (flux_normal_axis == 0) { scale_fac /= dx[0]; } + // When emission is in the z direction, the emitting surface is an annulus + // The factor 2*pi*r is added later below. + if (flux_normal_axis == 2) { scale_fac /= dx[1]; } + // When emission is in the theta direction (flux_normal_axis == 1), + // the emitting surface is a rectangle, within the plane of the simulation +#elif defined(WARPX_DIM_1D_Z) + if (flux_normal_axis == 2) { scale_fac /= dx[0]; } +#endif + return scale_fac; +} + +/* + These structs encapsulates several data structures needed for using the parser during plasma + injection. + */ +struct PlasmaParserWrapper +{ + PlasmaParserWrapper (std::size_t a_num_user_int_attribs, + std::size_t a_num_user_real_attribs, + const amrex::Vector< std::unique_ptr >& a_user_int_attrib_parser, + const amrex::Vector< std::unique_ptr >& a_user_real_attrib_parser); + + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_int_attrib_parserexec_pinned; + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > m_user_real_attrib_parserexec_pinned; +}; + +struct PlasmaParserHelper +{ + template + PlasmaParserHelper (SoAType& a_soa, std::size_t old_size, + const std::vector& a_user_int_attribs, + const std::vector& a_user_real_attribs, + const PlasmaParserWrapper& wrapper) : + m_wrapper_ptr(&wrapper) { + m_pa_user_int_pinned.resize(a_user_int_attribs.size()); + m_pa_user_real_pinned.resize(a_user_real_attribs.size()); + +#ifdef AMREX_USE_GPU + m_d_pa_user_int.resize(a_user_int_attribs.size()); + m_d_pa_user_real.resize(a_user_real_attribs.size()); + m_d_user_int_attrib_parserexec.resize(a_user_int_attribs.size()); + m_d_user_real_attrib_parserexec.resize(a_user_real_attribs.size()); +#endif + + for (std::size_t ia = 0; ia < a_user_int_attribs.size(); ++ia) { + m_pa_user_int_pinned[ia] = a_soa.GetIntData(a_user_int_attribs[ia]).data() + old_size; + } + for (std::size_t ia = 0; ia < a_user_real_attribs.size(); ++ia) { + m_pa_user_real_pinned[ia] = a_soa.GetRealData(a_user_real_attribs[ia]).data() + old_size; + } + +#ifdef AMREX_USE_GPU + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_pa_user_int_pinned.begin(), + m_pa_user_int_pinned.end(), m_d_pa_user_int.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_pa_user_real_pinned.begin(), + m_pa_user_real_pinned.end(), m_d_pa_user_real.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wrapper.m_user_int_attrib_parserexec_pinned.begin(), + wrapper.m_user_int_attrib_parserexec_pinned.end(), m_d_user_int_attrib_parserexec.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wrapper.m_user_real_attrib_parserexec_pinned.begin(), + wrapper.m_user_real_attrib_parserexec_pinned.end(), m_d_user_real_attrib_parserexec.begin()); +#endif + } + + int** getUserIntDataPtrs (); + amrex::ParticleReal** getUserRealDataPtrs (); + [[nodiscard]] amrex::ParserExecutor<7> const* getUserIntParserExecData () const; + [[nodiscard]] amrex::ParserExecutor<7> const* getUserRealParserExecData () const; + + amrex::Gpu::PinnedVector m_pa_user_int_pinned; + amrex::Gpu::PinnedVector m_pa_user_real_pinned; + +#ifdef AMREX_USE_GPU + // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, + // and them memcpy to device from host + amrex::Gpu::DeviceVector m_d_pa_user_int; + amrex::Gpu::DeviceVector m_d_pa_user_real; + amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > m_d_user_int_attrib_parserexec; + amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > m_d_user_real_attrib_parserexec; +#endif + const PlasmaParserWrapper* m_wrapper_ptr; +}; + +#ifdef WARPX_QED +struct QEDHelper +{ + template + QEDHelper (SoAType& a_soa, std::size_t old_size, + bool a_has_quantum_sync, bool a_has_breit_wheeler, + const std::shared_ptr& a_shr_p_qs_engine, + const std::shared_ptr& a_shr_p_bw_engine) + : has_quantum_sync(a_has_quantum_sync), has_breit_wheeler(a_has_breit_wheeler) + { + if(has_quantum_sync){ + quantum_sync_get_opt = + a_shr_p_qs_engine->build_optical_depth_functor(); + p_optical_depth_QSR = a_soa.GetRealData("opticalDepthQSR").data() + old_size; + } + if(has_breit_wheeler){ + breit_wheeler_get_opt = + a_shr_p_bw_engine->build_optical_depth_functor(); + p_optical_depth_BW = a_soa.GetRealData("opticalDepthBW").data() + old_size; + } + } + + amrex::ParticleReal* p_optical_depth_QSR = nullptr; + amrex::ParticleReal* p_optical_depth_BW = nullptr; + + bool has_quantum_sync; + bool has_breit_wheeler; + + QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; + BreitWheelerGetOpticalDepth breit_wheeler_get_opt; +}; +#endif + +#endif /*WARPX_ADDPLASMAUTILITIES_H_*/ diff --git a/Source/Particles/AddPlasmaUtilities.cpp b/Source/Particles/AddPlasmaUtilities.cpp new file mode 100644 index 00000000000..31066516477 --- /dev/null +++ b/Source/Particles/AddPlasmaUtilities.cpp @@ -0,0 +1,158 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + * Authors: Andrew Myers + */ +#include "AddPlasmaUtilities.H" + +#include + +bool find_overlap (const amrex::RealBox& tile_realbox, const amrex::RealBox& part_realbox, + const amrex::GpuArray& dx, + const amrex::GpuArray& prob_lo, + amrex::RealBox& overlap_realbox, amrex::Box& overlap_box, amrex::IntVect& shifted) +{ + using namespace amrex::literals; + + bool no_overlap = false; + for (int dir=0; dir= part_realbox.lo(dir) ) { + const amrex::Real ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); + overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0._rt) * dx[dir]); + } else { + no_overlap = true; break; + } + // Count the number of cells in this direction in overlap_realbox + overlap_box.setSmall( dir, 0 ); + overlap_box.setBig( dir, + int( std::round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir)) + /dx[dir] )) - 1); + shifted[dir] = + static_cast(std::round((overlap_realbox.lo(dir)-prob_lo[dir])/dx[dir])); + // shifted is exact in non-moving-window direction. That's all we care. + } + return no_overlap; +} + +bool find_overlap_flux (const amrex::RealBox& tile_realbox, const amrex::RealBox& part_realbox, + const amrex::GpuArray& dx, + const amrex::GpuArray& prob_lo, + const PlasmaInjector& plasma_injector, + amrex::RealBox& overlap_realbox, amrex::Box& overlap_box, amrex::IntVect& shifted) +{ + using namespace amrex::literals; + + bool no_overlap = false; + for (int dir=0; dir 0) { + if (plasma_injector.surface_flux_pos < tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos >= tile_realbox.hi(dir)) { + no_overlap = true; + break; + } + } else { + if (plasma_injector.surface_flux_pos <= tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos > tile_realbox.hi(dir)) { + no_overlap = true; + break; + } + } + overlap_realbox.setLo( dir, plasma_injector.surface_flux_pos ); + overlap_realbox.setHi( dir, plasma_injector.surface_flux_pos ); + overlap_box.setSmall( dir, 0 ); + overlap_box.setBig( dir, 0 ); + shifted[dir] = + static_cast(std::round((overlap_realbox.lo(dir)-prob_lo[dir])/dx[dir])); + } else { + if ( tile_realbox.lo(dir) <= part_realbox.hi(dir) ) { + const amrex::Real ncells_adjust = std::floor( (tile_realbox.lo(dir) - part_realbox.lo(dir))/dx[dir] ); + overlap_realbox.setLo( dir, part_realbox.lo(dir) + std::max(ncells_adjust, 0._rt) * dx[dir]); + } else { + no_overlap = true; break; + } + if ( tile_realbox.hi(dir) >= part_realbox.lo(dir) ) { + const amrex::Real ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); + overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0._rt) * dx[dir]); + } else { + no_overlap = true; break; + } + // Count the number of cells in this direction in overlap_realbox + overlap_box.setSmall( dir, 0 ); + overlap_box.setBig( dir, + int( std::round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir)) + /dx[dir] )) - 1); + shifted[dir] = + static_cast(std::round((overlap_realbox.lo(dir)-prob_lo[dir])/dx[dir])); + // shifted is exact in non-moving-window direction. That's all we care. + } + } + + return no_overlap; +} + +PlasmaParserWrapper::PlasmaParserWrapper (const std::size_t a_num_user_int_attribs, + const std::size_t a_num_user_real_attribs, + const amrex::Vector< std::unique_ptr >& a_user_int_attrib_parser, + const amrex::Vector< std::unique_ptr >& a_user_real_attrib_parser) + +{ + m_user_int_attrib_parserexec_pinned.resize(a_num_user_int_attribs); + m_user_real_attrib_parserexec_pinned.resize(a_num_user_real_attribs); + + for (std::size_t ia = 0; ia < a_num_user_int_attribs; ++ia) { + m_user_int_attrib_parserexec_pinned[ia] = a_user_int_attrib_parser[ia]->compile<7>(); + } + for (std::size_t ia = 0; ia < a_num_user_real_attribs; ++ia) { + m_user_real_attrib_parserexec_pinned[ia] = a_user_real_attrib_parser[ia]->compile<7>(); + } +} + +int** PlasmaParserHelper::getUserIntDataPtrs () { +#ifdef AMREX_USE_GPU + return m_d_pa_user_int.dataPtr(); +#else + return m_pa_user_int_pinned.dataPtr(); +#endif +} + +amrex::ParticleReal** PlasmaParserHelper::getUserRealDataPtrs () { +#ifdef AMREX_USE_GPU + return m_d_pa_user_real.dataPtr(); +#else + return m_pa_user_real_pinned.dataPtr(); +#endif +} + +amrex::ParserExecutor<7> const* PlasmaParserHelper::getUserIntParserExecData () const { +#ifdef AMREX_USE_GPU + return m_d_user_int_attrib_parserexec.dataPtr(); +#else + return m_wrapper_ptr->m_user_int_attrib_parserexec_pinned.dataPtr(); +#endif +} + +amrex::ParserExecutor<7> const* PlasmaParserHelper::getUserRealParserExecData () const { +#ifdef AMREX_USE_GPU + return m_d_user_real_attrib_parserexec.dataPtr(); +#else + return m_wrapper_ptr->m_user_real_attrib_parserexec_pinned.dataPtr(); +#endif +} diff --git a/Source/Particles/CMakeLists.txt b/Source/Particles/CMakeLists.txt index 67af14ef889..6b434c0a4e1 100644 --- a/Source/Particles/CMakeLists.txt +++ b/Source/Particles/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + AddPlasmaUtilities.cpp MultiParticleContainer.cpp ParticleBoundaries.cpp PhotonParticleContainer.cpp diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 80ce13744fd..8becd7d231a 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -106,6 +106,14 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const& collision_nam utils::parser::getWithParser( pp_collision_name, kw_energy.c_str(), energy); } + // if the scattering process is forward scattering get the energy + // associated with the process if it is given (this allows forward + // scattering to be used both with and without a fixed energy loss) + else if (scattering_process.find("forward") != std::string::npos) { + const std::string kw_energy = scattering_process + "_energy"; + utils::parser::queryWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } ScatteringProcess process(scattering_process, cross_section_file, energy); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 6f8f61e66b1..b64c6d4b1fa 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -262,6 +262,28 @@ public: } auto *tile_products_data = tile_products.data(); + amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); + auto const dV = AMREX_D_TERM(geom.CellSize(0), *geom.CellSize(1), *geom.CellSize(2)); +#if defined WARPX_DIM_RZ + amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box + auto const lo = lbound(cbx); + auto const hi = ubound(cbx); + int const nz = hi.y - lo.y + 1; + auto const dr = geom.CellSize(0); +#endif + + auto volume_factor = [=] AMREX_GPU_DEVICE(int i_cell) noexcept { +#if defined WARPX_DIM_RZ + // Return the radial factor for the volume element, dV + int const ri = (i_cell - i_cell%nz)/nz; + return MathConst::pi*(2.0_prt*ri + 1.0_prt)*dr; +#else + // No factor is needed for Cartesian + amrex::ignore_unused(i_cell); + return 1._prt; +#endif + }; + if ( m_isSameSpecies ) // species_1 == species_2 { // Extract particles in the tile that `mfi` points to @@ -282,22 +304,6 @@ public: const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); -#if defined WARPX_DIM_1D_Z - auto dV = geom.CellSize(0); -#elif defined WARPX_DIM_XZ - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined WARPX_DIM_RZ - amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box - const auto lo = lbound(cbx); - const auto hi = ubound(cbx); - int const nz = hi.y-lo.y+1; - auto dr = geom.CellSize(0); - auto dz = geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif - /* The following calculations are only required when creating product particles */ @@ -392,11 +398,8 @@ public: for (index_type i1=cell_start_1; i1(ptile_2, getpos_offset); - amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); -#if defined WARPX_DIM_1D_Z - auto dV = geom.CellSize(0); -#elif defined WARPX_DIM_XZ - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined WARPX_DIM_RZ - amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box - const auto lo = lbound(cbx); - const auto hi = ubound(cbx); - const int nz = hi.y-lo.y+1; - auto dr = geom.CellSize(0); - auto dz = geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif - /* The following calculations are only required when creating product particles */ @@ -655,12 +637,8 @@ public: for (index_type i2=cell_start_2; i2 0, * otherwise L will be calculated based on the algorithm. * @param[in] n12 = max(w1,w2)*min(N1,N2)/dV is the effective density used for s12 + * @param[in] sigma_max is the maximum cross section based on mfp = atomic spacing + * used for the normalized scattering length s12 (see Sec. II.C of Perez et al.) * To see if there are nan or inf updated velocities, * compile with USE_ASSERTION=TRUE. * @@ -34,27 +36,29 @@ template AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumPerezElastic ( T_PR& u1x, T_PR& u1y, T_PR& u1z, T_PR& u2x, T_PR& u2y, T_PR& u2z, - T_PR const n1, T_PR const n2, T_PR const n12, T_PR const q1, T_PR const m1, T_PR const w1, T_PR const q2, T_PR const m2, T_PR const w2, - T_R const dt, T_PR const L, T_PR const bmax, - amrex::RandomEngine const& engine) + T_PR const n12, T_PR const sigma_max, + T_PR const L, T_PR const bmax, + T_R const dt, amrex::RandomEngine const& engine ) { - T_PR const diffx = amrex::Math::abs(u1x-u2x); - T_PR const diffy = amrex::Math::abs(u1y-u2y); - T_PR const diffz = amrex::Math::abs(u1z-u2z); - T_PR const diffm = std::sqrt(diffx*diffx+diffy*diffy+diffz*diffz); - T_PR const summm = std::sqrt(u1x*u1x+u1y*u1y+u1z*u1z) + std::sqrt(u2x*u2x+u2y*u2y+u2z*u2z); - // If g = u1 - u2 = 0, do not collide. - // Or if the relative difference is less than 1.0e-10. - if ( diffm < std::numeric_limits::min() || diffm/summm < 1.0e-10 ) { return; } - T_PR constexpr inv_c2 = T_PR(1.0) / ( PhysConst::c * PhysConst::c ); // Compute Lorentz factor gamma - T_PR const g1 = std::sqrt( T_PR(1.0) + (u1x*u1x+u1y*u1y+u1z*u1z)*inv_c2 ); - T_PR const g2 = std::sqrt( T_PR(1.0) + (u2x*u2x+u2y*u2y+u2z*u2z)*inv_c2 ); + T_PR const gb1sq = (u1x*u1x + u1y*u1y + u1z*u1z)*inv_c2; + T_PR const gb2sq = (u2x*u2x + u2y*u2y + u2z*u2z)*inv_c2; + T_PR const g1 = std::sqrt( T_PR(1.0) + gb1sq ); + T_PR const g2 = std::sqrt( T_PR(1.0) + gb2sq ); + + T_PR const diffx = u1x-u2x; + T_PR const diffy = u1y-u2y; + T_PR const diffz = u1z-u2z; + T_PR const diffm = std::sqrt((diffx*diffx+diffy*diffy+diffz*diffz)*inv_c2); + T_PR const summm = std::sqrt(gb1sq) + std::sqrt(gb2sq); + // If g = u1 - u2 = 0, do not collide. + // Or if the relative difference is less than 1.0e-10. + if ( diffm < std::numeric_limits::min() || diffm/summm < 1.0e-10 ) { return; } // Compute momenta T_PR const p1x = u1x * m1; @@ -64,7 +68,7 @@ void UpdateMomentumPerezElastic ( T_PR const p2y = u2y * m2; T_PR const p2z = u2z * m2; - // Compute center-of-mass (COM) velocity and gamma + // Compute center-of-momentum (COM) velocity and gamma T_PR const mass_g = m1 * g1 + m2 * g2; T_PR const vcx = (p1x+p2x) / mass_g; T_PR const vcy = (p1y+p2y) / mass_g; @@ -77,84 +81,78 @@ void UpdateMomentumPerezElastic ( T_PR const vcDv2 = (vcx*u2x + vcy*u2y + vcz*u2z) / g2; // Compute p1 star - T_PR p1sx; - T_PR p1sy; - T_PR p1sz; - if ( vcms > std::numeric_limits::min() ) - { - /* lorentz_transform_factor = ( (gc-1.0)/vcms*vcDv1 - gc )*m1*g1; - * Rewrite to avoid loss of precision from subtracting similar - * numbers when gc is close to 1 */ - T_PR const lorentz_transform_factor = - ( (gc*gc*vcms*inv_c2/(T_PR(1.0) + gc))/vcms*vcDv1 - gc )*m1*g1; - p1sx = p1x + vcx*lorentz_transform_factor; - p1sy = p1y + vcy*lorentz_transform_factor; - p1sz = p1z + vcz*lorentz_transform_factor; - } - else // If vcms = 0, don't do Lorentz-transform. - { - p1sx = p1x; - p1sy = p1y; - p1sz = p1z; - } + // lorentz_transform_factor = ( (gc-1.0)/vcms*vcDv1 - gc )*m1*g1 + // Rewrite by multiplying and dividing first term by (gc+1) to + // avoid loss of precision when gc is close to 1 + T_PR const lorentz_transform_factor = + ( gc*gc*vcDv1*inv_c2/(T_PR(1.0) + gc) - gc )*m1*g1; + T_PR const p1sx = p1x + vcx*lorentz_transform_factor; + T_PR const p1sy = p1y + vcy*lorentz_transform_factor; + T_PR const p1sz = p1z + vcz*lorentz_transform_factor; T_PR const p1sm = std::sqrt( p1sx*p1sx + p1sy*p1sy + p1sz*p1sz ); // Compute gamma star T_PR const g1s = ( T_PR(1.0) - vcDv1*inv_c2 )*gc*g1; T_PR const g2s = ( T_PR(1.0) - vcDv2*inv_c2 )*gc*g2; - // Compute s - T_PR s = 0; + // Compute relative velocity in center-of-momentum frame + // (not a Lorentz invariant quantity) + T_PR const muRst = g1s*m1*g2s*m2/(g1s*m1 + g2s*m2); + T_PR const vrelst = p1sm/muRst; // |v1s - v2s| + + // Compute invariant relative velocity in center-of-momentum frame + // (Lorentz invariant quantity) + T_PR const denom = T_PR(1.0) + p1sm*p1sm/(m1*g1s*m2*g2s)*inv_c2; // (1.0 - v1s*v2s/c^2) + T_PR const vrelst_invar = vrelst/denom; // |v1s - v2s|/(1.0 - v1s*v2s/c^2) + + // Compute s12 + T_PR s12 = 0; if (p1sm > std::numeric_limits::min()) { - // s is non-zero (i.e. particles scatter) only if the relative - // motion between particles is not negligible (p1sm non-zero) + // Writing b0 in a form that is directly analagous to the well-known non-relativistic form. + // See Eq. 3.3.2 in Principles of Plasma Discharges and Material Processing by + // M. A. Lieberman and A. J. Lichtenberg. + // Note that b0 on Eq. 22 of Perez POP 19 (2012) is bmin = b0/2, + // Note: there is a typo in Eq 22 of Perez, the last square is incorrect! + // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html + // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 + T_PR const b0 = amrex::Math::abs(q1*q2) / + (T_PR(2.0)*MathConst::pi*PhysConst::ep0*muRst*vrelst*vrelst_invar); + // Compute the Coulomb log lnLmd first T_PR lnLmd; if ( L > T_PR(0.0) ) { lnLmd = L; } else { - // Compute b0 according to eq (22) from Perez et al., Phys.Plasmas.19.083104 (2012) - // Note: there is a typo in the equation, the last square is incorrect! - // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html - // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 - T_PR const b0 = amrex::Math::abs(q1*q2) * inv_c2 / - (T_PR(4.0)*MathConst::pi*PhysConst::ep0) * gc/mass_g * - ( m1*g1s*m2*g2s/(p1sm*p1sm*inv_c2) + T_PR(1.0) ); - - // Compute the minimal impact parameter - constexpr T_PR hbar_pi = static_cast(PhysConst::hbar*MathConst::pi); - const T_PR bmin = amrex::max(hbar_pi/p1sm, b0); + + // Compute the minimum impact parameter from quantum: bqm = lDB/(4*pi) = hbar/(2*p1sm) + // See NRL formulary. Also see "An introduction to the physics of the Coulomb logarithm, + // with emphasis on quantum-mechanical effects", J. Plasma Phys. vol. 85 (2019). by J.A. Krommes. + // Note: The formula in Perez 2012 and in Lee and More 1984 uses h rather than + // hbar for bqm. If this is used, then the transition energy where bmin goes from classical + // to quantum is only 2.5 eV for electrons; compared to 100 eV when using hbar. + const T_PR bmin_qm = static_cast(PhysConst::hbar*0.5/p1sm); + + // Set the minimum impact parameter + const T_PR bmin = amrex::max(bmin_qm, T_PR(0.5)*b0); // Compute the Coulomb log lnLmd lnLmd = amrex::max( T_PR(2.0), T_PR(0.5)*std::log(T_PR(1.0) + bmax*bmax/(bmin*bmin)) ); } - // Compute s - const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); - const auto tts2 = tts*tts; - s = n12 * dt*lnLmd*q1*q1*q2*q2 / - ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * - m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; - - // Compute s' - const auto cbrt_n1 = std::cbrt(n1); - const auto cbrt_n2 = std::cbrt(n2); - const auto coeff = static_cast( - std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); - T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n12 * dt * vrel * (m1+m2) / - amrex::max( m1*cbrt_n1*cbrt_n1, - m2*cbrt_n2*cbrt_n2); - - // Determine s - s = amrex::min(s,sp); + // Compute s12 with sigma limited by sigma_max where mfp = atomic spacing + // See https://github.com/user-attachments/files/16555064/CoulombScattering_s12.pdf + // for a proof that this expression for s12 is the same as Eq. 9 of Perez 2012 when + // sigma_eff = pi*b0^2*lnLmd + const T_PR sigma_eff = amrex::min(T_PR(MathConst::pi)*b0*b0*lnLmd,sigma_max); + s12 = sigma_eff * n12 * dt * vrelst * g1s*g2s/(g1*g2); + } - // Only modify momenta if is s is non-zero - if (s > std::numeric_limits::min()) { + // Only modify momenta if s12 is non-zero + if (s12 > std::numeric_limits::min()) { // Get random numbers T_PR r = amrex::Random(engine); @@ -162,27 +160,27 @@ void UpdateMomentumPerezElastic ( // Compute scattering angle T_PR cosXs; T_PR sinXs; - if ( s <= T_PR(0.1) ) + if ( s12 <= T_PR(0.1) ) { while ( true ) { - cosXs = T_PR(1.0) + s * std::log(r); + cosXs = T_PR(1.0) + s12 * std::log(r); // Avoid the bug when r is too small such that cosXs < -1 if ( cosXs >= T_PR(-1.0) ) { break; } r = amrex::Random(engine); } } - else if ( s > T_PR(0.1) && s <= T_PR(3.0) ) + else if ( s12 > T_PR(0.1) && s12 <= T_PR(3.0) ) { T_PR const Ainv = static_cast( - 0.0056958 + 0.9560202*s - 0.508139*s*s + - 0.47913906*s*s*s - 0.12788975*s*s*s*s + 0.02389567*s*s*s*s*s); + 0.0056958 + 0.9560202*s12 - 0.508139*s12*s12 + + 0.47913906*s12*s12*s12 - 0.12788975*s12*s12*s12*s12 + 0.02389567*s12*s12*s12*s12*s12); cosXs = Ainv * std::log( std::exp(T_PR(-1.0)/Ainv) + T_PR(2.0) * r * std::sinh(T_PR(1.0)/Ainv) ); } - else if ( s > T_PR(3.0) && s <= T_PR(6.0) ) + else if ( s12 > T_PR(3.0) && s12 <= T_PR(6.0) ) { - T_PR const A = T_PR(3.0) * std::exp(-s); + T_PR const A = T_PR(3.0) * std::exp(-s12); cosXs = T_PR(1.0)/A * std::log( std::exp(-A) + T_PR(2.0) * r * std::sinh(A) ); } @@ -241,33 +239,18 @@ void UpdateMomentumPerezElastic ( T_PR const p2fsz = -p1fsz; // Transform from COM to lab frame - T_PR p1fx; T_PR p2fx; - T_PR p1fy; T_PR p2fy; - T_PR p1fz; T_PR p2fz; - if ( vcms > std::numeric_limits::min() ) - { - T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; - T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; - /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ - T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); - T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; - T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; - p1fx = p1fsx + vcx * factor1; - p1fy = p1fsy + vcy * factor1; - p1fz = p1fsz + vcz * factor1; - p2fx = p2fsx + vcx * factor2; - p2fy = p2fsy + vcy * factor2; - p2fz = p2fsz + vcz * factor2; - } - else // If vcms = 0, don't do Lorentz-transform. - { - p1fx = p1fsx; - p1fy = p1fsy; - p1fz = p1fsz; - p2fx = p2fsx; - p2fy = p2fsy; - p2fz = p2fsz; - } + T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; + T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; + /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ + T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); + T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; + T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; + T_PR const p1fx = p1fsx + vcx * factor1; + T_PR const p1fy = p1fsy + vcy * factor1; + T_PR const p1fz = p1fsz + vcz * factor1; + T_PR const p2fx = p2fsx + vcx * factor2; + T_PR const p2fy = p2fsy + vcy * factor2; + T_PR const p2fz = p2fsz + vcz * factor2; // Rejection method r = amrex::Random(engine); diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index 46b228b049e..c5bd2e1cec6 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -34,7 +34,7 @@ * @param[in] scattering processes an array of scattering processes included for consideration. * @param[in] engine the random engine. */ -template +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleReal u1y, const amrex::ParticleReal u1z, const amrex::ParticleReal u2x, @@ -65,11 +65,11 @@ void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleRe // Evaluate the cross-section for each scattering process to determine // the total collision probability. - AMREX_ASSERT_WITH_MESSAGE( - (process_count < 4), "Too many scattering processes in DSMC routine." - ); - int coll_type[4] = {0, 0, 0, 0}; - amrex::ParticleReal sigma_sums[4] = {0._prt, 0._prt, 0._prt, 0._prt}; + + // The size of the arrays below is a compile-time constant (template parameter) + // for performance reasons: it avoids dynamic memory allocation on the GPU. + int coll_type[max_process_count] = {0}; + amrex::ParticleReal sigma_sums[max_process_count] = {0._prt}; for (int ii = 0; ii < process_count; ii++) { auto const& scattering_process = scattering_processes[ii]; coll_type[ii] = int(scattering_process.m_type); diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 30b466e2ec2..6051aab1b59 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -110,11 +110,13 @@ public: amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Number of macroparticles of each species const index_type NI1 = I1e - I1s; @@ -124,14 +126,9 @@ public: index_type pair_index = cell_start_pair + coll_idx; - // Because the number of particles of each species is not always equal (NI1 != NI2 - // in general), some macroparticles will be paired with multiple macroparticles of the - // other species and we need to decrease their weight accordingly. - // c1 corresponds to the minimum number of times a particle of species 1 will be paired - // with a particle of species 2. Same for c2. - // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 - const index_type c1 = amrex::max(NI2/NI1, index_type(1)); - const index_type c2 = amrex::max(NI1/NI2, index_type(1)); + // multiplier ratio to take into account unsampled pairs + const auto multiplier_ratio = static_cast( + m_isSameSpecies ? min_N + max_N - 1 : min_N); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; @@ -143,49 +140,61 @@ public: // stride (smaller set size) until we do all collisions (larger set size) for (index_type k = coll_idx; k < max_N; k += min_N) { - // c1k : how many times the current particle of species 1 is paired with a particle - // of species 2. Same for c2k. - const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + // do not check for collision if a particle's weight was + // reduced to zero from a previous collision + if (idcpu1[ I1[i1] ]==amrex::ParticleIdCpus::Invalid || + idcpu2[ I2[i2] ]==amrex::ParticleIdCpus::Invalid ) { + p_mask[pair_index] = false; + } else { #if (defined WARPX_DIM_RZ) - /* In RZ geometry, macroparticles can collide with other macroparticles - * in the same *cylindrical* cell. For this reason, collisions between macroparticles - * are actually not local in space. In this case, the underlying assumption is that - * particles within the same cylindrical cell represent a cylindrically-symmetry - * momentum distribution function. Therefore, here, we temporarily rotate the - * momentum of one of the macroparticles in agreement with this cylindrical symmetry. - * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; - * there is a corresponding assert statement at initialization.) - */ - amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; - amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); - u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) + */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); #endif - CollisionPairFilter( - u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], - u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], - m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, static_cast(pair_index), p_mask, - p_pair_reaction_weight, static_cast(max_N), - m_process_count, m_scattering_processes_data, engine); + const int max_process_count = 4; // Pre-defined value, for performance reasons + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + (m_process_count < max_process_count), "Too many scattering processes in DSMC routine (hardcoded to only allow 4). Update the max_process_count value in source code to allow more scattering processes." + ); + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ], w2[ I2[i2] ], + dt, dV, static_cast(pair_index), p_mask, + p_pair_reaction_weight, multiplier_ratio, + m_process_count, m_scattering_processes_data, engine); #if (defined WARPX_DIM_RZ) - amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); - u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); + /* Undo the earlier velocity rotation. */ + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif + // Remove pair reaction weight from the colliding particles' weights + if (p_mask[pair_index]) { + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[ I1[i1] ], idcpu1[ I1[i1] ], p_pair_reaction_weight[pair_index]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[ I2[i2] ], idcpu2[ I2[i2] ], p_pair_reaction_weight[pair_index]); + } + } + p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - if (max_N == NI1) { - i1 += min_N; - } - if (max_N == NI2) { - i2 += min_N; - } + if (max_N == NI1) { i1 += min_N; } + if (max_N == NI2) { i2 += min_N; } pair_index += min_N; } } @@ -193,6 +202,7 @@ public: int m_process_count; bool m_computeSpeciesDensities = false; bool m_computeSpeciesTemperatures = false; + bool m_isSameSpecies = false; ScatteringProcess::Executor* m_scattering_processes_data; }; @@ -201,6 +211,7 @@ public: private: amrex::Vector m_scattering_processes; amrex::Gpu::DeviceVector m_scattering_processes_exe; + bool m_isSameSpecies; Executor m_exe; }; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp index 9ee676bf002..cf5f8de8d3c 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -19,7 +19,7 @@ DSMCFunc::DSMCFunc ( const std::string& collision_name, [[maybe_unused]] MultiParticleContainer const * const mypc, - [[maybe_unused]] const bool isSameSpecies ) + const bool isSameSpecies ): m_isSameSpecies{isSameSpecies} { using namespace amrex::literals; @@ -46,6 +46,14 @@ DSMCFunc::DSMCFunc ( utils::parser::getWithParser( pp_collision_name, kw_energy.c_str(), energy); } + // if the scattering process is forward scattering get the energy + // associated with the process if it is given (this allows forward + // scattering to be used both with and without a fixed energy loss) + else if (scattering_process.find("forward") != std::string::npos) { + const std::string kw_energy = scattering_process + "_energy"; + utils::parser::queryWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } ScatteringProcess process(scattering_process, cross_section_file, energy); @@ -76,4 +84,5 @@ DSMCFunc::DSMCFunc ( // Link executor to appropriate ScatteringProcess executors m_exe.m_scattering_processes_data = m_scattering_processes_exe.data(); m_exe.m_process_count = process_count; + m_exe.m_isSameSpecies = m_isSameSpecies; } diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index a0c6ca8c8d4..e4b4d8a6a3a 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -94,11 +94,6 @@ public: const auto soa_1 = ptile1.getParticleTileData(); const auto soa_2 = ptile2.getParticleTileData(); - amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; - uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; - // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; for (int i = 0; i < m_num_product_species; i++) @@ -151,12 +146,6 @@ public: // Set the weight of the new particles to p_pair_reaction_weight[i] soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; - // Remove p_pair_reaction_weight[i] from the colliding particles' weights - BinaryCollisionUtils::remove_weight_from_colliding_particle( - w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); - BinaryCollisionUtils::remove_weight_from_colliding_particle( - w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); - // Set the child particle properties appropriately auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; auto& uy1 = soa_products_data[0].m_rdata[PIdx::uy][product1_index]; @@ -165,6 +154,25 @@ public: auto& uy2 = soa_products_data[1].m_rdata[PIdx::uy][product2_index]; auto& uz2 = soa_products_data[1].m_rdata[PIdx::uz][product2_index]; +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) + */ + amrex::ParticleReal const theta = ( + soa_products_data[1].m_rdata[PIdx::theta][product2_index] + - soa_products_data[0].m_rdata[PIdx::theta][product1_index] + ); + amrex::ParticleReal const ux1buf = ux1; + ux1 = ux1buf*std::cos(theta) - uy1*std::sin(theta); + uy1 = ux1buf*std::sin(theta) + uy1*std::cos(theta); +#endif + // for simplicity (for now) we assume non-relativistic particles // and simply calculate the center-of-momentum velocity from the // rest masses @@ -213,6 +221,8 @@ public: else { amrex::Abort("Uneven mass charge-exchange not implemented yet."); } + } else if (mask[i] == int(ScatteringProcessType::FORWARD)) { + amrex::Abort("Forward scattering with DSMC not implemented yet."); } else { amrex::Abort("Unknown scattering process."); @@ -224,6 +234,13 @@ public: ux2 += uCOM_x; uy2 += uCOM_y; uz2 += uCOM_z; + +#if (defined WARPX_DIM_RZ) + /* Undo the earlier velocity rotation. */ + amrex::ParticleReal const ux1buf_new = ux1; + ux1 = ux1buf_new*std::cos(-theta) - uy1*std::sin(-theta); + uy1 = ux1buf_new*std::sin(-theta) + uy1*std::cos(-theta); +#endif } }); @@ -235,7 +252,7 @@ public: ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], 0, 0, pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), - pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->GetRealSoANames(), pc_products[i]->GetIntSoANames(), pc_products[i]->getUserRealAttribParser(), pc_products[i]->getUserIntAttribParser(), #ifdef WARPX_QED diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 3113dc69839..14b1422db27 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -145,11 +145,13 @@ public: amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Number of macroparticles of each species const index_type NI1 = I1e - I1s; @@ -159,18 +161,9 @@ public: index_type pair_index = cell_start_pair + coll_idx; - // Because the number of particles of each species is not always equal (NI1 != NI2 - // in general), some macroparticles will be paired with multiple macroparticles of the - // other species and we need to decrease their weight accordingly. - // c1 corresponds to the minimum number of times a particle of species 1 will be paired - // with a particle of species 2. Same for c2. - // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 - const index_type c1 = amrex::max(NI2/NI1, index_type(1)); - const index_type c2 = amrex::max(NI1/NI2, index_type(1)); - // multiplier ratio to take into account unsampled pairs const auto multiplier_ratio = static_cast( - m_isSameSpecies ? 2*max_N - 1 : max_N); + m_isSameSpecies ? min_N + max_N - 1 : min_N); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; @@ -182,50 +175,60 @@ public: // stride (smaller set size) until we do all collisions (larger set size) for (index_type k = coll_idx; k < max_N; k += min_N) { - // c1k : how many times the current particle of species 1 is paired with a particle - // of species 2. Same for c2k. - const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + + // do not check for collision if a particle's weight was + // reduced to zero from a previous collision + if (idcpu1[ I1[i1] ]==amrex::ParticleIdCpus::Invalid || + idcpu2[ I2[i2] ]==amrex::ParticleIdCpus::Invalid ) { + p_mask[pair_index] = false; + } + else { #if (defined WARPX_DIM_RZ) - /* In RZ geometry, macroparticles can collide with other macroparticles - * in the same *cylindrical* cell. For this reason, collisions between macroparticles - * are actually not local in space. In this case, the underlying assumption is that - * particles within the same cylindrical cell represent a cylindrically-symmetry - * momentum distribution function. Therefore, here, we temporarily rotate the - * momentum of one of the macroparticles in agreement with this cylindrical symmetry. - * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; - * there is a corresponding assert statement at initialization.) */ - amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; - amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); - u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); #endif - SingleNuclearFusionEvent( - u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], - u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], - m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, static_cast(pair_index), p_mask, p_pair_reaction_weight, - m_fusion_multiplier, multiplier_ratio, - m_probability_threshold, - m_probability_target_value, - m_fusion_type, engine); + SingleNuclearFusionEvent( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ], w2[ I2[i2] ], + dt, dV, static_cast(pair_index), p_mask, p_pair_reaction_weight, + m_fusion_multiplier, multiplier_ratio, + m_probability_threshold, + m_probability_target_value, + m_fusion_type, engine); #if (defined WARPX_DIM_RZ) - amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); - u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif + // Remove pair reaction weight from the colliding particles' weights + if (p_mask[pair_index]) { + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[ I1[i1] ], idcpu1[ I1[i1] ], p_pair_reaction_weight[pair_index]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[ I2[i2] ], idcpu2[ I2[i2] ], p_pair_reaction_weight[pair_index]); + } + + } + p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - if (max_N == NI1) { - i1 += min_N; - } - if (max_N == NI2) { - i2 += min_N; - } + if (max_N == NI1) { i1 += min_N; } + if (max_N == NI2) { i2 += min_N; } pair_index += min_N; } } diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 375cc1e6d51..63f35d3d254 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -33,11 +33,11 @@ namespace { * \brief This function initializes the momentum of the alpha particles produced from * proton-boron fusion. The momentum is initialized by assuming that the fusion of a proton * with a boron nucleus into 3 alphas takes place in two steps. In the first step, the proton - * and the boron fuse into a beryllium nucleus and an alpha particle. In the second step, the - * beryllium decays into two alpha particles. The first step produces 8.59009 MeV of kinetic - * energy while the second step produces 91.8984 keV of kinetic energy. This two-step process + * and the boron fuse into an excited beryllium nucleus and an alpha particle. In the second step, the + * excited beryllium decays into two alpha particles. The first step produces ~5.56 MeV of kinetic + * energy while the second step produces ~3.12 MeV of kinetic energy. This two-step process * is considered to be the dominant process of proton+boron fusion into alphas (see - * Becker et al., Zeitschrift für Physik A Atomic Nuclei, 327(3), 341-355 (1987)). + * D.R. Tilley et al. / Nuclear Physics A 745 (2004) 155–362). * For each step, we assume in this function that the particles are emitted isotropically in * the corresponding center of mass frame (center of mass frame of proton + boron for the * creation of first alpha+beryllium and rest frame of beryllium for the creation of second and @@ -73,18 +73,21 @@ namespace { constexpr amrex::ParticleReal mev_to_joule = PhysConst::q_e*1.e6_prt; // Energy produced in the fusion reaction proton + boron11 -> Beryllium8 + alpha - // cf. Janis book of proton-induced cross-sections (2019) - constexpr amrex::ParticleReal E_fusion = 8.59009_prt*mev_to_joule; + // cf. https://doi.org/10.1016/j.radphyschem.2022.110727 + // Dominant reaction channel is p + B11 -> alpha_1 + 8Be* -> alpha_1 + alpha_11 + alpha_12 + 8.68 MeV + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_fusion = 5.55610759_prt*mev_to_joule; // Energy produced when Beryllium8 decays into two alphas - // cf. JEFF-3.3 radioactive decay data library (2017) - constexpr amrex::ParticleReal E_decay = 0.0918984_prt*mev_to_joule; + // cf. Kelley et al., (2017) http://dx.doi.org/10.1016/j.nuclphysa.2017.07.015 + constexpr amrex::ParticleReal E_decay = 3.12600414_prt*mev_to_joule; // The constexprs ma_sq and mBe_sq underflow in single precision because we use SI units, // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; - constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; + // mass of the Be8 excited state (3.03 MeV above ground state) + constexpr double m_beryllium = PhysConst::m_u*(8.0053095729_prt+0.00325283863_prt); constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 3cb7197b93a..59565c92516 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -131,11 +131,6 @@ public: const auto soa_1 = ptile1.getParticleTileData(); const auto soa_2 = ptile2.getParticleTileData(); - amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; - uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; - // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; for (int i = 0; i < m_num_product_species; i++) @@ -197,12 +192,6 @@ public: } } - // Remove p_pair_reaction_weight[i] from the colliding particles' weights - BinaryCollisionUtils::remove_weight_from_colliding_particle( - w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); - BinaryCollisionUtils::remove_weight_from_colliding_particle( - w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); - // Initialize the product particles' momentum, using a function depending on the // specific collision type if (t_collision_type == CollisionType::ProtonBoronToAlphasFusion) @@ -246,7 +235,7 @@ public: ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], 0, 0, pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), - pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->GetRealSoANames(), pc_products[i]->GetIntSoANames(), pc_products[i]->getUserRealAttribParser(), pc_products[i]->getUserIntAttribParser(), #ifdef WARPX_QED diff --git a/Source/Particles/Collision/ScatteringProcess.H b/Source/Particles/Collision/ScatteringProcess.H index 59ef7a02afb..0c3f2daf8c1 100644 --- a/Source/Particles/Collision/ScatteringProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -21,6 +21,7 @@ enum class ScatteringProcessType { CHARGE_EXCHANGE, EXCITATION, IONIZATION, + FORWARD, }; class ScatteringProcess diff --git a/Source/Particles/Collision/ScatteringProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp index ea1b4b40f54..ad3f179fa18 100644 --- a/Source/Particles/Collision/ScatteringProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -87,6 +87,8 @@ ScatteringProcess::parseProcessType(const std::string& scattering_process) return ScatteringProcessType::IONIZATION; } else if (scattering_process.find("excitation") != std::string::npos) { return ScatteringProcessType::EXCITATION; + } else if (scattering_process.find("forward") != std::string::npos) { + return ScatteringProcessType::FORWARD; } else { return ScatteringProcessType::INVALID; } diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index cb56c559bc0..bc870257d8f 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -638,6 +638,9 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param reduced_particle_shape_mask Array4 of int, Mask that indicates whether a particle + * should use its regular shape factor or a reduced, order-1 shape factor instead in a given cell. + * \param enable_reduced_shape Flag to indicate whether to use the reduced shape factor */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -656,7 +659,10 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - [[maybe_unused]]int n_rz_azimuthal_modes) + [[maybe_unused]] int n_rz_azimuthal_modes, + const amrex::Array4& reduced_particle_shape_mask, + bool enable_reduced_shape + ) { using namespace amrex; using namespace amrex::literals; @@ -680,9 +686,14 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr - amrex::ParallelFor( - np_to_deposit, - [=] AMREX_GPU_DEVICE (long const ip) { + + // (Compile 2 versions of the kernel: with and without reduced shape) + enum eb_flags : int { has_reduced_shape, no_reduced_shape }; + const int reduce_shape_runtime_flag = (enable_reduced_shape && (depos_order>1))? has_reduced_shape : no_reduced_shape; + + amrex::ParallelFor( TypeList>{}, + {reduce_shape_runtime_flag}, + np_to_deposit, [=] AMREX_GPU_DEVICE (long ip, auto reduce_shape_control) { // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -735,6 +746,43 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; + // Check whether the particle is close to the EB at the old and new position + bool reduce_shape_old, reduce_shape_new; +#ifdef AMREX_USE_CUDA + amrex::ignore_unused(reduced_particle_shape_mask, lo); // Needed to avoid compilation error with nvcc +#endif + if constexpr (reduce_shape_control == has_reduced_shape) { +#if defined(WARPX_DIM_3D) + reduce_shape_old = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(x_old)), + lo.y + int(amrex::Math::floor(y_old)), + lo.z + int(amrex::Math::floor(z_old))); + reduce_shape_new = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(x_new)), + lo.y + int(amrex::Math::floor(y_new)), + lo.z + int(amrex::Math::floor(z_new))); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + reduce_shape_old = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(x_old)), + lo.y + int(amrex::Math::floor(z_old)), + 0); + reduce_shape_new = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(x_new)), + lo.y + int(amrex::Math::floor(z_new)), + 0); +#elif defined(WARPX_DIM_1D_Z) + reduce_shape_old = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(z_old)), + 0, 0); + reduce_shape_new = reduced_particle_shape_mask( + lo.x + int(amrex::Math::floor(z_new)), + 0, 0); +#endif + } else { + reduce_shape_old = false; + reduce_shape_new = false; + } + #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; #elif defined(WARPX_DIM_XZ) @@ -749,6 +797,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // [ijk]_new: leftmost grid point that the particle touches const Compute_shape_factor< depos_order > compute_shape_factor; const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // In cells marked by reduced_particle_shape_mask, we need order 1 deposition + const Compute_shifted_shape_factor< 1 > compute_shifted_shape_factor_order1; + amrex::ignore_unused(compute_shifted_shape_factor_order1); // unused for `no_reduced_shape` // Shape factor arrays // Note that there are extra values above and below @@ -758,19 +809,58 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; - const int i_new = compute_shape_factor(sx_new+1, x_new); + const int i_new = compute_shape_factor(sx_new+1, x_new ); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); + // If particle is close to the embedded boundary, recompute deposition with order 1 shape + if constexpr (reduce_shape_control == has_reduced_shape) { + if (reduce_shape_new) { + for (int i=0; i m_get_position; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 514526374bd..f509f884c48 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -47,7 +47,7 @@ public: /** * \brief Constructor of the PhotonEmissionFilterFunc functor. * - * @param[in] opt_depth_runtime_comp Index of the optical depth component + * @param[in] opt_depth_runtime_comp Index of the optical depth component in the runtime real data */ PhotonEmissionFilterFunc(int const opt_depth_runtime_comp) : m_opt_depth_runtime_comp(opt_depth_runtime_comp) @@ -73,7 +73,7 @@ public: } private: - int m_opt_depth_runtime_comp; /*!< Index of the optical depth component of the source species*/ + int m_opt_depth_runtime_comp; /*!< Index of the optical depth runtime component of the source species */ }; /** @@ -178,12 +178,12 @@ public: } private: - const QuantumSynchrotronGetOpticalDepth + QuantumSynchrotronGetOpticalDepth m_opt_depth_functor; /*!< A copy of the functor to initialize the optical depth of the source species. */ - const int m_opt_depth_runtime_comp = 0; /*!< Index of the optical depth component of source species*/ + int m_opt_depth_runtime_comp = 0; /*!< Index of the optical depth component of source species*/ - const QuantumSynchrotronPhotonEmission + QuantumSynchrotronPhotonEmission m_emission_functor; /*!< A copy of the functor to generate photons. It contains only pointers to the lookup tables.*/ GetParticlePosition m_get_position; diff --git a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H index 32b58dc50dc..e7eb7e8be04 100644 --- a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H +++ b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H @@ -17,9 +17,9 @@ */ struct SchwingerFilterFunc { - const int m_threshold_poisson_gaussian; - const amrex::Real m_dV; - const amrex::Real m_dt; + int m_threshold_poisson_gaussian; + amrex::Real m_dV; + amrex::Real m_dt; /** Get the number of created pairs in a given cell at a given timestep. * @@ -59,8 +59,8 @@ struct SchwingerFilterFunc */ struct SchwingerTransformFunc { - const amrex::Real m_y_size; - const int m_weight_index; + amrex::Real m_y_size; + int m_weight_index; /** Assign a weight to particles created via the Schwinger process. * diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index fc68e8d9723..8980d1fe668 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -51,8 +51,8 @@ struct RandomFilter return ( (!m_is_active) || (amrex::Random(engine) < m_fraction) ); } private: - const bool m_is_active; //! select all particles if false - const amrex::Real m_fraction = 1.0; //! range: [0.0:1.0] where 0 is no & 1 is all particles + bool m_is_active; //! select all particles if false + amrex::Real m_fraction = 1.0; //! range: [0.0:1.0] where 0 is no & 1 is all particles }; /** @@ -78,8 +78,8 @@ struct UniformFilter return ( (!m_is_active) || ( p.id()%m_stride == 0 ) ); } private: - const bool m_is_active; //! select all particles if false - const int m_stride = 0; //! selection of every n-th particle + bool m_is_active; //! select all particles if false + int m_stride = 0; //! selection of every n-th particle }; /** @@ -171,7 +171,7 @@ struct ParserFilter } private: /** Whether this diagnostics is activated. Select all particles if false */ - const bool m_is_active; + bool m_is_active; /** Record indices of user-defined vars */ int m_ius; int m_itr; @@ -211,9 +211,9 @@ struct GeometryFilter } private: /** Whether this diagnostics is activated. Select all particles if false */ - const bool m_is_active; + bool m_is_active; /** Physical extent of the axis-aligned region used for particle check */ - const amrex::RealBox m_domain; + amrex::RealBox m_domain; }; #endif // WARPX_FILTERFUNCTORS_H diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 4b4590b8642..95a010031ec 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -1714,9 +1714,9 @@ void doGatherShapeNImplicit ( const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, - const int depos_type ) + const CurrentDepositionAlgo depos_type ) { - if (depos_type==0) { // CurrentDepositionAlgo::Esirkepov + if (depos_type == CurrentDepositionAlgo::Esirkepov) { if (nox == 1) { doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -1743,7 +1743,7 @@ void doGatherShapeNImplicit ( dinv, xyzmin, lo, n_rz_azimuthal_modes); } } - else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor + else if (depos_type == CurrentDepositionAlgo::Villasenor) { if (nox == 1) { doGatherPicnicShapeN<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -1770,7 +1770,7 @@ void doGatherShapeNImplicit ( dinv, xyzmin, lo, n_rz_azimuthal_modes); } } - else if (depos_type==1) { // CurrentDepositionAlgo::Direct + else if (depos_type == CurrentDepositionAlgo::Direct) { if (nox == 1) { doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 7000d6d7c26..90a61bd25db 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -112,9 +112,9 @@ struct GetExternalEBField lab_time = m_gamma_boost*m_time + m_uz_boost*z*inv_c2; z = m_gamma_boost*z + m_uz_boost*m_time; } - Bx = m_Bxfield_partparser(x, y, z, lab_time); - By = m_Byfield_partparser(x, y, z, lab_time); - Bz = m_Bzfield_partparser(x, y, z, lab_time); + Bx = m_Bxfield_partparser((amrex::ParticleReal) x, (amrex::ParticleReal) y, (amrex::ParticleReal) z, lab_time); + By = m_Byfield_partparser((amrex::ParticleReal) x, (amrex::ParticleReal) y, (amrex::ParticleReal) z, lab_time); + Bz = m_Bzfield_partparser((amrex::ParticleReal) x, (amrex::ParticleReal) y, (amrex::ParticleReal) z, lab_time); } if (m_Etype == RepeatedPlasmaLens || diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index bb55f79f394..207ef4a5a8b 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -50,19 +50,17 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function") { m_Etype = ExternalFieldInitType::Parser; - constexpr auto num_arguments = 4; //x,y,z,t - m_Exfield_partparser = mypc.m_Ex_particle_parser->compile(); - m_Eyfield_partparser = mypc.m_Ey_particle_parser->compile(); - m_Ezfield_partparser = mypc.m_Ez_particle_parser->compile(); + m_Exfield_partparser = mypc.m_Ex_particle_parser->compile<4>(); + m_Eyfield_partparser = mypc.m_Ey_particle_parser->compile<4>(); + m_Ezfield_partparser = mypc.m_Ez_particle_parser->compile<4>(); } if (mypc.m_B_ext_particle_s == "parse_b_ext_particle_function") { m_Btype = ExternalFieldInitType::Parser; - constexpr auto num_arguments = 4; //x,y,z,t - m_Bxfield_partparser = mypc.m_Bx_particle_parser->compile(); - m_Byfield_partparser = mypc.m_By_particle_parser->compile(); - m_Bzfield_partparser = mypc.m_Bz_particle_parser->compile(); + m_Bxfield_partparser = mypc.m_Bx_particle_parser->compile<4>(); + m_Byfield_partparser = mypc.m_By_particle_parser->compile<4>(); + m_Bzfield_partparser = mypc.m_Bz_particle_parser->compile<4>(); } if (mypc.m_E_ext_particle_s == "repeated_plasma_lens" || diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 5731bc047f4..a8c685eef8b 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -47,8 +47,21 @@ struct ScaleFields // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - const amrex::Real dtscale = 1._rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; - if (0._rt < dtscale && dtscale < 1._rt) + + // The scaling factor corresponds to the fraction of time that + // the particles spends to the right of the injection plane, + // between (n-1/2)*dt and (n+1/2)*dt, which is the interval + // over which the velocity is updated, in the leap-frog velocity push. + // (Note that here, `zp` is the particle position at time n*dt) + amrex::Real dtscale = 0.5_rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; + // If the particle stays to the left of the plane during the + // whole push, simply set the scaling factor to 0, and thus + // the velocity push leaves the velocity unchanged. + if (dtscale < 0._rt) { + dtscale = 0; + } + // Scale the fields. + if (dtscale < 1._rt) { Exp *= dtscale; Eyp *= dtscale; diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index 197cb897602..ea884a5f93f 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -64,16 +64,11 @@ public: void WriteHeader (std::ostream& os) const final; - void Evolve (int lev, - const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, - const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab*, amrex::MultiFab*, amrex::MultiFab*, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, - const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false, PushType push_type=PushType::Explicit) final; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index bd266ab368a..c79d1f675b5 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -10,6 +10,7 @@ #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" +#include "Fields.H" #include "Laser/LaserProfiles.H" #include "Particles/LaserParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -181,10 +182,11 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, if (WarpX::gamma_boost > 1.) { // Check that the laser direction is equal to the boost direction - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( m_nvec[0]*WarpX::boost_direction[0] - + m_nvec[1]*WarpX::boost_direction[1] - + m_nvec[2]*WarpX::boost_direction[2] - 1. < 1.e-12, - "The Lorentz boost should be in the same direction as the laser propagation"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + (m_nvec[0]-WarpX::boost_direction[0])*(m_nvec[0]-WarpX::boost_direction[0]) + + (m_nvec[1]-WarpX::boost_direction[1])*(m_nvec[1]-WarpX::boost_direction[1]) + + (m_nvec[2]-WarpX::boost_direction[2])*(m_nvec[2]-WarpX::boost_direction[2]) < 1.e-12, + "The Lorentz boost should be in the same direction as the laser propagation"); // Get the position of the plane, along the boost direction, in the lab frame // and convert the position of the antenna to the boosted frame m_Z0_lab = m_nvec[0]*m_position[0] + m_nvec[1]*m_position[1] + m_nvec[2]*m_position[2]; @@ -246,8 +248,10 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, windir[dir] = 1.0; #endif AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - (m_nvec[0]-windir[0]) + (m_nvec[1]-windir[1]) + (m_nvec[2]-windir[2]) - < 1.e-12, "do_continous_injection for laser particle only works" + + (m_nvec[0]-windir[0])*(m_nvec[0]-windir[0]) + + (m_nvec[1]-windir[1])*(m_nvec[1]-windir[1]) + + (m_nvec[2]-windir[2])*(m_nvec[2]-windir[2]) < 1.e-12, + "do_continous_injection for laser particle only works" + " if moving window direction and laser propagation direction are the same"); if ( WarpX::gamma_boost>1 ){ AMREX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -410,12 +414,10 @@ LaserParticleContainer::InitData (int lev) #if defined(WARPX_DIM_3D) return {m_u_X[0]*(pos[0]-m_position[0])+m_u_X[1]*(pos[1]-m_position[1])+m_u_X[2]*(pos[2]-m_position[2]), m_u_Y[0]*(pos[0]-m_position[0])+m_u_Y[1]*(pos[1]-m_position[1])+m_u_Y[2]*(pos[2]-m_position[2])}; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) -# if defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_RZ) return {pos[0]-m_position[0], 0.0_rt}; -# else +#elif defined(WARPX_DIM_XZ) return {m_u_X[0]*(pos[0]-m_position[0])+m_u_X[2]*(pos[2]-m_position[2]), 0.0_rt}; -# endif #else return {m_u_X[2]*(pos[2]-m_position[2]), 0.0_rt}; #endif @@ -555,16 +557,14 @@ LaserParticleContainer::InitData (int lev) } void -LaserParticleContainer::Evolve (int lev, - const MultiFab&, const MultiFab&, const MultiFab&, - const MultiFab&, const MultiFab&, const MultiFab&, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab*, const MultiFab*, const MultiFab*, - const MultiFab*, const MultiFab*, const MultiFab*, +LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); @@ -581,11 +581,12 @@ LaserParticleContainer::Evolve (int lev, // Update laser profile m_up_laser_profile->update(t_lab); - BL_ASSERT(OnSameGrids(lev,jx)); + BL_ASSERT(OnSameGrids(lev, *fields.get(FieldType::current_fp, Direction{0}, lev))); amrex::LayoutData* cost = WarpX::getCosts(lev); - const bool has_buffer = cjx; + const bool has_rho = fields.has(FieldType::rho_fp, lev); + const bool has_buffer = fields.has_vector(FieldType::current_buf, lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -625,11 +626,13 @@ LaserParticleContainer::Evolve (int lev, np_current = 0; } - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer) { + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -657,6 +660,7 @@ LaserParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_STOP(blp_pp); // Current Deposition + using ablastr::fields::Direction; if (!skip_deposition) { // Deposit at t_{n+1/2} @@ -664,13 +668,19 @@ LaserParticleContainer::Evolve (int lev, int* ion_lev = nullptr; // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, + amrex::MultiFab * jx = fields.get(current_fp_string, Direction{0}, lev); + amrex::MultiFab * jy = fields.get(current_fp_string, Direction{1}, lev); + amrex::MultiFab * jz = fields.get(current_fp_string, Direction{2}, lev); + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, jx, jy, jz, 0, np_current, thread_num, lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers + amrex::MultiFab * cjx = fields.get(FieldType::current_buf, Direction{0}, lev); + amrex::MultiFab * cjy = fields.get(FieldType::current_buf, Direction{1}, lev); + amrex::MultiFab * cjz = fields.get(FieldType::current_buf, Direction{2}, lev); DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt, relative_time, push_type); @@ -678,11 +688,13 @@ LaserParticleContainer::Evolve (int lev, } - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); if (has_buffer) { + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } @@ -731,13 +743,12 @@ LaserParticleContainer::ComputeSpacing (int lev, Real& Sx, Real& Sy) const Sy = std::min(std::min(dx[0]/(std::abs(m_u_Y[0])+eps), dx[1]/(std::abs(m_u_Y[1])+eps)), dx[2]/(std::abs(m_u_Y[2])+eps)); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) -# if defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_RZ) Sx = dx[0]; -# else + Sy = 1.0; +#elif defined(WARPX_DIM_XZ) Sx = std::min(dx[0]/(std::abs(m_u_X[0])+eps), dx[2]/(std::abs(m_u_X[2])+eps)); -# endif Sy = 1.0; #else Sx = 1.0; @@ -747,7 +758,7 @@ LaserParticleContainer::ComputeSpacing (int lev, Real& Sx, Real& Sy) const } void -LaserParticleContainer::ComputeWeightMobility (Real Sx, Real Sy) +LaserParticleContainer::ComputeWeightMobility ([[maybe_unused]] Real Sx, [[maybe_unused]] Real Sy) { // The mobility is the constant of proportionality between the field to // be emitted, and the corresponding velocity that the particles need to have. @@ -757,14 +768,7 @@ LaserParticleContainer::ComputeWeightMobility (Real Sx, Real Sy) m_mobility = eps/m_e_max; m_weight = PhysConst::ep0 / m_mobility; // Multiply by particle spacing -#if defined(WARPX_DIM_3D) - m_weight *= Sx * Sy; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - m_weight *= Sx; - amrex::ignore_unused(Sy); -#else - amrex::ignore_unused(Sx,Sy); -#endif + m_weight *= AMREX_D_TERM(1._rt, * Sx, * Sy); // When running in the boosted-frame, the input parameters (and in particular // the amplitude of the field) are given in the lab-frame. // Therefore, the mobility needs to be modified by a factor WarpX::gamma_boost. @@ -869,18 +873,18 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, #if (AMREX_SPACEDIM >= 2) ParticleReal* x_n = nullptr; if (push_type == PushType::Implicit) { - x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + x_n = pti.GetAttribs("x_n").dataPtr(); } #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal* y_n = nullptr; if (push_type == PushType::Implicit) { - y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + y_n = pti.GetAttribs("y_n").dataPtr(); } #endif ParticleReal* z_n = nullptr; if (push_type == PushType::Implicit) { - z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + z_n = pti.GetAttribs("z_n").dataPtr(); } // Copy member variables to tmp copies for GPU runs. @@ -913,11 +917,13 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, puyp[i] = gamma * vy; puzp[i] = gamma * vz; - // Push the the particle positions + // Push the particle positions // When using the implicit solver, this function is called multiple times per timestep // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset - // to the initial position (at the beginning of the timestep), before updating the particle position + // to the initial position (at the beginning of the timestep), before updating the particle position. + // Also, the current deposition schemes expect the particle positions to be time centered + // (cur_time + 0.5*dt) for PushType::Implicit. ParticleReal x=0., y=0., z=0.; if (push_type == PushType::Explicit) { @@ -926,20 +932,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, #if !defined(WARPX_DIM_1D_Z) if (push_type == PushType::Implicit) { - x = x_n[i]; + x = x_n[i] + vx * dt*0.5_prt; + } + else { + x += vx * dt; } - x += vx * dt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) if (push_type == PushType::Implicit) { - y = y_n[i]; + y = y_n[i] + vy * dt*0.5_prt; + } + else { + y += vy * dt; } - y += vy * dt; #endif if (push_type == PushType::Implicit) { - z = z_n[i]; + z = z_n[i] + vz * dt*0.5_prt; + } + else { + z += vz * dt; } - z += vz * dt; SetPosition(i, x, y, z); } diff --git a/Source/Particles/Make.package b/Source/Particles/Make.package index 58cbe11a980..69918f69940 100644 --- a/Source/Particles/Make.package +++ b/Source/Particles/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += AddPlasmaUtilities.cpp CEXE_sources += MultiParticleContainer.cpp CEXE_sources += WarpXParticleContainer.cpp CEXE_sources += RigidInjectedParticleContainer.cpp diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index f007d5088e3..0e33b6bac3c 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -26,6 +26,8 @@ #include "WarpXParticleContainer.H" #include "ParticleBoundaries.H" +#include + #include #include #include @@ -89,6 +91,8 @@ public: return allcontainers[index]->meanParticleVelocity(); } + amrex::ParticleReal maxParticleVelocity(); + void AllocData (); void InitData (); @@ -100,16 +104,16 @@ public: * field solve, and pushing the particles, for all the species in the MultiParticleContainer. * This is the electromagnetic version. */ - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, - PushType push_type=PushType::Explicit); + void Evolve ( + ablastr::fields::MultiFabRegister& fields, + int lev, + std::string const& current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit + ); /** * \brief This pushes the particle positions by one time step for all the species in the @@ -145,7 +149,7 @@ public: * the time of the deposition. */ void - DepositCharge (amrex::Vector >& rho, + DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, amrex::Real relative_time); /** @@ -160,7 +164,7 @@ public: * the time of the deposition. */ void - DepositCurrent (amrex::Vector, 3 > >& J, + DepositCurrent (ablastr::fields::MultiLevelVectorField const& J, amrex::Real dt, amrex::Real relative_time); /// @@ -296,7 +300,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - void ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb); + void ScrapeParticlesAtEB (ablastr::fields::MultiLevelScalarField const& distance_to_eb); std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index fc496217388..6c08dc6aa8d 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -11,7 +11,7 @@ */ #include "MultiParticleContainer.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" @@ -21,7 +21,6 @@ # include "Particles/ElementaryProcess/QEDPhotonEmission.H" #endif #include "Particles/LaserParticleContainer.H" -#include "Particles/NamedComponentParticleContainer.H" #include "Particles/ParticleCreation/FilterCopyTransform.H" #ifdef WARPX_QED # include "Particles/ParticleCreation/FilterCreateTransformFromFAB.H" @@ -39,13 +38,12 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" -#ifdef AMREX_USE_EB -# include "EmbeddedBoundary/ParticleScraper.H" -# include "EmbeddedBoundary/ParticleBoundaryProcess.H" -#endif +#include "EmbeddedBoundary/ParticleScraper.H" +#include "EmbeddedBoundary/ParticleBoundaryProcess.H" #include "WarpX.H" +#include #include #include @@ -82,14 +80,14 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { /** A little collection to transport six Array4 that point to the EM fields */ struct MyFieldList { - Array4< amrex::Real const > const Ex, Ey, Ez, Bx, By, Bz; + Array4< amrex::Real const > Ex, Ey, Ez, Bx, By, Bz; }; } @@ -224,7 +222,7 @@ MultiParticleContainer::ReadParameters () pp_particles, "repeated_plasma_lens_lengths", h_repeated_plasma_lens_lengths); - const int n_lenses = static_cast(h_repeated_plasma_lens_starts.size()); + const auto n_lenses = static_cast(h_repeated_plasma_lens_starts.size()); d_repeated_plasma_lens_starts.resize(n_lenses); d_repeated_plasma_lens_lengths.resize(n_lenses); amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, @@ -399,6 +397,15 @@ MultiParticleContainer::GetParticleContainerFromName (const std::string& name) c return *allcontainers[i]; } +amrex::ParticleReal +MultiParticleContainer::maxParticleVelocity() { + amrex::ParticleReal max_v = 0.0_prt; + for (const auto &pc : allcontainers) { + max_v = std::max(max_v, pc->maxParticleVelocity()); + } + return max_v; +} + void MultiParticleContainer::AllocData () { @@ -450,30 +457,26 @@ MultiParticleContainer::InitMultiPhysicsModules () } void -MultiParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string const& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { if (! skip_deposition) { - jx.setVal(0.0); - jy.setVal(0.0); - jz.setVal(0.0); - if (cjx) { cjx->setVal(0.0); } - if (cjy) { cjy->setVal(0.0); } - if (cjz) { cjz->setVal(0.0); } - if (rho) { rho->setVal(0.0); } - if (crho) { crho->setVal(0.0); } + using ablastr::fields::Direction; + + fields.get(current_fp_string, Direction{0}, lev)->setVal(0.0); + fields.get(current_fp_string, Direction{1}, lev)->setVal(0.0); + fields.get(current_fp_string, Direction{2}, lev)->setVal(0.0); + if (fields.has(FieldType::current_buf, Direction{0}, lev)) { fields.get(FieldType::current_buf, Direction{0}, lev)->setVal(0.0); } + if (fields.has(FieldType::current_buf, Direction{1}, lev)) { fields.get(FieldType::current_buf, Direction{1}, lev)->setVal(0.0); } + if (fields.has(FieldType::current_buf, Direction{2}, lev)) { fields.get(FieldType::current_buf, Direction{2}, lev)->setVal(0.0); } + if (fields.has(FieldType::rho_fp, lev)) { fields.get(FieldType::rho_fp, lev)->setVal(0.0); } + if (fields.has(FieldType::rho_buf, lev)) { fields.get(FieldType::rho_buf, lev)->setVal(0.0); } } for (auto& pc : allcontainers) { - pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); + pc->Evolve(fields, lev, current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } } @@ -522,11 +525,11 @@ MultiParticleContainer::GetZeroChargeDensity (const int lev) void MultiParticleContainer::DepositCurrent ( - amrex::Vector, 3 > >& J, + ablastr::fields::MultiLevelVectorField const & J, const amrex::Real dt, const amrex::Real relative_time) { // Reset the J arrays - for (auto& J_lev : J) + for (const auto& J_lev : J) { J_lev[0]->setVal(0.0_rt); J_lev[1]->setVal(0.0_rt); @@ -543,18 +546,18 @@ MultiParticleContainer::DepositCurrent ( for (int lev = 0; lev < J.size(); ++lev) { WarpX::GetInstance().ApplyInverseVolumeScalingToCurrentDensity( - J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), lev); + J[lev][0], J[lev][1], J[lev][2], lev); } #endif } void MultiParticleContainer::DepositCharge ( - amrex::Vector >& rho, + const ablastr::fields::MultiLevelScalarField& rho, const amrex::Real relative_time) { // Reset the rho array - for (auto& rho_lev : rho) + for (const auto& rho_lev : rho) { rho_lev->setVal(0.0_rt); } @@ -580,7 +583,7 @@ MultiParticleContainer::DepositCharge ( #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) { - WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev].get(), lev); + WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev], lev); } #endif } @@ -956,15 +959,12 @@ void MultiParticleContainer::CheckIonizationProductSpecies() } } -void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) +void MultiParticleContainer::ScrapeParticlesAtEB ( + ablastr::fields::MultiLevelScalarField const& distance_to_eb) { -#ifdef AMREX_USE_EB for (auto& pc : allcontainers) { scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); } -#else - amrex::ignore_unused(distance_to_eb); -#endif } #ifdef WARPX_QED @@ -1355,12 +1355,13 @@ MultiParticleContainer::doQEDSchwinger () pc_product_ele->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, level_0,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, level_0,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, level_0,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, level_0,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, level_0,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, level_0,2); + using ablastr::fields::Direction; + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, level_0); + const MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, level_0); + const MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, level_0); + const MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, level_0); + const MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, level_0); + const MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, level_0); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -1620,7 +1621,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, auto Transform = PhotonEmissionTransformFunc( m_shr_p_qs_engine->build_optical_depth_functor(), - pc_source->particle_runtime_comps["opticalDepthQSR"], + pc_source->GetRealCompIndex("opticalDepthQSR") - pc_source->NArrayReal, m_shr_p_qs_engine->build_phot_em_functor(), pti, lev, Ex.nGrowVect(), Ex[pti], Ey[pti], Ez[pti], diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H deleted file mode 100644 index 02f4c44314a..00000000000 --- a/Source/Particles/NamedComponentParticleContainer.H +++ /dev/null @@ -1,222 +0,0 @@ -/* Copyright 2022 Remi Lehe - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_NamedComponentParticleContainer_H_ -#define WARPX_NamedComponentParticleContainer_H_ - -#include "Utils/TextMsg.H" - -#include -#include -#include - -#include -#include -#include - - -/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array - */ -struct PIdx -{ - enum { -#if !defined (WARPX_DIM_1D_Z) - x, -#endif -#if defined (WARPX_DIM_3D) - y, -#endif - z, - w, ///< weight - ux, uy, uz, -#ifdef WARPX_DIM_RZ - theta, ///< RZ needs all three position components -#endif - nattribs ///< number of compile-time attributes - }; -}; - -/** Integer Particle Attributes stored in amrex::ParticleContainer's struct of array - */ -struct PIdxInt -{ - enum { - nattribs ///< number of compile-time attributes - }; -}; - -/** Particle Container class that allows to add/access particle components - * with a name (string) instead of doing so with an integer index. - * (The "components" are all the particle amrex::Real quantities.) - * - * This is done by storing maps that give the index of the component - * that corresponds to a given string. - * - * @tparam T_Allocator Mainly controls in which type of memory (e.g. device - * arena, pinned memory arena, etc.) the particle data will be stored - */ -template class T_Allocator=amrex::DefaultAllocator> -class NamedComponentParticleContainer : -public amrex::ParticleContainerPureSoA -{ -public: - /** Construct an empty NamedComponentParticleContainer **/ - NamedComponentParticleContainer () : amrex::ParticleContainerPureSoA() {} - - /** Construct a NamedComponentParticleContainer from an AmrParGDB object - * - * In this case, the only components are the default ones: - * weight, momentum and (in RZ geometry) theta. - * - * @param amr_pgdb A pointer to a ParGDBBase, which contains pointers to - * the Geometry, DistributionMapping, and BoxArray objects that define the - * AMR hierarchy. Usually, this is generated by an AmrCore or AmrLevel object. - */ - NamedComponentParticleContainer (amrex::AmrParGDB* amr_pgdb) - : amrex::ParticleContainerPureSoA(amr_pgdb) { - // build up the map of string names to particle component numbers -#if !defined (WARPX_DIM_1D_Z) - particle_comps["x"] = PIdx::x; -#endif -#if defined (WARPX_DIM_3D) - particle_comps["y"] = PIdx::y; -#endif - particle_comps["z"] = PIdx::z; - particle_comps["w"] = PIdx::w; - particle_comps["ux"] = PIdx::ux; - particle_comps["uy"] = PIdx::uy; - particle_comps["uz"] = PIdx::uz; -#ifdef WARPX_DIM_RZ - particle_comps["theta"] = PIdx::theta; -#endif - } - - /** Destructor for NamedComponentParticleContainer */ - ~NamedComponentParticleContainer() override = default; - - /** Construct a NamedComponentParticleContainer from a regular - * amrex::ParticleContainer, and additional name-to-index maps - * - * @param pc regular particle container, where components are not named (only indexed) - * @param p_comps name-to-index map for compile-time and run-time real components - * @param p_icomps name-to-index map for compile-time and run-time integer components - * @param p_rcomps name-to-index map for run-time real components - * @param p_ricomps name-to-index map for run-time integer components - */ - NamedComponentParticleContainer( - amrex::ParticleContainerPureSoA && pc, - std::map p_comps, - std::map p_icomps, - std::map p_rcomps, - std::map p_ricomps) - : amrex::ParticleContainerPureSoA(std::move(pc)), - particle_comps(std::move(p_comps)), - particle_icomps(std::move(p_icomps)), - particle_runtime_comps(std::move(p_rcomps)), - particle_runtime_icomps(std::move(p_ricomps)) {} - - /** Copy constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( const NamedComponentParticleContainer &) = delete; - /** Copy operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( const NamedComponentParticleContainer & ) = delete; - - /** Move constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( NamedComponentParticleContainer && ) noexcept = default; - /** Move operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) noexcept = default; - - /** Create an empty particle container - * - * This creates a new NamedComponentParticleContainer with same compile-time - * and run-time attributes. But it can change its allocator. - * - * This function overloads the corresponding function from the parent - * class (amrex::ParticleContainer) - */ - template class NewAllocator=amrex::DefaultAllocator> - NamedComponentParticleContainer - make_alike () const { - auto tmp = NamedComponentParticleContainer( - amrex::ParticleContainerPureSoA::template make_alike(), - particle_comps, - particle_icomps, - particle_runtime_comps, - particle_runtime_icomps); - - return tmp; - } - - using amrex::ParticleContainerPureSoA::NumRealComps; - using amrex::ParticleContainerPureSoA::NumIntComps; - using amrex::ParticleContainerPureSoA::AddRealComp; - using amrex::ParticleContainerPureSoA::AddIntComp; - - /** Allocate a new run-time real component - * - * @param name Name of the new component - * @param comm Whether to communicate this component, in the particle Redistribute - */ - void AddRealComp (const std::string& name, bool comm=true) - { - auto search = particle_comps.find(name); - if (search == particle_comps.end()) { - particle_comps[name] = NumRealComps(); - particle_runtime_comps[name] = NumRealComps() - PIdx::nattribs; - AddRealComp(comm); - } else { - amrex::Print() << Utils::TextMsg::Info( - name + " already exists in particle_comps, not adding."); - } - } - - /** Allocate a new run-time integer component - * - * @param name Name of the new component - * @param comm Whether to communicate this component, in the particle Redistribute - */ - void AddIntComp (const std::string& name, bool comm=true) - { - auto search = particle_icomps.find(name); - if (search == particle_icomps.end()) { - particle_icomps[name] = NumIntComps(); - particle_runtime_icomps[name] = NumIntComps() - 0; - AddIntComp(comm); - } else { - amrex::Print() << Utils::TextMsg::Info( - name + " already exists in particle_icomps, not adding."); - } - } - - void defineAllParticleTiles () noexcept - { - for (int lev = 0; lev <= amrex::ParticleContainerPureSoA::finestLevel(); ++lev) - { - for (auto mfi = amrex::ParticleContainerPureSoA::MakeMFIter(lev); mfi.isValid(); ++mfi) - { - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - amrex::ParticleContainerPureSoA::DefineAndReturnParticleTile(lev, grid_id, tile_id); - } - } - } - - /** Return the name-to-index map for the compile-time and runtime-time real components */ - [[nodiscard]] std::map getParticleComps () const noexcept { return particle_comps;} - /** Return the name-to-index map for the compile-time and runtime-time integer components */ - [[nodiscard]] std::map getParticleiComps () const noexcept { return particle_icomps;} - /** Return the name-to-index map for the runtime-time real components */ - [[nodiscard]] std::map getParticleRuntimeComps () const noexcept { return particle_runtime_comps;} - /** Return the name-to-index map for the runtime-time integer components */ - [[nodiscard]] std::map getParticleRuntimeiComps () const noexcept { return particle_runtime_icomps;} - -protected: - std::map particle_comps; - std::map particle_icomps; - std::map particle_runtime_comps; - std::map particle_runtime_icomps; -}; - -#endif //WARPX_NamedComponentParticleContainer_H_ diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 71b95aae95a..5dd340d9053 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -109,17 +109,10 @@ namespace ApplyParticleBoundaries { */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void - apply_boundaries ( -#ifndef WARPX_DIM_1D_Z - amrex::ParticleReal& x, amrex::Real xmin, amrex::Real xmax, -#endif -#if (defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ) - amrex::ParticleReal& y, -#endif -#if (defined WARPX_DIM_3D) - amrex::Real ymin, amrex::Real ymax, -#endif - amrex::ParticleReal& z, amrex::Real zmin, amrex::Real zmax, + apply_boundaries ([[maybe_unused]] amrex::ParticleReal& x, + [[maybe_unused]] amrex::ParticleReal& y, + [[maybe_unused]] amrex::ParticleReal& z, + amrex::XDim3 gridmin, amrex::XDim3 gridmax, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, bool& particle_lost, ParticleBoundaries::ParticleBoundariesData const& boundaries, @@ -131,7 +124,7 @@ namespace ApplyParticleBoundaries { #ifndef WARPX_DIM_1D_Z bool rethermalize_x = false; // stores if particle crosses x boundary and needs to be thermalized - apply_boundary(x, xmin, xmax, change_sign_ux, rethermalize_x, particle_lost, + apply_boundary(x, gridmin.x, gridmax.x, change_sign_ux, rethermalize_x, particle_lost, boundaries.xmin_bc, boundaries.xmax_bc, boundaries.reflection_model_xlo(-ux), boundaries.reflection_model_xhi(ux), engine); @@ -141,7 +134,7 @@ namespace ApplyParticleBoundaries { #endif #ifdef WARPX_DIM_3D bool rethermalize_y = false; // stores if particle crosses y boundary and needs to be thermalized - apply_boundary(y, ymin, ymax, change_sign_uy, rethermalize_y, particle_lost, + apply_boundary(y, gridmin.y, gridmax.y, change_sign_uy, rethermalize_y, particle_lost, boundaries.ymin_bc, boundaries.ymax_bc, boundaries.reflection_model_ylo(-uy), boundaries.reflection_model_yhi(uy), engine); @@ -150,7 +143,7 @@ namespace ApplyParticleBoundaries { } #endif bool rethermalize_z = false; // stores if particle crosses z boundary and needs to be thermalized - apply_boundary(z, zmin, zmax, change_sign_uz, rethermalize_z, particle_lost, + apply_boundary(z, gridmin.z, gridmax.z, change_sign_uz, rethermalize_z, particle_lost, boundaries.zmin_bc, boundaries.zmax_bc, boundaries.reflection_model_zlo(-uz), boundaries.reflection_model_zhi(uz), engine); diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index d33834309ab..c9589ac0c75 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -12,6 +12,8 @@ #include "Particles/PinnedMemoryParticleContainer.H" #include "Utils/export.H" +#include + #include @@ -30,9 +32,9 @@ public: /** Copy operator for ParticleBoundaryBuffer */ ParticleBoundaryBuffer& operator= ( const ParticleBoundaryBuffer & ) = delete; - /** Move constructor for NamedComponentParticleContainer */ + /** Move constructor for ParticleBoundaryBuffer */ ParticleBoundaryBuffer ( ParticleBoundaryBuffer && ) = default; - /** Move operator for NamedComponentParticleContainer */ + /** Move operator for ParticleBoundaryBuffer */ ParticleBoundaryBuffer& operator= ( ParticleBoundaryBuffer && ) = default; int numSpecies() const { return static_cast(getSpeciesNames().size()); } @@ -41,7 +43,7 @@ public: void gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc); void gatherParticlesFromEmbeddedBoundaries ( - MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb + MultiParticleContainer& mypc, ablastr::fields::MultiLevelScalarField const& distance_to_eb ); void redistribute (); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 25068b2e65c..048534bff6a 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -6,6 +6,7 @@ */ #include "WarpX.H" +#include "EmbeddedBoundary/Enabled.H" #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Particles/MultiParticleContainer.H" @@ -23,8 +24,10 @@ #include #include #include + using namespace amrex::literals; + struct IsOutsideDomainBoundary { amrex::GpuArray m_plo; amrex::GpuArray m_phi; @@ -46,13 +49,12 @@ struct IsOutsideDomainBoundary { } }; -#ifdef AMREX_USE_EB struct FindEmbeddedBoundaryIntersection { - const int m_step_index; - const int m_delta_index; - const int m_normal_index; - const int m_step; - const amrex::Real m_dt; + int m_step_index; + int m_delta_index; + int m_normal_index; + int m_step; + amrex::Real m_dt; amrex::Array4 m_phiarr; amrex::GpuArray m_dxi; amrex::GpuArray m_plo; @@ -86,13 +88,13 @@ struct FindEmbeddedBoundaryIntersection { amrex::ParticleReal const uz = dst.m_rdata[PIdx::uz][dst_i]; // Temporary variables to avoid implicit capture - amrex::Real dt = m_dt; - amrex::Array4 phiarr = m_phiarr; - amrex::GpuArray dxi = m_dxi; - amrex::GpuArray plo = m_plo; + amrex::Real const dt = m_dt; + amrex::Array4 const phiarr = m_phiarr; + amrex::GpuArray const dxi = m_dxi; + amrex::GpuArray const plo = m_plo; // Bisection algorithm to find the point where phi(x,y,z)=0 (i.e. on the embedded boundary) - amrex::Real dt_fraction = amrex::bisect( 0.0, 1.0, + amrex::Real const dt_fraction = amrex::bisect( 0.0, 1.0, [=] (amrex::Real dt_frac) { int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; @@ -100,7 +102,7 @@ struct FindEmbeddedBoundaryIntersection { UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); ablastr::particles::compute_weights( x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); - amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -146,7 +148,7 @@ struct FindEmbeddedBoundaryIntersection { dst.m_rdata[PIdx::z][dst_i] = z_temp; dst.m_rdata[PIdx::theta][dst_i] = std::atan2(y_temp, x_temp); //save normal components - amrex::Real theta=std::atan2(y_temp, x_temp); + amrex::Real const theta = std::atan2(y_temp, x_temp); dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]*std::cos(theta); dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[0]*std::sin(theta); dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; @@ -165,14 +167,13 @@ struct FindEmbeddedBoundaryIntersection { amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; -#endif struct CopyAndTimestamp { int m_step_index; int m_delta_index; int m_normal_index; int m_step; - const amrex::Real m_dt; + amrex::Real m_dt; int m_idim; int m_iside; @@ -239,6 +240,8 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () constexpr auto idx_zhi = 5; #endif + bool const eb_enabled = EB::enabled(); + for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { const amrex::ParmParse pp_species(getSpeciesNames()[ispecies]); @@ -258,9 +261,9 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () pp_species.query("save_particles_at_zlo", m_do_boundary_buffer[idx_zlo][ispecies]); pp_species.query("save_particles_at_zhi", m_do_boundary_buffer[idx_zhi][ispecies]); #endif -#ifdef AMREX_USE_EB - pp_species.query("save_particles_at_eb", m_do_boundary_buffer[AMREX_SPACEDIM*2][ispecies]); -#endif + + if (eb_enabled) { pp_species.query("save_particles_at_eb", m_do_boundary_buffer[AMREX_SPACEDIM*2][ispecies]); } + // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } @@ -283,10 +286,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () m_boundary_names[idx_zlo] = "zlo"; m_boundary_names[idx_zhi] = "zhi"; #endif -#ifdef AMREX_USE_EB - m_boundary_names[AMREX_SPACEDIM*2] = "eb"; -#endif - + if (eb_enabled) { m_boundary_names[AMREX_SPACEDIM*2] = "eb"; } } void ParticleBoundaryBuffer::printNumParticles () const { @@ -306,17 +306,17 @@ void ParticleBoundaryBuffer::printNumParticles () const { } } } -#ifdef AMREX_USE_EB - auto& buffer = m_particle_containers[2*AMREX_SPACEDIM]; - for (int i = 0; i < numSpecies(); ++i) - { - const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; - amrex::Print() << Utils::TextMsg::Info( - "Species " + getSpeciesNames()[i] + " has " - + std::to_string(np) + " particles in the EB boundary buffer" - ); + + if (EB::enabled()) { + auto const & buffer = m_particle_containers[2 * AMREX_SPACEDIM]; + for (int i = 0; i < numSpecies(); ++i) { + const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; + amrex::Print() << Utils::TextMsg::Info( + "Species " + getSpeciesNames()[i] + " has " + + std::to_string(np) + " particles in the EB boundary buffer" + ); + } } -#endif } void ParticleBoundaryBuffer::redistribute () { @@ -401,6 +401,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for (int lev = 0; lev < pc.numLevels(); ++lev) { + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + const auto& plevel = pc.GetParticles(lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -408,10 +413,10 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) { continue; } - auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( - lev, pti.index(), pti.LocalTileIndex()); + auto& ptile_buffer = + species_buffer.ParticlesAt(lev, pti.index(), pti.LocalTileIndex()); + const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); if (np == 0) { continue; } @@ -438,11 +443,10 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterAndTransform"); auto& warpx = WarpX::GetInstance(); const auto dt = warpx.getdt(pti.GetLevel()); - auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); - const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); - auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); - const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); - const int normal_index = string_to_index_realcomp.at("nx"); + auto & buf = buffer[i]; + const int step_scraped_index = buf.GetIntCompIndex("stepScraped") - PinnedMemoryParticleContainer::NArrayInt; + const int delta_index = buf.GetRealCompIndex("deltaTimeScraped") - PinnedMemoryParticleContainer::NArrayReal; + const int normal_index = buf.GetRealCompIndex("nx") - PinnedMemoryParticleContainer::NArrayReal; const int step = warpx_instance.getistep(0); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, @@ -457,109 +461,109 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC } void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( - MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) + MultiParticleContainer& mypc, ablastr::fields::MultiLevelScalarField const& distance_to_eb) { -#ifdef AMREX_USE_EB - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); + if (EB::enabled()) { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); - using PIter = amrex::ParConstIterSoA; - const auto& warpx_instance = WarpX::GetInstance(); - const amrex::Geometry& geom = warpx_instance.Geom(0); - auto plo = geom.ProbLoArray(); - auto& buffer = m_particle_containers[m_particle_containers.size()-1]; - for (int i = 0; i < numSpecies(); ++i) - { - if (!m_do_boundary_buffer[AMREX_SPACEDIM*2][i]) continue; - const auto& pc = mypc.GetParticleContainer(i); - if (!buffer[i].isDefined()) + using PIter = amrex::ParConstIterSoA; + const auto &warpx_instance = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx_instance.Geom(0); + auto plo = geom.ProbLoArray(); + + auto& buffer = m_particle_containers[m_particle_containers.size()-1]; + for (int i = 0; i < numSpecies(); ++i) { - buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("stepScraped", false); - buffer[i].AddRealComp("deltaTimeScraped", false); - buffer[i].AddRealComp("nx", false); - buffer[i].AddRealComp("ny", false); - buffer[i].AddRealComp("nz", false); + if (!m_do_boundary_buffer[AMREX_SPACEDIM*2][i]) { continue; } + const auto& pc = mypc.GetParticleContainer(i); + if (!buffer[i].isDefined()) + { + buffer[i] = pc.make_alike(); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); + buffer[i].AddRealComp("nx", false); + buffer[i].AddRealComp("ny", false); + buffer[i].AddRealComp("nz", false); - } + } - auto& species_buffer = buffer[i]; - for (int lev = 0; lev < pc.numLevels(); ++lev){ - for(PIter pti(pc, lev); pti.isValid(); ++pti){ - species_buffer.DefineAndReturnParticleTile( - lev, pti.index(), pti.LocalTileIndex()); + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev) { + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } } - } - for (int lev = 0; lev < pc.numLevels(); ++lev) - { - const auto& plevel = pc.GetParticles(lev); - auto dxi = warpx_instance.Geom(lev).InvCellSizeArray(); + for (int lev = 0; lev < pc.numLevels(); ++lev) + { + const auto& plevel = pc.GetParticles(lev); + auto dxi = warpx_instance.Geom(lev).InvCellSizeArray(); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for(PIter pti(pc, lev); pti.isValid(); ++pti) - { - auto phiarr = (*distance_to_eb[lev])[pti].array(); // signed distance function - auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; - - const auto getPosition = GetParticlePosition(pti); - auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile(lev, pti.index(), - pti.LocalTileIndex()); - const auto& ptile = plevel.at(index); - auto np = ptile.numParticles(); - if (np == 0) { continue; } - - using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; - auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) - /* NVCC 11.3.109 chokes in C++17 on this: noexcept */ - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - - amrex::Real phi_value = ablastr::particles::doGatherScalarFieldNodal( - xp, yp, zp, phiarr, dxi, plo - ); - return phi_value < 0.0 ? 1 : 0; - }; - - const auto ptile_data = ptile.getConstParticleTileData(); - - amrex::ReduceOps reduce_op; - amrex::ReduceData reduce_data(reduce_op); - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::count_out_of_boundsEB"); - reduce_op.eval(np, reduce_data, [=] AMREX_GPU_HOST_DEVICE (int ip) - { return predicate(ptile_data, ip) ? 1 : 0; }); - } + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + auto phiarr = (*distance_to_eb[lev])[pti].array(); // signed distance function + auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); + if (plevel.find(index) == plevel.end()) { continue; } + + const auto getPosition = GetParticlePosition(pti); + auto &ptile_buffer = species_buffer.DefineAndReturnParticleTile(lev, pti.index(), + pti.LocalTileIndex()); + const auto &ptile = plevel.at(index); + auto np = ptile.numParticles(); + if (np == 0) { continue; } + + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + auto predicate = [=] AMREX_GPU_HOST_DEVICE(const SrcData & /*src*/, const int ip) + /* NVCC 11.3.109 chokes in C++17 on this: noexcept */ + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); - auto dst_index = ptile_buffer.numParticles(); - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); - ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); - } - auto& warpx = WarpX::GetInstance(); - const auto dt = warpx.getdt(pti.GetLevel()); - auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); - const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); - auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); - const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); - const int normal_index = string_to_index_realcomp.at("nx"); - const int step = warpx_instance.getistep(0); + amrex::Real const phi_value = ablastr::particles::doGatherScalarFieldNodal( + xp, yp, zp, phiarr, dxi, plo + ); + return phi_value < 0.0 ? 1 : 0; + }; - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); - amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - FindEmbeddedBoundaryIntersection{step_scraped_index,delta_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + const auto ptile_data = ptile.getConstParticleTileData(); + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::count_out_of_boundsEB"); + reduce_op.eval(np, reduce_data, + [=] AMREX_GPU_HOST_DEVICE(int ip) { return predicate(ptile_data, ip) ? 1 : 0; }); + } + + auto dst_index = ptile_buffer.numParticles(); + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); + ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); + } + auto &warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + auto & buf = buffer[i]; + const int step_scraped_index = buf.GetIntCompIndex("stepScraped") - PinnedMemoryParticleContainer::NArrayInt; + const int delta_index = buf.GetRealCompIndex("deltaTimeScraped") - PinnedMemoryParticleContainer::NArrayReal; + const int normal_index = buf.GetRealCompIndex("nx") - PinnedMemoryParticleContainer::NArrayReal; + const int step = warpx_instance.getistep(0); + + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); + amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, + FindEmbeddedBoundaryIntersection{step_scraped_index, + delta_index, normal_index, + step, dt, phiarr, dxi, plo}, + 0, dst_index); + + } } } } } -#else - amrex::ignore_unused(mypc, distance_to_eb); -#endif } int ParticleBoundaryBuffer::getNumParticlesInContainer( diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 88b23905481..1922c829379 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -102,8 +102,8 @@ namespace ParticleCreation { * These are NOT initialized by this function. * @param[in] user_real_attribs The names of the real components for this particle tile * @param[in] user_int_attribs The names of the int components for this particle tile - * @param[in] particle_comps map between particle component index and component name for real comps - * @param[in] particle_icomps map between particle component index and component name for int comps + * @param[in] particle_comps particle component names for real comps + * @param[in] particle_icomps particle component names for int comps * @param[in] user_real_attrib_parser the parser functions used to initialize the user real components * @param[in] user_int_attrib_parser the parser functions used to initialize the user int components * @param[in] do_qed_comps whether to initialize the qed components (these are usually handled by @@ -120,8 +120,8 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, const int n_external_attr_int, const std::vector& user_real_attribs, const std::vector& user_int_attribs, - const std::map& particle_comps, - const std::map& particle_icomps, + const std::vector& particle_comps, + const std::vector& particle_icomps, const std::vector& user_real_attrib_parser, const std::vector& user_int_attrib_parser, #ifdef WARPX_QED @@ -151,8 +151,9 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, auto attr_ptr = ptile.GetStructOfArrays().GetRealData(j).data(); #ifdef WARPX_QED // Current runtime comp is quantum synchrotron optical depth - if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && - particle_comps.at("opticalDepthQSR") == j) + auto const it_qsr = std::find(particle_comps.begin(), particle_comps.end(), "opticalDepthQSR"); + if (it_qsr != particle_comps.end() && + std::distance(particle_comps.begin(), it_qsr) == j) { if (!do_qed_comps) { continue; } const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = @@ -172,9 +173,10 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, } } - // Current runtime comp is Breit-Wheeler optical depth - if (particle_comps.find("opticalDepthBW") != particle_comps.end() && - particle_comps.at("opticalDepthBW") == j) + // Current runtime comp is Breit-Wheeler optical depth + auto const it_bw = std::find(particle_comps.begin(), particle_comps.end(), "opticalDepthBW"); + if (it_bw != particle_comps.end() && + std::distance(particle_comps.begin(), it_bw) == j) { if (!do_qed_comps) { continue; } const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = @@ -198,8 +200,9 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, for (int ia = 0; ia < n_user_real_attribs; ++ia) { // Current runtime comp is ia-th user defined attribute - if (particle_comps.find(user_real_attribs[ia]) != particle_comps.end() && - particle_comps.at(user_real_attribs[ia]) == j) + auto const it_ura = std::find(particle_comps.begin(), particle_comps.end(), user_real_attribs[ia]); + if (it_ura != particle_comps.end() && + std::distance(particle_comps.begin(), it_ura) == j) { const amrex::ParserExecutor<7> user_real_attrib_parserexec = user_real_attrib_parser[ia]->compile<7>(); @@ -232,8 +235,9 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, auto attr_ptr = ptile.GetStructOfArrays().GetIntData(j).data(); // Current runtime comp is ionization level - if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && - particle_icomps.at("ionizationLevel") == j) + auto const it_ioniz = std::find(particle_icomps.begin(), particle_icomps.end(), "ionizationLevel"); + if (it_ioniz != particle_icomps.end() && + std::distance(particle_icomps.begin(), it_ioniz) == j) { if constexpr (amrex::RunOnGpu>::value) { amrex::ParallelFor(stop - start, @@ -251,8 +255,9 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, for (int ia = 0; ia < n_user_int_attribs; ++ia) { // Current runtime comp is ia-th user defined attribute - if (particle_icomps.find(user_int_attribs[ia]) != particle_icomps.end() && - particle_icomps.at(user_int_attribs[ia]) == j) + auto const it_uia = std::find(particle_icomps.begin(), particle_icomps.end(), user_int_attribs[ia]); + if (it_uia != particle_icomps.end() && + std::distance(particle_icomps.begin(), it_uia) == j) { const amrex::ParserExecutor<7> user_int_attrib_parserexec = user_int_attrib_parser[ia]->compile<7>(); diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 4815a98ca31..c05038fae2f 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -51,7 +51,7 @@ */ template ::value, int> foo = 0> + amrex::EnableIf_t, int> foo = 0> Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index* mask, Index dst_index, CopyFunc&& copy, TransFunc&& transform) noexcept @@ -88,7 +88,7 @@ Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, ParticleCreation::DefaultInitializeRuntimeAttributes(dst, 0, 0, pc.getUserRealAttribs(), pc.getUserIntAttribs(), - pc.getParticleComps(), pc.getParticleiComps(), + pc.GetRealSoANames(), pc.GetIntSoANames(), pc.getUserRealAttribParser(), pc.getUserIntAttribParser(), #ifdef WARPX_QED @@ -210,7 +210,7 @@ Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index */ template ::value, int> foo = 0> + amrex::EnableIf_t, int> foo = 0> Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, Index dst1_index, Index dst2_index, CopyFunc1&& copy1, CopyFunc2&& copy2, @@ -258,7 +258,7 @@ Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, 0, 0, pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), - pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.GetRealSoANames(), pc1.GetIntSoANames(), pc1.getUserRealAttribParser(), pc1.getUserIntAttribParser(), #ifdef WARPX_QED @@ -272,7 +272,7 @@ Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, 0, 0, pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), - pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.GetRealSoANames(), pc2.GetIntSoANames(), pc2.getUserRealAttribParser(), pc2.getUserIntAttribParser(), #ifdef WARPX_QED diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 8a83c60b221..266faae6322 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -45,7 +45,7 @@ */ template ::value, int> foo = 0> + amrex::EnableIf_t, int> foo = 0> Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, const amrex::Box box, const FAB *src_FAB, const Index* mask, @@ -136,7 +136,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, 0, 0, pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), - pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.GetRealSoANames(), pc1.GetIntSoANames(), pc1.getUserRealAttribParser(), pc1.getUserIntAttribParser(), #ifdef WARPX_QED @@ -150,7 +150,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, 0, 0, pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), - pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.GetRealSoANames(), pc2.GetIntSoANames(), pc2.getUserRealAttribParser(), pc2.getUserIntAttribParser(), #ifdef WARPX_QED diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index e1d944e9c30..6be363e6337 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -140,10 +140,10 @@ class SmartCopyFactory public: template SmartCopyFactory (const SrcPC& src, const DstPC& dst) noexcept : - m_tag_real{getSmartCopyTag(src.getParticleComps(), dst.getParticleComps())}, - m_tag_int{getSmartCopyTag(src.getParticleiComps(), dst.getParticleiComps())}, - m_policy_real{getPolicies(dst.getParticleComps())}, - m_policy_int{getPolicies(dst.getParticleiComps())}, + m_tag_real{getSmartCopyTag(src.GetRealSoANames(), dst.GetRealSoANames())}, + m_tag_int{getSmartCopyTag(src.GetIntSoANames(), dst.GetIntSoANames())}, + m_policy_real{getPolicies(dst.GetRealSoANames())}, + m_policy_int{getPolicies(dst.GetIntSoANames())}, m_defined{true} {} diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index fe4cb5929e0..688f1c3701f 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -35,7 +35,7 @@ struct SmartCreate { const InitializationPolicy* m_policy_real; const InitializationPolicy* m_policy_int; - const int m_weight_index = 0; + int m_weight_index = 0; template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -97,8 +97,8 @@ class SmartCreateFactory public: template SmartCreateFactory (const PartTileData& part) noexcept: - m_policy_real{getPolicies(part.getParticleComps())}, - m_policy_int{getPolicies(part.getParticleiComps())}, + m_policy_real{getPolicies(part.GetRealSoANames())}, + m_policy_int{getPolicies(part.GetIntSoANames())}, m_defined{true} {} diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index 652a3aecd17..358c2b1a7a9 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -35,9 +35,9 @@ struct SmartCopyTag [[nodiscard]] int size () const noexcept { return static_cast(common_names.size()); } }; -PolicyVec getPolicies (const NameMap& names) noexcept; +PolicyVec getPolicies (std::vector const & names) noexcept; -SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept; +SmartCopyTag getSmartCopyTag (std::vector const & src, std::vector const & dst) noexcept; /** * \brief Sets the ids of newly created particles to the next values. diff --git a/Source/Particles/ParticleCreation/SmartUtils.cpp b/Source/Particles/ParticleCreation/SmartUtils.cpp index 7e79f58c59e..19e5bee8b97 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.cpp +++ b/Source/Particles/ParticleCreation/SmartUtils.cpp @@ -13,8 +13,11 @@ #include #include -PolicyVec getPolicies (const NameMap& names) noexcept +PolicyVec getPolicies (std::vector const & names_vec) noexcept { + NameMap names; + for (auto i = 0u; i < names_vec.size(); ++i) { names.emplace(names_vec[i], i); } + std::vector h_policies; h_policies.resize(names.size()); for (const auto& kv : names) @@ -31,10 +34,16 @@ PolicyVec getPolicies (const NameMap& names) noexcept return policies; } -SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept +SmartCopyTag getSmartCopyTag (std::vector const & src_names, std::vector const & dst_names) noexcept { SmartCopyTag tag; + // We want to avoid running an NxM algorithm to find pairs, so sort the components first. + NameMap src; + NameMap dst; + for (auto i = 0u; i < src_names.size(); ++i) { src.emplace(src_names[i], i); } + for (auto i = 0u; i < dst_names.size(); ++i) { dst.emplace(dst_names[i], i); } + std::vector h_src_comps; std::vector h_dst_comps; diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index d5fc68f4097..8d3516e6890 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -90,6 +90,6 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* */ void storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, - int electrostatic_solver_id, bool is_full_diagnostic ); + ElectrostaticSolverAlgo electrostatic_solver_id, bool is_full_diagnostic ); #endif /* WARPX_PARTICLEIO_H_ */ diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 34afac53482..485f56dba43 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -46,34 +46,16 @@ public: void InitData() override; - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; - void PushPX(WarpXParIter& pti, + void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, amrex::FArrayBox const * eyfab, amrex::FArrayBox const * ezfab, diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 1f15d5210f5..ad0b3364eea 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -122,7 +122,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const bool local_has_breit_wheeler = has_breit_wheeler(); if (local_has_breit_wheeler) { evolve_opt = m_shr_p_bw_engine->build_evolve_functor(); - p_optical_depth_BW = pti.GetAttribs(particle_comps["opticalDepthBW"]).dataPtr() + offset; + p_optical_depth_BW = pti.GetAttribs("opticalDepthBW").dataPtr() + offset; } #endif @@ -229,27 +229,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, } void -PhotonParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +PhotonParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { // This does gather, push and deposit. // Push and deposit have been re-written for photons - PhysicalParticleContainer::Evolve (lev, - Ex, Ey, Ez, - Bx, By, Bz, - jx, jy, jz, - cjx, cjy, cjz, - rho, crho, - cEx, cEy, cEz, - cBx, cBy, cBz, + PhysicalParticleContainer::Evolve (fields, + lev, + current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 5d9b41b8b75..18880239183 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -81,27 +81,9 @@ public: * \brief Evolve is the central function PhysicalParticleContainer that * advances plasma particles for a time dt (typically one timestep). * + * \param fields the WarpX field register * \param lev level on which particles are living - * \param Ex MultiFab from which field Ex is gathered - * \param Ey MultiFab from which field Ey is gathered - * \param Ez MultiFab from which field Ez is gathered - * \param Bx MultiFab from which field Bx is gathered - * \param By MultiFab from which field By is gathered - * \param Bz MultiFab from which field Bz is gathered - * \param jx MultiFab to which the particles' current jx is deposited - * \param jy MultiFab to which the particles' current jy is deposited - * \param jz MultiFab to which the particles' current jz is deposited - * \param cjx Same as jx (coarser, from lev-1), when using deposition buffers - * \param cjy Same as jy (coarser, from lev-1), when using deposition buffers - * \param cjz Same as jz (coarser, from lev-1), when using deposition buffers - * \param rho MultiFab to which the particles' charge is deposited - * \param crho Same as rho (coarser, from lev-1), when using deposition buffers - * \param cEx Same as Ex (coarser, from lev-1), when using gather buffers - * \param cEy Same as Ey (coarser, from lev-1), when using gather buffers - * \param cEz Same as Ez (coarser, from lev-1), when using gather buffers - * \param cBx Same as Bx (coarser, from lev-1), when using gather buffers - * \param cBy Same as By (coarser, from lev-1), when using gather buffers - * \param cBz Same as Bz (coarser, from lev-1), when using gather buffers + * \param current_fp_string current coarse or fine patch identifier in fields * \param t current physical time * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) @@ -112,32 +94,14 @@ public: * field gather, particle push and current deposition for all particles * in the box. */ - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -392,6 +356,14 @@ public: } protected: + + /* + Finds the box defining the region where refine injection should be used, if that + option is enabled. Currently this only works for numLevels() == 2 and static mesh + refinement. + */ + bool findRefinedInjectionBox (amrex::Box& fine_injection_box, amrex::IntVect& rrfac); + std::string species_name; std::vector> plasma_injectors; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b7410920edb..e5654d97971 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -10,11 +10,13 @@ */ #include "PhysicalParticleContainer.H" +#include "Fields.H" #include "Filter/NCIGodfreyFilter.H" #include "Initialization/InjectorDensity.H" #include "Initialization/InjectorMomentum.H" #include "Initialization/InjectorPosition.H" #include "MultiParticleContainer.H" +#include "Particles/AddPlasmaUtilities.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" # include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" @@ -39,6 +41,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef AMREX_USE_EB # include "EmbeddedBoundary/ParticleBoundaryProcess.H" # include "EmbeddedBoundary/ParticleScraper.H" @@ -144,29 +147,6 @@ namespace return z0; } - struct PDim3 { - ParticleReal x, y, z; - - AMREX_GPU_HOST_DEVICE - PDim3(const amrex::XDim3& a): - x{static_cast(a.x)}, - y{static_cast(a.y)}, - z{static_cast(a.z)} - {} - - AMREX_GPU_HOST_DEVICE - ~PDim3() = default; - - AMREX_GPU_HOST_DEVICE - PDim3(PDim3 const &) = default; - AMREX_GPU_HOST_DEVICE - PDim3& operator=(PDim3 const &) = default; - AMREX_GPU_HOST_DEVICE - PDim3(PDim3&&) = default; - AMREX_GPU_HOST_DEVICE - PDim3& operator=(PDim3&&) = default; - }; - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE XDim3 getCellCoords (const GpuArray& lo_corner, const GpuArray& dx, @@ -177,16 +157,16 @@ namespace pos.x = lo_corner[0] + (iv[0]+r.x)*dx[0]; pos.y = lo_corner[1] + (iv[1]+r.y)*dx[1]; pos.z = lo_corner[2] + (iv[2]+r.z)*dx[2]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) pos.x = lo_corner[0] + (iv[0]+r.x)*dx[0]; pos.y = 0.0_rt; -#if defined WARPX_DIM_XZ pos.z = lo_corner[1] + (iv[1]+r.y)*dx[1]; -#elif defined WARPX_DIM_RZ +#elif defined(WARPX_DIM_RZ) // Note that for RZ, r.y will be theta + pos.x = lo_corner[0] + (iv[0]+r.x)*dx[0]; + pos.y = 0.0_rt; pos.z = lo_corner[1] + (iv[1]+r.z)*dx[1]; -#endif -#else +#elif defined(WARPX_DIM_1D_Z) pos.x = 0.0_rt; pos.y = 0.0_rt; pos.z = lo_corner[0] + (iv[0]+r.x)*dx[0]; @@ -216,29 +196,17 @@ namespace const GpuArray& pa, long& ip, const bool& do_field_ionization, int* pi #ifdef WARPX_QED - ,const bool& has_quantum_sync, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR - ,const bool& has_breit_wheeler, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_BW + ,const QEDHelper& qed_helper #endif ) noexcept { - pa[PIdx::z][ip] = 0._rt; -#if (AMREX_SPACEDIM >= 2) - pa[PIdx::x][ip] = 0._rt; -#endif -#if defined(WARPX_DIM_3D) - pa[PIdx::y][ip] = 0._rt; -#endif - pa[PIdx::w ][ip] = 0._rt; - pa[PIdx::ux][ip] = 0._rt; - pa[PIdx::uy][ip] = 0._rt; - pa[PIdx::uz][ip] = 0._rt; -#ifdef WARPX_DIM_RZ - pa[PIdx::theta][ip] = 0._rt; -#endif + for (int idx=0 ; idx < PIdx::nattribs ; idx++) { + pa[idx][ip] = 0._rt; + } if (do_field_ionization) {pi[ip] = 0;} #ifdef WARPX_QED - if (has_quantum_sync) {p_optical_depth_QSR[ip] = 0._rt;} - if (has_breit_wheeler) {p_optical_depth_BW[ip] = 0._rt;} + if (qed_helper.has_quantum_sync) {qed_helper.p_optical_depth_QSR[ip] = 0._rt;} + if (qed_helper.has_breit_wheeler) {qed_helper.p_optical_depth_BW[ip] = 0._rt;} #endif idcpu[ip] = amrex::ParticleIdCpus::Invalid; @@ -757,6 +725,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, const std::shared_ptr ptr_offset_z = ps["positionOffset"]["z"].loadChunk(); auto const position_unit_z = static_cast(ps["position"]["z"].unitSI()); auto const position_offset_unit_z = static_cast(ps["positionOffset"]["z"].unitSI()); + const std::shared_ptr ptr_ux = ps["momentum"]["x"].loadChunk(); auto const momentum_unit_x = static_cast(ps["momentum"]["x"].unitSI()); const std::shared_ptr ptr_uz = ps["momentum"]["z"].loadChunk(); @@ -844,7 +813,7 @@ PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, n_external_attr_real, n_external_attr_int, m_user_real_attribs, m_user_int_attribs, - particle_comps, particle_icomps, + GetRealSoANames(), GetIntSoANames(), amrex::GetVecOfPtrs(m_user_real_attrib_parser), amrex::GetVecOfPtrs(m_user_int_attrib_parser), #ifdef WARPX_QED @@ -973,22 +942,9 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int amrex::LayoutData* cost = WarpX::getCosts(lev); - const int nlevs = numLevels(); - static bool refine_injection = false; - static Box fine_injection_box; - static amrex::IntVect rrfac(AMREX_D_DECL(1,1,1)); - // This does not work if the mesh is dynamic. But in that case, we should - // not use refined injected either. We also assume there is only one fine level. - if (WarpX::moving_window_active(WarpX::GetInstance().getistep(0)+1) and WarpX::refine_plasma - and do_continuous_injection and nlevs == 2) - { - refine_injection = true; - fine_injection_box = ParticleBoxArray(1).minimalBox(); - fine_injection_box.setSmall(WarpX::moving_window_dir, std::numeric_limits::lowest()/2); - fine_injection_box.setBig(WarpX::moving_window_dir, std::numeric_limits::max()/2); - rrfac = m_gdb->refRatio(0); - fine_injection_box.coarsen(rrfac); - } + Box fine_injection_box; + amrex::IntVect rrfac(AMREX_D_DECL(1,1,1)); + const bool refine_injection = findRefinedInjectionBox(fine_injection_box, rrfac); bool refineplasma = false; amrex::ParticleLocator > refinepatch_locator; amrex::ParticleLocator > parent_locator; @@ -1024,18 +980,12 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const bool radially_weighted = plasma_injector.radially_weighted; #endif - - // User-defined integer and real attributes: prepare parsers - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); - for (int ia = 0; ia < n_user_int_attribs; ++ia) { - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); - } - for (int ia = 0; ia < n_user_real_attribs; ++ia) { - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); - } + auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + const PlasmaParserWrapper plasma_parser_wrapper (m_user_int_attribs.size(), + m_user_real_attribs.size(), + m_user_int_attrib_parser, + m_user_real_attrib_parser); MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { @@ -1061,30 +1011,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int RealBox overlap_realbox; Box overlap_box; IntVect shifted; - bool no_overlap = false; - - for (int dir=0; dir= part_realbox.lo(dir) ) { - const Real ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); - overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0._rt) * dx[dir]); - } else { - no_overlap = true; break; - } - // Count the number of cells in this direction in overlap_realbox - overlap_box.setSmall( dir, 0 ); - overlap_box.setBig( dir, - int( std::round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir)) - /dx[dir] )) - 1); - shifted[dir] = - static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); - // shifted is exact in non-moving-window direction. That's all we care. - } + const bool no_overlap = find_overlap(tile_realbox, part_realbox, dx, problo, overlap_realbox, overlap_box, shifted); if (no_overlap) { continue; // Go to the next tile } @@ -1101,7 +1028,6 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); - const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { fine_overlap_box = overlap_box & amrex::shift(fine_injection_box, -shifted); @@ -1123,7 +1049,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int in_refpatch = true; } const amrex::Long r = ( (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) || (refineplasma && in_refpatch) ) ? - (AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2])) : (1); + (AMREX_D_TERM(rrfac[0],*rrfac[1],*rrfac[2])) : (1); pcounts[index] = num_ppc*r; // update pcount by checking if cell-corners or cell-center // has non-zero density @@ -1143,19 +1069,9 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } return 0; }; - const int flag_pcount = checker(); - if (flag_pcount == 1) { - pcounts[index] = num_ppc*r; - } else { - pcounts[index] = 0; - } + pcounts[index] = checker() ? num_ppc*r : 0; } -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(k); -#endif -#if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); -#endif }); // Max number of new particles. All of them are created, @@ -1192,75 +1108,22 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int pa[ia] = soa.GetRealData(ia).data() + old_size; } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; - // user-defined integer and real attributes - amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - for (int ia = 0; ia < n_user_int_attribs; ++ia) { - pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - } - for (int ia = 0; ia < n_user_real_attribs; ++ia) { - pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - } -#ifdef AMREX_USE_GPU - // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, - // and them memcpy to device from host - amrex::Gpu::DeviceVector d_pa_user_int(n_user_int_attribs); - amrex::Gpu::DeviceVector d_pa_user_real(n_user_real_attribs); - amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > d_user_int_attrib_parserexec(n_user_int_attribs); - amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > d_user_real_attrib_parserexec(n_user_real_attribs); - amrex::Gpu::copyAsync(Gpu::hostToDevice, pa_user_int_pinned.begin(), - pa_user_int_pinned.end(), d_pa_user_int.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, pa_user_real_pinned.begin(), - pa_user_real_pinned.end(), d_pa_user_real.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, user_int_attrib_parserexec_pinned.begin(), - user_int_attrib_parserexec_pinned.end(), d_user_int_attrib_parserexec.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, user_real_attrib_parserexec_pinned.begin(), - user_real_attrib_parserexec_pinned.end(), d_user_real_attrib_parserexec.begin()); - int** pa_user_int_data = d_pa_user_int.dataPtr(); - ParticleReal** pa_user_real_data = d_pa_user_real.dataPtr(); - amrex::ParserExecutor<7> const* user_int_parserexec_data = d_user_int_attrib_parserexec.dataPtr(); - amrex::ParserExecutor<7> const* user_real_parserexec_data = d_user_real_attrib_parserexec.dataPtr(); -#else - int** pa_user_int_data = pa_user_int_pinned.dataPtr(); - ParticleReal** pa_user_real_data = pa_user_real_pinned.dataPtr(); - amrex::ParserExecutor<7> const* user_int_parserexec_data = user_int_attrib_parserexec_pinned.dataPtr(); - amrex::ParserExecutor<7> const* user_real_parserexec_data = user_real_attrib_parserexec_pinned.dataPtr(); -#endif + + PlasmaParserHelper plasma_parser_helper(soa, old_size, m_user_int_attribs, m_user_real_attribs, plasma_parser_wrapper); + int** pa_user_int_data = plasma_parser_helper.getUserIntDataPtrs(); + ParticleReal** pa_user_real_data = plasma_parser_helper.getUserRealDataPtrs(); + amrex::ParserExecutor<7> const* user_int_parserexec_data = plasma_parser_helper.getUserIntParserExecData(); + amrex::ParserExecutor<7> const* user_real_parserexec_data = plasma_parser_helper.getUserRealParserExecData(); int* pi = nullptr; if (do_field_ionization) { - pi = soa.GetIntData(particle_icomps["ionizationLevel"]).data() + old_size; + pi = soa.GetIntData("ionizationLevel").data() + old_size; } #ifdef WARPX_QED - //Pointer to the optical depth component - amrex::ParticleReal* p_optical_depth_QSR = nullptr; - amrex::ParticleReal* p_optical_depth_BW = nullptr; - - // If a QED effect is enabled, the corresponding optical depth - // has to be initialized - const bool loc_has_quantum_sync = has_quantum_sync(); - const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) { - p_optical_depth_QSR = soa.GetRealData( - particle_comps["opticalDepthQSR"]).data() + old_size; - } - if(loc_has_breit_wheeler) { - p_optical_depth_BW = soa.GetRealData( - particle_comps["opticalDepthBW"]).data() + old_size; - } - - //If needed, get the appropriate functors from the engines - QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; - BreitWheelerGetOpticalDepth breit_wheeler_get_opt; - if(loc_has_quantum_sync){ - quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor(); - } - if(loc_has_breit_wheeler){ - breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor(); - } + const QEDHelper qed_helper(soa, old_size, + has_quantum_sync(), has_breit_wheeler(), + m_shr_p_qs_engine, m_shr_p_bw_engine); #endif const bool loc_do_field_ionization = do_field_ionization; @@ -1278,6 +1141,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { const IntVect iv = IntVect(AMREX_D_DECL(i, j, k)); + amrex::ignore_unused(j,k); const auto index = overlap_box.index(iv); amrex::IntVect glo_iv = iv + tile_box.smallEnd(); bool in_refpatch = false; @@ -1289,60 +1153,34 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if (rz_random_theta) { theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; } #endif - Real scale_fac = 0.0_rt; - if( pcounts[index] != 0) { -#if defined(WARPX_DIM_3D) - scale_fac = dx[0]*dx[1]*dx[2]/pcounts[index]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - scale_fac = dx[0]*dx[1]/pcounts[index]; -#elif defined(WARPX_DIM_1D_Z) - scale_fac = dx[0]/pcounts[index]; -#endif - } - + const Real scale_fac = compute_scale_fac_volume(dx, pcounts[index]); for (int i_part = 0; i_part < pcounts[index]; ++i_part) { long ip = poffset[index] + i_part; pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); const XDim3 r = ( (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) || (refineplasma && in_refpatch)) ? // In the refined injection region: use refinement ratio `lrrfac` - inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : + inj_pos->getPositionUnitBox(i_part, rrfac, engine) : // Otherwise: use 1 as the refinement ratio inj_pos->getPositionUnitBox(i_part, amrex::IntVect::TheUnitVector(), engine); auto pos = getCellCoords(overlap_corner, dx, r, iv); #if defined(WARPX_DIM_3D) - if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi -#ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW -#endif - ); - continue; - } + bool const box_contains = tile_realbox.contains(XDim3{pos.x,pos.y,pos.z}); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); - if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi -#ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW -#endif - ); - continue; - } -#else + bool const box_contains = tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt}); +#elif defined(WARPX_DIM_1D_Z) amrex::ignore_unused(j,k); - if (!tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt})) { + bool const box_contains = tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt}); +#endif + if (!box_contains) { ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW + ,qed_helper #endif ); continue; } -#endif // Save the x and y values to use in the insideBounds checks. // This is needed with WARPX_DIM_RZ since x and y are modified. @@ -1375,8 +1213,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if (!inj_pos->insideBounds(xb, yb, z0)) { ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW + ,qed_helper #endif ); continue; @@ -1389,8 +1226,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if ( dens < density_min ){ ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW + ,qed_helper #endif ); continue; @@ -1407,8 +1243,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if (!inj_pos->insideBounds(xb, yb, z0_lab)) { ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW + ,qed_helper #endif ); continue; @@ -1419,8 +1254,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if ( dens < density_min ){ ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED - ,loc_has_quantum_sync, p_optical_depth_QSR - ,loc_has_breit_wheeler, p_optical_depth_BW + ,qed_helper #endif ); continue; @@ -1444,12 +1278,12 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } #ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth_QSR[ip] = quantum_sync_get_opt(engine); + if(qed_helper.has_quantum_sync){ + qed_helper.p_optical_depth_QSR[ip] = qed_helper.quantum_sync_get_opt(engine); } - if(loc_has_breit_wheeler){ - p_optical_depth_BW[ip] = breit_wheeler_get_opt(engine); + if(qed_helper.has_breit_wheeler){ + qed_helper.p_optical_depth_BW[ip] = qed_helper.breit_wheeler_get_opt(engine); } #endif // Initialize user-defined integers with user-defined parser @@ -1487,13 +1321,14 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int pa[PIdx::x][ip] = pos.x; pa[PIdx::y][ip] = pos.y; pa[PIdx::z][ip] = pos.z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) -#ifdef WARPX_DIM_RZ +#elif defined(WARPX_DIM_XZ) + pa[PIdx::x][ip] = pos.x; + pa[PIdx::z][ip] = pos.z; +#elif defined(WARPX_DIM_RZ) pa[PIdx::theta][ip] = theta; -#endif pa[PIdx::x][ip] = xb; pa[PIdx::z][ip] = pos.z; -#else +#elif defined(WARPX_DIM_1D_Z) pa[PIdx::z][ip] = pos.z; #endif } @@ -1510,8 +1345,15 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + if (EB::enabled()) + { + using warpx::fields::FieldType; + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + *this, + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); + } #endif // The function that calls this is responsible for redistributing particles. @@ -1533,23 +1375,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const auto dx = geom.CellSizeArray(); const auto problo = geom.ProbLoArray(); - Real scale_fac = 0._rt; - // Scale particle weight by the area of the emitting surface, within one cell -#if defined(WARPX_DIM_3D) - scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector.flux_normal_axis]/num_ppc_real; -#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - scale_fac = dx[0]*dx[1]/num_ppc_real; - // When emission is in the r direction, the emitting surface is a cylinder. - // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 0) { scale_fac /= dx[0]; } - // When emission is in the z direction, the emitting surface is an annulus - // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[1]; } - // When emission is in the theta direction (flux_normal_axis == 1), - // the emitting surface is a rectangle, within the plane of the simulation -#elif defined(WARPX_DIM_1D_Z) - scale_fac = dx[0]/num_ppc_real; - if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[0]; } +#ifdef AMREX_USE_EB + bool const inject_from_eb = plasma_injector.m_inject_from_eb; // whether to inject from EB or from a plane + // Extract data structures for embedded boundaries + amrex::EBFArrayBoxFactory const* eb_factory = nullptr; + amrex::FabArray const* eb_flag = nullptr; + if (inject_from_eb) { + eb_factory = &(WarpX::GetInstance().fieldEBFactory(0)); + eb_flag = &(eb_factory->getMultiEBCellFlagFab()); + } #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1562,19 +1396,9 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, for (int ic = 0; ic < NumRuntimeIntComps(); ++ic) { tmp_pc.AddIntComp(false); } tmp_pc.defineAllParticleTiles(); - const int nlevs = numLevels(); - static bool refine_injection = false; - static Box fine_injection_box; - static amrex::IntVect rrfac(AMREX_D_DECL(1,1,1)); - // This does not work if the mesh is dynamic. But in that case, we should - // not use refined injected either. We also assume there is only one fine level. - if (WarpX::refine_plasma && nlevs == 2) - { - refine_injection = true; - fine_injection_box = ParticleBoxArray(1).minimalBox(); - rrfac = m_gdb->refRatio(0); - fine_injection_box.coarsen(rrfac); - } + Box fine_injection_box; + amrex::IntVect rrfac(AMREX_D_DECL(1,1,1)); + const bool refine_injection = findRefinedInjectionBox(fine_injection_box, rrfac); InjectorPosition* flux_pos = plasma_injector.getInjectorFluxPosition(); InjectorFlux* inj_flux = plasma_injector.getInjectorFlux(); @@ -1588,6 +1412,13 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const bool radially_weighted = plasma_injector.radially_weighted; #endif + auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + const PlasmaParserWrapper plasma_parser_wrapper (m_user_int_attribs.size(), + m_user_real_attribs.size(), + m_user_int_attrib_parser, + m_user_real_attrib_parser); + MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); @@ -1612,63 +1443,20 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, RealBox overlap_realbox; Box overlap_box; IntVect shifted; - bool no_overlap = false; - - for (int dir=0; dir 0) { - if (plasma_injector.surface_flux_pos < tile_realbox.lo(dir) || - plasma_injector.surface_flux_pos >= tile_realbox.hi(dir)) { - no_overlap = true; - break; - } - } else { - if (plasma_injector.surface_flux_pos <= tile_realbox.lo(dir) || - plasma_injector.surface_flux_pos > tile_realbox.hi(dir)) { - no_overlap = true; - break; - } - } - overlap_realbox.setLo( dir, plasma_injector.surface_flux_pos ); - overlap_realbox.setHi( dir, plasma_injector.surface_flux_pos ); - overlap_box.setSmall( dir, 0 ); - overlap_box.setBig( dir, 0 ); - shifted[dir] = - static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); - } else { - if ( tile_realbox.lo(dir) <= part_realbox.hi(dir) ) { - const Real ncells_adjust = std::floor( (tile_realbox.lo(dir) - part_realbox.lo(dir))/dx[dir] ); - overlap_realbox.setLo( dir, part_realbox.lo(dir) + std::max(ncells_adjust, 0._rt) * dx[dir]); - } else { - no_overlap = true; break; - } - if ( tile_realbox.hi(dir) >= part_realbox.lo(dir) ) { - const Real ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); - overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0._rt) * dx[dir]); - } else { - no_overlap = true; break; - } - // Count the number of cells in this direction in overlap_realbox - overlap_box.setSmall( dir, 0 ); - overlap_box.setBig( dir, - int( std::round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir)) - /dx[dir] )) - 1); - shifted[dir] = - static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); - // shifted is exact in non-moving-window direction. That's all we care. - } - } - if (no_overlap) { - continue; // Go to the next tile + { + // Injection from a plane + const bool no_overlap = find_overlap_flux(tile_realbox, part_realbox, dx, problo, plasma_injector, overlap_realbox, overlap_box, shifted); + if (no_overlap) { continue; } // Go to the next tile } const int grid_id = mfi.index(); @@ -1683,35 +1471,50 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); - const amrex::IntVect lrrfac = rrfac; + const int flux_normal_axis = plasma_injector.flux_normal_axis; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { fine_overlap_box = overlap_box & amrex::shift(fine_injection_box, -shifted); } + +#ifdef AMREX_USE_EB + auto eb_flag_arr = eb_flag ? eb_flag->const_array(mfi) : Array4{}; + auto eb_data = eb_factory ? eb_factory->getEBData(mfi) : EBData{}; +#endif + amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { const IntVect iv(AMREX_D_DECL(i, j, k)); + amrex::ignore_unused(j,k); + + // Determine the number of macroparticles to inject in this cell (num_ppc_int) +#ifdef AMREX_USE_EB + amrex::Real num_ppc_real_in_this_cell = num_ppc_real; // user input: number of macroparticles per cell + if (inject_from_eb) { + // Injection from EB + // Skip cells that are not partially covered by the EB + if (eb_flag_arr(i,j,k).isRegular() || eb_flag_arr(i,j,k).isCovered()) { return; } + // Scale by the (normalized) area of the EB surface in this cell + num_ppc_real_in_this_cell *= eb_data.get(i,j,k); + } +#else + amrex::Real const num_ppc_real_in_this_cell = num_ppc_real; // user input: number of macroparticles per cell +#endif + // Skip cells that do not overlap with the bounds specified by the user (xmin/xmax, ymin/ymax, zmin/zmax) auto lo = getCellCoords(overlap_corner, dx, {0._rt, 0._rt, 0._rt}, iv); auto hi = getCellCoords(overlap_corner, dx, {1._rt, 1._rt, 1._rt}, iv); + if (!flux_pos->overlapsWith(lo, hi)) { return; } - const int num_ppc_int = static_cast(num_ppc_real + amrex::Random(engine)); - - if (flux_pos->overlapsWith(lo, hi)) - { - auto index = overlap_box.index(iv); - int r; - if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { - r = AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2]); - } else { - r = 1; - } - pcounts[index] = num_ppc_int*r; + auto index = overlap_box.index(iv); + // Take into account refined injection region + int r = 1; + if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { + r = compute_area_weights(rrfac, flux_normal_axis); } -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(k); -#elif defined(WARPX_DIM_1D_Z) + const int num_ppc_int = static_cast(num_ppc_real_in_this_cell*r + amrex::Random(engine)); + pcounts[index] = num_ppc_int; + amrex::ignore_unused(j,k); -#endif }); // Max number of new particles. All of them are created, @@ -1746,81 +1549,21 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; - // user-defined integer and real attributes - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); - amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); - for (int ia = 0; ia < n_user_int_attribs; ++ia) { - pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); - } - for (int ia = 0; ia < n_user_real_attribs; ++ia) { - pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); - } -#ifdef AMREX_USE_GPU - // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, - // and them memcpy to device from host - amrex::Gpu::DeviceVector d_pa_user_int(n_user_int_attribs); - amrex::Gpu::DeviceVector d_pa_user_real(n_user_real_attribs); - amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > d_user_int_attrib_parserexec(n_user_int_attribs); - amrex::Gpu::DeviceVector< amrex::ParserExecutor<7> > d_user_real_attrib_parserexec(n_user_real_attribs); - amrex::Gpu::copyAsync(Gpu::hostToDevice, pa_user_int_pinned.begin(), - pa_user_int_pinned.end(), d_pa_user_int.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, pa_user_real_pinned.begin(), - pa_user_real_pinned.end(), d_pa_user_real.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, user_int_attrib_parserexec_pinned.begin(), - user_int_attrib_parserexec_pinned.end(), d_user_int_attrib_parserexec.begin()); - amrex::Gpu::copyAsync(Gpu::hostToDevice, user_real_attrib_parserexec_pinned.begin(), - user_real_attrib_parserexec_pinned.end(), d_user_real_attrib_parserexec.begin()); - int** pa_user_int_data = d_pa_user_int.dataPtr(); - ParticleReal** pa_user_real_data = d_pa_user_real.dataPtr(); - amrex::ParserExecutor<7> const* user_int_parserexec_data = d_user_int_attrib_parserexec.dataPtr(); - amrex::ParserExecutor<7> const* user_real_parserexec_data = d_user_real_attrib_parserexec.dataPtr(); -#else - int** pa_user_int_data = pa_user_int_pinned.dataPtr(); - ParticleReal** pa_user_real_data = pa_user_real_pinned.dataPtr(); - amrex::ParserExecutor<7> const* user_int_parserexec_data = user_int_attrib_parserexec_pinned.dataPtr(); - amrex::ParserExecutor<7> const* user_real_parserexec_data = user_real_attrib_parserexec_pinned.dataPtr(); -#endif + PlasmaParserHelper plasma_parser_helper(soa, old_size, m_user_int_attribs, m_user_real_attribs, plasma_parser_wrapper); + int** pa_user_int_data = plasma_parser_helper.getUserIntDataPtrs(); + ParticleReal** pa_user_real_data = plasma_parser_helper.getUserRealDataPtrs(); + amrex::ParserExecutor<7> const* user_int_parserexec_data = plasma_parser_helper.getUserIntParserExecData(); + amrex::ParserExecutor<7> const* user_real_parserexec_data = plasma_parser_helper.getUserRealParserExecData(); int* p_ion_level = nullptr; if (do_field_ionization) { - p_ion_level = soa.GetIntData(particle_icomps["ionizationLevel"]).data() + old_size; + p_ion_level = soa.GetIntData("ionizationLevel").data() + old_size; } #ifdef WARPX_QED - //Pointer to the optical depth component - amrex::ParticleReal* p_optical_depth_QSR = nullptr; - amrex::ParticleReal* p_optical_depth_BW = nullptr; - - // If a QED effect is enabled, the corresponding optical depth - // has to be initialized - const bool loc_has_quantum_sync = has_quantum_sync(); - const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) { - p_optical_depth_QSR = soa.GetRealData( - particle_comps["opticalDepthQSR"]).data() + old_size; - } - if(loc_has_breit_wheeler) { - p_optical_depth_BW = soa.GetRealData( - particle_comps["opticalDepthBW"]).data() + old_size; - } - - //If needed, get the appropriate functors from the engines - QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; - BreitWheelerGetOpticalDepth breit_wheeler_get_opt; - if(loc_has_quantum_sync){ - quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor(); - } - if(loc_has_breit_wheeler){ - breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor(); - } + const QEDHelper qed_helper(soa, old_size, + has_quantum_sync(), has_breit_wheeler(), + m_shr_p_qs_engine, m_shr_p_bw_engine); #endif const bool loc_do_field_ionization = do_field_ionization; @@ -1838,19 +1581,58 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { const IntVect iv = IntVect(AMREX_D_DECL(i, j, k)); + amrex::ignore_unused(j,k); const auto index = overlap_box.index(iv); + + Real scale_fac; +#ifdef AMREX_USE_EB + if (inject_from_eb) { + scale_fac = compute_scale_fac_area_eb(dx, num_ppc_real, + AMREX_D_DECL(eb_data.get(i,j,k,0), + eb_data.get(i,j,k,1), + eb_data.get(i,j,k,2))); + } else +#endif + { + scale_fac = compute_scale_fac_area_plane(dx, num_ppc_real, flux_normal_axis); + } + + if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { + scale_fac /= compute_area_weights(rrfac, flux_normal_axis); + } + for (int i_part = 0; i_part < pcounts[index]; ++i_part) { const long ip = poffset[index] + i_part; pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); - // This assumes the flux_pos is of type InjectorPositionRandomPlane - const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? - // In the refined injection region: use refinement ratio `lrrfac` - flux_pos->getPositionUnitBox(i_part, lrrfac, engine) : - // Otherwise: use 1 as the refinement ratio - flux_pos->getPositionUnitBox(i_part, amrex::IntVect::TheUnitVector(), engine); - auto pos = getCellCoords(overlap_corner, dx, r, iv); + // Determine the position of the particle within the cell + XDim3 pos; + XDim3 r; +#ifdef AMREX_USE_EB + if (inject_from_eb) { + auto const& pt = eb_data.randomPointOnEB(i,j,k,engine); +#if defined(WARPX_DIM_3D) + pos.x = overlap_corner[0] + (iv[0] + 0.5_rt + pt[0])*dx[0]; + pos.y = overlap_corner[1] + (iv[1] + 0.5_rt + pt[1])*dx[1]; + pos.z = overlap_corner[2] + (iv[2] + 0.5_rt + pt[2])*dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + pos.x = overlap_corner[0] + (iv[0] + 0.5_rt + pt[0])*dx[0]; + pos.y = 0.0_rt; + pos.z = overlap_corner[1] + (iv[1] + 0.5_rt + pt[1])*dx[1]; +#endif + } else +#endif + { + // Injection from a plane + // This assumes the flux_pos is of type InjectorPositionRandomPlane + r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? + // In the refined injection region: use refinement ratio `rrfac` + flux_pos->getPositionUnitBox(i_part, rrfac, engine) : + // Otherwise: use 1 as the refinement ratio + flux_pos->getPositionUnitBox(i_part, amrex::IntVect::TheUnitVector(), engine); + pos = getCellCoords(overlap_corner, dx, r, iv); + } auto ppos = PDim3(pos); // inj_mom would typically be InjectorMomentumGaussianFlux @@ -1891,6 +1673,17 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, continue; } +#ifdef AMREX_USE_EB + if (inject_from_eb) { + // Injection from EB: rotate momentum according to the normal of the EB surface + // (The above code initialized the momentum by assuming that z is the direction + // normal to the EB surface. Thus we need to rotate from z to the normal.) + rotate_momentum_eb(pu, AMREX_D_DECL(eb_data.get(i,j,k,0), + eb_data.get(i,j,k,1), + eb_data.get(i,j,k,2))); + } +#endif + #ifdef WARPX_DIM_RZ // Conversion from cylindrical to Cartesian coordinates // Replace the x and y, setting an angle theta. @@ -1906,7 +1699,11 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const amrex::Real radial_position = ppos.x; ppos.x = radial_position*cos_theta; ppos.y = radial_position*sin_theta; - if (loc_flux_normal_axis != 2) { + if ((loc_flux_normal_axis != 2) +#ifdef AMREX_USE_EB + || (inject_from_eb) +#endif + ) { // Rotate the momentum // This because, when the flux direction is e.g. "r" // the `inj_mom` objects generates a v*Gaussian distribution @@ -1930,14 +1727,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } #ifdef WARPX_QED - if (loc_has_quantum_sync) { - p_optical_depth_QSR[ip] = quantum_sync_get_opt(engine); + if(qed_helper.has_quantum_sync){ + qed_helper.p_optical_depth_QSR[ip] = qed_helper.quantum_sync_get_opt(engine); } - if(loc_has_breit_wheeler){ - p_optical_depth_BW[ip] = breit_wheeler_get_opt(engine); + if(qed_helper.has_breit_wheeler){ + qed_helper.p_optical_depth_BW[ip] = qed_helper.breit_wheeler_get_opt(engine); } #endif + // Initialize user-defined integers with user-defined parser for (int ia = 0; ia < n_user_int_attribs; ++ia) { pa_user_int_data[ia][ip] = static_cast(user_int_parserexec_data[ia](pos.x, pos.y, pos.z, u.x, u.y, u.z, t)); @@ -1990,7 +1788,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, #elif defined(WARPX_DIM_XZ) pa[PIdx::x][ip] = ppos.x; pa[PIdx::z][ip] = ppos.z; -#else +#elif defined(WARPX_DIM_1D_Z) pa[PIdx::z][ip] = ppos.z; #endif } @@ -2007,8 +1805,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + if (EB::enabled()) + { + using warpx::fields::FieldType; + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + tmp_pc, + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); + } #endif // Redistribute the new particles that were added to the temporary container. @@ -2016,50 +1821,41 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // are in the right tile.) tmp_pc.Redistribute(); - // Add the particles to the current container, tile by tile - for (int lev=0; levaddParticles(tmp_pc, true); } void -PhysicalParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("PhysicalParticleContainer::Evolve::GatherAndPush", blp_fg); - BL_ASSERT(OnSameGrids(lev,jx)); + BL_ASSERT(OnSameGrids(lev, *fields.get(FieldType::current_fp, Direction{0}, lev))); + amrex::LayoutData* cost = WarpX::getCosts(lev); const iMultiFab* current_masks = WarpX::CurrentBufferMasks(lev); const iMultiFab* gather_masks = WarpX::GatherBufferMasks(lev); - const bool has_buffer = cEx || cjx; + + const bool has_rho = fields.has(FieldType::rho_fp, lev); + const bool has_J_buf = fields.has_vector(FieldType::current_buf, lev); + const bool has_E_cax = fields.has_vector(FieldType::Efield_cax, lev); + const bool has_buffer = has_E_cax || has_J_buf; + + amrex::MultiFab & Ex = *fields.get(FieldType::Efield_aux, Direction{0}, lev); + amrex::MultiFab & Ey = *fields.get(FieldType::Efield_aux, Direction{1}, lev); + amrex::MultiFab & Ez = *fields.get(FieldType::Efield_aux, Direction{2}, lev); + amrex::MultiFab & Bx = *fields.get(FieldType::Bfield_aux, Direction{0}, lev); + amrex::MultiFab & By = *fields.get(FieldType::Bfield_aux, Direction{1}, lev); + amrex::MultiFab & Bz = *fields.get(FieldType::Bfield_aux, Direction{2}, lev); if (m_do_back_transformed_particles) { @@ -2148,17 +1944,19 @@ PhysicalParticleContainer::Evolve (int lev, pti, lev, current_masks, gather_masks ); } - const long np_current = (cjx) ? nfine_current : np; + const long np_current = has_J_buf ? nfine_current : np; - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? - pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; + pti.GetiAttribs("ionizationLevel").dataPtr():nullptr; + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); - if ((np-np_current)> 0 ){ + if (has_buffer){ + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -2166,7 +1964,7 @@ PhysicalParticleContainer::Evolve (int lev, if (! do_not_push) { - const long np_gather = (cEx) ? nfine_gather : np; + const long np_gather = has_E_cax ? nfine_gather : np; int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); @@ -2193,13 +1991,20 @@ PhysicalParticleContainer::Evolve (int lev, const IntVect& ref_ratio = WarpX::RefRatio(lev-1); const Box& cbox = amrex::coarsen(box,ref_ratio); + amrex::MultiFab & cEx = *fields.get(FieldType::Efield_cax, Direction{0}, lev); + amrex::MultiFab & cEy = *fields.get(FieldType::Efield_cax, Direction{1}, lev); + amrex::MultiFab & cEz = *fields.get(FieldType::Efield_cax, Direction{2}, lev); + amrex::MultiFab & cBx = *fields.get(FieldType::Bfield_cax, Direction{0}, lev); + amrex::MultiFab & cBy = *fields.get(FieldType::Bfield_cax, Direction{1}, lev); + amrex::MultiFab & cBz = *fields.get(FieldType::Bfield_cax, Direction{2}, lev); + // Data on the grid - FArrayBox const* cexfab = &(*cEx)[pti]; - FArrayBox const* ceyfab = &(*cEy)[pti]; - FArrayBox const* cezfab = &(*cEz)[pti]; - FArrayBox const* cbxfab = &(*cBx)[pti]; - FArrayBox const* cbyfab = &(*cBy)[pti]; - FArrayBox const* cbzfab = &(*cBz)[pti]; + FArrayBox const* cexfab = &cEx[pti]; + FArrayBox const* ceyfab = &cEy[pti]; + FArrayBox const* cezfab = &cEz[pti]; + FArrayBox const* cbxfab = &cBx[pti]; + FArrayBox const* cbyfab = &cBy[pti]; + FArrayBox const* cbzfab = &cBz[pti]; if (WarpX::use_fdtd_nci_corr) { @@ -2211,22 +2016,22 @@ PhysicalParticleContainer::Evolve (int lev, applyNCIFilter(lev-1, cbox, exeli, eyeli, ezeli, bxeli, byeli, bzeli, filtered_Ex, filtered_Ey, filtered_Ez, filtered_Bx, filtered_By, filtered_Bz, - (*cEx)[pti], (*cEy)[pti], (*cEz)[pti], - (*cBx)[pti], (*cBy)[pti], (*cBz)[pti], + cEx[pti], cEy[pti], cEz[pti], + cBx[pti], cBy[pti], cBz[pti], cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab); } // Field gather and push for particles in gather buffers - e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); + e_is_nodal = cEx.is_nodal() and cEy.is_nodal() and cEz.is_nodal(); if (push_type == PushType::Explicit) { PushPX(pti, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, + cEx.nGrowVect(), e_is_nodal, nfine_gather, np-nfine_gather, lev, lev-1, dt, ScaleFields(false), a_dt_type); } else if (push_type == PushType::Implicit) { ImplicitPushXP(pti, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, + cEx.nGrowVect(), e_is_nodal, nfine_gather, np-nfine_gather, lev, lev-1, dt, ScaleFields(false), a_dt_type); } @@ -2237,44 +2042,47 @@ PhysicalParticleContainer::Evolve (int lev, if (!skip_deposition) { //// Deposit at t_{n+1/2} with explicit push - //const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); + const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? - pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; + pti.GetiAttribs("ionizationLevel").dataPtr():nullptr; // Deposit inside domains - for (int i=0; i0) - { - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt / WarpX::n_subcycle_current, relative_time, push_type); - } + lev, lev, dt, relative_time, push_type); + + if (has_buffer) + { + // Deposit in buffers + amrex::MultiFab * cjx = fields.get(FieldType::current_buf, Direction{0}, lev); + amrex::MultiFab * cjy = fields.get(FieldType::current_buf, Direction{1}, lev); + amrex::MultiFab * cjz = fields.get(FieldType::current_buf, Direction{2}, lev); + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt, relative_time, push_type); } } // end of "if electrostatic_solver_id == ElectrostaticSolverAlgo::None" } // end of "if do_not_push" - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, "Cannot deposit charge in rho component 1: only component 0 is allocated!"); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? - pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; + pti.GetiAttribs("ionizationLevel").dataPtr():nullptr; DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); - if ((np-np_current)>0 ){ + if (has_buffer){ + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } @@ -2390,13 +2198,7 @@ PhysicalParticleContainer::SplitParticles (int lev) long np_split; if(split_type==0) { - #if defined(WARPX_DIM_3D) - np_split = 8; - #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - np_split = 4; - #else - np_split = 2; - #endif + np_split = amrex::Math::powi(2); } else { np_split = 2*AMREX_SPACEDIM; } @@ -2649,7 +2451,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr(); + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr(); } // Loop over the particles and update their momentum @@ -2845,7 +2647,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr() + offset; } const bool save_previous_position = m_save_previous_position; @@ -2854,16 +2656,13 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, ParticleReal* z_old = nullptr; if (save_previous_position) { #if (AMREX_SPACEDIM >= 2) - x_old = pti.GetAttribs(particle_comps["prev_x"]).dataPtr() + offset; -#else - amrex::ignore_unused(x_old); + x_old = pti.GetAttribs("prev_x").dataPtr() + offset; #endif #if defined(WARPX_DIM_3D) - y_old = pti.GetAttribs(particle_comps["prev_y"]).dataPtr() + offset; -#else - amrex::ignore_unused(y_old); + y_old = pti.GetAttribs("prev_y").dataPtr() + offset; #endif - z_old = pti.GetAttribs(particle_comps["prev_z"]).dataPtr() + offset; + z_old = pti.GetAttribs("prev_z").dataPtr() + offset; + amrex::ignore_unused(x_old, y_old); } // Loop over the particles and update their momentum @@ -2882,7 +2681,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const bool local_has_quantum_sync = has_quantum_sync(); if (local_has_quantum_sync) { evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); - p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + p_optical_depth_QSR = pti.GetAttribs("opticalDepthQSR").dataPtr() + offset; } #endif @@ -3063,7 +2862,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const Dim3 lo = lbound(box); - const int depos_type = WarpX::current_deposition_algo; + const auto depos_type = WarpX::current_deposition_algo; const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; @@ -3087,15 +2886,15 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr() + offset; #if (AMREX_SPACEDIM >= 2) - ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + ParticleReal* x_n = pti.GetAttribs("x_n").dataPtr(); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + ParticleReal* y_n = pti.GetAttribs("y_n").dataPtr(); #endif - ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + ParticleReal* z_n = pti.GetAttribs("z_n").dataPtr(); + ParticleReal* ux_n = pti.GetAttribs("ux_n").dataPtr(); + ParticleReal* uy_n = pti.GetAttribs("uy_n").dataPtr(); + ParticleReal* uz_n = pti.GetAttribs("uz_n").dataPtr(); const int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); CopyParticleAttribs copyAttribs; @@ -3105,7 +2904,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr() + offset; } // Loop over the particles and update their momentum @@ -3124,7 +2923,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const bool local_has_quantum_sync = has_quantum_sync(); if (local_has_quantum_sync) { evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); - p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + p_optical_depth_QSR = pti.GetAttribs("opticalDepthQSR").dataPtr() + offset; } #endif @@ -3289,10 +3088,10 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, #if !defined(AMREX_USE_GPU) std::stringstream convergenceMsg; convergenceMsg << "Picard solver for particle failed to converge after " << - iter << " iterations. " << std::endl; + iter << " iterations.\n"; convergenceMsg << "Position step norm is " << step_norm << - " and the tolerance is " << particle_tolerance << std::endl; - convergenceMsg << " ux = " << ux[ip] << ", uy = " << uy[ip] << ", uz = " << uz[ip] << std::endl; + " and the tolerance is " << particle_tolerance << "\n"; + convergenceMsg << " ux = " << ux[ip] << ", uy = " << uy[ip] << ", uz = " << uz[ip] << "\n"; convergenceMsg << " xp = " << xp << ", yp = " << yp << ", zp = " << zp; ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", convergenceMsg.str()); #endif @@ -3421,7 +3220,7 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, adk_exp_prefactor.dataPtr(), adk_power.dataPtr(), adk_correction_factors.dataPtr(), - particle_icomps["ionizationLevel"], + GetIntCompIndex("ionizationLevel"), ion_atomic_number, do_adk_correction}; } @@ -3473,6 +3272,28 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose WARPX_PROFILE_VAR_STOP(blp_resample_actual); } +bool +PhysicalParticleContainer::findRefinedInjectionBox (amrex::Box& a_fine_injection_box, amrex::IntVect& a_rrfac) +{ + WARPX_PROFILE("PhysicalParticleContainer::findRefinedInjectionBox"); + + // This does not work if the mesh is dynamic. But in that case, we should + // not use refined injected either. We also assume there is only one fine level. + static bool refine_injection = false; + static Box fine_injection_box; + static amrex::IntVect rrfac(AMREX_D_DECL(1,1,1)); + if (!refine_injection and WarpX::moving_window_active(WarpX::GetInstance().getistep(0)+1) and WarpX::refine_plasma and do_continuous_injection and numLevels() == 2) { + refine_injection = true; + fine_injection_box = ParticleBoxArray(1).minimalBox(); + fine_injection_box.setSmall(WarpX::moving_window_dir, std::numeric_limits::lowest()/2); + fine_injection_box.setBig(WarpX::moving_window_dir, std::numeric_limits::max()/2); + rrfac = m_gdb->refRatio(0); + fine_injection_box.coarsen(rrfac); + } + a_fine_injection_box = fine_injection_box; + a_rrfac = rrfac; + return refine_injection; +} #ifdef WARPX_QED @@ -3505,14 +3326,14 @@ PhotonEmissionFilterFunc PhysicalParticleContainer::getPhotonEmissionFilterFunc () { WARPX_PROFILE("PhysicalParticleContainer::getPhotonEmissionFunc()"); - return PhotonEmissionFilterFunc{particle_runtime_comps["opticalDepthQSR"]}; + return PhotonEmissionFilterFunc{GetRealCompIndex("opticalDepthQSR") - NArrayReal}; } PairGenerationFilterFunc PhysicalParticleContainer::getPairGenerationFilterFunc () { WARPX_PROFILE("PhysicalParticleContainer::getPairGenerationFunc()"); - return PairGenerationFilterFunc{particle_runtime_comps["opticalDepthBW"]}; + return PairGenerationFilterFunc{GetRealCompIndex("opticalDepthBW") - NArrayReal}; } #endif diff --git a/Source/Particles/PinnedMemoryParticleContainer.H b/Source/Particles/PinnedMemoryParticleContainer.H index 402c621eb9a..b9fc4bbe79e 100644 --- a/Source/Particles/PinnedMemoryParticleContainer.H +++ b/Source/Particles/PinnedMemoryParticleContainer.H @@ -1,8 +1,8 @@ #ifndef WARPX_PinnedMemoryParticleContainer_H_ #define WARPX_PinnedMemoryParticleContainer_H_ -#include "NamedComponentParticleContainer.H" +#include "WarpXParticleContainer.H" -using PinnedMemoryParticleContainer = NamedComponentParticleContainer; +using PinnedMemoryParticleContainer = amrex::ParticleContainerPureSoA; #endif //WARPX_PinnedMemoryParticleContainer_H_ diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index 44641557756..d2a223c57d8 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -9,7 +9,6 @@ #define WARPX_PARTICLES_PUSHER_GETANDSETPOSITION_H_ #include "Particles/WarpXParticleContainer.H" -#include "Particles/NamedComponentParticleContainer.H" #include #include @@ -63,25 +62,16 @@ struct GetParticlePosition { using RType = amrex::ParticleReal; -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - const RType* AMREX_RESTRICT m_x = nullptr; - const RType* AMREX_RESTRICT m_z = nullptr; -#elif defined(WARPX_DIM_3D) const RType* AMREX_RESTRICT m_x = nullptr; const RType* AMREX_RESTRICT m_y = nullptr; const RType* AMREX_RESTRICT m_z = nullptr; -#elif defined(WARPX_DIM_1D_Z) - const RType* AMREX_RESTRICT m_z = nullptr; -#endif #if defined(WARPX_DIM_RZ) const RType* m_theta = nullptr; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - static constexpr RType m_y_default = RType(0.0); -#elif defined(WARPX_DIM_1D_Z) +#endif + static constexpr RType m_x_default = RType(0.0); static constexpr RType m_y_default = RType(0.0); -#endif GetParticlePosition () = default; @@ -118,20 +108,20 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { -#ifdef WARPX_DIM_RZ +#if defined(WARPX_DIM_RZ) RType const r = m_x[i]; x = r*std::cos(m_theta[i]); y = r*std::sin(m_theta[i]); z = m_z[i]; -#elif WARPX_DIM_3D +#elif defined(WARPX_DIM_3D) x = m_x[i]; y = m_y[i]; z = m_z[i]; -#elif WARPX_DIM_XZ +#elif defined(WARPX_DIM_XZ) x = m_x[i]; y = m_y_default; z = m_z[i]; -#else +#elif defined(WARPX_DIM_1D_Z) x = m_x_default; y = m_y_default; z = m_z[i]; @@ -146,19 +136,19 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { -#ifdef WARPX_DIM_RZ +#if defined(WARPX_DIM_RZ) x = m_x[i]; y = m_theta[i]; z = m_z[i]; -#elif WARPX_DIM_3D +#elif defined(WARPX_DIM_3D) x = m_x[i]; y = m_y[i]; z = m_z[i]; -#elif WARPX_DIM_XZ +#elif defined(WARPX_DIM_XZ) x = m_x[i]; y = m_y_default; z = m_z[i]; -#else +#elif defined(WARPX_DIM_1D_Z) x = m_x_default; y = m_y_default; z = m_z[i]; @@ -178,16 +168,17 @@ struct SetParticlePosition { using RType = amrex::ParticleReal; -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) +#if defined(WARPX_DIM_3D) RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_y; RType* AMREX_RESTRICT m_z; -#elif defined(WARPX_DIM_3D) +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) RType* AMREX_RESTRICT m_x; - RType* AMREX_RESTRICT m_y; RType* AMREX_RESTRICT m_z; #elif defined(WARPX_DIM_1D_Z) RType* AMREX_RESTRICT m_z; #endif + #if defined(WARPX_DIM_RZ) RType* AMREX_RESTRICT m_theta; #endif @@ -216,24 +207,19 @@ struct SetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType x, RType y, RType z) const noexcept { -#if defined(WARPX_DIM_XZ) - amrex::ignore_unused(y); -#endif -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(x,y); -#endif -#ifdef WARPX_DIM_RZ + amrex::ignore_unused(x,y,z); +#if defined(WARPX_DIM_RZ) m_theta[i] = std::atan2(y, x); m_x[i] = std::sqrt(x*x + y*y); m_z[i] = z; -#elif WARPX_DIM_3D +#elif defined(WARPX_DIM_3D) m_x[i] = x; m_y[i] = y; m_z[i] = z; -#elif WARPX_DIM_XZ +#elif defined(WARPX_DIM_XZ) m_x[i] = x; m_z[i] = z; -#else +#elif defined(WARPX_DIM_1D_Z) m_z[i] = z; #endif } @@ -245,24 +231,19 @@ struct SetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType x, RType y, RType z) const noexcept { -#if defined(WARPX_DIM_XZ) - amrex::ignore_unused(y); -#endif -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(x,y); -#endif -#ifdef WARPX_DIM_RZ + amrex::ignore_unused(x,y,z); +#if defined(WARPX_DIM_RZ) m_x[i] = x; m_theta[i] = y; m_z[i] = z; -#elif WARPX_DIM_3D +#elif defined(WARPX_DIM_3D) m_x[i] = x; m_y[i] = y; m_z[i] = z; -#elif WARPX_DIM_XZ +#elif defined(WARPX_DIM_XZ) m_x[i] = x; m_z[i] = z; -#else +#elif defined(WARPX_DIM_1D_Z) m_z[i] = z; #endif } diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index 4a82e582bfb..d256a1a5e40 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -49,7 +49,7 @@ void doParticleMomentumPush(amrex::ParticleReal& ux, const int ion_lev, const amrex::ParticleReal m, const amrex::ParticleReal a_q, - const int pusher_algo, + const ParticlePusherAlgo pusher_algo, const int do_crr, #ifdef WARPX_QED const amrex::Real t_chi_max, diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 89c2de88e47..d11ba6fe21f 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -22,7 +22,9 @@ * x^{n+1} - x^{n} = dt*u^{n+1/2}/gamma^{n+1/2} */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, +void UpdatePosition([[maybe_unused]] amrex::ParticleReal& x, + [[maybe_unused]] amrex::ParticleReal& y, + [[maybe_unused]] amrex::ParticleReal& z, const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, const amrex::Real dt ) { @@ -35,13 +37,9 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti // Update positions over one time step #if (AMREX_SPACEDIM >= 2) x += ux * inv_gamma * dt; -#else - amrex::ignore_unused(x); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D y += uy * inv_gamma * dt; -#else - amrex::ignore_unused(y); #endif z += uz * inv_gamma * dt; } @@ -53,10 +51,12 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti * See Eqs. 15 and 17 in Chen, JCP 407 (2020) 109228. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, - const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, - const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, - const amrex::Real dt ) +void UpdatePositionImplicit ([[maybe_unused]] amrex::ParticleReal& x, + [[maybe_unused]] amrex::ParticleReal& y, + [[maybe_unused]] amrex::ParticleReal& z, + const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, + const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, + const amrex::Real dt ) { using namespace amrex::literals; @@ -74,13 +74,9 @@ void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amre // Update positions over one time step #if (AMREX_SPACEDIM >= 2) x += ux * inv_gamma * dt; -#else - amrex::ignore_unused(x); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D y += uy * inv_gamma * dt; -#else - amrex::ignore_unused(y); #endif z += uz * inv_gamma * dt; } @@ -90,20 +86,19 @@ void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amre * of the particles for given electric and magnetic fields on the grid. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void PositionNorm( amrex::ParticleReal dxp, amrex::ParticleReal dyp, amrex::ParticleReal dzp, - amrex::ParticleReal& dxp_save, amrex::ParticleReal& dyp_save, amrex::ParticleReal& dzp_save, - amrex::ParticleReal idxg2, amrex::ParticleReal idyg2, amrex::ParticleReal idzg2, +void PositionNorm ([[maybe_unused]] amrex::ParticleReal dxp, + [[maybe_unused]] amrex::ParticleReal dyp, + [[maybe_unused]] amrex::ParticleReal dzp, + [[maybe_unused]] amrex::ParticleReal& dxp_save, + [[maybe_unused]] amrex::ParticleReal& dyp_save, + [[maybe_unused]] amrex::ParticleReal& dzp_save, + [[maybe_unused]] amrex::ParticleReal idxg2, + [[maybe_unused]] amrex::ParticleReal idyg2, + [[maybe_unused]] amrex::ParticleReal idzg2, amrex::ParticleReal& step_norm, const int iter ) { using namespace amrex::literals; -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(dxp, dxp_save, idxg2); -#endif -#if !defined(WARPX_DIM_3D) - amrex::ignore_unused(dyp, dyp_save, idyg2); -#endif - if (iter==0) { step_norm = 1.0_prt; } else { step_norm = (dzp - dzp_save)*(dzp - dzp_save)*idzg2; diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index bb325734777..d55aed99bcd 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -14,6 +14,8 @@ #include "Utils/Parser/ParserUtils.H" #include "Utils/ParticleUtils.H" +#include + /** * \brief This class implements a particle merging scheme wherein particles * are clustered in phase space and particles in the same cluster is merged @@ -66,14 +68,6 @@ public: */ struct HeapSort { - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void swap(int &a, int &b) const - { - const auto temp = b; - b = a; - a = temp; - } - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (int index_array[], const int bin_array[], const int start, const int n) const { @@ -84,7 +78,7 @@ public: // move child through heap if it is bigger than its parent while (j > 0 && bin_array[index_array[j+start]] > bin_array[index_array[(j - 1)/2 + start]]) { // swap child and parent until branch is properly ordered - swap(index_array[j+start], index_array[(j - 1)/2 + start]); + amrex::Swap(index_array[j+start], index_array[(j - 1)/2 + start]); j = (j - 1) / 2; } } @@ -92,7 +86,7 @@ public: for (int i = n - 1; i > 0; i--) { // swap value of first (now the largest value) to the new end point - swap(index_array[start], index_array[i+start]); + amrex::Swap(index_array[start], index_array[i+start]); // remake the max heap int j = 0, index; @@ -105,7 +99,7 @@ public: } // if parent is smaller than child, swap parent with child having higher value if (index < i && bin_array[index_array[j+start]] < bin_array[index_array[index+start]]) { - swap(index_array[j+start], index_array[index+start]); + amrex::Swap(index_array[j+start], index_array[index+start]); } j = index; } @@ -123,7 +117,7 @@ public: void labelOnSphericalVelocityGrid (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], const amrex::ParticleReal uz[], - const unsigned int indices[], + const int indices[], int bin_array[], int index_array[], const int cell_start, const int cell_stop ) const { @@ -151,7 +145,7 @@ public: void labelOnCartesianVelocityGrid (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], const amrex::ParticleReal uz[], - const unsigned int indices[], + const int indices[], int bin_array[], int index_array[], const int cell_start, const int cell_stop ) const { @@ -168,7 +162,7 @@ public: AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], - const amrex::ParticleReal uz[], const unsigned int indices[], + const amrex::ParticleReal uz[], const int indices[], int bin_array[], int index_array[], const int cell_start, const int cell_stop) const { diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index bc20420ea6e..d3565dd2df6 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -61,32 +61,14 @@ public: virtual void RemapParticles(); - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index c3ec4c41131..420d7599ecb 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -176,10 +176,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, // Save the position, momentum and optical depth, making copies amrex::Gpu::DeviceVector xp_save, yp_save, zp_save; - amrex::Gpu::DeviceVector uxp_save, uyp_save, uzp_save; -#ifdef WARPX_QED - amrex::Gpu::DeviceVector optical_depth_save; -#endif const auto GetPosition = GetParticlePosition(pti, offset); auto SetPosition = SetParticlePosition(pti, offset); @@ -188,12 +184,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal* const AMREX_RESTRICT uy = uyp.dataPtr() + offset; amrex::ParticleReal* const AMREX_RESTRICT uz = uzp.dataPtr() + offset; -#ifdef WARPX_QED - const bool loc_has_quantum_sync = has_quantum_sync(); - amrex::ParticleReal* AMREX_RESTRICT p_optical_depth = nullptr; - amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_save = nullptr; -#endif - if (!done_injecting_lev) { // If the old values are not already saved, create copies here. @@ -201,27 +191,10 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, yp_save.resize(np_to_push); zp_save.resize(np_to_push); - uxp_save.resize(np_to_push); - uyp_save.resize(np_to_push); - uzp_save.resize(np_to_push); - amrex::ParticleReal* const AMREX_RESTRICT xp_save_ptr = xp_save.dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT yp_save_ptr = yp_save.dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT zp_save_ptr = zp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uxp_save_ptr = uxp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uyp_save_ptr = uyp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uzp_save_ptr = uzp_save.dataPtr(); - -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() - + offset; - optical_depth_save.resize(np_to_push); - p_optical_depth_save = optical_depth_save.dataPtr(); - } -#endif - amrex::ParallelFor( np_to_push, [=] AMREX_GPU_DEVICE (long i) { amrex::ParticleReal xp, yp, zp; @@ -229,13 +202,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, xp_save_ptr[i] = xp; yp_save_ptr[i] = yp; zp_save_ptr[i] = zp; - uxp_save_ptr[i] = ux[i]; - uyp_save_ptr[i] = uy[i]; - uzp_save_ptr[i] = uz[i]; -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth_save[i] = p_optical_depth[i];} -#endif }); } @@ -252,9 +218,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal* AMREX_RESTRICT x_save = xp_save.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT y_save = yp_save.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT z_save = zp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT ux_save = uxp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT uy_save = uyp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT uz_save = uzp_save.dataPtr(); // Undo the push for particles not injected yet. // The zp are advanced a fixed amount. @@ -267,9 +230,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); if (zp <= z_plane_lev) { - ux[i] = ux_save[i]; - uy[i] = uy_save[i]; - uz[i] = uz_save[i]; xp = x_save[i]; yp = y_save[i]; if (rigid) { @@ -281,24 +241,15 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, zp = z_save[i] + dt*uz[i]*gi; } SetPosition(i, xp, yp, zp); -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth[i] = p_optical_depth_save[i];} -#endif } }); } } void -RigidInjectedParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +RigidInjectedParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { @@ -317,14 +268,9 @@ RigidInjectedParticleContainer::Evolve (int lev, done_injecting_lev = ((zinject_plane_levels[lev] < plo[WARPX_ZINDEX] && WarpX::moving_window_v + WarpX::beta_boost*PhysConst::c >= 0.) || (zinject_plane_levels[lev] > phi[WARPX_ZINDEX] && WarpX::moving_window_v + WarpX::beta_boost*PhysConst::c <= 0.)); - PhysicalParticleContainer::Evolve (lev, - Ex, Ey, Ez, - Bx, By, Bz, - jx, jy, jz, - cjx, cjy, cjz, - rho, crho, - cEx, cEy, cEz, - cBx, cBy, cBz, + PhysicalParticleContainer::Evolve (fields, + lev, + current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } @@ -399,7 +345,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr(); + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr(); } // Save the position and momenta, making copies diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index ba7761bf48a..49366e888ae 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -174,9 +174,9 @@ class fillBufferFlagRemainingParticles amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; - WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType const m_ptd; + WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType m_ptd; amrex::Array4 m_buffer_mask; - int const m_start_index; + int m_start_index; int const* m_indices_ptr; }; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index d4c325fbfb8..a4581d4415d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -23,7 +23,8 @@ # include "ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper_fwd.H" #endif #include "MultiParticleContainer_fwd.H" -#include "NamedComponentParticleContainer.H" + +#include #include #include @@ -47,6 +48,55 @@ #include #include +/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array + */ +struct PIdx +{ + enum { +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + w, ///< weight + ux, uy, uz, +#ifdef WARPX_DIM_RZ + theta, ///< RZ needs all three position components +#endif + nattribs ///< number of compile-time attributes + }; + + //! component names + static constexpr auto names = { +#if !defined (WARPX_DIM_1D_Z) + "x", +#endif +#if defined (WARPX_DIM_3D) + "y", +#endif + "z", + "w", + "ux", + "uy", + "uz", +#ifdef WARPX_DIM_RZ + "theta" +#endif + }; + + static_assert(names.size() == nattribs); +}; + +struct IntIdx { + enum + { + nattribs ///< the number of attributes above (always last) + }; + + static constexpr std::initializer_list names = {}; +}; class WarpXParIter : public amrex::ParIterSoA @@ -78,10 +128,35 @@ public: return GetStructOfArrays().GetRealData(comp); } + [[nodiscard]] const IntVector& GetiAttribs (int comp) const + { + return GetStructOfArrays().GetIntData(comp); + } + [[nodiscard]] IntVector& GetiAttribs (int comp) { return GetStructOfArrays().GetIntData(comp); } + + [[nodiscard]] const RealVector& GetAttribs (const std::string& name) const + { + return GetStructOfArrays().GetRealData(name); + } + + [[nodiscard]] RealVector& GetAttribs (const std::string& name) + { + return GetStructOfArrays().GetRealData(name); + } + + [[nodiscard]] const IntVector& GetiAttribs (const std::string& name) const + { + return GetStructOfArrays().GetIntData(name); + } + + [[nodiscard]] IntVector& GetiAttribs (const std::string& name) + { + return GetStructOfArrays().GetIntData(name); + } }; /** @@ -107,7 +182,7 @@ public: * derived classes, e.g., Evolve) or actual functions (e.g. CurrentDeposition). */ class WarpXParticleContainer - : public NamedComponentParticleContainer + : public amrex::ParticleContainerPureSoA { public: friend MultiParticleContainer; @@ -145,14 +220,9 @@ public: * particles for a time dt (typically one timestep). It is a pure virtual * function for flexibility. */ - virtual void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, + virtual void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, PushType push_type=PushType::Explicit) = 0; @@ -199,7 +269,7 @@ public: * the particle position will be temporarily modified to match * the time of the deposition. */ - void DepositCurrent (amrex::Vector, 3 > >& J, + void DepositCurrent (ablastr::fields::MultiLevelVectorField const & J, amrex::Real dt, amrex::Real relative_time); /** @@ -212,12 +282,12 @@ public: * \param[in] interpolate_across_levels whether to average down from the fine patch to the coarse patch * \param[in] icomp component of the MultiFab where rho is deposited (old, new) */ - void DepositCharge (amrex::Vector >& rho, + void DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, bool interpolate_across_levels = true, int icomp = 0); - void DepositCharge (std::unique_ptr& rho, int lev, + void DepositCharge (amrex::MultiFab* rho, int lev, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, int icomp = 0); @@ -354,7 +424,7 @@ public: amrex::Vector m_E_external_particle; //! Current injection position - amrex::Real m_current_injection_position; + amrex::Real m_current_injection_position = 0.; // split along diagonals (0) or axes (1) int split_type = 0; @@ -433,7 +503,7 @@ protected: amrex::ParticleReal charge; amrex::ParticleReal mass; - PhysicalSpecies physical_species; + PhysicalSpecies physical_species = PhysicalSpecies::unspecified; // Controls boundaries for particles exiting the domain ParticleBoundaries m_boundary_conditions; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index bdce18b7b2b..8e91093d95b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -13,6 +13,8 @@ #include "Deposition/ChargeDeposition.H" #include "Deposition/CurrentDeposition.H" #include "Deposition/SharedDepositionUtils.H" +#include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Pusher/GetAndSetPosition.H" #include "Pusher/UpdatePosition.H" #include "ParticleBoundaries_K.H" @@ -87,10 +89,14 @@ WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) } WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) - : NamedComponentParticleContainer(amr_core->GetParGDB()) + : amrex::ParticleContainerPureSoA(amr_core->GetParGDB()) , species_id(ispecies) { SetParticleSize(); + SetSoACompileTimeNames( + {PIdx::names.begin(), PIdx::names.end()}, + {IntIdx::names.begin(), IntIdx::names.end()} + ); ReadParameters(); // Reading the external fields needs to be here since ReadParameters @@ -172,6 +178,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, int uniqueparticles, amrex::Long id) { using namespace amrex::literals; + using warpx::fields::FieldType; WARPX_ALWAYS_ASSERT_WITH_MESSAGE((PIdx::nattribs + nattr_real - 1) <= NumRealComps(), "Too many real attributes specified"); @@ -300,9 +307,14 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); - deleteInvalidParticles(); + if (EB::enabled()) { + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + *this, + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); + deleteInvalidParticles(); + } #endif } @@ -577,49 +589,64 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (push_type == PushType::Explicit) { - if (WarpX::nox == 1){ + + amrex::Array4 eb_reduce_particle_shape; + if (EB::enabled()) { + eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); + } + + if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); } + } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) - auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + auto& xp_n = pti.GetAttribs("x_n"); const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; #else const ParticleReal* xp_n_data = nullptr; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + auto& yp_n = pti.GetAttribs("y_n"); const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; #else const ParticleReal* yp_n_data = nullptr; #endif - auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + auto& zp_n = pti.GetAttribs("z_n"); const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; - auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); - auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); - auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + auto& uxp_n = pti.GetAttribs("ux_n"); + auto& uyp_n = pti.GetAttribs("uy_n"); + auto& uzp_n = pti.GetAttribs("uz_n"); if (WarpX::nox == 1){ doChargeConservingDepositionShapeNImplicit<1>( xp_n_data, yp_n_data, zp_n_data, @@ -657,22 +684,22 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) - auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + auto& xp_n = pti.GetAttribs("x_n"); const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; #else const ParticleReal* xp_n_data = nullptr; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + auto& yp_n = pti.GetAttribs("y_n"); const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; #else const ParticleReal* yp_n_data = nullptr; #endif - auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + auto& zp_n = pti.GetAttribs("z_n"); const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; - auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); - auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); - auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + auto& uxp_n = pti.GetAttribs("ux_n"); + auto& uyp_n = pti.GetAttribs("uy_n"); + auto& uzp_n = pti.GetAttribs("uz_n"); if (WarpX::nox == 1){ doVillasenorDepositionShapeNImplicit<1>( xp_n_data, yp_n_data, zp_n_data, @@ -767,9 +794,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { - auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); - auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); - auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + auto& uxp_n = pti.GetAttribs("ux_n"); + auto& uyp_n = pti.GetAttribs("uy_n"); + auto& uzp_n = pti.GetAttribs("uz_n"); if (WarpX::nox == 1){ doDepositionShapeNImplicit<1>( GetPosition, wp.dataPtr() + offset, @@ -820,7 +847,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, void WarpXParticleContainer::DepositCurrent ( - amrex::Vector, 3 > >& J, + ablastr::fields::MultiLevelVectorField const & J, const amrex::Real dt, const amrex::Real relative_time) { // Loop over the refinement levels @@ -846,11 +873,11 @@ WarpXParticleContainer::DepositCurrent ( int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr(); + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr(); } DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, - J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), + J[lev][0], J[lev][1], J[lev][2], 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP @@ -1167,7 +1194,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } void -WarpXParticleContainer::DepositCharge (amrex::Vector >& rho, +WarpXParticleContainer::DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const bool interpolate_across_levels, @@ -1208,7 +1235,7 @@ WarpXParticleContainer::DepositCharge (amrex::Vector& rho, +WarpXParticleContainer::DepositCharge (amrex::MultiFab* rho, const int lev, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const int icomp) @@ -1239,10 +1266,10 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization) { - ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr(); + ion_lev = pti.GetiAttribs("ionizationLevel").dataPtr(); } - DepositCharge(pti, wp, ion_lev, rho.get(), icomp, 0, np, thread_num, lev, lev); + DepositCharge(pti, wp, ion_lev, rho, icomp, 0, np, thread_num, lev, lev); } #ifdef AMREX_USE_OMP } @@ -1251,7 +1278,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, #ifdef WARPX_DIM_RZ if (apply_boundary_and_scale_volume) { - WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho.get(), lev); + WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho, lev); } #endif @@ -1270,7 +1297,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, if (apply_boundary_and_scale_volume) { // Reflect density over PEC boundaries, if needed. - WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho.get(), PatchType::fine); + WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho, PatchType::fine); } #endif } @@ -1297,7 +1324,7 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) const int ng_rho = warpx.get_ng_depos_rho().max(); auto rho = std::make_unique(nba, dm, WarpX::ncomps,ng_rho); - DepositCharge(rho, lev, local, true, true, 0); + DepositCharge(rho.get(), lev, local, true, true, 0); return rho; } @@ -1422,26 +1449,33 @@ std::array WarpXParticleContainer::meanParticleVelocity(bool lo amrex::ParticleReal WarpXParticleContainer::maxParticleVelocity(bool local) { - amrex::ParticleReal max_v = 0.0; + const amrex::ParticleReal inv_clight_sq = 1.0_prt/(PhysConst::c*PhysConst::c); + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); const int nLevels = finestLevel(); - for (int lev = 0; lev <= nLevels; ++lev) - { #ifdef AMREX_USE_OMP -#pragma omp parallel reduction(max:max_v) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif + for (int lev = 0; lev <= nLevels; ++lev) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - auto& ux = pti.GetAttribs(PIdx::ux); - auto& uy = pti.GetAttribs(PIdx::uy); - auto& uz = pti.GetAttribs(PIdx::uz); - for (unsigned long i = 0; i < ux.size(); i++) { - max_v = std::max(max_v, std::sqrt(ux[i]*ux[i] + uy[i]*uy[i] + uz[i]*uz[i])); - } + auto *const ux = pti.GetAttribs(PIdx::ux).data(); + auto *const uy = pti.GetAttribs(PIdx::uy).data(); + auto *const uz = pti.GetAttribs(PIdx::uz).data(); + + reduce_op.eval(pti.numParticles(), reduce_data, + [=] AMREX_GPU_DEVICE (int ip) + { return (ux[ip]*ux[ip] + uy[ip]*uy[ip] + uz[ip]*uz[ip]) * inv_clight_sq; }); } } + const amrex::ParticleReal max_usq = amrex::get<0>(reduce_data.value()); + + const amrex::ParticleReal gaminv = 1.0_prt/std::sqrt(1.0_prt + max_usq); + amrex::ParticleReal max_v = gaminv * std::sqrt(max_usq) * PhysConst::c; + if (!local) { ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); } return max_v; } @@ -1516,8 +1550,16 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) // without runtime component). void WarpXParticleContainer::defineAllParticleTiles () noexcept { - // Call the parent class's method - NamedComponentParticleContainer::defineAllParticleTiles(); + for (int lev = 0; lev <= finestLevel(); ++lev) + { + for (auto mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) + { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + } + // Resize the tmp_particle_data (no present in parent class) tmp_particle_data.resize(finestLevel()+1); @@ -1540,7 +1582,7 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, { if (not do_splitting) { return; } - // Tag particle if goes to higher level. + // Tag particle if it goes to a higher level. // It will be split later in the loop if (pld.m_lev == lev+1 and p.id() != amrex::LongParticleIds::NoSplitParticleID @@ -1573,16 +1615,18 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ { auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); + amrex::XDim3 gridmin; + amrex::XDim3 gridmax; #ifndef WARPX_DIM_1D_Z - const Real xmin = Geom(lev).ProbLo(0); - const Real xmax = Geom(lev).ProbHi(0); + gridmin.x = Geom(lev).ProbLo(0); + gridmax.x = Geom(lev).ProbHi(0); #endif #ifdef WARPX_DIM_3D - const Real ymin = Geom(lev).ProbLo(1); - const Real ymax = Geom(lev).ProbHi(1); + gridmin.y = Geom(lev).ProbLo(1); + gridmax.y = Geom(lev).ProbHi(1); #endif - const Real zmin = Geom(lev).ProbLo(WARPX_ZINDEX); - const Real zmax = Geom(lev).ProbHi(WARPX_ZINDEX); + gridmin.z = Geom(lev).ProbLo(WARPX_ZINDEX); + gridmax.z = Geom(lev).ProbHi(WARPX_ZINDEX); ParticleTileType& ptile = ParticlesAt(lev, pti); @@ -1605,17 +1649,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ // Note that for RZ, (x, y, z) is actually (r, theta, z). bool particle_lost = false; - ApplyParticleBoundaries::apply_boundaries( -#ifndef WARPX_DIM_1D_Z - x, xmin, xmax, -#endif -#if (defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ) - y, -#endif -#if (defined WARPX_DIM_3D) - ymin, ymax, -#endif - z, zmin, zmax, + ApplyParticleBoundaries::apply_boundaries(x, y, z, gridmin, gridmax, ux[i], uy[i], uz[i], particle_lost, boundary_conditions, engine); diff --git a/Source/Python/CMakeLists.txt b/Source/Python/CMakeLists.txt index 17a75301306..1b4ab90aade 100644 --- a/Source/Python/CMakeLists.txt +++ b/Source/Python/CMakeLists.txt @@ -13,6 +13,7 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(pyWarpX_${SD} PRIVATE # pybind11 + MultiFabRegister.cpp WarpX.cpp ) endif() diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp new file mode 100644 index 00000000000..fcf38a1a6db --- /dev/null +++ b/Source/Python/MultiFabRegister.cpp @@ -0,0 +1,164 @@ +/* Copyright 2024 The WarpX Community + * + * Authors: Axel Huebl + * License: BSD-3-Clause-LBNL + */ +#include "Python/pyWarpX.H" + +#include + +#include +#include +#include + +void init_MultiFabRegister (py::module & m) +{ + using namespace ablastr::fields; + + py::class_(m, "Direction") + .def(py::init()); + + py::class_(m, "MultiFabRegister") + + .def("alloc_init", + py::overload_cast< + std::string, + int, + amrex::BoxArray const &, + amrex::DistributionMapping const &, + int, + amrex::IntVect const &, + std::optional, + bool, + bool + >(&MultiFabRegister::alloc_init), + py::arg("name"), + py::arg("level"), + py::arg("ba"), + py::arg("dm"), + py::arg("ncomp"), + py::arg("ngrow"), + py::arg("initial_value"), + py::arg("redistribute"), + py::arg("redistribute_on_remake") + ) + + .def("alloc_init", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int, + amrex::BoxArray const &, + amrex::DistributionMapping const &, + int, + amrex::IntVect const &, + std::optional, + bool, + bool + >(&MultiFabRegister::alloc_init), + py::arg("name"), + py::arg("dir"), + py::arg("level"), + py::arg("ba"), + py::arg("dm"), + py::arg("ncomp"), + py::arg("ngrow"), + py::arg("initial_value"), + py::arg("redistribute"), + py::arg("redistribute_on_remake") + ) + + .def("alias_init", + py::overload_cast< + std::string, + std::string, + int, + std::optional + >(&MultiFabRegister::alias_init), + py::arg("new_name"), + py::arg("alias_name"), + py::arg("level"), + py::arg("initial_value") + ) + + .def("alias_init", + py::overload_cast< + std::string, + std::string, + ablastr::fields::Direction, + int, + std::optional + >(&MultiFabRegister::alias_init), + py::arg("new_name"), + py::arg("alias_name"), + py::arg("dir"), + py::arg("level"), + py::arg("initial_value") + ) + + .def("has", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::has, py::const_), + py::arg("name"), + py::arg("level") + ) + + .def("has", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::has, py::const_), + py::arg("name"), + py::arg("dir"), + py::arg("level") + ) + + .def("get", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::get), + py::arg("name"), + py::arg("level") + ) + + .def("get", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::get), + py::arg("name"), + py::arg("dir"), + py::arg("level") + ) + + //.def("list", + // &MultiFabRegister::list + // // "..." + //) + + .def("erase", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::erase), + py::arg("name"), + py::arg("level") + ) + + .def("erase", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::erase), + py::arg("name"), + py::arg("dir"), + py::arg("level") + ) + ; +} diff --git a/Source/Python/Particles/CMakeLists.txt b/Source/Python/Particles/CMakeLists.txt index eed1bb07c74..6b7754fdf2d 100644 --- a/Source/Python/Particles/CMakeLists.txt +++ b/Source/Python/Particles/CMakeLists.txt @@ -10,7 +10,6 @@ foreach(D IN LISTS WarpX_DIMS) # pybind11 ParticleBoundaryBuffer.cpp MultiParticleContainer.cpp - PinnedMemoryParticleContainer.cpp WarpXParticleContainer.cpp ) endif() diff --git a/Source/Python/Particles/MultiParticleContainer.cpp b/Source/Python/Particles/MultiParticleContainer.cpp index e709f0950b4..7b3b114b080 100644 --- a/Source/Python/Particles/MultiParticleContainer.cpp +++ b/Source/Python/Particles/MultiParticleContainer.cpp @@ -42,5 +42,12 @@ i_lens: int strength_E, strength_B: floats The electric and magnetic focusing strength of the lens)pbdoc" ) + + .def("get_charge_density", + [](MultiParticleContainer& mpc, int lev, bool local) { + return mpc.GetChargeDensity(lev, local); + }, + py::arg("lev"), py::arg("local") + ) ; } diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp deleted file mode 100644 index 21dd6a9d364..00000000000 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2021-2023 The WarpX Community - * - * Authors: Axel Huebl, Remi Lehe, Roelof Groenewald - * License: BSD-3-Clause-LBNL - */ - -#include "Python/pyWarpX.H" - -#include - - -void init_PinnedMemoryParticleContainer (py::module& m) -{ - py::class_< - PinnedMemoryParticleContainer, - amrex::ParticleContainerPureSoA - > pmpc (m, "PinnedMemoryParticleContainer"); - pmpc - .def_property_readonly("real_comp_names", - [](PinnedMemoryParticleContainer& pc) - { - return pc.getParticleComps(); - } - ) - .def_property_readonly("int_comp_names", - [](PinnedMemoryParticleContainer& pc) - { - return pc.getParticleiComps(); - } - ); -} diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index aa2cd7a2091..73e0a8b0db0 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -85,19 +85,19 @@ void init_WarpXParticleContainer (py::module& m) py::arg("nattr_int"), py::arg("attr_int"), py::arg("uniqueparticles"), py::arg("id")=-1 ) - .def("get_comp_index", + .def("get_comp_index", // deprecated: use pyAMReX get_real_comp_index [](WarpXParticleContainer& pc, std::string comp_name) { - auto particle_comps = pc.getParticleComps(); - return particle_comps.at(comp_name); + py::print("get_comp_index is deprecated. Use get_real_comp_index instead."); + return pc.GetRealCompIndex(comp_name); }, py::arg("comp_name") ) - .def("get_icomp_index", + .def("get_icomp_index", // deprecated: use pyAMReX get_int_comp_index [](WarpXParticleContainer& pc, std::string comp_name) { - auto particle_comps = pc.getParticleiComps(); - return particle_comps.at(comp_name); + py::print("get_icomp_index is deprecated. Use get_int_comp_index instead."); + return pc.GetIntCompIndex(comp_name); }, py::arg("comp_name") ) diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index b3bd5414303..5b4b07af07b 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -7,6 +7,7 @@ #include // see WarpX.cpp - full includes for _fwd.H headers +#include #include #include #include @@ -36,7 +37,7 @@ #include #include #include - +#include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" #include #include #include @@ -55,6 +56,8 @@ namespace warpx { void init_WarpX (py::module& m) { + using ablastr::fields::Direction; + // Expose the WarpX instance m.def("get_instance", [] () { return &WarpX::GetInstance(); }, @@ -111,16 +114,80 @@ void init_WarpX (py::module& m) py::arg("lev") ) .def("multifab", - [](WarpX const & wx, std::string const multifab_name) { - if (wx.multifab_map.count(multifab_name) > 0) { - return wx.multifab_map.at(multifab_name); + [](WarpX & wx, std::string internal_name) { + if (wx.m_fields.internal_has(internal_name)) { + return wx.m_fields.internal_get(internal_name); + } else { + throw std::runtime_error("MultiFab '" + internal_name + "' is unknown or is not allocated!"); + } + }, + py::arg("internal_name"), + py::return_value_policy::reference_internal, + R"doc(Return a MultiFab by its internal name (deprecated). + +The multifab('internal_name') signature is deprecated. +Please use: +- multifab('prefix', level=...) for scalar fields +- multifab('prefix', dir=..., level=...) for vector field components +where 'prefix' is the part of 'internal_name';' before the [])doc" + ) + .def("multifab", + [](WarpX & wx, std::string scalar_name, int level) { + if (wx.m_fields.has(scalar_name, level)) { + return wx.m_fields.get(scalar_name, level); + } else { + throw std::runtime_error("The scalar field '" + scalar_name + "' is unknown or is not allocated!"); + } + }, + py::arg("scalar_name"), + py::arg("level"), + py::return_value_policy::reference_internal, + R"doc(Return scalar fields (MultiFabs) by name and level. The name is in the form like``\"rho_fp\"``, ``\"phi_fp"``. The level is an integer with 0 being the lowest level. + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" + ) + .def("multifab", + [](WarpX & wx, std::string vector_name, Direction dir, int level) { + if (wx.m_fields.has(vector_name, dir, level)) { + return wx.m_fields.get(vector_name, dir, level); + } else { + throw std::runtime_error("The vector field '" + vector_name + "' is unknown or is not allocated!"); + } + }, + py::arg("vector_name"), + py::arg("dir"), + py::arg("level"), + py::return_value_policy::reference_internal, + R"doc(Return the component of a vector field (MultiFab) by name, direction, and level. The name is in the form like ``\"Efield_aux\"``, ``\"Efield_fp"``, etc. The direction is a Direction instance, Direction(idir) where idir is an integer 0, 1, or 2. The level is an integer with 0 being the lowest level. + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" + ) + .def("multifab", + [](WarpX & wx, std::string vector_name, int idir, int level) { + Direction const dir{idir}; + if (wx.m_fields.has(vector_name, dir, level)) { + return wx.m_fields.get(vector_name, dir, level); } else { - throw std::runtime_error("The MultiFab '" + multifab_name + "' is unknown or is not allocated!"); + throw std::runtime_error("The vector field '" + vector_name + "' is unknown or is not allocated!"); } }, - py::arg("multifab_name"), + py::arg("vector_name"), + py::arg("idir"), + py::arg("level"), py::return_value_policy::reference_internal, - R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + R"doc(Return the component of a vector field (MultiFab) by name, direction, and level. The name is in the form like ``\"Efield_aux\"``, ``\"Efield_fp"``, etc. The direction is an integer 0, 1, or 2. The level is an integer with 0 being the lowest level. The physical fields in WarpX have the following naming: @@ -176,13 +243,13 @@ The physical fields in WarpX have the following naming: std::string potential_lo_y, std::string potential_hi_y, std::string potential_lo_z, std::string potential_hi_z) { - if (potential_lo_x != "") wx.m_poisson_boundary_handler.potential_xlo_str = potential_lo_x; - if (potential_hi_x != "") wx.m_poisson_boundary_handler.potential_xhi_str = potential_hi_x; - if (potential_lo_y != "") wx.m_poisson_boundary_handler.potential_ylo_str = potential_lo_y; - if (potential_hi_y != "") wx.m_poisson_boundary_handler.potential_yhi_str = potential_hi_y; - if (potential_lo_z != "") wx.m_poisson_boundary_handler.potential_zlo_str = potential_lo_z; - if (potential_hi_z != "") wx.m_poisson_boundary_handler.potential_zhi_str = potential_hi_z; - wx.m_poisson_boundary_handler.buildParsers(); + if (potential_lo_x != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_xlo_str = potential_lo_x; + if (potential_hi_x != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_xhi_str = potential_hi_x; + if (potential_lo_y != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_ylo_str = potential_lo_y; + if (potential_hi_y != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_yhi_str = potential_hi_y; + if (potential_lo_z != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_zlo_str = potential_lo_z; + if (potential_hi_z != "") wx.GetElectrostaticSolver().m_poisson_boundary_handler->potential_zhi_str = potential_hi_z; + wx.GetElectrostaticSolver().m_poisson_boundary_handler->BuildParsers(); }, py::arg("potential_lo_x") = "", py::arg("potential_hi_x") = "", @@ -194,11 +261,23 @@ The physical fields in WarpX have the following naming: ) .def("set_potential_on_eb", [](WarpX& wx, std::string potential) { - wx.m_poisson_boundary_handler.setPotentialEB(potential); + wx.GetElectrostaticSolver().m_poisson_boundary_handler->setPotentialEB(potential); }, py::arg("potential"), "Sets the EB potential string and updates the function parser." ) + .def("run_div_cleaner", + [] (WarpX& wx) { wx.ProjectionCleanDivB(); }, + "Executes projection based divergence cleaner on loaded Bfield_fp_external." + ) + .def_static("calculate_hybrid_external_curlA", + [] (WarpX& wx) { wx.CalculateExternalCurlA(); }, + "Executes calculation of the curl of the external A in the hybrid solver." + ) + .def("synchronize", + [] (WarpX& wx) { wx.Synchronize(); }, + "Synchronize particle velocities and positions." + ) ; py::class_(m, "Config") diff --git a/Source/Python/callbacks.cpp b/Source/Python/callbacks.cpp index 79f15c62835..81d379b189a 100644 --- a/Source/Python/callbacks.cpp +++ b/Source/Python/callbacks.cpp @@ -33,8 +33,8 @@ void ExecutePythonCallback ( const std::string& name ) try { warpx_callback_py_map[name](); } catch (std::exception &e) { - std::cerr << "Python callback '" << name << "' failed!" << std::endl; - std::cerr << e.what() << std::endl; + std::cerr << "Python callback '" << name << "' failed!" << "\n"; + std::cerr << e.what() << "\n"; std::exit(3); // note: NOT amrex::Abort(), to avoid hangs with MPI // future note: diff --git a/Source/Python/pyWarpX.cpp b/Source/Python/pyWarpX.cpp index 26f4c77502d..8ae174b4d3e 100644 --- a/Source/Python/pyWarpX.cpp +++ b/Source/Python/pyWarpX.cpp @@ -32,8 +32,8 @@ // forward declarations of exposed classes void init_BoundaryBufferParIter (py::module&); void init_MultiParticleContainer (py::module&); +void init_MultiFabRegister (py::module&); void init_ParticleBoundaryBuffer (py::module&); -void init_PinnedMemoryParticleContainer (py::module&); void init_WarpXParIter (py::module&); void init_WarpXParticleContainer (py::module&); void init_WarpX(py::module&); @@ -59,7 +59,7 @@ PYBIND11_MODULE(PYWARPX_MODULE_NAME, m) { )pbdoc"; // note: order from parent to child classes - init_PinnedMemoryParticleContainer(m); + init_MultiFabRegister(m); init_WarpXParticleContainer(m); init_WarpXParIter(m); init_BoundaryBufferParIter(m); @@ -93,7 +93,7 @@ PYBIND11_MODULE(PYWARPX_MODULE_NAME, m) { // TODO broken numpy if not at least v1.15.0: raise warning // auto numpy = py::module::import("numpy"); // auto npversion = numpy.attr("__version__"); - // std::cout << "numpy version: " << py::str(npversion) << std::endl; + // std::cout << "numpy version: " << py::str(npversion) << "\n"; m.def("amrex_init", [](const py::list args) { diff --git a/Source/Utils/Algorithms/IsIn.H b/Source/Utils/Algorithms/IsIn.H index c9d2f477ef8..a4a9f955eec 100644 --- a/Source/Utils/Algorithms/IsIn.H +++ b/Source/Utils/Algorithms/IsIn.H @@ -26,7 +26,7 @@ namespace utils::algorithms * @return true if elem is in vect, false otherwise */ template ::value>::type> + class = typename std::enable_if_t>> bool is_in(const std::vector& vect, const TE& elem) { @@ -46,7 +46,7 @@ namespace utils::algorithms * @return true if any element of elems is in vect, false otherwise */ template ::value>::type> + class = typename std::enable_if_t>> bool any_of_is_in(const std::vector& vect, const std::vector& elems) { diff --git a/Source/Utils/CMakeLists.txt b/Source/Utils/CMakeLists.txt index 3d804fe9cde..82053bfc88a 100644 --- a/Source/Utils/CMakeLists.txt +++ b/Source/Utils/CMakeLists.txt @@ -6,7 +6,6 @@ foreach(D IN LISTS WarpX_DIMS) ParticleUtils.cpp SpeciesUtils.cpp RelativeCellPosition.cpp - WarpXAlgorithmSelection.cpp WarpXMovingWindow.cpp WarpXTagging.cpp WarpXUtil.cpp diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index dd7e61ff4fa..dc1f1da5c4c 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -2,7 +2,6 @@ CEXE_sources += WarpXMovingWindow.cpp CEXE_sources += WarpXTagging.cpp CEXE_sources += WarpXUtil.cpp CEXE_sources += WarpXVersion.cpp -CEXE_sources += WarpXAlgorithmSelection.cpp CEXE_sources += Interpolate.cpp CEXE_sources += IntervalsParser.cpp CEXE_sources += RelativeCellPosition.cpp diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index 96937abdbbe..19f976c3a6c 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -88,6 +88,20 @@ namespace utils::parser std::string& stored_string); + /** + * \brief If the input is provided, parse the string (typically a mathematical expression) from the + * input file and store it into a variable, replacing its contents. + * + * \param pp used to read the query_string `pp.=string` + * \param query_string ParmParse.query will look for this string + * \param stored_string variable in which the string to parse is stored + */ + bool Query_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string); + + /** Parse a string and return as a double precision floating point number * * In case the string cannot be interpreted as a double, diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index 0339b766e38..d017a6e019c 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -51,6 +51,19 @@ void utils::parser::Store_parserString( } } +bool utils::parser::Query_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string) +{ + bool const input_specified = pp.contains(query_string.c_str()); + if (input_specified) { + stored_string.clear(); + utils::parser::Store_parserString(pp, query_string, stored_string); + } + return input_specified; +} + int utils::parser::query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) { const bool is_specified_without_group = a_pp.contains(str); diff --git a/Source/Utils/Physics/write_atomic_data_cpp.py b/Source/Utils/Physics/write_atomic_data_cpp.py index 11cd3b2c0c5..e1572871ada 100644 --- a/Source/Utils/Physics/write_atomic_data_cpp.py +++ b/Source/Utils/Physics/write_atomic_data_cpp.py @@ -7,84 +7,93 @@ # # License: BSD-3-Clause-LBNL -''' +""" This python script reads ionization tables in atomic_data.txt (generated from the NIST website) and extracts ionization levels into C++ file IonizationEnergiesTable.H, which contains tables + metadata. -''' +""" import os import re import numpy as np -filename = os.path.join( '.', 'atomic_data.txt' ) +filename = os.path.join(".", "atomic_data.txt") with open(filename) as f: text_data = f.read() # Read full table from file and get names, atomic numbers and offsets # position in table of ionization energies for all species -regex_command = '\n\s+(\d+)\s+\|\s+([A-Z]+[a-z]*)\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)' -list_of_tuples = re.findall( regex_command, text_data ) -ion_atom_numbers = [int(i) for i in list(dict.fromkeys( [x[0] for x in list_of_tuples] ))] -ion_names = list(dict.fromkeys( [x[1] for x in list_of_tuples] )) +regex_command = ( + "\n\s+(\d+)\s+\|\s+([A-Z]+[a-z]*)\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)" +) +list_of_tuples = re.findall(regex_command, text_data) +ion_atom_numbers = [int(i) for i in list(dict.fromkeys([x[0] for x in list_of_tuples]))] +ion_names = list(dict.fromkeys([x[1] for x in list_of_tuples])) ion_offsets = np.concatenate(([0], np.cumsum(np.array(ion_atom_numbers)[:-1])), axis=0) # Head of CPP file -cpp_string = '// This script was automatically generated!\n' -cpp_string += '// Edit dev/Source/Utils/Physics/write_atomic_data_cpp.py instead!\n\n' -cpp_string += '#ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n' -cpp_string += '#define WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n\n' -cpp_string += '#include \n\n' -cpp_string += '#include \n' -cpp_string += '#include \n\n' -cpp_string += 'namespace utils::physics\n' -cpp_string += '{\n' +cpp_string = "// This script was automatically generated!\n" +cpp_string += "// Edit dev/Source/Utils/Physics/write_atomic_data_cpp.py instead!\n\n" +cpp_string += "#ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n" +cpp_string += "#define WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n\n" +cpp_string += "#include \n\n" +cpp_string += "#include \n" +cpp_string += "#include \n\n" +cpp_string += "namespace utils::physics\n" +cpp_string += "{\n" # Map each element to ID in table -cpp_string += ' static std::map const ion_map_ids = {' +cpp_string += " static std::map const ion_map_ids = {" for count, name in enumerate(ion_names): - cpp_string += '\n {"' + name + '", ' + str(count) + '},' + cpp_string += '\n {"' + name + '", ' + str(count) + "}," cpp_string = cpp_string[:-1] -cpp_string += ' };\n\n' +cpp_string += " };\n\n" # Atomic number of each species -cpp_string += ' constexpr int nelements = ' + str(len(ion_names)) + ';\n\n' -cpp_string += ' constexpr int ion_atomic_numbers[nelements] = {\n ' +cpp_string += " constexpr int nelements = " + str(len(ion_names)) + ";\n\n" +cpp_string += " constexpr int ion_atomic_numbers[nelements] = {\n " for count, atom_num in enumerate(ion_atom_numbers): - if count%10==0 and count>0: cpp_string = cpp_string[:-2] + ',\n ' - cpp_string += str(atom_num) + ', ' + if count % 10 == 0 and count > 0: + cpp_string = cpp_string[:-2] + ",\n " + cpp_string += str(atom_num) + ", " cpp_string = cpp_string[:-2] -cpp_string += '};\n\n' +cpp_string += "};\n\n" # Offset of each element in table of ionization energies -cpp_string += ' constexpr int ion_energy_offsets[nelements] = {\n ' +cpp_string += " constexpr int ion_energy_offsets[nelements] = {\n " for count, offset in enumerate(ion_offsets): - if count%10==0 and count>0: cpp_string = cpp_string[:-2] + ',\n ' - cpp_string += str(offset) + ', ' + if count % 10 == 0 and count > 0: + cpp_string = cpp_string[:-2] + ",\n " + cpp_string += str(offset) + ", " cpp_string = cpp_string[:-2] -cpp_string += '};\n\n' +cpp_string += "};\n\n" # Table of ionization energies -cpp_string += ' constexpr int energies_tab_length = ' + str(len(list_of_tuples)) + ';\n\n' -cpp_string += ' constexpr amrex::Real table_ionization_energies[energies_tab_length]{' +cpp_string += ( + " constexpr int energies_tab_length = " + str(len(list_of_tuples)) + ";\n\n" +) +cpp_string += ( + " constexpr amrex::Real table_ionization_energies[energies_tab_length]{" +) for element in ion_names: - cpp_string += '\n // ' + element + '\n ' - regex_command = \ - '\n\s+(\d+)\s+\|\s+%s\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)' \ - %element - list_of_tuples = re.findall( regex_command, text_data ) + cpp_string += "\n // " + element + "\n " + regex_command = ( + "\n\s+(\d+)\s+\|\s+%s\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)" % element + ) + list_of_tuples = re.findall(regex_command, text_data) for count, energy in enumerate([x[2] for x in list_of_tuples]): - if count%3==0 and count>0: cpp_string = cpp_string[:-2] + ',\n ' - cpp_string += "amrex::Real(" + energy + '), ' + if count % 3 == 0 and count > 0: + cpp_string = cpp_string[:-2] + ",\n " + cpp_string += "amrex::Real(" + energy + "), " cpp_string = cpp_string[:-1] cpp_string = cpp_string[:-1] -cpp_string += '\n };\n\n' +cpp_string += "\n };\n\n" -cpp_string += '}\n\n' +cpp_string += "}\n\n" # Write the string to file -cpp_string += '#endif // #ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n' -f= open("IonizationEnergiesTable.H","w") +cpp_string += "#endif // #ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_\n" +f = open("IonizationEnergiesTable.H", "w") f.write(cpp_string) f.close() diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index b9105557462..278088e16b6 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -9,6 +9,7 @@ #define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include +#include #include #include @@ -20,23 +21,20 @@ using namespace ablastr::utils::enums; // NOLINT(google-global-names-in-headers * \brief struct to determine the computational medium, i.e., vacuum or material/macroscopic default is vacuum. */ -struct MediumForEM { - enum { - Vacuum = 0, - Macroscopic = 1 - }; -}; +AMREX_ENUM(MediumForEM, + Vacuum, + Macroscopic, + Default = Vacuum); /** * \brief struct to select the overall evolve scheme */ -struct EvolveScheme { - enum { - Explicit = 0, - ThetaImplicitEM = 1, - SemiImplicitEM = 2 - }; -}; +AMREX_ENUM(EvolveScheme, + Explicit, + ThetaImplicitEM, + SemiImplicitEM, + StrangImplicitSpectralEM, + Default = Explicit); /** * \brief struct to select algorithm for macroscopic Maxwell solver @@ -44,158 +42,118 @@ struct EvolveScheme { Backward Euler (fully-implicit) represents sigma*E = sigma*E^(n+1) default is Backward Euler as it is more robust. */ -struct MacroscopicSolverAlgo { - enum { - BackwardEuler = 0, - LaxWendroff = 1 - }; -}; - -struct ElectromagneticSolverAlgo { - enum { - None = 0, - Yee = 1, - CKC = 2, - PSATD = 3, - ECT = 4, - HybridPIC = 5 - }; -}; - -struct ElectrostaticSolverAlgo { - enum { - None = 0, - Relativistic = 1, - LabFrameElectroMagnetostatic = 2, - LabFrame = 3 // Non relativistic - }; -}; - -struct PoissonSolverAlgo { - enum { - Multigrid = 1, - IntegratedGreenFunction = 2, - }; -}; - -struct ParticlePusherAlgo { - enum { - Boris = 0, - Vay = 1, - HigueraCary = 2 - }; -}; - -struct CurrentDepositionAlgo { - enum { - Esirkepov = 0, - Direct = 1, - Vay = 2, - Villasenor = 3 - }; -}; - -struct ChargeDepositionAlgo { - // Only the Standard algorithm is implemented - enum { - Standard = 0 - }; -}; - -struct GatheringAlgo { - enum { - EnergyConserving = 0, - MomentumConserving - }; -}; - -struct PSATDSolutionType { - enum { - FirstOrder = 0, - SecondOrder = 1 - }; -}; - -struct JInTime { - enum { - Constant = 0, - Linear = 1 - }; -}; - -struct RhoInTime { - enum { - Constant = 0, - Linear = 1 - }; -}; +AMREX_ENUM(MacroscopicSolverAlgo, + BackwardEuler, + LaxWendroff, + Default = BackwardEuler); + +AMREX_ENUM(ElectromagneticSolverAlgo, + None, + Yee, + CKC, + PSATD, + ECT, + HybridPIC, + hybrid = HybridPIC, + Default = Yee); + +AMREX_ENUM(ElectrostaticSolverAlgo, + None, + Relativistic, + LabFrameElectroMagnetostatic, + LabFrame, + LabFrameEffectivePotential, + Default = None); + +AMREX_ENUM(PoissonSolverAlgo, + Multigrid, + IntegratedGreenFunction, + fft = IntegratedGreenFunction, + Default = Multigrid); + +AMREX_ENUM(ParticlePusherAlgo, + Boris, + Vay, + HigueraCary, + higuera = HigueraCary, + Default = Boris); + +AMREX_ENUM(CurrentDepositionAlgo, + Esirkepov, + Direct, + Vay, + Villasenor, + Default = Esirkepov); + +AMREX_ENUM(ChargeDepositionAlgo, + Standard, + Default = Standard); + +AMREX_ENUM(GatheringAlgo, + EnergyConserving, + MomentumConserving, + Default = EnergyConserving); + +AMREX_ENUM(PSATDSolutionType, + FirstOrder, + SecondOrder, + Default = SecondOrder); + +AMREX_ENUM(JInTime, + Constant, + Linear, + Default = Constant); + +AMREX_ENUM(RhoInTime, + Constant, + Linear, + Default = Linear); /** Strategy to compute weights for use in load balance. */ -struct LoadBalanceCostsUpdateAlgo { - enum { - Timers = 0, //!< load balance according to in-code timer-based weights (i.e., with `costs`) - Heuristic = 1 /**< load balance according to weights computed from number of cells - and number of particles per box (i.e., with `costs_heuristic`) */ - }; -}; +AMREX_ENUM(LoadBalanceCostsUpdateAlgo, + Timers, //!< load balance according to in-code timer-based weights (i.e., with `costs`) + Heuristic, /**< load balance according to weights computed from number of cells + and number of particles per box (i.e., with `costs_heuristic`) */ + Default = Timers); /** Field boundary conditions at the domain boundary */ -enum struct FieldBoundaryType { - PML = 0, - Periodic = 1, - PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 - PMC = 3, //!< perfect magnetic conductor (PMC) with B_tangential=0 - Damped = 4, // Fields in the guard cells are damped for PSATD - //in the moving window direction - Absorbing_SilverMueller = 5, // Silver-Mueller boundary condition - Neumann = 6, // For electrostatic, the normal E is set to zero - None = 7, // The fields values at the boundary are not updated. This is - // useful for RZ simulations, at r=0. - Open = 8 // Used in the Integrated Green Function Poisson solver - // Note that the solver implicitely assumes open BCs: - // no need to enforce them separately -}; +AMREX_ENUM(FieldBoundaryType, + PML, + Periodic, + PEC, //!< perfect electric conductor (PEC) with E_tangential=0 + PMC, //!< perfect magnetic conductor (PMC) with B_tangential=0 + Neumann = PMC, // For electrostatic, the normal E is set to zero + Damped, // Fields in the guard cells are damped for PSATD + //in the moving window direction + Absorbing_SilverMueller, // Silver-Mueller boundary condition + absorbingsilvermueller = Absorbing_SilverMueller, + None, // The fields values at the boundary are not updated. This is + // useful for RZ simulations, at r=0. + Open, // Used in the Integrated Green Function Poisson solver + // Note that the solver implicitely assumes open BCs: + // no need to enforce them separately + PECInsulator, // Mixed boundary with PEC and insulator + Default = PML); /** Particle boundary conditions at the domain boundary */ -enum struct ParticleBoundaryType { - Absorbing = 0, //!< particles crossing domain boundary are removed - Open = 1, //!< particles cross domain boundary leave with damped j - Reflecting = 2, //!< particles are reflected - Periodic = 3, //!< particles are introduced from the periodic boundary - Thermal = 4, - None = 5 //!< For r=0 boundary with RZ simulations -}; +AMREX_ENUM(ParticleBoundaryType, + Absorbing, //!< particles crossing domain boundary are removed + Open, //!< particles cross domain boundary leave with damped j + Reflecting, //!< particles are reflected + Periodic, //!< particles are introduced from the periodic boundary + Thermal, + None, //!< For r=0 boundary with RZ simulations + Default = Absorbing); /** MPI reductions */ -struct ReductionType { - enum { - Maximum = 0, - Minimum = 1, - Sum = 2 - }; -}; - -int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ); - -/** Select BC Type for fields, if field=true - * else select BCType for particles. - */ -FieldBoundaryType -GetFieldBCTypeInteger( std::string BCType ); - -/** Select BC Type for particles. - */ -ParticleBoundaryType -GetParticleBCTypeInteger( std::string BCType ); - -/** Find the name associated with a BC type - */ -std::string -GetFieldBCTypeString( FieldBoundaryType fb_type ); +AMREX_ENUM(ReductionType, + Maximum, + Minimum, + Sum, + Integral = Sum); #endif // WARPX_UTILS_WARPXALGORITHMSELECTION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp deleted file mode 100644 index 9e2360315b8..00000000000 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/* Copyright 2019-2020 Axel Huebl, David Grote, Luca Fedeli - * Remi Lehe, Weiqun Zhang, Yinjian Zhao - * - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX.H" -#include "WarpXAlgorithmSelection.H" -#include "Utils/TextMsg.H" - -#include - -#include -#include - -#include -#include -#include -#include -#include - -// Define dictionary with correspondence between user-input strings, -// and corresponding integer for use inside the code - -const std::map evolve_scheme_to_int = { - {"explicit", EvolveScheme::Explicit }, - {"theta_implicit_em", EvolveScheme::ThetaImplicitEM }, - {"semi_implicit_em", EvolveScheme::SemiImplicitEM }, - {"default", EvolveScheme::Explicit } -}; - -const std::map grid_to_int = { - {"collocated", static_cast(ablastr::utils::enums::GridType::Collocated)}, - {"staggered", static_cast(ablastr::utils::enums::GridType::Staggered)}, - {"hybrid", static_cast(ablastr::utils::enums::GridType::Hybrid)}, - {"default", static_cast(ablastr::utils::enums::GridType::Staggered)} -}; - -const std::map electromagnetic_solver_algo_to_int = { - {"none", ElectromagneticSolverAlgo::None }, - {"yee", ElectromagneticSolverAlgo::Yee }, - {"ckc", ElectromagneticSolverAlgo::CKC }, - {"psatd", ElectromagneticSolverAlgo::PSATD }, - {"ect", ElectromagneticSolverAlgo::ECT }, - {"hybrid", ElectromagneticSolverAlgo::HybridPIC }, - {"default", ElectromagneticSolverAlgo::Yee } -}; - -const std::map electrostatic_solver_algo_to_int = { - {"none", ElectrostaticSolverAlgo::None }, - {"relativistic", ElectrostaticSolverAlgo::Relativistic}, - {"labframe-electromagnetostatic", ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic}, - {"labframe", ElectrostaticSolverAlgo::LabFrame}, - {"default", ElectrostaticSolverAlgo::None } -}; - -const std::map poisson_solver_algo_to_int = { - {"multigrid", PoissonSolverAlgo::Multigrid}, - {"fft", PoissonSolverAlgo::IntegratedGreenFunction}, - {"default", PoissonSolverAlgo::Multigrid } -}; - -const std::map particle_pusher_algo_to_int = { - {"boris", ParticlePusherAlgo::Boris }, - {"vay", ParticlePusherAlgo::Vay }, - {"higuera", ParticlePusherAlgo::HigueraCary }, - {"default", ParticlePusherAlgo::Boris } -}; - -const std::map current_deposition_algo_to_int = { - {"esirkepov", CurrentDepositionAlgo::Esirkepov }, - {"direct", CurrentDepositionAlgo::Direct }, - {"vay", CurrentDepositionAlgo::Vay }, - {"villasenor", CurrentDepositionAlgo::Villasenor }, - {"default", CurrentDepositionAlgo::Esirkepov } // NOTE: overwritten for PSATD and Hybrid-PIC below -}; - -const std::map charge_deposition_algo_to_int = { - {"standard", ChargeDepositionAlgo::Standard }, - {"default", ChargeDepositionAlgo::Standard } -}; - -const std::map gathering_algo_to_int = { - {"energy-conserving", GatheringAlgo::EnergyConserving }, - {"momentum-conserving", GatheringAlgo::MomentumConserving }, - {"default", GatheringAlgo::EnergyConserving } -}; - -const std::map psatd_solution_type_to_int = { - {"first-order", PSATDSolutionType::FirstOrder}, - {"second-order", PSATDSolutionType::SecondOrder}, - {"default", PSATDSolutionType::SecondOrder} -}; - -const std::map J_in_time_to_int = { - {"constant", JInTime::Constant}, - {"linear", JInTime::Linear}, - {"default", JInTime::Constant} -}; - -const std::map rho_in_time_to_int = { - {"constant", RhoInTime::Constant}, - {"linear", RhoInTime::Linear}, - {"default", RhoInTime::Linear} -}; - -const std::map load_balance_costs_update_algo_to_int = { - {"timers", LoadBalanceCostsUpdateAlgo::Timers }, - {"heuristic", LoadBalanceCostsUpdateAlgo::Heuristic }, - {"default", LoadBalanceCostsUpdateAlgo::Timers } -}; - -const std::map MaxwellSolver_medium_algo_to_int = { - {"vacuum", MediumForEM::Vacuum}, - {"macroscopic", MediumForEM::Macroscopic}, - {"default", MediumForEM::Vacuum} -}; - -const std::map MacroscopicSolver_algo_to_int = { - {"backwardeuler", MacroscopicSolverAlgo::BackwardEuler}, - {"laxwendroff", MacroscopicSolverAlgo::LaxWendroff}, - {"default", MacroscopicSolverAlgo::BackwardEuler} -}; - -const std::map FieldBCType_algo_to_enum = { - {"pml", FieldBoundaryType::PML}, - {"periodic", FieldBoundaryType::Periodic}, - {"pec", FieldBoundaryType::PEC}, - {"pmc", FieldBoundaryType::PMC}, - {"damped", FieldBoundaryType::Damped}, - {"absorbing_silver_mueller", FieldBoundaryType::Absorbing_SilverMueller}, - {"neumann", FieldBoundaryType::Neumann}, - {"open", FieldBoundaryType::Open}, - {"none", FieldBoundaryType::None}, - {"default", FieldBoundaryType::PML} -}; - -const std::map ParticleBCType_algo_to_enum = { - {"absorbing", ParticleBoundaryType::Absorbing}, - {"open", ParticleBoundaryType::Open}, - {"reflecting", ParticleBoundaryType::Reflecting}, - {"periodic", ParticleBoundaryType::Periodic}, - {"thermal", ParticleBoundaryType::Thermal}, - {"none", ParticleBoundaryType::None}, - {"default", ParticleBoundaryType::Absorbing} -}; - -const std::map ReductionType_algo_to_int = { - {"maximum", ReductionType::Maximum}, - {"minimum", ReductionType::Minimum}, - {"integral", ReductionType::Sum} -}; - -int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ) -{ - // Read user input ; use "default" if it is not found - std::string algo = "default"; - pp.query( pp_search_key, algo ); - // Convert to lower case - std::transform(algo.begin(), algo.end(), algo.begin(), ::tolower); - - // Pick the right dictionary - std::map algo_to_int; - if (0 == std::strcmp(pp_search_key, "evolve_scheme")) { - algo_to_int = evolve_scheme_to_int; - } else if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { - algo_to_int = electromagnetic_solver_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "grid_type")) { - algo_to_int = grid_to_int; - } else if (0 == std::strcmp(pp_search_key, "do_electrostatic")) { - algo_to_int = electrostatic_solver_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "poisson_solver")) { - algo_to_int = poisson_solver_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "particle_pusher")) { - algo_to_int = particle_pusher_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "current_deposition")) { - algo_to_int = current_deposition_algo_to_int; - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || - WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) { - algo_to_int["default"] = CurrentDepositionAlgo::Direct; - } - } else if (0 == std::strcmp(pp_search_key, "charge_deposition")) { - algo_to_int = charge_deposition_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "field_gathering")) { - algo_to_int = gathering_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "solution_type")) { - algo_to_int = psatd_solution_type_to_int; - } else if (0 == std::strcmp(pp_search_key, "J_in_time")) { - algo_to_int = J_in_time_to_int; - } else if (0 == std::strcmp(pp_search_key, "rho_in_time")) { - algo_to_int = rho_in_time_to_int; - } else if (0 == std::strcmp(pp_search_key, "load_balance_costs_update")) { - algo_to_int = load_balance_costs_update_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "em_solver_medium")) { - algo_to_int = MaxwellSolver_medium_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "macroscopic_sigma_method")) { - algo_to_int = MacroscopicSolver_algo_to_int; - } else if (0 == std::strcmp(pp_search_key, "reduction_type")) { - algo_to_int = ReductionType_algo_to_int; - } else { - std::string const pp_search_string = pp_search_key; - WARPX_ABORT_WITH_MESSAGE("Unknown algorithm type: " + pp_search_string); - } - - // Check if the user-input is a valid key for the dictionary - if (algo_to_int.count(algo) == 0) { - // Not a valid key ; print error message - const std::string pp_search_string = pp_search_key; - std::string error_message = "Invalid string for algo." + pp_search_string - + ": " + algo + ".\nThe valid values are:\n"; - for ( const auto &valid_pair : algo_to_int ) { - if (valid_pair.first != "default"){ - error_message += " - " + valid_pair.first + "\n"; - } - } - WARPX_ABORT_WITH_MESSAGE(error_message); - } - - // If the input is a valid key, return the value - return algo_to_int[algo]; -} - -FieldBoundaryType -GetFieldBCTypeInteger( std::string BCType ){ - std::transform(BCType.begin(), BCType.end(), BCType.begin(), ::tolower); - - if (FieldBCType_algo_to_enum.count(BCType) == 0) { - std::string error_message = "Invalid string for field/particle BC. : " + BCType + "\nThe valid values are : \n"; - for (const auto &valid_pair : FieldBCType_algo_to_enum) { - if (valid_pair.first != "default"){ - error_message += " - " + valid_pair.first + "\n"; - } - } - WARPX_ABORT_WITH_MESSAGE(error_message); - } - // return FieldBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map - return FieldBCType_algo_to_enum.at(BCType); -} - -ParticleBoundaryType -GetParticleBCTypeInteger( std::string BCType ){ - std::transform(BCType.begin(), BCType.end(), BCType.begin(), ::tolower); - - if (ParticleBCType_algo_to_enum.count(BCType) == 0) { - std::string error_message = "Invalid string for particle BC. : " + BCType + "\nThe valid values are : \n"; - for (const auto &valid_pair : ParticleBCType_algo_to_enum) { - if (valid_pair.first != "default"){ - error_message += " - " + valid_pair.first + "\n"; - } - } - WARPX_ABORT_WITH_MESSAGE(error_message); - } - // return ParticleBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map - return ParticleBCType_algo_to_enum.at(BCType); -} - -std::string -GetFieldBCTypeString( FieldBoundaryType fb_type ) { - std::string boundary_name; - for (const auto &valid_pair : FieldBCType_algo_to_enum) { - if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ - boundary_name = valid_pair.first; - break; - } - } - return boundary_name; -} diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 73696838cd4..281aa5e75ba 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -14,6 +14,7 @@ #endif #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" +#include "Fields.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" #include "Utils/TextMsg.H" @@ -56,6 +57,199 @@ using namespace amrex; +namespace +{ + + /** This function shifts a MultiFab in a given direction + * + * \param[in,out] mf the MultiFab to be shifted + * \param[in] geom the Geometry object associated to the level of the MultiFab mf + * \param[in] num_shift magnitude of the shift (cell number) + * \param[in] dir direction of the shift + * \param[in] safe_guard_cells flag to enable "safe mode" data exchanges with more guard cells + * \param[in] do_single_precision_comms flag to enable single precision communications + * \param[in,out] cost the pointer to the data structure holding costs for timer-based load-balance + * \param[in] external_field the external field (used to initialize EM fields) + * \param[in] useparser flag to enable the use of a field parser to initialize EM fields + * \param[in] field_parser the field parser + * \param[in] PMLRZ_flag flag to enable a special treatment for PML in RZ simulations + */ + void shiftMF ( + amrex::MultiFab& mf, const amrex::Geometry& geom, + int num_shift, int dir, + bool safe_guard_cells, bool do_single_precision_comms, + amrex::LayoutData* cost, + amrex::Real external_field=0.0, bool useparser = false, + amrex::ParserExecutor<3> const& field_parser={}, + const bool PMLRZ_flag = false) + { + using namespace amrex::literals; + WARPX_PROFILE("warpx::shiftMF()"); + const amrex::BoxArray& ba = mf.boxArray(); + const amrex::DistributionMapping& dm = mf.DistributionMap(); + const int nc = mf.nComp(); + const amrex::IntVect& ng = mf.nGrowVect(); + + AMREX_ALWAYS_ASSERT(ng[dir] >= std::abs(num_shift)); + + amrex::MultiFab tmpmf(ba, dm, nc, ng); + amrex::MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); + + if ( safe_guard_cells ) { + // Fill guard cells. + ablastr::utils::communication::FillBoundary(tmpmf, do_single_precision_comms, geom.periodicity()); + } else { + amrex::IntVect ng_mw = amrex::IntVect::TheUnitVector(); + // Enough guard cells in the MW direction + ng_mw[dir] = std::abs(num_shift); + // Make sure we don't exceed number of guard cells allocated + ng_mw = ng_mw.min(ng); + // Fill guard cells. + ablastr::utils::communication::FillBoundary(tmpmf, ng_mw, do_single_precision_comms, geom.periodicity()); + } + + // Make a box that covers the region that the window moved into + const amrex::IndexType& typ = ba.ixType(); + const amrex::Box& domainBox = geom.Domain(); + amrex::Box adjBox; + if (num_shift > 0) { + adjBox = adjCellHi(domainBox, dir, ng[dir]); + } else { + adjBox = adjCellLo(domainBox, dir, ng[dir]); + } + adjBox = amrex::convert(adjBox, typ); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (idim == dir and typ.nodeCentered(dir)) { + if (num_shift > 0) { + adjBox.growLo(idim, -1); + } else { + adjBox.growHi(idim, -1); + } + } else if (idim != dir) { + adjBox.growLo(idim, ng[idim]); + adjBox.growHi(idim, ng[idim]); + } + } + + amrex::IntVect shiftiv(0); + shiftiv[dir] = num_shift; + const amrex::Dim3 shift = shiftiv.dim3(); + + const amrex::RealBox& real_box = geom.ProbDomain(); + const auto dx = geom.CellSizeArray(); + +#ifdef AMREX_USE_OMP + #pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(tmpmf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + if (cost) + { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + auto const& dstfab = mf.array(mfi); + auto const& srcfab = tmpmf.array(mfi); + + const amrex::Box& outbox = mfi.growntilebox() & adjBox; + + if (outbox.ok()) { + if (!useparser) { + AMREX_PARALLEL_FOR_4D ( outbox, nc, i, j, k, n, + { + srcfab(i,j,k,n) = external_field; + }) + } else { + // index type of the src mf + auto const& mf_IndexType = (tmpmf).ixType(); + amrex::IntVect mf_type(AMREX_D_DECL(0,0,0)); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + mf_type[idim] = mf_IndexType.nodeCentered(idim); + } + + amrex::ParallelFor (outbox, nc, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + { + // Compute x,y,z co-ordinates based on index type of mf +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0.0_rt; + const amrex::Real y = 0.0_rt; + const amrex::Real fac_z = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; + const amrex::Real z = i*dx[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; + const amrex::Real x = i*dx[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0.0; + const amrex::Real fac_z = (1.0_rt - mf_type[1]) * dx[1]*0.5_rt; + const amrex::Real z = j*dx[1] + real_box.lo(1) + fac_z; +#else + const amrex::Real fac_x = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; + const amrex::Real x = i*dx[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1.0_rt - mf_type[1]) * dx[1]*0.5_rt; + const amrex::Real y = j*dx[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1.0_rt - mf_type[2]) * dx[2]*0.5_rt; + const amrex::Real z = k*dx[2] + real_box.lo(2) + fac_z; +#endif + srcfab(i,j,k,n) = field_parser(x,y,z); + }); + } + + } + + amrex::Box dstBox = mf[mfi].box(); + if (num_shift > 0) { + dstBox.growHi(dir, -num_shift); + } else { + dstBox.growLo(dir, num_shift); + } + AMREX_PARALLEL_FOR_4D ( dstBox, nc, i, j, k, n, + { + dstfab(i,j,k,n) = srcfab(i+shift.x,j+shift.y,k+shift.z,n); + }) + + if (cost) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) + if (PMLRZ_flag) { + // This does the exchange of data in the corner guard cells, the cells that are in the + // guard region both radially and longitudinally. These are the PML cells in the overlapping + // longitudinal region. FillBoundary normally does not update these cells. + // This update is needed so that the cells at the end of the FABs are updated appropriately + // with the data shifted from the neighboring FAB. Without this update, the RZ PML becomes + // unstable with the moving grid. + // This code creates a temporary MultiFab using a BoxList where the radial size of all of + // its boxes is increased so that the radial guard cells are included in the boxes valid domain. + // The temporary MultiFab is setup to refer to the data of the original Multifab (this can + // be done since the shape of the data is all the same, just the indexing is different). + amrex::BoxList bl; + const auto ba_size = static_cast(ba.size()); + for (int i = 0; i < ba_size; ++i) { + bl.push_back(amrex::grow(ba[i], 0, mf.nGrowVect()[0])); + } + const amrex::BoxArray rba(std::move(bl)); + amrex::MultiFab rmf(rba, dm, mf.nComp(), IntVect(0,mf.nGrowVect()[1]), MFInfo().SetAlloc(false)); + + for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { + rmf.setFab(mfi, FArrayBox(mf[mfi], amrex::make_alias, 0, mf.nComp())); + } + rmf.FillBoundary(false); + } +#else + amrex::ignore_unused(PMLRZ_flag); +#endif + + } + +} + void WarpX::UpdateInjectionPosition (const amrex::Real a_dt) { @@ -139,6 +333,11 @@ WarpX::MoveWindow (const int step, bool move_j) { WARPX_PROFILE("WarpX::MoveWindow"); + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + bool const skip_lev0_coarse_patch = true; + if (step == start_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Starting moving window"); } @@ -202,9 +401,6 @@ WarpX::MoveWindow (const int step, bool move_j) int num_shift = num_shift_base; int num_shift_crse = num_shift; - constexpr auto do_update_cost = true; - constexpr auto dont_update_cost = false; //We can't update cost for PML - // Shift the mesh fields for (int lev = 0; lev <= finest_level; ++lev) { @@ -213,6 +409,11 @@ WarpX::MoveWindow (const int step, bool move_j) num_shift *= refRatio(lev-1)[dir]; } + auto* cost_lev = + (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) ? getCosts(lev) : nullptr; + + amrex::LayoutData* no_cost = nullptr ; //We can't update cost for PML + // Shift each component of vector fields (E, B, j) for (int dim = 0; dim < 3; ++dim) { // Fine grid @@ -234,69 +435,74 @@ WarpX::MoveWindow (const int step, bool move_j) if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + ::shiftMF(*m_fields.get(FieldType::Bfield_fp, Direction{dim}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + ::shiftMF(*m_fields.get(FieldType::Efield_fp, Direction{dim}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { - shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); + ::shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + ::shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::current_fp, Direction{dim}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); } if (pml[lev] && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_fp(); - const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_B = m_fields.get(FieldType::pml_B_fp, Direction{dim}, lev); + amrex::MultiFab* pml_E = m_fields.get(FieldType::pml_E_fp, Direction{dim}, lev); + ::shiftMF(*pml_B, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); + ::shiftMF(*pml_E, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) + const bool PMLRZ_flag = getPMLRZ(); if (pml_rz[lev] && dim < 2) { - const std::array& pml_rz_B = pml_rz[lev]->GetB_fp(); - const std::array& pml_rz_E = pml_rz[lev]->GetE_fp(); - shiftMF(*pml_rz_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); - shiftMF(*pml_rz_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_rz_B = m_fields.get(FieldType::pml_B_fp, Direction{dim}, lev); + amrex::MultiFab* pml_rz_E = m_fields.get(FieldType::pml_E_fp, Direction{dim}, lev); + ::shiftMF(*pml_rz_B, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost, 0.0_rt, false, amrex::ParserExecutor<3>{}, PMLRZ_flag); + ::shiftMF(*pml_rz_E, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost, 0.0_rt, false, amrex::ParserExecutor<3>{}, PMLRZ_flag); } #endif if (lev > 0) { // coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + ::shiftMF(*m_fields.get(FieldType::Bfield_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + ::shiftMF(*m_fields.get(FieldType::Efield_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); + ::shiftMF(*m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); if (fft_do_time_averaging) { - shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_lev0_coarse_patch); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_lev0_coarse_patch); + ::shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + ::shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::current_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); } if (do_pml && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_cp(); - const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); + amrex::MultiFab* pml_B_cp = m_fields.get(FieldType::pml_B_cp, Direction{dim}, lev); + amrex::MultiFab* pml_E_cp = m_fields.get(FieldType::pml_E_cp, Direction{dim}, lev); + ::shiftMF(*pml_B_cp, geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); + ::shiftMF(*pml_E_cp, geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } } } // Shift scalar field F with div(E) cleaning in valid domain // TODO: shift F from pml_rz for RZ geometry with PSATD, once implemented - if (F_fp[lev]) + if (m_fields.has(FieldType::F_fp, lev)) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::F_fp, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::F_cp, lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); } } @@ -306,30 +512,30 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_F = m_fields.get(FieldType::pml_F_fp, lev); + ::shiftMF(*pml_F, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } if (lev > 0) { // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); + amrex::MultiFab* pml_F = m_fields.get(FieldType::pml_F_cp, lev); + ::shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } } } // Shift scalar field G with div(B) cleaning in valid domain // TODO: shift G from pml_rz for RZ geometry with PSATD, once implemented - if (G_fp[lev]) + if (m_fields.has(FieldType::G_fp, lev)) { // Fine grid - shiftMF(*G_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::G_fp, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); if (lev > 0) { // Coarse grid - shiftMF(*G_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::G_cp, lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); } } @@ -339,28 +545,28 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = pml[lev]->GetG_fp(); - shiftMF(*pml_G, geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_G = m_fields.get(FieldType::pml_G_fp, lev); + ::shiftMF(*pml_G, geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } if (lev > 0) { // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = pml[lev]->GetG_cp(); - shiftMF(*pml_G, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); + amrex::MultiFab* pml_G = m_fields.get(FieldType::pml_G_cp, lev); + ::shiftMF(*pml_G, geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, no_cost); } } } // Shift scalar component rho if (move_j) { - if (rho_fp[lev]){ + if (m_fields.has(FieldType::rho_fp, lev)) { // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::rho_fp,lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + ::shiftMF(*m_fields.get(FieldType::rho_cp,lev), geom[lev-1], num_shift_crse, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev); } } } @@ -369,11 +575,11 @@ WarpX::MoveWindow (const int step, bool move_j) if (do_fluid_species) { const int n_fluid_species = myfl->nSpecies(); for (int i=0; iGetFluidContainer(i); - shiftMF( *fl.N[lev], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][0], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][1], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][2], geom[lev], num_shift, dir, lev, do_update_cost ); + WarpXFluidContainer const& fl = myfl->GetFluidContainer(i); + ::shiftMF( *m_fields.get(fl.name_mf_N, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev ); + ::shiftMF( *m_fields.get(fl.name_mf_NU, Direction{0}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev ); + ::shiftMF( *m_fields.get(fl.name_mf_NU, Direction{1}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev ); + ::shiftMF( *m_fields.get(fl.name_mf_NU, Direction{2}, lev), geom[lev], num_shift, dir, m_safe_guard_cells, do_single_precision_comms, cost_lev ); } } } @@ -449,196 +655,24 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real cur_time = t_new[0]; for (int i=0; iGetFluidContainer(i); - fl.InitData( lev, injection_box, cur_time ); + fl.InitData( m_fields, injection_box, cur_time, lev ); } } // Recompute macroscopic properties of the medium - if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + if (m_em_solver_medium == MediumForEM::Macroscopic) { const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + m_fields.get(FieldType::Efield_fp, Direction{0}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev_zero)->ixType().toIntVect() ); } return num_shift_base; } -void -WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, - int num_shift, int dir, const int lev, bool update_cost_flag, - amrex::Real external_field, bool useparser, - amrex::ParserExecutor<3> const& field_parser) -{ - using namespace amrex::literals; - WARPX_PROFILE("WarpX::shiftMF()"); - const amrex::BoxArray& ba = mf.boxArray(); - const amrex::DistributionMapping& dm = mf.DistributionMap(); - const int nc = mf.nComp(); - const amrex::IntVect& ng = mf.nGrowVect(); - - AMREX_ALWAYS_ASSERT(ng[dir] >= num_shift); - - amrex::MultiFab tmpmf(ba, dm, nc, ng); - amrex::MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); - - if ( WarpX::safe_guard_cells ) { - // Fill guard cells. - ablastr::utils::communication::FillBoundary(tmpmf, WarpX::do_single_precision_comms, geom.periodicity()); - } else { - amrex::IntVect ng_mw = amrex::IntVect::TheUnitVector(); - // Enough guard cells in the MW direction - ng_mw[dir] = num_shift; - // Make sure we don't exceed number of guard cells allocated - ng_mw = ng_mw.min(ng); - // Fill guard cells. - ablastr::utils::communication::FillBoundary(tmpmf, ng_mw, WarpX::do_single_precision_comms, geom.periodicity()); - } - - // Make a box that covers the region that the window moved into - const amrex::IndexType& typ = ba.ixType(); - const amrex::Box& domainBox = geom.Domain(); - amrex::Box adjBox; - if (num_shift > 0) { - adjBox = adjCellHi(domainBox, dir, ng[dir]); - } else { - adjBox = adjCellLo(domainBox, dir, ng[dir]); - } - adjBox = amrex::convert(adjBox, typ); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (idim == dir and typ.nodeCentered(dir)) { - if (num_shift > 0) { - adjBox.growLo(idim, -1); - } else { - adjBox.growHi(idim, -1); - } - } else if (idim != dir) { - adjBox.growLo(idim, ng[idim]); - adjBox.growHi(idim, ng[idim]); - } - } - - amrex::IntVect shiftiv(0); - shiftiv[dir] = num_shift; - const amrex::Dim3 shift = shiftiv.dim3(); - - const amrex::RealBox& real_box = geom.ProbDomain(); - const auto dx = geom.CellSizeArray(); - - amrex::LayoutData* cost = WarpX::getCosts(lev); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - - for (amrex::MFIter mfi(tmpmf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - } - auto wt = static_cast(amrex::second()); - - auto const& dstfab = mf.array(mfi); - auto const& srcfab = tmpmf.array(mfi); - - const amrex::Box& outbox = mfi.growntilebox() & adjBox; - - if (outbox.ok()) { - if (!useparser) { - AMREX_PARALLEL_FOR_4D ( outbox, nc, i, j, k, n, - { - srcfab(i,j,k,n) = external_field; - }) - } else { - // index type of the src mf - auto const& mf_IndexType = (tmpmf).ixType(); - amrex::IntVect mf_type(AMREX_D_DECL(0,0,0)); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - mf_type[idim] = mf_IndexType.nodeCentered(idim); - } - - amrex::ParallelFor (outbox, nc, - [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept - { - // Compute x,y,z co-ordinates based on index type of mf -#if defined(WARPX_DIM_1D_Z) - const amrex::Real x = 0.0_rt; - const amrex::Real y = 0.0_rt; - const amrex::Real fac_z = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; - const amrex::Real z = i*dx[0] + real_box.lo(0) + fac_z; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real fac_x = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; - const amrex::Real x = i*dx[0] + real_box.lo(0) + fac_x; - const amrex::Real y = 0.0; - const amrex::Real fac_z = (1.0_rt - mf_type[1]) * dx[1]*0.5_rt; - const amrex::Real z = j*dx[1] + real_box.lo(1) + fac_z; -#else - const amrex::Real fac_x = (1.0_rt - mf_type[0]) * dx[0]*0.5_rt; - const amrex::Real x = i*dx[0] + real_box.lo(0) + fac_x; - const amrex::Real fac_y = (1.0_rt - mf_type[1]) * dx[1]*0.5_rt; - const amrex::Real y = j*dx[1] + real_box.lo(1) + fac_y; - const amrex::Real fac_z = (1.0_rt - mf_type[2]) * dx[2]*0.5_rt; - const amrex::Real z = k*dx[2] + real_box.lo(2) + fac_z; -#endif - srcfab(i,j,k,n) = field_parser(x,y,z); - }); - } - - } - - amrex::Box dstBox = mf[mfi].box(); - if (num_shift > 0) { - dstBox.growHi(dir, -num_shift); - } else { - dstBox.growLo(dir, num_shift); - } - AMREX_PARALLEL_FOR_4D ( dstBox, nc, i, j, k, n, - { - dstfab(i,j,k,n) = srcfab(i+shift.x,j+shift.y,k+shift.z,n); - }) - - if (cost && update_cost_flag && - WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - wt = static_cast(amrex::second()) - wt; - amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); - } - } - -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) - if (WarpX::GetInstance().getPMLRZ()) { - // This does the exchange of data in the corner guard cells, the cells that are in the - // guard region both radially and longitudinally. These are the PML cells in the overlapping - // longitudinal region. FillBoundary normally does not update these cells. - // This update is needed so that the cells at the end of the FABs are updated appropriately - // with the data shifted from the neighboring FAB. Without this update, the RZ PML becomes - // unstable with the moving grid. - // This code creates a temporary MultiFab using a BoxList where the radial size of all of - // its boxes is increased so that the radial guard cells are included in the boxes valid domain. - // The temporary MultiFab is setup to refer to the data of the original Multifab (this can - // be done since the shape of the data is all the same, just the indexing is different). - amrex::BoxList bl; - const auto ba_size = static_cast(ba.size()); - for (int i = 0; i < ba_size; ++i) { - bl.push_back(amrex::grow(ba[i], 0, mf.nGrowVect()[0])); - } - const amrex::BoxArray rba(std::move(bl)); - amrex::MultiFab rmf(rba, dm, mf.nComp(), IntVect(0,mf.nGrowVect()[1]), MFInfo().SetAlloc(false)); - - for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { - rmf.setFab(mfi, FArrayBox(mf[mfi], amrex::make_alias, 0, mf.nComp())); - } - rmf.FillBoundary(false); - } -#endif - -} - void WarpX::ShiftGalileanBoundary () { diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index 1de03eb61f0..f76db974f9d 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -8,6 +8,8 @@ #ifndef WARPX_UTILS_H_ #define WARPX_UTILS_H_ +#include + #include #include #include @@ -26,11 +28,13 @@ #include #include -void ParseGeometryInput(); - void ReadBoostedFrameParameters(amrex::Real& gamma_boost, amrex::Real& beta_boost, amrex::Vector& boost_direction); +void ReadMovingWindowParameters( + int& do_moving_window, int& start_moving_window_step, int& end_moving_window_step, + int& moving_window_dir, amrex::Real& moving_window_v); + void ConvertLabParamsToBoost(); /** @@ -38,10 +42,6 @@ void ConvertLabParamsToBoost(); */ void ReadBCParams (); -/** Check the warpx.dims matches the binary name - */ -void CheckDims (); - /** Check the warpx.dims matches the binary name & set up RZ gridding * * Ensures that the blocks are setup correctly for the RZ spectral solver @@ -53,9 +53,53 @@ void CheckDims (); */ void CheckGriddingForRZSpectral (); -void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, - amrex::Real zmax); - +/** Function that sets the value of MultiFab MF to zero. + * + * \param[in] mf Pointer to the MultiFab + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified + */ +void NullifyMFinstance ( + amrex::MultiFab *mf, + int lev, + amrex::Real zmin, + amrex::Real zmax +); + +/** Function that sets the value of MultiFab MF to zero. + * + * \param[in] multifab_map Multifab registry + * \param[in] nf_name Name of Multifab to modify + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified + */ +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +); + +/** Function that sets the value of MultiFab MF to zero. + * + * \param[in] multifab_map Multifab registry + * \param[in] nf_name Name of Multifab to modify + * \param[in] dir Direction, for Multifabs that are components of vectors + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified + */ +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + ablastr::fields::Direction dir, + int lev, + amrex::Real zmin, + amrex::Real zmax +); namespace WarpXUtilIO{ /** @@ -68,44 +112,6 @@ bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector const mf_type, - amrex::GpuArray const domain_lo, - amrex::GpuArray const dx, - amrex::Real &x, amrex::Real &y, amrex::Real &z) -{ - using namespace amrex::literals; - x = domain_lo[0] + i*dx[0] + (1._rt - mf_type[0]) * dx[0]*0.5_rt; -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(j); - y = 0._rt; - z = domain_lo[1] + k*dx[1] + (1._rt - mf_type[1]) * dx[1]*0.5_rt; -#else - y = domain_lo[1] + j*dx[1] + (1._rt - mf_type[1]) * dx[1]*0.5_rt; - z = domain_lo[2] + k*dx[2] + (1._rt - mf_type[2]) * dx[2]*0.5_rt; -#endif -} - -} - - namespace WarpXUtilLoadBalance { /** \brief We only want to update the cost data if the grids we are working on diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 2ef4ee55d6e..ae2adfac043 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -14,6 +14,7 @@ #include "WarpXProfilerWrapper.H" #include "WarpXUtil.H" +#include #include #include @@ -42,73 +43,6 @@ using namespace amrex; -void PreparseAMReXInputIntArray(amrex::ParmParse& a_pp, char const * const input_str, const bool replace) -{ - const int cnt = a_pp.countval(input_str); - if (cnt > 0) { - Vector input_array; - utils::parser::getArrWithParser(a_pp, input_str, input_array); - if (replace) { - a_pp.remove(input_str); - } - a_pp.addarr(input_str, input_array); - } -} - -void ParseGeometryInput() -{ - // Ensure that geometry.dims is set properly. - CheckDims(); - - // Parse prob_lo and hi, evaluating any expressions since geometry does not - // parse its input - ParmParse pp_geometry("geometry"); - - Vector prob_lo(AMREX_SPACEDIM); - Vector prob_hi(AMREX_SPACEDIM); - - utils::parser::getArrWithParser( - pp_geometry, "prob_lo", prob_lo, 0, AMREX_SPACEDIM); - AMREX_ALWAYS_ASSERT(prob_lo.size() == AMREX_SPACEDIM); - utils::parser::getArrWithParser( - pp_geometry, "prob_hi", prob_hi, 0, AMREX_SPACEDIM); - AMREX_ALWAYS_ASSERT(prob_hi.size() == AMREX_SPACEDIM); - -#ifdef WARPX_DIM_RZ - const ParmParse pp_algo("algo"); - const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(prob_lo[0] == 0., - "Lower bound of radial coordinate (prob_lo[0]) with RZ PSATD solver must be zero"); - } - else - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(prob_lo[0] >= 0., - "Lower bound of radial coordinate (prob_lo[0]) with RZ FDTD solver must be non-negative"); - } -#endif - - pp_geometry.addarr("prob_lo", prob_lo); - pp_geometry.addarr("prob_hi", prob_hi); - - // Parse amr input, evaluating any expressions since amr does not parse its input - ParmParse pp_amr("amr"); - - // Note that n_cell is replaced so that only the parsed version is written out to the - // warpx_job_info file. This must be done since yt expects to be able to parse - // the value of n_cell from that file. For the rest, this doesn't matter. - PreparseAMReXInputIntArray(pp_amr, "n_cell", true); - PreparseAMReXInputIntArray(pp_amr, "max_grid_size", false); - PreparseAMReXInputIntArray(pp_amr, "max_grid_size_x", false); - PreparseAMReXInputIntArray(pp_amr, "max_grid_size_y", false); - PreparseAMReXInputIntArray(pp_amr, "max_grid_size_z", false); - PreparseAMReXInputIntArray(pp_amr, "blocking_factor", false); - PreparseAMReXInputIntArray(pp_amr, "blocking_factor_x", false); - PreparseAMReXInputIntArray(pp_amr, "blocking_factor_y", false); - PreparseAMReXInputIntArray(pp_amr, "blocking_factor_z", false); -} - void ReadBoostedFrameParameters(Real& gamma_boost, Real& beta_boost, Vector& boost_direction) { @@ -138,6 +72,43 @@ void ReadBoostedFrameParameters(Real& gamma_boost, Real& beta_boost, } } +void ReadMovingWindowParameters( + int& do_moving_window, int& start_moving_window_step, int& end_moving_window_step, + int& moving_window_dir, amrex::Real& moving_window_v) +{ + const ParmParse pp_warpx("warpx"); + pp_warpx.query("do_moving_window", do_moving_window); + if (do_moving_window) { + utils::parser::queryWithParser( + pp_warpx, "start_moving_window_step", start_moving_window_step); + utils::parser::queryWithParser( + pp_warpx, "end_moving_window_step", end_moving_window_step); + std::string s; + pp_warpx.get("moving_window_dir", s); + + if (s == "z" || s == "Z") { + moving_window_dir = WARPX_ZINDEX; + } +#if defined(WARPX_DIM_3D) + else if (s == "y" || s == "Y") { + moving_window_dir = 1; + } +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) + else if (s == "x" || s == "X") { + moving_window_dir = 0; + } +#endif + else { + WARPX_ABORT_WITH_MESSAGE("Unknown moving_window_dir: "+s); + } + + utils::parser::getWithParser( + pp_warpx, "moving_window_v", moving_window_v); + moving_window_v *= PhysConst::c; + } +} + void ConvertLabParamsToBoost() { Real gamma_boost = 1., beta_boost = 0.; @@ -194,8 +165,11 @@ void ConvertLabParamsToBoost() { if (boost_direction[dim_map[idim]]) { amrex::Real convert_factor; - // Assume that the window travels with speed +c - convert_factor = 1._rt/( gamma_boost * ( 1 - beta_boost ) ); + amrex::Real beta_window = beta_boost; + if (WarpX::do_moving_window && idim == WarpX::moving_window_dir) { + beta_window = WarpX::moving_window_v / PhysConst::c; + } + convert_factor = 1._rt/( gamma_boost * ( 1 - beta_boost * beta_window ) ); prob_lo[idim] *= convert_factor; prob_hi[idim] *= convert_factor; if (max_level > 0){ @@ -220,16 +194,18 @@ void ConvertLabParamsToBoost() } -/* \brief Function that sets the value of MultiFab MF to zero for z between - * zmin and zmax. - */ -void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax){ - WARPX_PROFILE("WarpXUtil::NullifyMF()"); - int const ncomp = mf.nComp(); +void NullifyMFinstance ( + amrex::MultiFab *mf, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ + int const ncomp = mf->nComp(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for(amrex::MFIter mfi(mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ + for(amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.tilebox(); // Get box lower and upper physical z bound, and dz const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt).z; @@ -245,7 +221,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) #endif // Check if box intersect with [zmin, zmax] if ( (zmax>zmin_box && zmin<=zmax_box) ){ - const Array4 arr = mf[mfi].array(); + const Array4 arr = (*mf)[mfi].array(); // Set field to 0 between zmin and zmax ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept{ @@ -265,6 +241,39 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) } } +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ + WARPX_PROFILE("WarpXUtil::NullifyMF()"); + if (!multifab_map.has(mf_name, lev)) { return; } + + auto * mf = multifab_map.get(mf_name, lev); + + NullifyMFinstance ( mf, lev, zmin, zmax); +} + +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + ablastr::fields::Direction dir, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ + WARPX_PROFILE("WarpXUtil::NullifyMF()"); + if (!multifab_map.has(mf_name, dir, lev)) { return; } + + auto * mf = multifab_map.get(mf_name, dir, lev); + + NullifyMFinstance ( mf, lev, zmin, zmax); +} + namespace WarpXUtilIO{ bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data) { @@ -275,42 +284,12 @@ namespace WarpXUtilIO{ } } -void CheckDims () -{ - // Ensure that geometry.dims is set properly. -#if defined(WARPX_DIM_3D) - std::string const dims_compiled = "3"; -#elif defined(WARPX_DIM_XZ) - std::string const dims_compiled = "2"; -#elif defined(WARPX_DIM_1D_Z) - std::string const dims_compiled = "1"; -#elif defined(WARPX_DIM_RZ) - std::string const dims_compiled = "RZ"; -#endif - const ParmParse pp_geometry("geometry"); - std::string dims; - std::string dims_error = "The selected WarpX executable was built as '"; - dims_error.append(dims_compiled).append("'-dimensional, but the "); - if (pp_geometry.contains("dims")) { - pp_geometry.get("dims", dims); - dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); - dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); - } else { - dims = "Not specified"; - dims_error.append("inputs file does not declare 'geometry.dims'. Please add 'geometry.dims = "); - dims_error.append(dims_compiled).append("' to inputs file."); - } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(dims == dims_compiled, dims_error); -} - void CheckGriddingForRZSpectral () { #ifdef WARPX_DIM_RZ - // Ensure that geometry.dims is set properly. - CheckDims(); - const ParmParse pp_algo("algo"); - const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); + auto electromagnetic_solver_id = ElectromagneticSolverAlgo::Default; + pp_algo.query_enum_sloppy("maxwell_solver", electromagnetic_solver_id, "-_"); // only check for PSATD in RZ if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { @@ -395,16 +374,14 @@ void CheckGriddingForRZSpectral () void ReadBCParams () { - amrex::Vector field_BC_lo(AMREX_SPACEDIM,"default"); - amrex::Vector field_BC_hi(AMREX_SPACEDIM,"default"); - amrex::Vector particle_BC_lo(AMREX_SPACEDIM,"default"); - amrex::Vector particle_BC_hi(AMREX_SPACEDIM,"default"); amrex::Vector geom_periodicity(AMREX_SPACEDIM,0); ParmParse pp_geometry("geometry"); const ParmParse pp_warpx("warpx"); const ParmParse pp_algo("algo"); - const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); - const int poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); + auto electromagnetic_solver_id = ElectromagneticSolverAlgo::Default; + pp_algo.query_enum_sloppy("maxwell_solver", electromagnetic_solver_id, "-_"); + auto poisson_solver_id = PoissonSolverAlgo::Default; + pp_warpx.query_enum_sloppy("poisson_solver", poisson_solver_id, "-_"); if (pp_geometry.queryarr("is_periodic", geom_periodicity)) { @@ -419,26 +396,21 @@ void ReadBCParams () // particle boundary may not be explicitly specified for some applications bool particle_boundary_specified = false; const ParmParse pp_boundary("boundary"); - pp_boundary.queryarr("field_lo", field_BC_lo, 0, AMREX_SPACEDIM); - pp_boundary.queryarr("field_hi", field_BC_hi, 0, AMREX_SPACEDIM); - if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) { - particle_boundary_specified = true; - } - if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) { - particle_boundary_specified = true; - } - AMREX_ALWAYS_ASSERT(field_BC_lo.size() == AMREX_SPACEDIM); - AMREX_ALWAYS_ASSERT(field_BC_hi.size() == AMREX_SPACEDIM); - AMREX_ALWAYS_ASSERT(particle_BC_lo.size() == AMREX_SPACEDIM); - AMREX_ALWAYS_ASSERT(particle_BC_hi.size() == AMREX_SPACEDIM); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // Get field boundary type - WarpX::field_boundary_lo[idim] = GetFieldBCTypeInteger(field_BC_lo[idim]); - WarpX::field_boundary_hi[idim] = GetFieldBCTypeInteger(field_BC_hi[idim]); + pp_boundary.query_enum_sloppy("field_lo", + WarpX::field_boundary_lo[idim], "-_", idim); + pp_boundary.query_enum_sloppy("field_hi", + WarpX::field_boundary_hi[idim], "-_", idim); // Get particle boundary type - WarpX::particle_boundary_lo[idim] = GetParticleBCTypeInteger(particle_BC_lo[idim]); - WarpX::particle_boundary_hi[idim] = GetParticleBCTypeInteger(particle_BC_hi[idim]); + if (pp_boundary.query_enum_sloppy("particle_lo", + WarpX::particle_boundary_lo[idim], "-_", idim)) { + particle_boundary_specified = true; + } + if (pp_boundary.query_enum_sloppy("particle_hi", + WarpX::particle_boundary_hi[idim], "-_", idim)) { + particle_boundary_specified = true; + } if (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Periodic || WarpX::field_boundary_hi[idim] == FieldBoundaryType::Periodic || @@ -471,6 +443,15 @@ void ReadBCParams () "PEC boundary not implemented for PSATD, yet!" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) || + ( + WarpX::field_boundary_lo[idim] != FieldBoundaryType::PMC && + WarpX::field_boundary_hi[idim] != FieldBoundaryType::PMC + ), + "PMC boundary not implemented for PSATD, yet!" + ); + if(WarpX::field_boundary_lo[idim] == FieldBoundaryType::Open && WarpX::field_boundary_hi[idim] == FieldBoundaryType::Open){ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/Utils/check_interp_points_and_weights.py b/Source/Utils/check_interp_points_and_weights.py index 8bf2cf08490..2f5a6e13b96 100644 --- a/Source/Utils/check_interp_points_and_weights.py +++ b/Source/Utils/check_interp_points_and_weights.py @@ -4,7 +4,7 @@ # # License: BSD-3-Clause-LBNL -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Compute interpolation points and weights for coarsening and refinement in IO # and MR applications in 1D (extensions to 2D and 3D are trivial). Weights are # computed in order to guarantee total charge conservation for both cell-centered @@ -23,7 +23,7 @@ # while terms multiplied by sf*sc are ON for nodal data and OFF for cell-centered # data. C++ implementation in Source/ablastr/coarsen/average.(H/.cpp) and # Source/ablastr/coarsen/sample.(H/.cpp) -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- import sys @@ -31,57 +31,67 @@ # Fine grid limits (without ghost cells) -def fine_grid_limits( sf ): - if ( sf == 0 ): # cell-centered - iimin = 0 - iimax = 7 - elif ( sf == 1 ): # nodal - iimin = 0 - iimax = 8 - return [ iimin, iimax ] +def fine_grid_limits(sf): + if sf == 0: # cell-centered + iimin = 0 + iimax = 7 + elif sf == 1: # nodal + iimin = 0 + iimax = 8 + return [iimin, iimax] + # Coarse grid limits (without ghost cells) -def coarse_grid_limits( sc, sf, iimin, iimax ): - imin = int( iimin/cr ) - imax = int( iimax/cr )-(1-sc)*sf+(1-sf)*sc - return [ imin, imax ] +def coarse_grid_limits(sc, sf, iimin, iimax): + imin = int(iimin / cr) + imax = int(iimax / cr) - (1 - sc) * sf + (1 - sf) * sc + return [imin, imax] + # Coarsening for MR: interpolation points and weights -def coarsening_points_and_weights( i, sc, sf, cr ): - if ( cr==1 ): +def coarsening_points_and_weights(i, sc, sf, cr): + if cr == 1: numpts = 1 idxmin = i - elif ( cr>=2 ): - numpts = cr*(1-sf)*(1-sc)+(2*(cr-1)+1)*sf*sc - idxmin = i*cr*(1-sf)*(1-sc)+(i*cr-cr+1)*sf*sc - weights = np.zeros( numpts ) - for ir in range( numpts ): - ii = idxmin+ir - weights[ir] = (1/cr)*(1-sf)*(1-sc)+((abs(cr-abs(ii-i*cr)))/(cr*cr))*sf*sc - return [ numpts, idxmin, weights ] + elif cr >= 2: + numpts = cr * (1 - sf) * (1 - sc) + (2 * (cr - 1) + 1) * sf * sc + idxmin = i * cr * (1 - sf) * (1 - sc) + (i * cr - cr + 1) * sf * sc + weights = np.zeros(numpts) + for ir in range(numpts): + ii = idxmin + ir + weights[ir] = (1 / cr) * (1 - sf) * (1 - sc) + ( + (abs(cr - abs(ii - i * cr))) / (cr * cr) + ) * sf * sc + return [numpts, idxmin, weights] + # Refinement for MR: interpolation points and weights -def refinement_points_and_weights( ii, sc, sf, cr ): - if ( cr==1 ): +def refinement_points_and_weights(ii, sc, sf, cr): + if cr == 1: numpts = 1 idxmin = ii - elif ( cr>=2 ): - if ( ii%cr==0 ): - numpts = (1-sf)*(1-sc)+sf*sc - elif ( ii%cr!=0 ): - numpts = (1-sf)*(1-sc)+2*sf*sc - idxmin = (ii//cr)*(1-sf)*(1-sc)+(ii//cr)*sf*sc - weights = np.zeros( numpts ) - for ir in range( numpts ): - i = idxmin+ir - if ( ii==iimin or ii==iimax ): - weights[ir] = (1-sf)*(1-sc)+((abs(cr-abs(ii-i*cr)))/(cr)+(cr/2-0.5))*sf*sc + elif cr >= 2: + if ii % cr == 0: + numpts = (1 - sf) * (1 - sc) + sf * sc + elif ii % cr != 0: + numpts = (1 - sf) * (1 - sc) + 2 * sf * sc + idxmin = (ii // cr) * (1 - sf) * (1 - sc) + (ii // cr) * sf * sc + weights = np.zeros(numpts) + for ir in range(numpts): + i = idxmin + ir + if ii == iimin or ii == iimax: + weights[ir] = (1 - sf) * (1 - sc) + ( + (abs(cr - abs(ii - i * cr))) / (cr) + (cr / 2 - 0.5) + ) * sf * sc else: - weights[ir] = (1-sf)*(1-sc)+((abs(cr-abs(ii-i*cr)))/(cr))*sf*sc - return [ numpts, idxmin, weights ] + weights[ir] = (1 - sf) * (1 - sc) + ( + (abs(cr - abs(ii - i * cr))) / (cr) + ) * sf * sc + return [numpts, idxmin, weights] + ## TODO Coarsening for IO: interpolation points and weights -#def coarsening_points_and_weights_for_IO( i, sf, sc, cr ): +# def coarsening_points_and_weights_for_IO( i, sf, sc, cr ): # if ( cr==1 ): # numpts = 1+abs(sf-sc) # idxmin = i-sc*(1-sf) @@ -93,98 +103,113 @@ def refinement_points_and_weights( ii, sc, sf, cr ): # weights[ir] = (1/numpts)*(1-sf)*(1-sc)+(1/numpts)*sf*sc # return [ numpts, idxmin, weights ] -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Main -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Input coarsening ratio -cr = int( input( "\n Select coarsening ratio (cr=1,2,4): cr=" ) ) -if ( cr!=1 and cr!=2 and cr!=4 ): +cr = int(input("\n Select coarsening ratio (cr=1,2,4): cr=")) +if cr != 1 and cr != 2 and cr != 4: print() - sys.exit( 'coarsening ratio cr={} is not valid'.format( cr ) ) + sys.exit("coarsening ratio cr={} is not valid".format(cr)) # Loop over possible staggering of coarse and fine grid (cell-centered or nodal) -for sc in [0,1]: - for sf in [0,1]: - - print( '\n **************************************************' ) - print( ' * Staggering of coarse grid: sc={}'.format( sc ), end='' ) - if ( sc == 0 ): - print( ' cell-centered *' ) - elif ( sc == 1 ): - print( ' nodal *' ) - print( ' * Staggering of fine grid: sf={}'.format( sf ), end='' ) - if ( sf == 0 ): - print( ' cell-centered *' ) - elif ( sf == 1 ): - print( ' nodal *' ) - print( ' **************************************************' ) - - iimin,iimax = fine_grid_limits( sf ) - imin ,imax = coarse_grid_limits( sc, sf, iimin, iimax ) - - print( '\n Min and max index on coarse grid: imin={} imax={}'.format( imin, imax ) ) - print( ' Min and max index on fine grid: iimin={} iimax={}'.format( iimin, iimax ) ) +for sc in [0, 1]: + for sf in [0, 1]: + print("\n **************************************************") + print(" * Staggering of coarse grid: sc={}".format(sc), end="") + if sc == 0: + print(" cell-centered *") + elif sc == 1: + print(" nodal *") + print(" * Staggering of fine grid: sf={}".format(sf), end="") + if sf == 0: + print(" cell-centered *") + elif sf == 1: + print(" nodal *") + print(" **************************************************") + + iimin, iimax = fine_grid_limits(sf) + imin, imax = coarse_grid_limits(sc, sf, iimin, iimax) + + print( + "\n Min and max index on coarse grid: imin={} imax={}".format(imin, imax) + ) + print( + " Min and max index on fine grid: iimin={} iimax={}".format(iimin, iimax) + ) # Number of grid points - nc = imax-imin+1 - nf = iimax-iimin+1 - - print( '\n Number of points on coarse grid: nc={}'.format( nc ) ) - print( ' Number of points on fine grid: nf={}'.format( nf ) ) - - if ( sf!=sc ): - print( '\n WARNING: sc={} not equal to sf={}, not implemented for MR, continue ...'.format( sc, sf ) ) + nc = imax - imin + 1 + nf = iimax - iimin + 1 + + print("\n Number of points on coarse grid: nc={}".format(nc)) + print(" Number of points on fine grid: nf={}".format(nf)) + + if sf != sc: + print( + "\n WARNING: sc={} not equal to sf={}, not implemented for MR, continue ...".format( + sc, sf + ) + ) continue - print( '\n Coarsening for MR: check interpolation points and weights' ) - print( ' ---------------------------------------------------------' ) + print("\n Coarsening for MR: check interpolation points and weights") + print(" ---------------------------------------------------------") # Coarsening for MR: interpolation points and weights - for i in range ( nc ): # index on coarse grid - numpts,idxmin,weights = coarsening_points_and_weights( i, sc, sf, cr ) - print( '\n Find value at i={} by interpolating over the following points and weights:'.format( i ) ) - for ir in range( numpts ): # interpolation points and weights - ii = idxmin+ir - print( ' ({},{})'.format( ii, weights[ir] ), end='' ) - if not ( ir == numpts-1 ): - print( ' ', end='' ) + for i in range(nc): # index on coarse grid + numpts, idxmin, weights = coarsening_points_and_weights(i, sc, sf, cr) + print( + "\n Find value at i={} by interpolating over the following points and weights:".format( + i + ) + ) + for ir in range(numpts): # interpolation points and weights + ii = idxmin + ir + print(" ({},{})".format(ii, weights[ir]), end="") + if not (ir == numpts - 1): + print(" ", end="") print() # Coarsening for MR: check conservation properties - for ii in range( nf ): # index on fine grid + for ii in range(nf): # index on fine grid ws = 0.0 - for i in range( nc ): # index on coarse grid - numpts,idxmin,weights = coarsening_points_and_weights( i, sc, sf, cr ) - for ir in range( numpts ): # interpolation points and weights - jj = idxmin+ir - if ( jj==ii ): # interpolation point matches point on fine grid - ws += weights[ir] - if ( ws!=1.0/cr ): - print( '\n ERROR: sum of weights ws={} should be 1/cr'.format( ws ) ) - - print( '\n Refinement for MR: check interpolation points and weights' ) - print( ' ---------------------------------------------------------' ) + for i in range(nc): # index on coarse grid + numpts, idxmin, weights = coarsening_points_and_weights(i, sc, sf, cr) + for ir in range(numpts): # interpolation points and weights + jj = idxmin + ir + if jj == ii: # interpolation point matches point on fine grid + ws += weights[ir] + if ws != 1.0 / cr: + print("\n ERROR: sum of weights ws={} should be 1/cr".format(ws)) + + print("\n Refinement for MR: check interpolation points and weights") + print(" ---------------------------------------------------------") # Refinement for MR: interpolation points and weights - for ii in range ( nf ): # index on fine grid - numpts,idxmin,weights = refinement_points_and_weights( ii, sc, sf, cr ) - print( '\n Find value at ii={} by interpolating over the following points and weights:'.format( ii ) ) - for ir in range( numpts ): # interpolation points and weights - i = idxmin+ir - print( ' ({},{})'.format( i, weights[ir] ), end='' ) - if not ( ir == numpts-1 ): - print( ' ', end='' ) + for ii in range(nf): # index on fine grid + numpts, idxmin, weights = refinement_points_and_weights(ii, sc, sf, cr) + print( + "\n Find value at ii={} by interpolating over the following points and weights:".format( + ii + ) + ) + for ir in range(numpts): # interpolation points and weights + i = idxmin + ir + print(" ({},{})".format(i, weights[ir]), end="") + if not (ir == numpts - 1): + print(" ", end="") print() # Refinement for MR: check conservation properties - for i in range( nc ): # index on coarse grid + for i in range(nc): # index on coarse grid ws = 0.0 - for ii in range( nf ): # index on fine grid - numpts,idxmin,weights = refinement_points_and_weights( ii, sc, sf, cr ) - for ir in range( numpts ): # interpolation points and weights - jj = idxmin+ir - if ( jj==i ): # interpolation point matches point on coarse grid - ws += weights[ir] - if ( ws!=cr ): - print( '\n ERROR: sum of weights ws={} should be cr'.format( ws ) ) + for ii in range(nf): # index on fine grid + numpts, idxmin, weights = refinement_points_and_weights(ii, sc, sf, cr) + for ir in range(numpts): # interpolation points and weights + jj = idxmin + ir + if jj == i: # interpolation point matches point on coarse grid + ws += weights[ir] + if ws != cr: + print("\n ERROR: sum of weights ws={} should be cr".format(ws)) diff --git a/Source/WarpX.H b/Source/WarpX.H index 39dc429698b..018640bbe25 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -12,10 +12,12 @@ #ifndef WARPX_H_ #define WARPX_H_ +#include "BoundaryConditions/PEC_Insulator_fwd.H" #include "BoundaryConditions/PML_fwd.H" #include "Diagnostics/MultiDiagnostics_fwd.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags_fwd.H" #include "EmbeddedBoundary/WarpXFaceInfoBox_fwd.H" +#include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel_fwd.H" @@ -38,8 +40,7 @@ #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" -#include "FieldSolver/Fields.H" -#include "FieldSolver/ElectrostaticSolver.H" +#include "Fields.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "FieldSolver/ImplicitSolvers/ImplicitSolver.H" #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" @@ -49,6 +50,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" +#include #include #include @@ -84,7 +86,6 @@ class WARPX_EXPORT WarpX : public amrex::AmrCore { public: - static WarpX& GetInstance (); static void ResetInstance (); @@ -104,37 +105,52 @@ public: WarpX& operator= ( WarpX const & ) = delete; /** Move constructor */ - WarpX ( WarpX && ) = default; + WarpX ( WarpX && ) = delete; /** Move operator */ - WarpX& operator= ( WarpX && ) = default; + WarpX& operator= ( WarpX && ) = delete; static std::string Version (); //!< Version of WarpX executable static std::string PicsarVersion (); //!< Version of PICSAR dependency [[nodiscard]] int Verbose () const { return verbose; } + [[nodiscard]] const amrex::Array& GetFieldBoundaryLo () const + { + return field_boundary_lo; + } + + [[nodiscard]] const amrex::Array& GetFieldBoundaryHi () const + { + return field_boundary_hi; + } + void InitData (); void Evolve (int numsteps = -1); + /** Push momentum one half step forward to synchronize with position. + * Also sets is_synchronized to `true`. + */ + void Synchronize (); + // // Functions used by implicit solvers // - void ImplicitPreRHSOp ( amrex::Real cur_time, - amrex::Real a_full_dt, - int a_nl_iter, - bool a_from_jacobian ); + void ImplicitPreRHSOp ( amrex::Real cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ); void SaveParticlesAtImplicitStepStart (); void FinishImplicitParticleUpdate (); - void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); - void UpdateMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_thetadt ); - void ApplyMagneticFieldBCs (); - void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_theta ); - void FinishImplicitField ( amrex::Vector, 3 > >& Field_fp, - const amrex::Vector, 3 > >& Field_n, - amrex::Real theta ); + void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E, amrex::Real a_time ); + void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_thetadt, amrex::Real start_time ); + void SpectralSourceFreeFieldAdvance ( amrex::Real start_time); + void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_theta, amrex::Real a_time ); + void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, + const ablastr::fields::MultiLevelVectorField& Field_n, + amrex::Real theta ); void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); void ImplicitComputeRHSE (int lev, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); void ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); @@ -142,18 +158,14 @@ public: MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } + ElectrostaticSolver& GetElectrostaticSolver () {return *m_electrostatic_solver;} HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} -#ifdef AMREX_USE_EB - amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} -#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } - - static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, - int num_shift, int dir, int lev, bool update_cost_flag, - amrex::Real external_field=0.0, bool useparser = false, - amrex::ParserExecutor<3> const& field_parser={}); + amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } + amrex::Vector,3 > >& GetEBUpdateBFlag() { return m_eb_update_B; } + amrex::Vector< std::unique_ptr > const & GetEBReduceParticleShapeFlag() const { return m_eb_reduce_particle_shape; } /** * \brief @@ -170,17 +182,17 @@ public: // Algorithms //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay, Villasenor) - static short current_deposition_algo; + static inline auto current_deposition_algo = CurrentDepositionAlgo::Default; //! Integer that corresponds to the charge deposition algorithm (only standard deposition) - static short charge_deposition_algo; + static inline auto charge_deposition_algo = ChargeDepositionAlgo::Default; //! Integer that corresponds to the field gathering algorithm (energy-conserving, momentum-conserving) - static short field_gathering_algo; + static inline auto field_gathering_algo = GatheringAlgo::Default; //! Integer that corresponds to the particle push algorithm (Boris, Vay, Higuera-Cary) - static short particle_pusher_algo; + static inline auto particle_pusher_algo = ParticlePusherAlgo::Default; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) - static short electromagnetic_solver_id; + static inline auto electromagnetic_solver_id = ElectromagneticSolverAlgo::Default; //! Integer that corresponds to the evolve scheme (explicit, semi_implicit_em, theta_implicit_em) - static short evolve_scheme; + EvolveScheme evolve_scheme = EvolveScheme::Default; //! Maximum iterations used for self-consistent particle update in implicit particle-suppressed evolve schemes static int max_particle_its_in_implicit_scheme; //! Relative tolerance used for self-consistent particle update in implicit particle-suppressed evolve schemes @@ -188,47 +200,46 @@ public: /** Records a number corresponding to the load balance cost update strategy * being used (0 or 1 corresponding to timers or heuristic). */ - static short load_balance_costs_update_algo; - //! Integer that corresponds to electromagnetic Maxwell solver (vacuum - 0, macroscopic - 1) - static int em_solver_medium; + static inline auto load_balance_costs_update_algo = LoadBalanceCostsUpdateAlgo::Default; /** Integer that correspond to macroscopic Maxwell solver algorithm * (BackwardEuler - 0, Lax-Wendroff - 1) */ - static int macroscopic_solver_algo; - //! Bool whether or not to subcycle current deposition - static bool do_subcycle_current; - //! Integer that corresponds to the number of times current deposition is subcycled - static int n_subcycle_current; - /** Integers that correspond to boundary condition applied to fields at the - * lower domain boundaries - * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) - */ - static amrex::Vector field_boundary_lo; - /** Integers that correspond to boundary condition applied to fields at the - * upper domain boundaries - * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) - */ - static amrex::Vector field_boundary_hi; + static inline auto macroscopic_solver_algo = MacroscopicSolverAlgo::Default; + /** Boundary conditions applied to fields at the lower domain boundaries + * (Possible values PML, Periodic, PEC, PMC, Neumann, Damped, Absorbing Silver-Mueller, None) + */ + static inline amrex::Array + field_boundary_lo {AMREX_D_DECL(FieldBoundaryType::Default, + FieldBoundaryType::Default, + FieldBoundaryType::Default)}; + /** Boundary conditions applied to fields at the upper domain boundaries + * (Possible values PML, Periodic, PEC, PMC, Neumann, Damped, Absorbing Silver-Mueller, None) + */ + static inline amrex::Array + field_boundary_hi {AMREX_D_DECL(FieldBoundaryType::Default, + FieldBoundaryType::Default, + FieldBoundaryType::Default)}; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ - static amrex::Vector particle_boundary_lo; + static inline amrex::Array + particle_boundary_lo {AMREX_D_DECL(ParticleBoundaryType::Default, + ParticleBoundaryType::Default, + ParticleBoundaryType::Default)}; /** Integers that correspond to boundary condition applied to particles at the * upper domain boundaries * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ - static amrex::Vector particle_boundary_hi; - - //! Integer that corresponds to the order of the PSATD solution - //! (whether the PSATD equations are derived from first-order or - //! second-order solution) - static short psatd_solution_type; + static inline amrex::Array + particle_boundary_hi {AMREX_D_DECL(ParticleBoundaryType::Default, + ParticleBoundaryType::Default, + ParticleBoundaryType::Default)}; //! Integers that correspond to the time dependency of J (constant, linear) //! and rho (linear, quadratic) for the PSATD algorithm - static short J_in_time; - static short rho_in_time; + static inline auto J_in_time = JInTime::Default; + static inline auto rho_in_time = RhoInTime::Default; //! If true, the current is deposited on a nodal grid and then centered onto a staggered grid //! using finite centering of order given by #current_centering_nox, #current_centering_noy, @@ -313,10 +324,6 @@ public: //! small time steps. static bool galerkin_interpolation; - //! Flag whether the Verboncoeur correction is applied to the current and charge density - //! on the axis when using RZ. - static bool verboncoeur_axis_correction; - //! If true, a bilinear filter is used to smooth charge and currents static bool use_filter; //! If true, the bilinear filtering of charge and currents is done in Fourier space @@ -333,14 +340,7 @@ public: static amrex::Real beta_boost; //! Direction of the Lorentz transform that defines the boosted frame of the simulation static amrex::Vector boost_direction; - //! If specified, the maximum number of iterations is computed automatically so that - //! the lower end of the simulation domain along z reaches #zmax_plasma_to_compute_max_step - //! in the boosted frame - static amrex::Real zmax_plasma_to_compute_max_step; - //! Set to true if #zmax_plasma_to_compute_max_step is specified, in which case - //! the maximum number of iterations is computed automatically so that the lower end of the - //! simulation domain along z reaches #zmax_plasma_to_compute_max_step in the boosted frame - static bool do_compute_max_step_from_zmax; + //! store initial value of zmin_domain_boost because WarpX::computeMaxStepBoostAccelerator //! needs the initial value of zmin_domain_boost, even if restarting from a checkpoint file static amrex::Real zmin_domain_boost_step_0; @@ -361,13 +361,9 @@ public: //! Specifies the type of grid used for the above sorting, i.e. cell-centered, nodal, or mixed static amrex::IntVect sort_idx_type; - static bool do_subcycling; static bool do_multi_J; static int do_multi_J_n_depositions; - static bool do_device_synchronize; - static bool safe_guard_cells; - //! With mesh refinement, particles located inside a refinement patch, but within //! #n_field_gather_buffer cells of the edge of the patch, will gather the fields //! from the lower refinement level instead of the refinement patch itself @@ -382,36 +378,11 @@ public: //! Integer that corresponds to the type of grid used in the simulation //! (collocated, staggered, hybrid) - static ablastr::utils::enums::GridType grid_type; + static inline auto grid_type = ablastr::utils::enums::GridType::Default; // Global rho nodal flag to know about rho index type when rho MultiFab is not allocated amrex::IntVect m_rho_nodal_flag; - /** - * \brief - * Allocate and optionally initialize the MultiFab. This also adds the MultiFab - * to the map of MultiFabs (used to ease the access to MultiFabs from the Python - * interface - * - * \param[out] mf The MultiFab unique pointer to be allocated - * \param[in] ba The BoxArray describing the MultiFab - * \param[in] dm The DistributionMapping describing the MultiFab - * \param[in] ncomp The number of components in the MultiFab - * \param[in] ngrow The number of guard cells in the MultiFab - * \param[in] level The refinement level - * \param[in] name The name of the MultiFab to use in the map - * \param[in] initial_value The optional initial value - */ - static void AllocInitMultiFab ( - std::unique_ptr& mf, - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm, - int ncomp, - const amrex::IntVect& ngrow, - int level, - const std::string& name, - std::optional initial_value = {}); - /** * \brief * Allocate and optionally initialize the iMultiFab. This also adds the iMultiFab @@ -427,7 +398,7 @@ public: * \param[in] name The name of the iMultiFab to use in the map * \param[in] initial_value The optional initial value */ - static void AllocInitMultiFab ( + void AllocInitMultiFab ( std::unique_ptr& mf, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, @@ -437,112 +408,17 @@ public: const std::string& name, std::optional initial_value = {}); - /** - * \brief - * Create an alias of a MultiFab, adding the alias to the MultiFab map - * \param[out] mf The MultiFab to create - * \param[in] mf_to_alias The MultiFab to alias - * \param[in] scomp The starting component to be aliased - * \param[in] ncomp The number of components to alias - * \param[in] level The refinement level - * \param[in] name The name of the MultiFab to use in the map - * \param[in] initial_value optional initial value for MultiFab - */ - static void AliasInitMultiFab ( - std::unique_ptr& mf, - const amrex::MultiFab& mf_to_alias, - int scomp, - int ncomp, - int level, - const std::string& name, - std::optional initial_value); - - /** - * \brief - * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) - * and optionally initialize it. This also adds the MultiFab - * to the map of MultiFabs (used to ease the access to MultiFabs from the Python - * interface - * - * \param mf[out] The MultiFab unique pointer to be allocated - * \param mf_model[in] The MultiFab to model - * \param name[in] The name of the MultiFab to use in the map - * \param initial_value[in] The optional initial value - */ - static void AllocInitMultiFabFromModel ( - std::unique_ptr& mf, - amrex::MultiFab& mf_model, - int level, - const std::string& name, - std::optional initial_value = {}); - - // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) - // This is a convenience for the Python interface, allowing all MultiFabs + // Maps of all of the iMultiFabs used (this can include MFs from other classes) + // This is a convenience for the Python interface, allowing all iMultiFabs // to be easily referenced from Python. - static std::map multifab_map; - static std::map imultifab_map; - - /** - * \brief - * Check if a field is initialized. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return true if the field is initialized, false otherwise - */ - [[nodiscard]] bool - isFieldInitialized (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - - /** - * \brief - * Get a pointer to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return the pointer to an amrex::MultiFab containing the field data - */ - [[nodiscard]] amrex::MultiFab* - getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; + std::map imultifab_map; /** * \brief - * For vector fields, get an array of three pointers to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * - * \return an array of three pointers amrex::MultiFab* containing the field data - */ - [[nodiscard]] std::array - getFieldPointerArray (warpx::fields::FieldType field_type, int lev) const; - - /** - * \brief - * Get a constant reference to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return a constant refernce to an amrex::MultiFab containing the field data - */ - [[nodiscard]] const amrex::MultiFab& - getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; - - /** - * \brief - * Get a constant reference to the specified vector field on the different MR levels - * - * \param field_type[in] the field type - * - * \return a vector (which one element per MR level) of arrays of three pointers (for 3 vector components) amrex::MultiFab* containing the field data + * Get pointer to the amrex::MultiFab containing the dotMask for the specified field */ - [[nodiscard]] const amrex::Vector,3>>& - getMultiLevelField(warpx::fields::FieldType field_type) const; + [[nodiscard]] const amrex::iMultiFab* + getFieldDotMaskPointer (warpx::fields::FieldType field_type, int lev, ablastr::fields::Direction dir) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -571,11 +447,6 @@ public: amrex::Vector m_v_comoving = amrex::Vector(3, amrex::Real(0.)); - static int num_mirrors; - amrex::Vector mirror_z; - amrex::Vector mirror_z_width; - amrex::Vector mirror_z_npoints; - /// object with all reduced diagnostics, similar to MultiParticleContainer for species. std::unique_ptr reduced_diags; @@ -584,15 +455,18 @@ public: /** Determine the timestep of the simulation. */ void ComputeDt (); + /** + * Determine the simulation timestep from the maximum speed of all particles + * Sets timestep so that a particle can only cross cfl*dx cells per timestep. + */ + void UpdateDtFromParticleSpeeds (); + /** Print main PIC parameters to stdout */ void PrintMainPICparameters (); /** Write a file that record all inputs: inputs file + command line options */ void WriteUsedInputsFile () const; - /** Print dt and dx,dy,dz */ - void PrintDtDxDyDz (); - /** * \brief * Compute the last time step of the simulation @@ -622,22 +496,22 @@ public: void UpdateInjectionPosition (amrex::Real dt); void ResetProbDomain (const amrex::RealBox& rb); - void EvolveE ( amrex::Real dt); - void EvolveE (int lev, amrex::Real dt); - void EvolveB ( amrex::Real dt, DtType dt_type); - void EvolveB (int lev, amrex::Real dt, DtType dt_type); + void EvolveE ( amrex::Real dt, amrex::Real start_time); + void EvolveE (int lev, amrex::Real dt, amrex::Real start_time); + void EvolveB ( amrex::Real dt, DtType dt_type, amrex::Real start_time); + void EvolveB (int lev, amrex::Real dt, DtType dt_type, amrex::Real start_time); void EvolveF ( amrex::Real dt, DtType dt_type); void EvolveF (int lev, amrex::Real dt, DtType dt_type); void EvolveG ( amrex::Real dt, DtType dt_type); void EvolveG (int lev, amrex::Real dt, DtType dt_type); - void EvolveB (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); - void EvolveE (int lev, PatchType patch_type, amrex::Real dt); + void EvolveB (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type, amrex::Real start_time); + void EvolveE (int lev, PatchType patch_type, amrex::Real dt, amrex::Real start_time); void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); void EvolveG (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); - void MacroscopicEvolveE ( amrex::Real dt); - void MacroscopicEvolveE (int lev, amrex::Real dt); - void MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real dt); + void MacroscopicEvolveE ( amrex::Real dt, amrex::Real start_time); + void MacroscopicEvolveE (int lev, amrex::Real dt, amrex::Real start_time); + void MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real dt, amrex::Real start_time); /** * \brief Hybrid-PIC field evolve function. @@ -676,7 +550,7 @@ public: */ void Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real dt); - static amrex::Real quantum_xi_c2; + amrex::Real m_quantum_xi_c2; /** Check and potentially compute load balancing */ @@ -713,8 +587,8 @@ public: * when FieldBoundaryType is set to damped. Vector version. */ void DampFieldsInGuards (int lev, - const std::array,3>& Efield, - const std::array,3>& Bfield); + const ablastr::fields::VectorField& Efield, + const ablastr::fields::VectorField& Bfield); /** * \brief Private function for spectral solver @@ -723,16 +597,16 @@ public: * can appear in parallel simulations. This will be called * when FieldBoundaryType is set to damped. Scalar version. */ - void DampFieldsInGuards (int lev, std::unique_ptr& mf); + void DampFieldsInGuards (int lev, amrex::MultiFab* mf); #ifdef WARPX_DIM_RZ void ApplyInverseVolumeScalingToCurrentDensity(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, - int lev); + int lev) const; void ApplyInverseVolumeScalingToChargeDensity(amrex::MultiFab* Rho, - int lev); + int lev) const; #endif void ApplySubcyclingScalingToCurrentDensity (amrex::MultiFab* Jx, @@ -757,8 +631,8 @@ public: amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type); - void ApplyEfieldBoundary (int lev, PatchType patch_type); - void ApplyBfieldBoundary (int lev, PatchType patch_type, DtType dt_type); + void ApplyEfieldBoundary (int lev, PatchType patch_type, amrex::Real cur_time); + void ApplyBfieldBoundary (int lev, PatchType patch_type, DtType dt_type, amrex::Real cur_time); #ifdef WARPX_DIM_RZ // Applies the boundary conditions that are specific to the axis when in RZ. @@ -785,7 +659,7 @@ public: void DampJPML (int lev, PatchType patch_type); void CopyJPML (); - bool isAnyBoundaryPML(); + /** True if any of the particle boundary condition type is Thermal */ static bool isAnyParticleBoundaryThermal(); @@ -796,18 +670,10 @@ public: /** Run the ionization module on all species */ void doFieldIonization (); - /** Run the ionization module on all species at level lev - * \param lev level - */ - void doFieldIonization (int lev); #ifdef WARPX_QED /** Run the QED module on all species */ void doQEDEvents (); - /** Run the QED module on all species at level lev - * \param lev level - */ - void doQEDEvents (int lev); #endif void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false, @@ -865,21 +731,16 @@ public: * Then, for each MR level, including level 0, apply filter and sum guard * cells across levels. * - * \param[in,out] J_fp reference to fine-patch current \c MultiFab (all MR levels) - * \param[in,out] J_cp reference to coarse-patch current \c MultiFab (all MR levels) - * \param[in,out] J_buffer reference to buffer current \c MultiFab (all MR levels) + * \param[in] current_fp_string the coarse of fine patch to use for current */ - void SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer); + void SyncCurrent (const std::string& current_fp_string); void SyncRho (); void SyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const amrex::Vector>& charge_buffer); + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_buffer); [[nodiscard]] amrex::Vector getnsubsteps () const {return nsubsteps;} [[nodiscard]] int getnsubsteps (int lev) const {return nsubsteps[lev];} @@ -902,14 +763,6 @@ public: [[nodiscard]] amrex::Real stopTime () const {return stop_time;} void updateStopTime (const amrex::Real new_stop_time) {stop_time = new_stop_time;} - void AverageAndPackFields( amrex::Vector& varnames, - amrex::Vector& mf_avg, amrex::IntVect ngrow) const; - - void prepareFields( int step, amrex::Vector& varnames, - amrex::Vector& mf_avg, - amrex::Vector& output_mf, - amrex::Vector& output_geom ) const; - static std::array CellSize (int lev); static amrex::XDim3 InvCellSize (int lev); static amrex::RealBox getRealBox(const amrex::Box& bx, int lev); @@ -942,14 +795,8 @@ public: static const amrex::iMultiFab* CurrentBufferMasks (int lev); static const amrex::iMultiFab* GatherBufferMasks (int lev); - static int electrostatic_solver_id; - static int poisson_solver_id; - - // Parameters for lab frame electrostatic - static amrex::Real self_fields_required_precision; - static amrex::Real self_fields_absolute_tolerance; - static int self_fields_max_iters; - static int self_fields_verbosity; + static inline auto electrostatic_solver_id = ElectrostaticSolverAlgo::Default; + static inline auto poisson_solver_id = PoissonSolverAlgo::Default; static int do_moving_window; // boolean static int start_moving_window_step; // the first step to move window @@ -970,15 +817,18 @@ public: // these should be private, but can't due to Cuda limitations static void ComputeDivB (amrex::MultiFab& divB, int dcomp, - const std::array& B, + ablastr::fields::VectorField const & B, const std::array& dx); static void ComputeDivB (amrex::MultiFab& divB, int dcomp, - const std::array& B, + ablastr::fields::VectorField const & B, const std::array& dx, amrex::IntVect ngrow); void ComputeDivE(amrex::MultiFab& divE, int lev); + void ProjectionCleanDivB (); + void CalculateExternalCurlA (); + [[nodiscard]] amrex::IntVect getngEB() const { return guard_cells.ng_alloc_EB; } [[nodiscard]] amrex::IntVect getngF() const { return guard_cells.ng_alloc_F; } [[nodiscard]] amrex::IntVect getngUpdateAux() const { return guard_cells.ng_UpdateAux; } @@ -995,74 +845,96 @@ public: */ [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;} - bool m_boundary_potential_specified = false; - ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler; + /** Electrostatic solve call */ void ComputeSpaceChargeField (bool reset_fields); - void AddBoundaryField (); - void AddSpaceChargeField (WarpXParticleContainer& pc); - void AddSpaceChargeFieldLabFrame (); - void computePhi (const amrex::Vector >& rho, - amrex::Vector >& phi, - std::array beta = {{0,0,0}}, - amrex::Real required_precision=amrex::Real(1.e-11), - amrex::Real absolute_tolerance=amrex::Real(0.0), - int max_iters=200, - int verbosity=2) const; - - void setPhiBC (amrex::Vector >& phi ) const; - - void computeE (amrex::Vector, 3> >& E, - const amrex::Vector >& phi, - std::array beta = {{0,0,0}} ) const; - void computeB (amrex::Vector, 3> >& B, - const amrex::Vector >& phi, - std::array beta = {{0,0,0}} ) const; - void computePhiTriDiagonal (const amrex::Vector >& rho, - amrex::Vector >& phi) const; // Magnetostatic Solver Interface MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler; + int magnetostatic_solver_max_iters = 200; + int magnetostatic_solver_verbosity = 2; void ComputeMagnetostaticField (); void AddMagnetostaticFieldLabFrame (); - void computeVectorPotential (const amrex::Vector, 3> >& curr, - amrex::Vector, 3> >& A, + void computeVectorPotential (ablastr::fields::MultiLevelVectorField const& curr, + ablastr::fields::MultiLevelVectorField const& A, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, - int verbosity=2) const; + int verbosity=2); // const; - void setVectorPotentialBC (amrex::Vector, 3> >& A) const; + void setVectorPotentialBC (ablastr::fields::MultiLevelVectorField const& A) const; /** * \brief - * This function initializes the E and B fields on each level + * This function computes the E, B, and J fields on each level + * using the parser and the user-defined function for the external fields. + * The subroutine will parse the x_/y_z_external_grid_function and + * then, the field multifab is initialized based on the (x,y,z) position + * on the staggered yee-grid or cell-centered grid, in the interior cells + * and guard cells. + * + * \param[in] field FieldType to grab from register to write into + * \param[in] fx_parser parser function to initialize x-field + * \param[in] fy_parser parser function to initialize y-field + * \param[in] fz_parser parser function to initialize z-field + * \param[in] lev level of the Multifabs that is initialized + * \param[in] patch_type PatchType on which the field is initialized (fine or coarse) + * \param[in] eb_update_field flag indicating which gridpoints should be modified by this functions + * \param[in] use_eb_flags (default:true) flag indicating if eb points should be excluded or not + */ + void ComputeExternalFieldOnGridUsingParser ( + warpx::fields::FieldType field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field, + bool use_eb_flags); + + void ComputeExternalFieldOnGridUsingParser ( + warpx::fields::FieldType field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field); + + /** + * \brief + * This function computes the E, B, and J fields on each level * using the parser and the user-defined function for the external fields. * The subroutine will parse the x_/y_z_external_grid_function and * then, the field multifab is initialized based on the (x,y,z) position * on the staggered yee-grid or cell-centered grid, in the interior cells * and guard cells. * - * \param[in] mfx x-component of the field to be initialized - * \param[in] mfy y-component of the field to be initialized - * \param[in] mfz z-component of the field to be initialized - * \param[in] xfield_parser parser function to initialize x-field - * \param[in] yfield_parser parser function to initialize y-field - * \param[in] zfield_parser parser function to initialize z-field + * \param[in] field string containing field name to grab from register + * \param[in] fx_parser parser function to initialize x-field + * \param[in] fy_parser parser function to initialize y-field + * \param[in] fz_parser parser function to initialize z-field * \param[in] edge_lengths edge lengths information * \param[in] face_areas face areas information - * \param[in] field flag indicating which field is being initialized ('E' for electric, 'B' for magnetic) + * \param[in] topology flag indicating if field is edge-based or face-based * \param[in] lev level of the Multifabs that is initialized * \param[in] patch_type PatchType on which the field is initialized (fine or coarse) - */ - void InitializeExternalFieldsOnGridUsingParser ( - amrex::MultiFab *mfx, amrex::MultiFab *mfy, amrex::MultiFab *mfz, - amrex::ParserExecutor<3> const& xfield_parser, - amrex::ParserExecutor<3> const& yfield_parser, - amrex::ParserExecutor<3> const& zfield_parser, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, - char field, - int lev, PatchType patch_type); + * \param[in] eb_update_field flag indicating which gridpoints should be modified by this functions + * \param[in] use_eb_flags (default:true) flag indicating if eb points should be excluded or not + */ + void ComputeExternalFieldOnGridUsingParser ( + std::string const& field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector< std::array< std::unique_ptr,3> > const& eb_update_field, + bool use_eb_flags); + + void ComputeExternalFieldOnGridUsingParser ( + std::string const& field, + amrex::ParserExecutor<4> const& fx_parser, + amrex::ParserExecutor<4> const& fy_parser, + amrex::ParserExecutor<4> const& fz_parser, + int lev, PatchType patch_type, + amrex::Vector< std::array< std::unique_ptr,3> > const& eb_update_field); /** * \brief Load field values from a user-specified openPMD file, @@ -1097,16 +969,6 @@ public: void ApplyFilterandSumBoundaryRho (int lev, int glev, amrex::MultiFab& rho, int icomp, int ncomp); - /** - * \brief Returns an array of coefficients (Fornberg coefficients), corresponding - * to the weight of each point in a finite-difference approximation of a derivative - * (up to order \c n_order). - * - * \param[in] n_order order of the finite-difference approximation - * \param[in] a_grid_type type of grid (collocated or not) - */ - static amrex::Vector getFornbergStencilCoefficients (int n_order, ablastr::utils::enums::GridType a_grid_type); - // Device vectors of stencil coefficients used for finite-order centering of fields amrex::Gpu::DeviceVector device_field_centering_stencil_coeffs_x; amrex::Gpu::DeviceVector device_field_centering_stencil_coeffs_y; @@ -1129,48 +991,13 @@ public: const amrex::IArrayBox &guard_mask, const int ng ); #ifdef AMREX_USE_EB - amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { + [[nodiscard]] amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { return static_cast(*m_field_factory[lev]); } #endif void InitEB (); -#ifdef AMREX_USE_EB - /** - * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. - * An edge of length 0 is fully covered. - */ - static void ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, - const amrex::EBFArrayBoxFactory& eb_fact); - /** - * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. - * An edge of area 0 is fully covered. - */ - static void ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, - const amrex::EBFArrayBoxFactory& eb_fact); - - /** - * \brief Scale the edges lengths by the mesh width to obtain the real lengths. - */ - static void ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, - const std::array& cell_size); - /** - * \brief Scale the edges areas by the mesh width to obtain the real areas. - */ - static void ScaleAreas (std::array< std::unique_ptr, 3 >& face_areas, - const std::array& cell_size); - /** - * \brief Initialize information for cell extensions. - * The flags convention for m_flag_info_face is as follows - * - 0 for unstable cells - * - 1 for stable cells which have not been intruded - * - 2 for stable cells which have been intruded - * Here we cannot know if a cell is intruded or not so we initialize all stable cells with 1 - */ - void MarkCells(); -#endif - /** * \brief Compute the level set function used for particle-boundary interaction. */ @@ -1217,17 +1044,14 @@ public: void PSATDSubtractCurrentPartialSumsAvg (); #ifdef WARPX_USE_FFT - -# ifdef WARPX_DIM_RZ - SpectralSolverRZ& -# else - SpectralSolver& -# endif - get_spectral_solver_fp (int lev) {return *spectral_solver_fp[lev];} + auto& get_spectral_solver_fp (int lev) {return *spectral_solver_fp[lev];} #endif FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); } + // Field container + ablastr::fields::MultiFabRegister m_fields; + protected: /** @@ -1261,6 +1085,10 @@ protected: //! This function is called in amrex::AmrCore::InitFromScratch. void PostProcessBaseGrids (amrex::BoxArray& ba0) const final; + //! Use this function to override how DistributionMapping is made. + [[nodiscard]] amrex::DistributionMapping + MakeDistributionMap (int lev, amrex::BoxArray const& ba) final; + //! Make a new level from scratch using provided BoxArray and //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. @@ -1323,61 +1151,46 @@ private: void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, int lev); void AddCurrentFromFineLevelandSumBoundary ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer, int lev); void StoreCurrent (int lev); void RestoreCurrent (int lev); - void ApplyFilterJ ( - const amrex::Vector,3>>& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev, int idim); - void ApplyFilterJ ( - const amrex::Vector,3>>& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev); void SumBoundaryJ ( - const amrex::Vector,3>>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev, int idim, const amrex::Periodicity& period); void SumBoundaryJ ( - const amrex::Vector,3>>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev, const amrex::Periodicity& period); - void NodalSyncJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - int lev, - PatchType patch_type); - void RestrictRhoFromFineToCoarsePatch ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - int lev); + void RestrictRhoFromFineToCoarsePatch (int lev ); void ApplyFilterandSumBoundaryRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, int lev, PatchType patch_type, int icomp, int ncomp); void AddRhoFromFineLevelandSumBoundary ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const amrex::Vector>& charge_buffer, - int lev, - int icomp, - int ncomp); - void NodalSyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_buffer, int lev, - PatchType patch_type, int icomp, int ncomp); @@ -1418,9 +1231,6 @@ private: */ void CheckKnownIssues(); - /** Check the requested resources and write performance hints */ - void PerformanceHints (); - void BuildBufferMasks (); [[nodiscard]] const amrex::iMultiFab* getCurrentBufferMasks (int lev) const { @@ -1433,18 +1243,6 @@ private: } - /** - * \brief Re-orders the Fornberg coefficients so that they can be used more conveniently for - * finite-order centering operations. For example, for finite-order centering of order 6, - * the Fornberg coefficients \c (c_0,c_1,c_2) are re-ordered as \c (c_2,c_1,c_0,c_0,c_1,c_2). - * - * \param[in,out] ordered_coeffs host vector where the re-ordered Fornberg coefficients will be stored - * \param[in] unordered_coeffs host vector storing the original sequence of Fornberg coefficients - * \param[in] order order of the finite-order centering along a given direction - */ - void ReorderFornbergCoefficients (amrex::Vector& ordered_coeffs, - amrex::Vector& unordered_coeffs, - int order); /** * \brief Allocates and initializes the stencil coefficients used for the finite-order centering * of fields and currents, and stores them in the given device vectors. @@ -1496,6 +1294,9 @@ private: amrex::Vector t_new; amrex::Vector t_old; amrex::Vector dt; + utils::parser::IntervalsParser m_dt_update_interval = utils::parser::IntervalsParser{}; // How often to update the timestep when using adaptive timestepping + + bool m_safe_guard_cells = false; // Particle container std::unique_ptr mypc; @@ -1505,101 +1306,54 @@ private: bool do_fluid_species = false; std::unique_ptr myfl; + //! Integer that corresponds to electromagnetic Maxwell solver (vacuum - 0, macroscopic - 1) + MediumForEM m_em_solver_medium = MediumForEM::Default; + // // Fields: First array for level, second for direction // - // Full solution - amrex::Vector, 3 > > Efield_aux; - amrex::Vector, 3 > > Bfield_aux; - - // Fine patch - amrex::Vector< std::unique_ptr > F_fp; - amrex::Vector< std::unique_ptr > G_fp; - amrex::Vector< std::unique_ptr > rho_fp; - amrex::Vector< std::unique_ptr > phi_fp; - amrex::Vector, 3 > > current_fp; - amrex::Vector, 3 > > current_fp_vay; - amrex::Vector, 3 > > Efield_fp; - amrex::Vector, 3 > > Bfield_fp; - amrex::Vector, 3 > > Efield_avg_fp; - amrex::Vector, 3 > > Bfield_avg_fp; - - // Memory buffers for computing magnetostatic fields - // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order - amrex::Vector, 3 > > vector_potential_fp_nodal; - amrex::Vector, 3 > > vector_potential_grad_buf_e_stag; - amrex::Vector, 3 > > vector_potential_grad_buf_b_stag; - - // Same as Bfield_fp/Efield_fp for reading external field data - amrex::Vector, 3 > > Efield_fp_external; - amrex::Vector, 3 > > Bfield_fp_external; - amrex::Vector, 3 > > E_external_particle_field; - amrex::Vector, 3 > > B_external_particle_field; - - //! EB: Lengths of the mesh edges - amrex::Vector, 3 > > m_edge_lengths; - //! EB: Areas of the mesh faces - amrex::Vector, 3 > > m_face_areas; + // Masks for computing dot product and global moments of fields when using grids that + // have shared locations across different ranks (e.g., a Yee grid) + mutable amrex::Vector,3 > > Efield_dotMask; + mutable amrex::Vector,3 > > Bfield_dotMask; + mutable amrex::Vector,3 > > Afield_dotMask; + mutable amrex::Vector< std::unique_ptr > phi_dotMask; + + /** EB: Flag to indicate whether a gridpoint is inside the embedded boundary and therefore + * whether the E or B should not be updated. (One array per level and per direction, due to staggering) + */ + amrex::Vector,3 > > m_eb_update_E; + amrex::Vector,3 > > m_eb_update_B; + + /** EB: Mask that indicates whether a particle should use its regular shape factor (mask set to 0) + * or a reduced, order-1 shape factor instead (mask set to 1) in a given cell, when depositing charge/current. + * The flag is typically set to 1 in cells that are close to the embedded boundary, in order to avoid + * errors in charge conservation when a particle is too close to the embedded boundary. + */ + amrex::Vector< std::unique_ptr > m_eb_reduce_particle_shape; /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces * * 2 if the face is actually intruded by other face - * It is initialized in WarpX::MarkCells + * It is initialized in WarpX::MarkExtensionCells * This is only used for the ECT solver.*/ amrex::Vector, 3 > > m_flag_info_face; /** EB: for every mesh face face flag_ext_face contains a: * * 1 if the face needs to be extended * * 0 otherwise - * It is initialized in WarpX::MarkCells and then modified in WarpX::ComputeOneWayExtensions + * It is initialized in WarpX::MarkExtensionCells and then modified in WarpX::ComputeOneWayExtensions * and in WarpX::ComputeEightWaysExtensions * This is only used for the ECT solver.*/ amrex::Vector, 3 > > m_flag_ext_face; - /** EB: m_area_mod contains the modified areas of the mesh faces, i.e. if a face is enlarged it - * contains the area of the enlarged face - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > m_area_mod; + /** EB: m_borrowing contains the info about the enlarged cells, i.e. for every enlarged cell it * contains the info of which neighbors are being intruded (and the amount of borrowed area). * This is only used for the ECT solver.*/ amrex::Vector >, 3 > > m_borrowing; - /** ECTRhofield is needed only by the ect - * solver and it contains the electromotive force density for every mesh face. - * The name ECTRhofield has been used to comply with the notation of the paper - * https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=4463918 (page 9, equation 4 - * and below). - * Although it's called rho it has nothing to do with the charge density! - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > ECTRhofield; - /** Venl contains the electromotive force for every mesh face, i.e. every entry is - * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > Venl; - - //EB level set - amrex::Vector > m_distance_to_eb; - - // store fine patch - amrex::Vector, 3 > > current_store; - - // Nodal MultiFab for nodal current deposition if warpx.do_current_centering = 1 - amrex::Vector,3>> current_fp_nodal; - - // Coarse patch - amrex::Vector< std::unique_ptr > F_cp; - amrex::Vector< std::unique_ptr > G_cp; - amrex::Vector< std::unique_ptr > rho_cp; - amrex::Vector, 3 > > current_cp; - amrex::Vector, 3 > > Efield_cp; - amrex::Vector, 3 > > Bfield_cp; - amrex::Vector, 3 > > Efield_avg_cp; - amrex::Vector, 3 > > Bfield_avg_cp; - // Copy of the coarse aux - amrex::Vector, 3 > > Efield_cax; - amrex::Vector, 3 > > Bfield_cax; amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; // /** Multifab that stores weights for interpolating fine and coarse solutions @@ -1608,24 +1362,6 @@ private: // */ // amrex::Vector > interp_weight_gbuffer; - // If charge/current deposition buffers are used - amrex::Vector, 3 > > current_buf; - amrex::Vector > charge_buf; - - /** - * \brief - * Get a pointer to the field data. Does not check if the pointer - * is not nullptr. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return the pointer to an amrex::MultiFab containing the field data - */ - [[nodiscard]] amrex::MultiFab* - getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - // PML int do_pml = 0; int do_silver_mueller = 0; @@ -1648,21 +1384,42 @@ private: #endif amrex::Real v_particle_pml; + // Insulator boundary conditions + std::unique_ptr pec_insulator_boundary; + // External fields parameters std::unique_ptr m_p_ext_field_params; amrex::Real moving_window_x = std::numeric_limits::max(); + // Mirrors + int m_num_mirrors = 0; + amrex::Vector m_mirror_z; + amrex::Vector m_mirror_z_width; + amrex::Vector m_mirror_z_npoints; + // Plasma injection parameters int warpx_do_continuous_injection = 0; int num_injected_species = -1; amrex::Vector injected_plasma_species; + // Timestepping parameters std::optional m_const_dt; + std::optional m_max_dt; + + // whether to use subcycling + bool m_do_subcycling = false; + + //! Flag whether the Verboncoeur correction is applied to the current and charge density + //! on the axis when using RZ. + bool m_verboncoeur_axis_correction = true; // Macroscopic properties std::unique_ptr m_macroscopic_properties; + // Electrostatic solver + std::unique_ptr m_electrostatic_solver; + // Hybrid PIC algorithm parameters std::unique_ptr m_hybrid_pic_model; @@ -1716,6 +1473,11 @@ private: int max_step = std::numeric_limits::max(); amrex::Real stop_time = std::numeric_limits::max(); + //! If specified, the maximum number of iterations is computed automatically so that + //! the lower end of the simulation domain along z reaches #zmax_plasma_to_compute_max_step + //! in the boosted frame + std::optional m_zmax_plasma_to_compute_max_step = std::nullopt; + int regrid_int = -1; amrex::Real cfl = amrex::Real(0.999); @@ -1746,23 +1508,21 @@ private: guardCellManager guard_cells; - //Slice Parameters + // Slice Parameters int slice_max_grid_size; int slice_plot_int = -1; amrex::RealBox slice_realbox; amrex::IntVect slice_cr_ratio; - amrex::Vector< std::unique_ptr > F_slice; - amrex::Vector< std::unique_ptr > G_slice; - amrex::Vector< std::unique_ptr > rho_slice; - amrex::Vector, 3 > > current_slice; - amrex::Vector, 3 > > Efield_slice; - amrex::Vector, 3 > > Bfield_slice; bool fft_periodic_single_box = false; int nox_fft = 16; int noy_fft = 16; int noz_fft = 16; + //! Solve Poisson equation when loading an external magnetic field to clean divergence + //! This is useful to remove errors that could lead to non-zero B field divergence + bool m_do_divb_cleaning_external = false; + //! Domain decomposition on Level 0 amrex::IntVect numprocs{0}; @@ -1794,13 +1554,6 @@ private: [[nodiscard]] bool checkStopSimulation (amrex::Real cur_time); - /** Print Unused Parameter Warnings after Step 1 - * - * Instead of waiting for a simulation to end, we already do an early "unused parameter check" - * after step 1 to inform users early of potential issues with their simulation setup. - */ - void checkEarlyUnusedParams (); - /** Perform essential particle house keeping at boundaries * * Inject, communicate, scrape and sort particles. @@ -1811,8 +1564,6 @@ private: */ void HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved); - void ScrapeParticles (); - /** Update the E and B fields in the explicit em PIC scheme. * * At the beginning, we have B^{n} and E^{n}. @@ -1820,46 +1571,25 @@ private: */ void ExplicitFillBoundaryEBUpdateAux (); - void PushPSATD (); + //! Integer that corresponds to the order of the PSATD solution + //! (whether the PSATD equations are derived from first-order or + //! second-order solution) + PSATDSolutionType m_psatd_solution_type = PSATDSolutionType::Default; + + void PushPSATD (amrex::Real start_time); #ifdef WARPX_USE_FFT /** * \brief Forward FFT of E,B on all mesh refinement levels - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDForwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp); + */ + void PSATDForwardTransformEB (); /** * \brief Backward FFT of E,B on all mesh refinement levels, * with field damping in the guard cells (if needed) - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDBackwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp); + */ + void PSATDBackwardTransformEB (); /** * \brief Backward FFT of averaged E,B on all mesh refinement levels @@ -1874,10 +1604,10 @@ private: * storing the coarse patch averaged magnetic field to be transformed */ void PSATDBackwardTransformEBavg ( - const amrex::Vector,3>>& E_avg_fp, - const amrex::Vector,3>>& B_avg_fp, - const amrex::Vector,3>>& E_avg_cp, - const amrex::Vector,3>>& B_avg_cp); + ablastr::fields::MultiLevelVectorField const& E_avg_fp, + ablastr::fields::MultiLevelVectorField const& B_avg_fp, + ablastr::fields::MultiLevelVectorField const& E_avg_cp, + ablastr::fields::MultiLevelVectorField const& B_avg_cp); /** * \brief Forward FFT of J on all mesh refinement levels, @@ -1891,8 +1621,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, bool apply_kspace_filter=true); /** @@ -1904,8 +1634,8 @@ private: * storing the coarse patch current to be transformed */ void PSATDBackwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp); + std::string const & J_fp_string, + std::string const & J_cp_string); /** * \brief Forward FFT of rho on all mesh refinement levels, @@ -1919,8 +1649,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, int icomp, int dcomp, bool apply_kspace_filter=true); /** diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cf1aa908202..6dbd8250ed8 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -11,10 +11,16 @@ */ #include "WarpX.H" +#include "BoundaryConditions/PEC_Insulator.H" #include "BoundaryConditions/PML.H" #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "EmbeddedBoundary/Enabled.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" +#include "FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H" +#include "FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H" +#include "FieldSolver/ElectrostaticSolvers/EffectivePotentialES.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" @@ -30,6 +36,7 @@ #include "FieldSolver/WarpX_FDTD.H" #include "Filter/NCIGodfreyFilter.H" #include "Initialization/ExternalField.H" +#include "Initialization/WarpXInit.H" #include "Particles/MultiParticleContainer.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" @@ -43,6 +50,7 @@ #include "FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H" +#include #include #include @@ -83,11 +91,12 @@ #include #include #include +#include #include #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; @@ -100,34 +109,19 @@ bool WarpX::fft_do_time_averaging = false; amrex::IntVect WarpX::m_fill_guards_fields = amrex::IntVect(0); amrex::IntVect WarpX::m_fill_guards_current = amrex::IntVect(0); -Real WarpX::quantum_xi_c2 = PhysConst::xi_c2; Real WarpX::gamma_boost = 1._rt; Real WarpX::beta_boost = 0._rt; Vector WarpX::boost_direction = {0,0,0}; -bool WarpX::do_compute_max_step_from_zmax = false; bool WarpX::compute_max_step_from_btd = false; -Real WarpX::zmax_plasma_to_compute_max_step = 0._rt; Real WarpX::zmin_domain_boost_step_0 = 0._rt; -short WarpX::current_deposition_algo; -short WarpX::charge_deposition_algo; -short WarpX::field_gathering_algo; -short WarpX::particle_pusher_algo; -short WarpX::electromagnetic_solver_id; -short WarpX::evolve_scheme; int WarpX::max_particle_its_in_implicit_scheme = 21; ParticleReal WarpX::particle_tol_in_implicit_scheme = 1.e-10; -short WarpX::psatd_solution_type; -short WarpX::J_in_time; -short WarpX::rho_in_time; -short WarpX::load_balance_costs_update_algo; bool WarpX::do_dive_cleaning = false; bool WarpX::do_divb_cleaning = false; -int WarpX::em_solver_medium; -int WarpX::macroscopic_solver_algo; bool WarpX::do_single_precision_comms = false; -bool WarpX::do_subcycle_current = false; -int WarpX::n_subcycle_current = 1; +//bool WarpX::do_subcycle_current = false; +//int WarpX::n_subcycle_current = 1; bool WarpX::do_abc_in_pml = false; int WarpX::load_balance_startlevel = 0; @@ -143,11 +137,6 @@ amrex::IntVect WarpX::shared_tilesize(AMREX_D_DECL(1,1,1)); #endif int WarpX::shared_mem_current_tpb = 128; -amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,FieldBoundaryType::PML); -amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBoundaryType::PML); -amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); -amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); - int WarpX::n_rz_azimuthal_modes = 1; int WarpX::ncomps = 1; @@ -169,8 +158,6 @@ int WarpX::current_centering_noz = 2; bool WarpX::use_fdtd_nci_corr = false; bool WarpX::galerkin_interpolation = true; -bool WarpX::verboncoeur_axis_correction = true; - bool WarpX::use_filter = true; bool WarpX::use_kspace_filter = true; bool WarpX::use_filter_compensation = false; @@ -180,8 +167,6 @@ bool WarpX::refine_plasma = false; bool WarpX::refineAddplasma = false; amrex::IntVect WarpX::AddplasmaRefRatio(AMREX_D_DECL(1,1,1)); -int WarpX::num_mirrors = 0; - utils::parser::IntervalsParser WarpX::sort_intervals; amrex::IntVect WarpX::sort_bin_size(AMREX_D_DECL(1,1,1)); @@ -195,20 +180,8 @@ amrex::IntVect WarpX::sort_idx_type(AMREX_D_DECL(0,0,0)); bool WarpX::do_dynamic_scheduling = true; -int WarpX::electrostatic_solver_id; -int WarpX::poisson_solver_id; -Real WarpX::self_fields_required_precision = 1.e-11_rt; -Real WarpX::self_fields_absolute_tolerance = 0.0_rt; -int WarpX::self_fields_max_iters = 200; -int WarpX::self_fields_verbosity = 2; - -bool WarpX::do_subcycling = false; bool WarpX::do_multi_J = false; int WarpX::do_multi_J_n_depositions; -bool WarpX::safe_guard_cells = false; - -std::map WarpX::multifab_map; -std::map WarpX::imultifab_map; IntVect WarpX::filter_npass_each_dir(1); @@ -217,14 +190,54 @@ amrex::IntVect WarpX::n_current_deposition_buffer_each_dir(-1); int WarpX::n_field_gather_buffer = -1; int WarpX::n_current_deposition_buffer = -1; -ablastr::utils::enums::GridType WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; WarpX* WarpX::m_instance = nullptr; +namespace +{ + + [[nodiscard]] bool + isAnyBoundaryPML( + const amrex::Array& field_boundary_lo, + const amrex::Array& field_boundary_hi) + { + constexpr auto is_pml = [](const FieldBoundaryType fbt) {return (fbt == FieldBoundaryType::PML);}; + const auto is_any_pml = + std::any_of(field_boundary_lo.begin(), field_boundary_lo.end(), is_pml) || + std::any_of(field_boundary_hi.begin(), field_boundary_hi.end(), is_pml); + return is_any_pml; + } + + /** + * \brief + * Set the dotMask container + */ + void SetDotMask( std::unique_ptr& field_dotMask, + ablastr::fields::ConstScalarField const& field, + amrex::Periodicity const& periodicity) + + { + // Define the dot mask for this field_type needed to properly compute dotProduct() + // for field values that have shared locations on different MPI ranks + if (field_dotMask != nullptr) { return; } + + const auto& this_ba = field->boxArray(); + const auto tmp = amrex::MultiFab{ + this_ba, field->DistributionMap(), + 1, 0, amrex::MFInfo().SetAlloc(false)}; + + field_dotMask = tmp.OwnerMask(periodicity); + } +} + void WarpX::MakeWarpX () { - ParseGeometryInput(); + warpx::initialization::check_dims(); + + ReadMovingWindowParameters( + do_moving_window, start_moving_window_step, end_moving_window_step, + moving_window_dir, moving_window_v); ConvertLabParamsToBoost(); ReadBCParams(); @@ -262,11 +275,13 @@ WarpX::Finalize() WarpX::WarpX () { + warpx::initialization::initialize_warning_manager(); + ReadParameters(); BackwardCompatibility(); - InitEB(); + if (EB::enabled()) { InitEB(); } ablastr::utils::SignalHandling::InitSignalHandling(); @@ -278,12 +293,6 @@ WarpX::WarpX () istep.resize(nlevs_max, 0); nsubsteps.resize(nlevs_max, 1); -#if 0 - // no subcycling yet - for (int lev = 1; lev < nlevs_max; ++lev) { - nsubsteps[lev] = MaxRefRatio(lev-1); - } -#endif t_new.resize(nlevs_max, 0.0); t_old.resize(nlevs_max, std::numeric_limits::lowest()); @@ -320,88 +329,48 @@ WarpX::WarpX () // Fluid Container if (do_fluid_species) { - myfl = std::make_unique(nlevs_max); + myfl = std::make_unique(); } - Efield_aux.resize(nlevs_max); - Bfield_aux.resize(nlevs_max); - - F_fp.resize(nlevs_max); - G_fp.resize(nlevs_max); - rho_fp.resize(nlevs_max); - phi_fp.resize(nlevs_max); - current_fp.resize(nlevs_max); - Efield_fp.resize(nlevs_max); - Bfield_fp.resize(nlevs_max); - - // Only allocate vector potential arrays when using the Magnetostatic Solver - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - vector_potential_fp_nodal.resize(nlevs_max); - vector_potential_grad_buf_e_stag.resize(nlevs_max); - vector_potential_grad_buf_b_stag.resize(nlevs_max); - } - - if (fft_do_time_averaging) - { - Efield_avg_fp.resize(nlevs_max); - Bfield_avg_fp.resize(nlevs_max); - } + Efield_dotMask.resize(nlevs_max); + Bfield_dotMask.resize(nlevs_max); + Afield_dotMask.resize(nlevs_max); + phi_dotMask.resize(nlevs_max); - // Same as Bfield_fp/Efield_fp for reading external field data - Bfield_fp_external.resize(nlevs_max); - Efield_fp_external.resize(nlevs_max); - B_external_particle_field.resize(1); - E_external_particle_field.resize(1); + m_eb_update_E.resize(nlevs_max); + m_eb_update_B.resize(nlevs_max); + m_eb_reduce_particle_shape.resize(nlevs_max); - m_edge_lengths.resize(nlevs_max); - m_face_areas.resize(nlevs_max); - m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); - m_area_mod.resize(nlevs_max); - - ECTRhofield.resize(nlevs_max); - Venl.resize(nlevs_max); - current_store.resize(nlevs_max); - - if (do_current_centering) + // Create Electrostatic Solver object if needed + if ((WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) + || (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic)) { - current_fp_nodal.resize(nlevs_max); + m_electrostatic_solver = std::make_unique(nlevs_max); } - - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) + // Initialize the effective potential electrostatic solver if required + else if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameEffectivePotential) { - current_fp_vay.resize(nlevs_max); + m_electrostatic_solver = std::make_unique(nlevs_max); } - - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) + else { - // Create hybrid-PIC model object if needed - m_hybrid_pic_model = std::make_unique(nlevs_max); + m_electrostatic_solver = std::make_unique(nlevs_max); } - F_cp.resize(nlevs_max); - G_cp.resize(nlevs_max); - rho_cp.resize(nlevs_max); - current_cp.resize(nlevs_max); - Efield_cp.resize(nlevs_max); - Bfield_cp.resize(nlevs_max); - - if (fft_do_time_averaging) + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - Efield_avg_cp.resize(nlevs_max); - Bfield_avg_cp.resize(nlevs_max); + // Create hybrid-PIC model object if needed + m_hybrid_pic_model = std::make_unique(); } - Efield_cax.resize(nlevs_max); - Bfield_cax.resize(nlevs_max); current_buffer_masks.resize(nlevs_max); gather_buffer_masks.resize(nlevs_max); - current_buf.resize(nlevs_max); - charge_buf.resize(nlevs_max); +// current_buf.resize(nlevs_max); +// charge_buf.resize(nlevs_max); // interp_weight_gbuffer.resize(nlevs_max); pml.resize(nlevs_max); @@ -417,7 +386,7 @@ WarpX::WarpX () m_field_factory.resize(nlevs_max); - if (em_solver_medium == MediumForEM::Macroscopic) { + if (m_em_solver_medium == MediumForEM::Macroscopic) { // create object for macroscopic solver m_macroscopic_properties = std::make_unique(); } @@ -500,7 +469,7 @@ WarpX::WarpX () // (e.g., use_fdtd_nci_corr) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { AMREX_ALWAYS_ASSERT(use_fdtd_nci_corr == 0); - AMREX_ALWAYS_ASSERT(do_subcycling == 0); + AMREX_ALWAYS_ASSERT(m_do_subcycling == 0); } if (WarpX::current_deposition_algo != CurrentDepositionAlgo::Esirkepov) { @@ -524,9 +493,6 @@ WarpX::~WarpX () void WarpX::ReadParameters () { - // Ensure that geometry.dims is set properly. - CheckDims(); - { const ParmParse pp;// Traditionally, max_step and stop_time do not have prefix. utils::parser::queryWithParser(pp, "max_step", max_step); @@ -542,37 +508,28 @@ WarpX::ReadParameters () { const ParmParse pp_algo("algo"); - electromagnetic_solver_id = static_cast(GetAlgorithmInteger(pp_algo, "maxwell_solver")); + pp_algo.query_enum_sloppy("maxwell_solver", electromagnetic_solver_id, "-_"); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT && !EB::enabled()) { + throw std::runtime_error("ECP Solver requires to enable embedded boundaries at runtime."); + } +#ifdef WARPX_DIM_RZ + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).ProbLo(0) == 0., + "Lower bound of radial coordinate (prob_lo[0]) with RZ PSATD solver must be zero"); + } + else + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).ProbLo(0) >= 0., + "Lower bound of radial coordinate (prob_lo[0]) with RZ FDTD solver must be non-negative"); + } +#endif + + pp_algo.query_enum_sloppy("evolve_scheme", evolve_scheme, "-_"); } { - const ParmParse pp_warpx("warpx"); - - //"Synthetic" warning messages may be injected in the Warning Manager via - // inputfile for debug&testing purposes. - ablastr::warn_manager::GetWMInstance().debug_read_warnings_from_input(pp_warpx); - - // Set the flag to control if WarpX has to emit a warning message as soon as a warning is recorded - bool always_warn_immediately = false; - pp_warpx.query("always_warn_immediately", always_warn_immediately); - ablastr::warn_manager::GetWMInstance().SetAlwaysWarnImmediately(always_warn_immediately); - - // Set the WarnPriority threshold to decide if WarpX has to abort when a warning is recorded - if(std::string str_abort_on_warning_threshold; - pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ - std::optional abort_on_warning_threshold = std::nullopt; - if (str_abort_on_warning_threshold == "high") { - abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; - } else if (str_abort_on_warning_threshold == "medium" ) { - abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; - } else if (str_abort_on_warning_threshold == "low") { - abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; - } else { - WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold - +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); - } - ablastr::warn_manager::GetWMInstance().SetAbortThreshold(abort_on_warning_threshold); - } + ParmParse const pp_warpx("warpx"); std::vector numprocs_in; utils::parser::queryArrWithParser( @@ -670,7 +627,7 @@ WarpX::ReadParameters () utils::parser::queryWithParser(pp_warpx, "cfl", cfl); pp_warpx.query("verbose", verbose); utils::parser::queryWithParser(pp_warpx, "regrid_int", regrid_int); - pp_warpx.query("do_subcycling", do_subcycling); + pp_warpx.query("do_subcycling", m_do_subcycling); pp_warpx.query("do_multi_J", do_multi_J); if (do_multi_J) { @@ -678,62 +635,31 @@ WarpX::ReadParameters () pp_warpx, "do_multi_J_n_depositions", do_multi_J_n_depositions); } pp_warpx.query("use_hybrid_QED", use_hybrid_QED); - pp_warpx.query("safe_guard_cells", safe_guard_cells); + pp_warpx.query("safe_guard_cells", m_safe_guard_cells); std::vector override_sync_intervals_string_vec = {"1"}; pp_warpx.queryarr("override_sync_intervals", override_sync_intervals_string_vec); override_sync_intervals = utils::parser::IntervalsParser(override_sync_intervals_string_vec); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1, + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_do_subcycling != 1 || max_level <= 1, "Subcycling method 1 only works for 2 levels."); ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); // queryWithParser returns 1 if argument zmax_plasma_to_compute_max_step is // specified by the user, 0 otherwise. - do_compute_max_step_from_zmax = utils::parser::queryWithParser( - pp_warpx, "zmax_plasma_to_compute_max_step", - zmax_plasma_to_compute_max_step); + if(auto temp = 0.0_rt; utils::parser::queryWithParser(pp_warpx, "zmax_plasma_to_compute_max_step",temp)){ + m_zmax_plasma_to_compute_max_step = temp; + } pp_warpx.query("compute_max_step_from_btd", compute_max_step_from_btd); - pp_warpx.query("do_moving_window", do_moving_window); - if (do_moving_window) - { - utils::parser::queryWithParser( - pp_warpx, "start_moving_window_step", start_moving_window_step); - utils::parser::queryWithParser( - pp_warpx, "end_moving_window_step", end_moving_window_step); - std::string s; - pp_warpx.get("moving_window_dir", s); - - if (s == "z" || s == "Z") { - moving_window_dir = WARPX_ZINDEX; - } -#if defined(WARPX_DIM_3D) - else if (s == "y" || s == "Y") { - moving_window_dir = 1; - } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) - else if (s == "x" || s == "X") { - moving_window_dir = 0; - } -#endif - - else { - WARPX_ABORT_WITH_MESSAGE("Unknown moving_window_dir: "+s); - } - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).isPeriodic(moving_window_dir) == 0, - "The problem must be non-periodic in the moving window direction"); - + if (do_moving_window) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + Geom(0).isPeriodic(moving_window_dir) == 0, + "The problem must be non-periodic in the moving window direction"); moving_window_x = geom[0].ProbLo(moving_window_dir); - - utils::parser::getWithParser( - pp_warpx, "moving_window_v", moving_window_v); - moving_window_v *= PhysConst::c; } m_p_ext_field_params = std::make_unique(pp_warpx); @@ -746,27 +672,13 @@ WarpX::ReadParameters () maxlevel_extEMfield_init = maxLevel(); pp_warpx.query("maxlevel_extEMfield_init", maxlevel_extEMfield_init); - electrostatic_solver_id = GetAlgorithmInteger(pp_warpx, "do_electrostatic"); + pp_warpx.query_enum_sloppy("do_electrostatic", electrostatic_solver_id, "-_"); // if an electrostatic solver is used, set the Maxwell solver to None if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { electromagnetic_solver_id = ElectromagneticSolverAlgo::None; } - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || - electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - // Note that with the relativistic version, these parameters would be - // input for each species. - utils::parser::queryWithParser( - pp_warpx, "self_fields_required_precision", self_fields_required_precision); - utils::parser::queryWithParser( - pp_warpx, "self_fields_absolute_tolerance", self_fields_absolute_tolerance); - utils::parser::queryWithParser( - pp_warpx, "self_fields_max_iters", self_fields_max_iters); - pp_warpx.query("self_fields_verbosity", self_fields_verbosity); - } - - poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); + pp_warpx.query_enum_sloppy("poisson_solver", poisson_solver_id, "-_"); #ifndef WARPX_DIM_3D WARPX_ALWAYS_ASSERT_WITH_MESSAGE( poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, @@ -777,6 +689,8 @@ WarpX::ReadParameters () poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, "To use the FFT Poisson solver, compile with WARPX_USE_FFT=ON."); #endif + utils::parser::queryWithParser(pp_warpx, "self_fields_max_iters", magnetostatic_solver_max_iters); + utils::parser::queryWithParser(pp_warpx, "self_fields_verbosity", magnetostatic_solver_verbosity); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ( @@ -786,42 +700,30 @@ WarpX::ReadParameters () "The FFT Poisson solver is not implemented in labframe-electromagnetostatic mode yet." ); - // Parse the input file for domain boundary potentials - const ParmParse pp_boundary("boundary"); - bool potential_specified = false; - // When reading the potential at the boundary from the input file, set this flag to true if any of the potential is specified - potential_specified |= pp_boundary.query("potential_lo_x", m_poisson_boundary_handler.potential_xlo_str); - potential_specified |= pp_boundary.query("potential_hi_x", m_poisson_boundary_handler.potential_xhi_str); - potential_specified |= pp_boundary.query("potential_lo_y", m_poisson_boundary_handler.potential_ylo_str); - potential_specified |= pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); - potential_specified |= pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); - potential_specified |= pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); -#if defined(AMREX_USE_EB) - potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); + [[maybe_unused]] bool const eb_enabled = EB::enabled(); +#if !defined(AMREX_USE_EB) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !eb_enabled, + "Embedded boundaries are requested via warpx.eb_enabled but were not compiled!" + ); #endif - m_boundary_potential_specified = potential_specified; - if (potential_specified & (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC)) { - ablastr::warn_manager::WMRecordWarning( - "Algorithms", - "The input script specifies the electric potential (phi) at the boundary, but \ - also uses the hybrid PIC solver based on Ohm’s law. When using this solver, the \ - electric potential does not have any impact on the simulation.", - ablastr::warn_manager::WarnPriority::low); - } - else if (potential_specified & (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None)) { - ablastr::warn_manager::WMRecordWarning( - "Algorithms", - "The input script specifies the electric potential (phi) at the boundary so \ - an initial Poisson solve will be performed.", - ablastr::warn_manager::WarnPriority::low); - } - m_poisson_boundary_handler.buildParsers(); #ifdef WARPX_DIM_RZ - pp_boundary.query("verboncoeur_axis_correction", verboncoeur_axis_correction); + const ParmParse pp_boundary("boundary"); + pp_boundary.query("verboncoeur_axis_correction", m_verboncoeur_axis_correction); #endif + // Read timestepping options utils::parser::queryWithParser(pp_warpx, "const_dt", m_const_dt); + utils::parser::queryWithParser(pp_warpx, "max_dt", m_max_dt); + std::vector dt_interval_vec = {"-1"}; + pp_warpx.queryarr("dt_update_interval", dt_interval_vec); + m_dt_update_interval = utils::parser::IntervalsParser(dt_interval_vec); + + // Filter defaults to true for the explicit scheme, and false for the implicit schemes + if (evolve_scheme != EvolveScheme::Explicit) { + use_filter = false; + } // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) @@ -852,27 +754,37 @@ WarpX::ReadParameters () use_kspace_filter = use_filter; use_filter = false; } - else // FDTD + else { - // Filter currently not working with FDTD solver in RZ geometry along R - // (see https://github.com/ECP-WarpX/WarpX/issues/1943) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!use_filter || filter_npass_each_dir[0] == 0, - "In RZ geometry with FDTD, filtering can only be apply along z. This can be controlled by setting warpx.filter_npass_each_dir"); + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::HybridPIC) { + // Filter currently not working with FDTD solver in RZ geometry along R + // (see https://github.com/ECP-WarpX/WarpX/issues/1943) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!use_filter || filter_npass_each_dir[0] == 0, + "In RZ geometry with FDTD, filtering can only be applied along z. This can be controlled by setting warpx.filter_npass_each_dir"); + } else { + if (use_filter && filter_npass_each_dir[0] > 0) { + ablastr::warn_manager::WMRecordWarning( + "HybridPIC ElectromagneticSolver", + "Radial Filtering in RZ is not currently using radial geometric weighting to conserve charge. Use at your own risk.", + ablastr::warn_manager::WarnPriority::low + ); + } + } } #endif utils::parser::queryWithParser( - pp_warpx, "num_mirrors", num_mirrors); - if (num_mirrors>0){ - mirror_z.resize(num_mirrors); + pp_warpx, "num_mirrors", m_num_mirrors); + if (m_num_mirrors>0){ + m_mirror_z.resize(m_num_mirrors); utils::parser::getArrWithParser( - pp_warpx, "mirror_z", mirror_z, 0, num_mirrors); - mirror_z_width.resize(num_mirrors); + pp_warpx, "mirror_z", m_mirror_z, 0, m_num_mirrors); + m_mirror_z_width.resize(m_num_mirrors); utils::parser::getArrWithParser( - pp_warpx, "mirror_z_width", mirror_z_width, 0, num_mirrors); - mirror_z_npoints.resize(num_mirrors); + pp_warpx, "mirror_z_width", m_mirror_z_width, 0, m_num_mirrors); + m_mirror_z_npoints.resize(m_num_mirrors); utils::parser::getArrWithParser( - pp_warpx, "mirror_z_npoints", mirror_z_npoints, 0, num_mirrors); + pp_warpx, "mirror_z_npoints", m_mirror_z_npoints, 0, m_num_mirrors); } pp_warpx.query("do_single_precision_comms", do_single_precision_comms); @@ -924,6 +836,7 @@ WarpX::ReadParameters () pp_warpx.query("do_dive_cleaning", do_dive_cleaning); pp_warpx.query("do_divb_cleaning", do_divb_cleaning); + utils::parser::queryWithParser( pp_warpx, "n_field_gather_buffer", n_field_gather_buffer); amrex::Vector nfieldgatherbuffer_eachdir(AMREX_SPACEDIM,n_field_gather_buffer); @@ -942,12 +855,15 @@ WarpX::ReadParameters () n_current_deposition_buffer_each_dir[i] = ncurrentdepositionbuffer_eachdir[i]; } + //Default value for the quantum parameter used in Maxwell’s QED equations + m_quantum_xi_c2 = PhysConst::xi_c2; + amrex::Real quantum_xi_tmp; const auto quantum_xi_is_specified = utils::parser::queryWithParser(pp_warpx, "quantum_xi", quantum_xi_tmp); if (quantum_xi_is_specified) { double const quantum_xi = quantum_xi_tmp; - quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); + m_quantum_xi_c2 = static_cast(quantum_xi * PhysConst::c * PhysConst::c); } const auto at_least_one_boundary_is_pml = @@ -1041,7 +957,7 @@ WarpX::ReadParameters () } #ifdef WARPX_DIM_RZ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( isAnyBoundaryPML() == false || electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ::isAnyBoundaryPML(field_boundary_lo, field_boundary_hi) == false || electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, "PML are not implemented in RZ geometry with FDTD; please set a different boundary condition using boundary.field_lo and boundary.field_hi."); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( field_boundary_lo[1] != FieldBoundaryType::PML && field_boundary_hi[1] != FieldBoundaryType::PML, "PML are not implemented in RZ geometry along z; please set a different boundary condition using boundary.field_lo and boundary.field_hi."); @@ -1108,7 +1024,7 @@ WarpX::ReadParameters () // Integer that corresponds to the type of grid used in the simulation // (collocated, staggered, hybrid) - grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); + pp_warpx.query_enum_sloppy("grid_type", grid_type, "-_"); // Use same shape factors in all directions, for gathering if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } @@ -1160,6 +1076,22 @@ WarpX::ReadParameters () "warpx.grid_type=hybrid is not implemented in RZ geometry"); #endif + // Update default to external projection divb cleaner if external fields are loaded, + // the grids are staggered, and the solver is compatible with the cleaner + if (!do_divb_cleaning + && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero + && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant + && grid_type != GridType::Collocated + && (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee + || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC + || ( (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame + || WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + && WarpX::poisson_solver_id == PoissonSolverAlgo::Multigrid))) + { + m_do_divb_cleaning_external = true; + } + pp_warpx.query("do_divb_cleaning_external", m_do_divb_cleaning_external); + // If true, the current is deposited on a nodal grid and centered onto // a staggered grid. Setting warpx.do_current_centering=1 makes sense // only if warpx.grid_type=hybrid. Instead, if warpx.grid_type=nodal or @@ -1204,7 +1136,10 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD, "algo.maxwell_solver = psatd is not supported because WarpX was built without spectral solvers"); #endif - +#if defined(WARPX_DIM_1D_Z) && defined(WARPX_USE_FFT) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD, + "algo.maxwell_solver = psatd is not available for 1D geometry"); +#endif #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).isPeriodic(0) == 0, "The problem must not be periodic in the radial direction"); @@ -1235,10 +1170,14 @@ WarpX::ReadParameters () // note: current_deposition must be set after maxwell_solver (electromagnetic_solver_id) or // do_electrostatic (electrostatic_solver_id) are already determined, // because its default depends on the solver selection - current_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "current_deposition")); - charge_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "charge_deposition")); - particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); - evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || + electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || + electrostatic_solver_id != ElectrostaticSolverAlgo::None) { + current_deposition_algo = CurrentDepositionAlgo::Direct; + } + pp_algo.query_enum_sloppy("current_deposition", current_deposition_algo, "-_"); + pp_algo.query_enum_sloppy("charge_deposition", charge_deposition_algo, "-_"); + pp_algo.query_enum_sloppy("particle_pusher", particle_pusher_algo, "-_"); // check for implicit evolve scheme if (evolve_scheme == EvolveScheme::SemiImplicitEM) { @@ -1247,11 +1186,14 @@ WarpX::ReadParameters () else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { m_implicit_solver = std::make_unique(); } + else if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + m_implicit_solver = std::make_unique(); + } // implicit evolve schemes not setup to use mirrors if (evolve_scheme == EvolveScheme::SemiImplicitEM || evolve_scheme == EvolveScheme::ThetaImplicitEM) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( num_mirrors == 0, + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_num_mirrors == 0, "Mirrors cannot be used with Implicit evolve schemes."); } @@ -1292,14 +1234,15 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM, + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } // Query algo.field_gathering from input, set field_gathering_algo to // "default" if not found (default defined in Utils/WarpXAlgorithmSelection.cpp) - field_gathering_algo = static_cast(GetAlgorithmInteger(pp_algo, "field_gathering")); + pp_algo.query_enum_sloppy("field_gathering", field_gathering_algo, "-_"); // Set default field gathering algorithm for hybrid grids (momentum-conserving) std::string tmp_algo; @@ -1356,25 +1299,28 @@ WarpX::ReadParameters () " combined with mesh refinement is currently not implemented"); } - em_solver_medium = GetAlgorithmInteger(pp_algo, "em_solver_medium"); - if (em_solver_medium == MediumForEM::Macroscopic ) { - macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); + pp_algo.query_enum_sloppy("em_solver_medium", m_em_solver_medium, "-_"); + if (m_em_solver_medium == MediumForEM::Macroscopic ) { + pp_algo.query_enum_sloppy("macroscopic_sigma_method", + macroscopic_solver_algo, "-_"); } - utils::parser::queryWithParser(pp_algo, "do_subcycle_current", do_subcycle_current); - if (do_subcycle_current) { - bool found_subcycle_current = utils::parser::queryWithParser(pp_algo, "n_subcycle_current", n_subcycle_current); - if (! found_subcycle_current) { - amrex::Abort(Utils::TextMsg::Err("If do_subcycle_current is true," - "n_subcycle_current must be specified")); - } - if (n_subcycle_current < 1) { - amrex::Abort(Utils::TextMsg::Err("n_subcycle_current must be greater than or equal to 1")); - } - } +// utils::parser::queryWithParser(pp_algo, "do_subcycle_current", do_subcycle_current); +// if (do_subcycle_current) { +// bool found_subcycle_current = utils::parser::queryWithParser(pp_algo, "n_subcycle_current", n_subcycle_current); +// if (! found_subcycle_current) { +// amrex::Abort(Utils::TextMsg::Err("If do_subcycle_current is true," +// "n_subcycle_current must be specified")); +// } +// if (n_subcycle_current < 1) { +// amrex::Abort(Utils::TextMsg::Err("n_subcycle_current must be greater than or equal to 1")); +// } +// } if (evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM) { + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || current_deposition_algo == CurrentDepositionAlgo::Villasenor || @@ -1383,8 +1329,9 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || - electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, - "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "Only the Yee, CKC, and PSATD EM solvers are supported with the implicit and semi-implicit schemes"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( particle_pusher_algo == ParticlePusherAlgo::Boris || @@ -1395,6 +1342,11 @@ WarpX::ReadParameters () field_gathering_algo != GatheringAlgo::MomentumConserving, "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "With the strang_implicit_spectral_em evolve scheme, the algo.maxwell_solver must be psatd"); + } // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; @@ -1411,7 +1363,7 @@ WarpX::ReadParameters () } utils::parser::queryWithParser(pp_algo, "load_balance_efficiency_ratio_threshold", load_balance_efficiency_ratio_threshold); - load_balance_costs_update_algo = static_cast(GetAlgorithmInteger(pp_algo, "load_balance_costs_update")); + pp_algo.query_enum_sloppy("load_balance_costs_update", load_balance_costs_update_algo, "-_"); if (WarpX::load_balance_costs_update_algo==LoadBalanceCostsUpdateAlgo::Heuristic) { utils::parser::queryWithParser( pp_algo, "costs_heuristic_cells_wt", costs_heuristic_cells_wt); @@ -1515,8 +1467,9 @@ WarpX::ReadParameters () // Instead, if warpx.grid_type=collocated, the momentum-conserving and // energy conserving field gathering algorithms are equivalent (forces // gathered from the collocated grid) and no fields centering occurs. - if (WarpX::field_gathering_algo == GatheringAlgo::MomentumConserving && - WarpX::grid_type != GridType::Collocated) + if ((WarpX::field_gathering_algo == GatheringAlgo::MomentumConserving + && WarpX::grid_type != GridType::Collocated) + || WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { utils::parser::queryWithParser( pp_warpx, "field_centering_nox", field_centering_nox); @@ -1582,14 +1535,14 @@ WarpX::ReadParameters () // Integer that corresponds to the order of the PSATD solution // (whether the PSATD equations are derived from first-order or // second-order solution) - psatd_solution_type = static_cast(GetAlgorithmInteger(pp_psatd, "solution_type")); + pp_psatd.query_enum_sloppy("solution_type", m_psatd_solution_type, "-_"); // Integers that correspond to the time dependency of J (constant, linear) // and rho (linear, quadratic) for the PSATD algorithm - J_in_time = static_cast(GetAlgorithmInteger(pp_psatd, "J_in_time")); - rho_in_time = static_cast(GetAlgorithmInteger(pp_psatd, "rho_in_time")); + pp_psatd.query_enum_sloppy("J_in_time", J_in_time, "-_"); + pp_psatd.query_enum_sloppy("rho_in_time", rho_in_time, "-_"); - if (psatd_solution_type != PSATDSolutionType::FirstOrder || !do_multi_J) + if (m_psatd_solution_type != PSATDSolutionType::FirstOrder || !do_multi_J) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( rho_in_time == RhoInTime::Linear, @@ -1823,6 +1776,9 @@ WarpX::ReadParameters () } } + // Setup pec_insulator boundary conditions + pec_insulator_boundary = std::make_unique(); + // for slice generation // { const ParmParse pp_slice("slice"); @@ -2115,60 +2071,15 @@ WarpX::MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::B void WarpX::ClearLevel (int lev) { - for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i].reset(); - Bfield_aux[lev][i].reset(); - - current_fp[lev][i].reset(); - Efield_fp [lev][i].reset(); - Bfield_fp [lev][i].reset(); - - current_store[lev][i].reset(); - - if (do_current_centering) - { - current_fp_nodal[lev][i].reset(); - } + m_fields.clear_level(lev); - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) - { - current_fp_vay[lev][i].reset(); - } - - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - vector_potential_fp_nodal[lev][i].reset(); - vector_potential_grad_buf_e_stag[lev][i].reset(); - vector_potential_grad_buf_b_stag[lev][i].reset(); - } - - current_cp[lev][i].reset(); - Efield_cp [lev][i].reset(); - Bfield_cp [lev][i].reset(); - - Efield_cax[lev][i].reset(); - Bfield_cax[lev][i].reset(); - current_buf[lev][i].reset(); - } - - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) - { - m_hybrid_pic_model->ClearLevel(lev); + for (int i = 0; i < 3; ++i) { + Efield_dotMask [lev][i].reset(); + Bfield_dotMask [lev][i].reset(); + Afield_dotMask [lev][i].reset(); } - charge_buf[lev].reset(); - - current_buffer_masks[lev].reset(); - gather_buffer_masks[lev].reset(); -// interp_weight_gbuffer[lev].reset(); - - F_fp [lev].reset(); - G_fp [lev].reset(); - rho_fp[lev].reset(); - phi_fp[lev].reset(); - F_cp [lev].reset(); - G_cp [lev].reset(); - rho_cp[lev].reset(); + phi_dotMask[lev].reset(); #ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -2198,7 +2109,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d guard_cells.Init( dt[lev], dx, - do_subcycling, + m_do_subcycling, WarpX::use_fdtd_nci_corr, grid_type, do_moving_window, @@ -2210,25 +2121,28 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d maxLevel(), WarpX::m_v_galilean, WarpX::m_v_comoving, - safe_guard_cells, + m_safe_guard_cells, WarpX::do_multi_J, WarpX::fft_do_time_averaging, - WarpX::isAnyBoundaryPML(), + ::isAnyBoundaryPML(field_boundary_lo, field_boundary_hi), WarpX::do_pml_in_domain, WarpX::pml_ncell, this->refRatio(), use_filter, bilinear_filter.stencil_length_each_dir); - #ifdef AMREX_USE_EB - int max_guard = guard_cells.ng_FieldSolver.max(); + bool const eb_enabled = EB::enabled(); + if (eb_enabled) { + int const max_guard = guard_cells.ng_FieldSolver.max(); m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, {max_guard, max_guard, max_guard}, amrex::EBSupport::full); -#else - m_field_factory[lev] = std::make_unique(); + } else #endif + { + m_field_factory[lev] = std::make_unique(); + } if (mypc->nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { @@ -2260,7 +2174,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); m_accelerator_lattice[lev] = std::make_unique(); - m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); + m_accelerator_lattice[lev]->InitElementFinder(lev, gamma_boost, ba, dm); } @@ -2269,6 +2183,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const IntVect& ngEB, IntVect& ngJ, const IntVect& ngRho, const IntVect& ngF, const IntVect& ngG, const bool aux_is_nodal) { + using ablastr::fields::Direction; + // Declare nodal flags IntVect Ex_nodal_flag, Ey_nodal_flag, Ez_nodal_flag; IntVect Bx_nodal_flag, By_nodal_flag, Bz_nodal_flag; @@ -2370,76 +2286,71 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // const std::array dx = CellSize(lev); - AllocInitMultiFab(Bfield_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(current_fp[lev][0], amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[x]", 0.0_rt); - AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); - AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - AllocInitMultiFab(current_fp_nodal[lev][0], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[x]", 0.0_rt); - AllocInitMultiFab(current_fp_nodal[lev][1], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[y]", 0.0_rt); - AllocInitMultiFab(current_fp_nodal[lev][2], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[z]", 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{0}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{1}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{2}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - AllocInitMultiFab(current_fp_vay[lev][0], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[x]", 0.0_rt); - AllocInitMultiFab(current_fp_vay[lev][1], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[y]", 0.0_rt); - AllocInitMultiFab(current_fp_vay[lev][2], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[z]", 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - AllocInitMultiFab(vector_potential_fp_nodal[lev][0], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_fp_nodal[lev][1], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_fp_nodal[lev][2], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[z]", 0.0_rt); - - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][0], amrex::convert(ba, Ex_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][1], amrex::convert(ba, Ey_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][2], amrex::convert(ba, Ez_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[z]", 0.0_rt); - - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][0], amrex::convert(ba, Bx_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][1], amrex::convert(ba, By_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][2], amrex::convert(ba, Bz_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[z]", 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + + // Memory buffers for computing magnetostatic fields + // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Allocate extra multifabs needed by the kinetic-fluid hybrid algorithm. if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { m_hybrid_pic_model->AllocateLevelMFs( - lev, ba, dm, ncomps, ngJ, ngRho, jx_nodal_flag, jy_nodal_flag, - jz_nodal_flag, rho_nodal_flag + m_fields, + lev, ba, dm, ncomps, ngJ, ngRho, ngEB, jx_nodal_flag, jy_nodal_flag, + jz_nodal_flag, rho_nodal_flag, Ex_nodal_flag, Ey_nodal_flag, Ez_nodal_flag, + Bx_nodal_flag, By_nodal_flag, Bz_nodal_flag ); } // Allocate extra multifabs needed for fluids if (do_fluid_species) { - myfl->AllocateLevelMFs(lev, ba, dm); + myfl->AllocateLevelMFs(m_fields, ba, dm, lev); auto & warpx = GetInstance(); const amrex::Real cur_time = warpx.gett_new(lev); - myfl->InitData(lev, geom[lev].Domain(),cur_time); + myfl->InitData(m_fields, geom[lev].Domain(), cur_time, lev); } // Allocate extra multifabs for macroscopic properties of the medium - if (em_solver_medium == MediumForEM::Macroscopic) { + if (m_em_solver_medium == MediumForEM::Macroscopic) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( lev==0, "Macroscopic properties are not supported with mesh refinement."); m_macroscopic_properties->AllocateLevelMFs(ba, dm, ngEB); @@ -2447,64 +2358,122 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[z]", 0.0_rt); - - AllocInitMultiFab(Efield_avg_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[x]", 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[y]", 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[z]", 0.0_rt); - } - -#ifdef AMREX_USE_EB - constexpr int nc_ls = 1; - amrex::IntVect ng_ls(2); - AllocInitMultiFab(m_distance_to_eb[lev], amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, lev, "m_distance_to_eb"); - - // EB info are needed only at the finest level - if (lev == maxLevel()) - { - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); - } - if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); - AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); - AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[y]"); - AllocInitMultiFab(m_flag_info_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[z]"); - AllocInitMultiFab(m_flag_ext_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[x]"); - AllocInitMultiFab(m_flag_ext_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[y]"); - AllocInitMultiFab(m_flag_ext_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[z]"); - AllocInitMultiFab(m_area_mod[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[x]"); - AllocInitMultiFab(m_area_mod[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[y]"); - AllocInitMultiFab(m_area_mod[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[z]"); - m_borrowing[lev][0] = std::make_unique>(amrex::convert(ba, Bx_nodal_flag), dm); - m_borrowing[lev][1] = std::make_unique>(amrex::convert(ba, By_nodal_flag), dm); - m_borrowing[lev][2] = std::make_unique>(amrex::convert(ba, Bz_nodal_flag), dm); - AllocInitMultiFab(Venl[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[x]"); - AllocInitMultiFab(Venl[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[y]"); - AllocInitMultiFab(Venl[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[z]"); - - AllocInitMultiFab(ECTRhofield[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[x]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[y]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + } + + if (EB::enabled()) { + constexpr int nc_ls = 1; + amrex::IntVect const ng_ls(2); + //EB level set + m_fields.alloc_init(FieldType::distance_to_eb, lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); + // Whether to reduce the particle shape to order 1 when close to the EB + AllocInitMultiFab(m_eb_reduce_particle_shape[lev], amrex::convert(ba, IntVect::TheCellVector()), dm, ncomps, + ngRho, lev, "m_eb_reduce_particle_shape"); + + // EB info are needed only at the finest level + if (lev == maxLevel()) { + + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + + AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[x]"); + AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[y]"); + AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); + + AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[x]"); + AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[y]"); + AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[z]"); + } + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + + //! EB: Lengths of the mesh edges + m_fields.alloc_init(FieldType::edge_lengths, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::edge_lengths, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::edge_lengths, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + + //! EB: Areas of the mesh faces + m_fields.alloc_init(FieldType::face_areas, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::face_areas, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::face_areas, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + + AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); + AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[y]"); + AllocInitMultiFab(m_flag_info_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[z]"); + AllocInitMultiFab(m_flag_ext_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[x]"); + AllocInitMultiFab(m_flag_ext_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[y]"); + AllocInitMultiFab(m_flag_ext_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[z]"); + + /** EB: area_mod contains the modified areas of the mesh faces, i.e. if a face is enlarged it + * contains the area of the enlarged face + * This is only used for the ECT solver.*/ + m_fields.alloc_init(FieldType::area_mod, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::area_mod, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::area_mod, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + + m_borrowing[lev][0] = std::make_unique>( + amrex::convert(ba, Bx_nodal_flag), dm); + m_borrowing[lev][1] = std::make_unique>( + amrex::convert(ba, By_nodal_flag), dm); + m_borrowing[lev][2] = std::make_unique>( + amrex::convert(ba, Bz_nodal_flag), dm); + + /** Venl contains the electromotive force for every mesh face, i.e. every entry is + * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) + * This is only used for the ECT solver.*/ + m_fields.alloc_init(FieldType::Venl, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::Venl, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::Venl, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + + /** ECTRhofield is needed only by the ect + * solver and it contains the electromotive force density for every mesh face. + * The name ECTRhofield has been used to comply with the notation of the paper + * https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=4463918 (page 9, equation 4 + * and below). + * Although it's called rho it has nothing to do with the charge density! + * This is only used for the ECT solver.*/ + m_fields.alloc_init(FieldType::ECTRhofield, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::ECTRhofield, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init(FieldType::ECTRhofield, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + } } } -#endif int rho_ncomps = 0; if( (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) || + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameEffectivePotential) || (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) ) { rho_ncomps = ncomps; } @@ -2519,31 +2488,39 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (rho_ncomps > 0) { - AllocInitMultiFab(rho_fp[lev], amrex::convert(ba, rho_nodal_flag), dm, rho_ncomps, ngRho, lev, "rho_fp", 0.0_rt); + m_fields.alloc_init(FieldType::rho_fp, + lev, amrex::convert(ba, rho_nodal_flag), dm, + rho_ncomps, ngRho, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || - electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic || + electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameEffectivePotential ) { const IntVect ngPhi = IntVect( AMREX_D_DECL(1,1,1) ); - AllocInitMultiFab(phi_fp[lev], amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, lev, "phi_fp", 0.0_rt); + m_fields.alloc_init(FieldType::phi_fp, lev, amrex::convert(ba, phi_nodal_flag), dm, + ncomps, ngPhi, 0.0_rt ); } - if (do_subcycling && lev == 0) + if (m_do_subcycling && lev == 0) { - AllocInitMultiFab(current_store[lev][0], amrex::convert(ba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_store[x]"); - AllocInitMultiFab(current_store[lev][1], amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_store[y]"); - AllocInitMultiFab(current_store[lev][2], amrex::convert(ba,jz_nodal_flag),dm,ncomps,ngJ,lev, "current_store[z]"); + m_fields.alloc_init(FieldType::current_store, Direction{0}, lev, amrex::convert(ba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_store, Direction{1}, lev, amrex::convert(ba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_store, Direction{2}, lev, amrex::convert(ba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (do_dive_cleaning) { - AllocInitMultiFab(F_fp[lev], amrex::convert(ba, F_nodal_flag), dm, ncomps, ngF, lev, "F_fp", 0.0_rt); + m_fields.alloc_init(FieldType::F_fp, + lev, amrex::convert(ba, F_nodal_flag), dm, + ncomps, ngF, 0.0_rt); } if (do_divb_cleaning) { - AllocInitMultiFab(G_fp[lev], amrex::convert(ba, G_nodal_flag), dm, ncomps, ngG, lev, "G_fp", 0.0_rt); + m_fields.alloc_init(FieldType::G_fp, + lev, amrex::convert(ba, G_nodal_flag), dm, + ncomps, ngG, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -2612,90 +2589,103 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // Create aux multifabs on Nodal Box Array BoxArray const nba = amrex::convert(ba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_aux[lev][0], nba, dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], nba, dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); } else if (lev == 0) { if (WarpX::fft_do_time_averaging) { - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_avg_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_avg_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_avg_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{2}, lev, 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{2}, lev, 0.0_rt); } else { if (mypc->m_B_ext_particle_s == "read_from_file") { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{2}, lev, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{2}, lev, 0.0_rt); } } } else { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // The external fields that are read from file if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, Bfield_fp[lev][1]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bfield_fp[lev][2]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{0}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{0},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{1}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{1},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{2}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{2},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux[lev][0]->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux[lev][1]->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); + auto *Bfield_aux_levl_0 = m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + auto *Bfield_aux_levl_1 = m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + auto *Bfield_aux_levl_2 = m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); + + // Same as Bfield_fp for reading external field data + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{1}, lev, amrex::convert(ba, Bfield_aux_levl_1->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{2}, lev, amrex::convert(ba, Bfield_aux_levl_2->ixType()), + dm, ncomps, ngEB, 0.0_rt); } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Efield_fp[lev][1]->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Efield_fp[lev][2]->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{0}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{1}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{2}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux[lev][0]->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux[lev][1]->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux[lev][2]->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); + auto *Efield_aux_levl_0 = m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + auto *Efield_aux_levl_1 = m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + auto *Efield_aux_levl_2 = m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + + // Same as Efield_fp for reading external field data + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{1}, lev, amrex::convert(ba, Efield_aux_levl_1->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{2}, lev, amrex::convert(ba, Efield_aux_levl_2->ixType()), + dm, ncomps, ngEB, 0.0_rt); } // @@ -2708,49 +2698,57 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[x]", 0.0_rt); - AllocInitMultiFab(Efield_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[y]", 0.0_rt); - AllocInitMultiFab(Efield_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[x]", 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[y]", 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Create the MultiFabs for the current - AllocInitMultiFab(current_cp[lev][0], amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[x]", 0.0_rt); - AllocInitMultiFab(current_cp[lev][1], amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[y]", 0.0_rt); - AllocInitMultiFab(current_cp[lev][2], amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[z]", 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{0}, lev, amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{1}, lev, amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{2}, lev, amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (rho_ncomps > 0) { - AllocInitMultiFab(rho_cp[lev], amrex::convert(cba, rho_nodal_flag), dm, rho_ncomps, ngRho, lev, "rho_cp", 0.0_rt); + m_fields.alloc_init(FieldType::rho_cp, + lev, amrex::convert(cba, rho_nodal_flag), dm, + rho_ncomps, ngRho, 0.0_rt); } if (do_dive_cleaning) { - AllocInitMultiFab(F_cp[lev], amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngF, lev, "F_cp", 0.0_rt); + m_fields.alloc_init(FieldType::F_cp, + lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, + ncomps, ngF, 0.0_rt); } if (do_divb_cleaning) { if (grid_type == GridType::Collocated) { - AllocInitMultiFab(G_cp[lev], amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngG, lev, "G_cp", 0.0_rt); + m_fields.alloc_init(FieldType::G_cp, + lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, + ncomps, ngG, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { - AllocInitMultiFab(G_cp[lev], amrex::convert(cba, IntVect::TheZeroVector()), dm, ncomps, ngG, lev, "G_cp", 0.0_rt); + m_fields.alloc_init(FieldType::G_cp, + lev, amrex::convert(cba, IntVect::TheZeroVector()), dm, + ncomps, ngG, 0.0_rt); } } @@ -2809,22 +2807,25 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (n_field_gather_buffer > 0 || mypc->nSpeciesGatherFromMainGrid() > 0) { if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); + // Create the MultiFabs for B + m_fields.alloc_init(FieldType::Bfield_cax, Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + + // Create the MultiFabs for E + m_fields.alloc_init(FieldType::Efield_cax, Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); } else { // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{0}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{1}, lev, amrex::convert(cba,Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{2}, lev, amrex::convert(cba,Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } amrex::Print() << " ba for allocating gahter buffer mask " << "\n"; // Gather buffer masks have 1 ghost cell, because of the fact @@ -2834,12 +2835,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (n_current_deposition_buffer > 0) { - amrex::Print() << " current dep buffer : " << n_current_deposition_buffer << "\n"; - AllocInitMultiFab(current_buf[lev][0], amrex::convert(cba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[x]"); - AllocInitMultiFab(current_buf[lev][1], amrex::convert(cba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[y]"); - AllocInitMultiFab(current_buf[lev][2], amrex::convert(cba,jz_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[z]"); - if (rho_cp[lev]) { - AllocInitMultiFab(charge_buf[lev], amrex::convert(cba,rho_nodal_flag),dm,2*ncomps,ngRho,lev, "charge_buf"); + m_fields.alloc_init(FieldType::current_buf, Direction{0}, lev, amrex::convert(cba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_buf, Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_buf, Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + if (m_fields.has(FieldType::rho_cp, lev)) { + m_fields.alloc_init(FieldType::rho_buf, lev, amrex::convert(cba,rho_nodal_flag), dm, 2*ncomps, ngRho, 0.0_rt); } amrex::Print() << " allocate current buffer mask \n"; AllocInitMultiFab(current_buffer_masks[lev], ba, dm, ncomps, ngJ, lev, "current_buffer_masks"); @@ -2877,6 +2877,10 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, @@ -2887,7 +2891,7 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, @@ -2946,7 +2954,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector& B, + ablastr::fields::VectorField const& B, const std::array& dx) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(grid_type != GridType::Collocated, - "ComputeDivB not implemented with warpx.grid_type=Collocated."); - - const Real dxinv = 1._rt/dx[0], dyinv = 1._rt/dx[1], dzinv = 1._rt/dx[2]; - -#ifdef WARPX_DIM_RZ - const Real rmin = GetInstance().Geom(0).ProbLo(0); -#endif - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(divB, TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); - amrex::Array4 const& Bxfab = B[0]->array(mfi); - amrex::Array4 const& Byfab = B[1]->array(mfi); - amrex::Array4 const& Bzfab = B[2]->array(mfi); - amrex::Array4 const& divBfab = divB.array(mfi); - - ParallelFor(bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept - { - warpx_computedivb(i, j, k, dcomp, divBfab, Bxfab, Byfab, Bzfab, dxinv, dyinv, dzinv -#ifdef WARPX_DIM_RZ - ,rmin -#endif - ); - }); - } + ComputeDivB(divB, dcomp, B, dx, IntVect::TheZeroVector()); } void WarpX::ComputeDivB (amrex::MultiFab& divB, int const dcomp, - const std::array& B, + ablastr::fields::VectorField const& B, const std::array& dx, IntVect const ngrow) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(grid_type != GridType::Collocated, @@ -3120,13 +3099,15 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { #ifdef WARPX_USE_FFT - spectral_solver_fp[lev]->ComputeSpectralDivE( lev, Efield_aux[lev], divE ); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs(FieldType::Efield_aux, lev); + spectral_solver_fp[lev]->ComputeSpectralDivE(lev, Efield_aux_lev, divE); #else WARPX_ABORT_WITH_MESSAGE( "ComputeDivE: PSATD requested but not compiled"); #endif } else { - m_fdtd_solver_fp[lev]->ComputeDivE( Efield_aux[lev], divE ); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs(FieldType::Efield_aux, lev); + m_fdtd_solver_fp[lev]->ComputeDivE(Efield_aux_lev, divE); } } @@ -3272,62 +3253,6 @@ WarpX::BuildBufferMasksInBox ( const amrex::Box tbx, amrex::IArrayBox &buffer_ma }); } -amrex::Vector WarpX::getFornbergStencilCoefficients (const int n_order, ablastr::utils::enums::GridType a_grid_type) -{ - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n_order % 2 == 0, "n_order must be even"); - - const int m = n_order / 2; - amrex::Vector coeffs; - coeffs.resize(m); - - // There are closed-form formula for these coefficients, but they result in - // an overflow when evaluated numerically. One way to avoid the overflow is - // to calculate the coefficients by recurrence. - - // Coefficients for collocated (nodal) finite-difference approximation - if (a_grid_type == GridType::Collocated) - { - // First coefficient - coeffs.at(0) = m * 2._rt / (m+1); - // Other coefficients by recurrence - for (int n = 1; n < m; n++) - { - coeffs.at(n) = - (m-n) * 1._rt / (m+n+1) * coeffs.at(n-1); - } - } - // Coefficients for staggered finite-difference approximation - else - { - Real prod = 1.; - for (int k = 1; k < m+1; k++) - { - prod *= (m + k) / (4._rt * k); - } - // First coefficient - coeffs.at(0) = 4_rt * m * prod * prod; - // Other coefficients by recurrence - for (int n = 1; n < m; n++) - { - coeffs.at(n) = - ((2_rt*n-1) * (m-n)) * 1._rt / ((2_rt*n+1) * (m+n)) * coeffs.at(n-1); - } - } - - return coeffs; -} - -void WarpX::ReorderFornbergCoefficients (amrex::Vector& ordered_coeffs, - amrex::Vector& unordered_coeffs, - const int order) -{ - const int n = order / 2; - for (int i = 0; i < n; i++) { - ordered_coeffs[i] = unordered_coeffs[n-1-i]; - } - for (int i = n; i < order; i++) { - ordered_coeffs[i] = unordered_coeffs[i-n]; - } -} - void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector& device_centering_stencil_coeffs_x, amrex::Gpu::DeviceVector& device_centering_stencil_coeffs_y, amrex::Gpu::DeviceVector& device_centering_stencil_coeffs_z, @@ -3346,9 +3271,9 @@ void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector amrex::Vector host_centering_stencil_coeffs_y; amrex::Vector host_centering_stencil_coeffs_z; - Fornberg_stencil_coeffs_x = getFornbergStencilCoefficients(centering_nox, a_grid_type); - Fornberg_stencil_coeffs_y = getFornbergStencilCoefficients(centering_noy, a_grid_type); - Fornberg_stencil_coeffs_z = getFornbergStencilCoefficients(centering_noz, a_grid_type); + Fornberg_stencil_coeffs_x = ablastr::math::getFornbergStencilCoefficients(centering_nox, a_grid_type); + Fornberg_stencil_coeffs_y = ablastr::math::getFornbergStencilCoefficients(centering_noy, a_grid_type); + Fornberg_stencil_coeffs_z = ablastr::math::getFornbergStencilCoefficients(centering_noz, a_grid_type); host_centering_stencil_coeffs_x.resize(centering_nox); host_centering_stencil_coeffs_y.resize(centering_noy); @@ -3356,12 +3281,21 @@ void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector // Re-order Fornberg stencil coefficients: // example for order 6: (c_0,c_1,c_2) becomes (c_2,c_1,c_0,c_0,c_1,c_2) - ReorderFornbergCoefficients(host_centering_stencil_coeffs_x, - Fornberg_stencil_coeffs_x, centering_nox); - ReorderFornbergCoefficients(host_centering_stencil_coeffs_y, - Fornberg_stencil_coeffs_y, centering_noy); - ReorderFornbergCoefficients(host_centering_stencil_coeffs_z, - Fornberg_stencil_coeffs_z, centering_noz); + ablastr::math::ReorderFornbergCoefficients( + host_centering_stencil_coeffs_x, + Fornberg_stencil_coeffs_x, + centering_nox + ); + ablastr::math::ReorderFornbergCoefficients( + host_centering_stencil_coeffs_y, + Fornberg_stencil_coeffs_y, + centering_noy + ); + ablastr::math::ReorderFornbergCoefficients( + host_centering_stencil_coeffs_z, + Fornberg_stencil_coeffs_z, + centering_noz + ); // Device vectors of stencil coefficients used for finite-order centering @@ -3401,10 +3335,12 @@ WarpX::GatherBufferMasks (int lev) void WarpX::StoreCurrent (int lev) { + using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { - MultiFab::Copy(*current_store[lev][idim], *current_fp[lev][idim], - 0, 0, 1, current_store[lev][idim]->nGrowVect()); + if (m_fields.has(FieldType::current_store, Direction{idim},lev)) { + MultiFab::Copy(*m_fields.get(FieldType::current_store, Direction{idim}, lev), + *m_fields.get(FieldType::current_fp, Direction{idim}, lev), + 0, 0, 1, m_fields.get(FieldType::current_store, Direction{idim}, lev)->nGrowVect()); } } } @@ -3412,23 +3348,19 @@ WarpX::StoreCurrent (int lev) void WarpX::RestoreCurrent (int lev) { + using ablastr::fields::Direction; + using warpx::fields::FieldType; + for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { - std::swap(current_fp[lev][idim], current_store[lev][idim]); + if (m_fields.has(FieldType::current_store, Direction{idim}, lev)) { + std::swap( + *m_fields.get(FieldType::current_fp, Direction{idim}, lev), + *m_fields.get(FieldType::current_store, Direction{idim}, lev) + ); } } } -bool -WarpX::isAnyBoundaryPML() -{ - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) { return true; } - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) { return true; } - } - return false; -} - bool WarpX::isAnyParticleBoundaryThermal () { @@ -3439,34 +3371,6 @@ WarpX::isAnyParticleBoundaryThermal () return false; } -std::string -TagWithLevelSuffix (std::string name, int const level) -{ - // Add the suffix "[level=level]" - name.append("[level=").append(std::to_string(level)).append("]"); - return name; -} - -void -WarpX::AllocInitMultiFab ( - std::unique_ptr& mf, - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm, - const int ncomp, - const amrex::IntVect& ngrow, - const int level, - const std::string& name, - std::optional initial_value) -{ - const auto name_with_suffix = TagWithLevelSuffix(name, level); - const auto tag = amrex::MFInfo().SetTag(name_with_suffix); - mf = std::make_unique(ba, dm, ncomp, ngrow, tag); - if (initial_value) { - mf->setVal(*initial_value); - } - multifab_map[name_with_suffix] = mf.get(); -} - void WarpX::AllocInitMultiFab ( std::unique_ptr& mf, @@ -3478,7 +3382,8 @@ WarpX::AllocInitMultiFab ( const std::string& name, std::optional initial_value) { - const auto name_with_suffix = TagWithLevelSuffix(name, level); + // Add the suffix "[level=level]" + const auto name_with_suffix = name + "[level=" + std::to_string(level) + "]"; const auto tag = amrex::MFInfo().SetTag(name_with_suffix); mf = std::make_unique(ba, dm, ncomp, ngrow, tag); if (initial_value) { @@ -3487,191 +3392,52 @@ WarpX::AllocInitMultiFab ( imultifab_map[name_with_suffix] = mf.get(); } -void -WarpX::AliasInitMultiFab ( - std::unique_ptr& mf, - const amrex::MultiFab& mf_to_alias, - const int scomp, - const int ncomp, - const int level, - const std::string& name, - std::optional initial_value) +amrex::DistributionMapping +WarpX::MakeDistributionMap (int lev, amrex::BoxArray const& ba) { - const auto name_with_suffix = TagWithLevelSuffix(name, level); - mf = std::make_unique(mf_to_alias, amrex::make_alias, scomp, ncomp); - if (initial_value) { - mf->setVal(*initial_value); - } - multifab_map[name_with_suffix] = mf.get(); -} - -void -WarpX::AllocInitMultiFabFromModel ( - std::unique_ptr& mf, - amrex::MultiFab& mf_model, - const int level, - const std::string& name, - std::optional initial_value) -{ - const auto name_with_suffix = TagWithLevelSuffix(name, level); - const auto tag = amrex::MFInfo().SetTag(name_with_suffix); - mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), - mf_model.nComp(), mf_model.nGrowVect(), tag); - if (initial_value) { - mf->setVal(*initial_value); - } - multifab_map[name_with_suffix] = mf.get(); -} - -amrex::MultiFab* -WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const -{ - // This function does *not* check if the returned field pointer is != nullptr - - amrex::MultiFab* field_pointer = nullptr; - - switch(field_type) - { - case FieldType::Efield_aux : - field_pointer = Efield_aux[lev][direction].get(); - break; - case FieldType::Bfield_aux : - field_pointer = Bfield_aux[lev][direction].get(); - break; - case FieldType::Efield_fp : - field_pointer = Efield_fp[lev][direction].get(); - break; - case FieldType::Bfield_fp : - field_pointer = Bfield_fp[lev][direction].get(); - break; - case FieldType::current_fp : - field_pointer = current_fp[lev][direction].get(); - break; - case FieldType::current_fp_nodal : - field_pointer = current_fp_nodal[lev][direction].get(); - break; - case FieldType::rho_fp : - field_pointer = rho_fp[lev].get(); - break; - case FieldType::F_fp : - field_pointer = F_fp[lev].get(); - break; - case FieldType::G_fp : - field_pointer = G_fp[lev].get(); - break; - case FieldType::phi_fp : - field_pointer = phi_fp[lev].get(); - break; - case FieldType::vector_potential_fp : - field_pointer = vector_potential_fp_nodal[lev][direction].get(); - break; - case FieldType::Efield_cp : - field_pointer = Efield_cp[lev][direction].get(); - break; - case FieldType::Bfield_cp : - field_pointer = Bfield_cp[lev][direction].get(); - break; - case FieldType::current_cp : - field_pointer = current_cp[lev][direction].get(); - break; - case FieldType::rho_cp : - field_pointer = rho_cp[lev].get(); - break; - case FieldType::F_cp : - field_pointer = F_cp[lev].get(); - break; - case FieldType::G_cp : - field_pointer = G_cp[lev].get(); - break; - case FieldType::edge_lengths : - field_pointer = m_edge_lengths[lev][direction].get(); - break; - case FieldType::face_areas : - field_pointer = m_face_areas[lev][direction].get(); - break; - case FieldType::Efield_avg_fp : - field_pointer = Efield_avg_fp[lev][direction].get(); - break; - case FieldType::Bfield_avg_fp : - field_pointer = Bfield_avg_fp[lev][direction].get(); - break; - case FieldType::Efield_avg_cp : - field_pointer = Efield_avg_cp[lev][direction].get(); - break; - case FieldType::Bfield_avg_cp : - field_pointer = Bfield_avg_cp[lev][direction].get(); - break; - default: - WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - break; + bool roundrobin_sfc = false; + const ParmParse pp("warpx"); + pp.query("roundrobin_sfc", roundrobin_sfc); + + // If this is true, AMReX's RRSFC strategy is used to make + // DistributionMapping. Note that the DistributionMapping made by the + // here could still be overridden by load balancing. In the RRSFC + // strategy, the Round robin method is used to distribute Boxes orderd + // by the space filling curve. This might help avoid some processes + // running out of memory due to having too many particles during + // initialization. + + if (roundrobin_sfc) { + auto old_strategy = amrex::DistributionMapping::strategy(); + amrex::DistributionMapping::strategy(amrex::DistributionMapping::RRSFC); + amrex::DistributionMapping dm(ba); + amrex::DistributionMapping::strategy(old_strategy); + return dm; + } else { + return amrex::AmrCore::MakeDistributionMap(lev, ba); } - - return field_pointer; -} - -bool -WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const -{ - const bool is_field_init = (getFieldPointerUnchecked(field_type, lev, direction) != nullptr); - return is_field_init; -} - -amrex::MultiFab* -WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const -{ - auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - field_pointer != nullptr, "Requested field is not initialized!"); - return field_pointer; -} - -std::array -WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const -{ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (field_type == FieldType::Efield_aux) || (field_type == FieldType::Bfield_aux) || - (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || - (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || - (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || - (field_type == FieldType::current_cp), "Requested field type is not a vector."); - - return std::array{ - getFieldPointer(field_type, lev, 0), - getFieldPointer(field_type, lev, 1), - getFieldPointer(field_type, lev, 2)}; -} - -const amrex::MultiFab& -WarpX::getField(FieldType field_type, const int lev, const int direction) const -{ - return *getFieldPointer(field_type, lev, direction); } -const amrex::Vector,3>>& -WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const +const amrex::iMultiFab* +WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, ablastr::fields::Direction dir ) const { + const auto periodicity = Geom(lev).periodicity(); switch(field_type) { - case FieldType::Efield_aux : - return Efield_aux; - case FieldType::Bfield_aux : - return Bfield_aux; case FieldType::Efield_fp : - return Efield_fp; + ::SetDotMask( Efield_dotMask[lev][dir], m_fields.get("Efield_fp", dir, lev), periodicity); + return Efield_dotMask[lev][dir].get(); case FieldType::Bfield_fp : - return Bfield_fp; - case FieldType::current_fp : - return current_fp; - case FieldType::current_fp_nodal : - return current_fp_nodal; - case FieldType::Efield_cp : - return Efield_cp; - case FieldType::Bfield_cp : - return Bfield_cp; - case FieldType::current_cp : - return current_cp; + ::SetDotMask( Bfield_dotMask[lev][dir], m_fields.get("Bfield_fp", dir, lev), periodicity); + return Bfield_dotMask[lev][dir].get(); + case FieldType::vector_potential_fp : + ::SetDotMask( Afield_dotMask[lev][dir], m_fields.get("vector_potential_fp", dir, lev), periodicity); + return Afield_dotMask[lev][dir].get(); + case FieldType::phi_fp : + ::SetDotMask( phi_dotMask[lev], m_fields.get("phi_fp", dir, lev), periodicity); + return phi_dotMask[lev].get(); default: - WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - return Efield_fp; + WARPX_ABORT_WITH_MESSAGE("Invalid field type for dotMask"); + return Efield_dotMask[lev][dir].get(); } } diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package index b9ff3c72560..edbf43b7802 100644 --- a/Source/ablastr/Make.package +++ b/Source/ablastr/Make.package @@ -1,4 +1,3 @@ -#CEXE_sources += ParticleBoundaries.cpp include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package include $(WARPX_HOME)/Source/ablastr/math/Make.package diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index 56acc678217..011d765a6bb 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,5 +1,11 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) + + target_sources(ablastr_${SD} + PRIVATE + MultiFabRegister.cpp + ) + if(ABLASTR_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE diff --git a/Source/ablastr/fields/EffectivePotentialPoissonSolver.H b/Source/ablastr/fields/EffectivePotentialPoissonSolver.H new file mode 100644 index 00000000000..80e899df027 --- /dev/null +++ b/Source/ablastr/fields/EffectivePotentialPoissonSolver.H @@ -0,0 +1,274 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +/* + * This file was copied and edited from PoissonSolver.H in the same directory. + */ +#ifndef ABLASTR_EFFECTIVE_POTENTIAL_POISSON_SOLVER_H +#define ABLASTR_EFFECTIVE_POTENTIAL_POISSON_SOLVER_H + +#include +#include +#include +#include +#include +#include +#include +#include "PoissonSolver.H" + +#if defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D) +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef AMREX_USE_EB +# include +#endif + +#include +#include + + +namespace ablastr::fields { + +/** Compute the potential `phi` by solving the Poisson equation with a modifed dielectric function + * + * Uses `rho` as a source. This uses the AMReX solver. + * + * More specifically, this solves the equation + * \f[ + * \nabla \cdot \sigma \nabla \phi = - \rho/\epsilon_0 + * \f] + * + * \tparam T_PostPhiCalculationFunctor a calculation per level directly after phi was calculated + * \tparam T_BoundaryHandler handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler + * \tparam T_FArrayBoxFactory usually nothing or an amrex::EBFArrayBoxFactory (EB ONLY) + * \param[in] rho The charge density a given species + * \param[out] phi The potential to be computed by this function + * \param[in] sigma The matrix representing the mass operator used to lower the local plasma frequency + * \param[in] relative_tolerance The relative convergence threshold for the MLMG solver + * \param[in] absolute_tolerance The absolute convergence threshold for the MLMG solver + * \param[in] max_iters The maximum number of iterations allowed for the MLMG solver + * \param[in] verbosity The verbosity setting for the MLMG solver + * \param[in] geom the geometry per level (e.g., from AmrMesh) + * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) + * \param[in] grids the grids per level (e.g., from AmrMesh) + * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels + * \param[in] do_single_precision_comms perform communications in single precision + * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) + * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) + * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler + * \param[in] current_time the current time; required for embedded boundaries (default: none) + * \param[in] eb_farray_box_factory a factory for field data, @see amrex::EBFArrayBoxFactory; required for embedded boundaries (default: none) + */ +template< + typename T_PostPhiCalculationFunctor = std::nullopt_t, + typename T_BoundaryHandler = std::nullopt_t, + typename T_FArrayBoxFactory = void +> +void +computeEffectivePotentialPhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + amrex::MultiFab const & sigma, + amrex::Real relative_tolerance, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, + [[maybe_unused]] utils::enums::GridType grid_type, + bool is_solver_igf_on_lev0, + bool eb_enabled = false, + bool do_single_precision_comms = false, + std::optional > rel_ref_ratio = std::nullopt, + [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, + [[maybe_unused]] T_BoundaryHandler const boundary_handler = std::nullopt, + [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB + [[maybe_unused]] std::optional > eb_farray_box_factory = std::nullopt // only used for EB +) { + using namespace amrex::literals; + + ABLASTR_PROFILE("computeEffectivePotentialPhi"); + + if (!rel_ref_ratio.has_value()) { + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, + "rel_ref_ratio must be set if mesh-refinement is used"); + rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; + } + +#if !defined(AMREX_USE_EB) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, + "Embedded boundary solve requested but not compiled in"); +#endif + if (eb_enabled && std::is_same_v) { + throw std::runtime_error("EB requested by eb_farray_box_factory not provided!"); + } + + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "FFT solver cannot be used with effective potential Poisson solve"); + +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; +#endif + + auto const finest_level = static_cast(rho.size() - 1); + + // determine if rho is zero everywhere + const amrex::Real max_norm_b = getMaxNormRho( + amrex::GetVecOfConstPtrs(rho), finest_level, absolute_tolerance); + + const amrex::LPInfo info; + + for (int lev=0; lev<=finest_level; lev++) { + + // Use the Multigrid (MLMG) solver but first scale rho appropriately + using namespace ablastr::constant::SI; + rho[lev]->mult(-1._rt/ep0); + + std::unique_ptr linop; + // In the presence of EB or RZ the EB enabled linear solver is used + if (eb_enabled) + { +#if defined(AMREX_USE_EB) + auto linop_nodelap = std::make_unique(); + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info, + amrex::Vector{eb_farray_box_factory.value()[lev]} + ); + if constexpr (!std::is_same_v) { + // if the EB potential only depends on time, the potential can be passed + // as a float instead of a callable + if (boundary_handler.phi_EB_only_t) { + linop_nodelap->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); + } else { + linop_nodelap->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); + } + } + linop_nodelap->setSigma(lev, sigma); + linop = std::move(linop_nodelap); +#endif + } + else if (is_rz) + { + auto linop_nodelap = std::make_unique(); + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + linop_nodelap->setRZ(true); + linop_nodelap->setSigma(lev, sigma); + linop = std::move(linop_nodelap); + } + else + { + auto linop_nodelap = std::make_unique(); + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + linop_nodelap->setSigma(lev, sigma); + linop = std::move(linop_nodelap); + } + + // Set domain boundary conditions + if constexpr (std::is_same_v) { + amrex::Array const lobc = {AMREX_D_DECL( + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet + )}; + amrex::Array const hibc = lobc; + linop->setDomainBC(lobc, hibc); + } else { + linop->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + } + + // Solve the Poisson equation + amrex::MLMG mlmg(*linop); // actual solver defined here + mlmg.setVerbose(verbosity); + mlmg.setMaxIter(max_iters); + mlmg.setAlwaysUseBNorm((max_norm_b > 0)); + + const int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells + if (ng) { + // In this case, computeE needs to use ghost nodes data. So we + // ask MLMG to fill BC for us after it solves the problem. + mlmg.setFinalFillBC(true); + } + + // Solve Poisson equation at lev + mlmg.solve( {phi[lev]}, {rho[lev]}, + relative_tolerance, absolute_tolerance ); + + // needed for solving the levels by levels: + // - coarser level is initial guess for finer level + // - coarser level provides boundary values for finer level patch + // Interpolation from phi[lev] to phi[lev+1] + // (This provides both the boundary conditions and initial guess for phi[lev+1]) + if (lev < finest_level) { + const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; + const int ncomp = linop->getNComp(); + interpolatePhiBetweenLevels(phi[lev], + phi[lev+1], + geom[lev], + do_single_precision_comms, + refratio, + ncomp, + ng); + } + + // Run additional operations, such as calculation of the E field for embedded boundaries + if constexpr (!std::is_same_v) { + if (post_phi_calculation.has_value()) { + post_phi_calculation.value()(mlmg, lev); + } + } + rho[lev]->mult(-ep0); // Multiply rho by epsilon again + } // loop over lev(els) +} + +} // namespace ablastr::fields + +#endif // ABLASTR_EFFECTIVE_POTENTIAL_POISSON_SOLVER_H diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H old mode 100644 new mode 100755 index 97ffdb5ac36..b34678055a8 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H @@ -7,19 +7,22 @@ #ifndef ABLASTR_IGF_SOLVER_H #define ABLASTR_IGF_SOLVER_H +#include + #include #include #include #include #include - #include #include namespace ablastr::fields { + using namespace amrex::literals; + /** @brief Implements equation 2 in https://doi.org/10.1103/PhysRevSTAB.10.129901 * with some modification to symmetrize the function. @@ -28,25 +31,92 @@ namespace ablastr::fields * @param[in] y y-coordinate of given location * @param[in] z z-coordinate of given location * - * @return the integrated Green function G + * @return the integrated Green function G in 3D */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real - IntegratedPotential (amrex::Real x, amrex::Real y, amrex::Real z) + IntegratedPotential3D (amrex::Real x, amrex::Real y, amrex::Real z) { - using namespace amrex::literals; - amrex::Real const r = std::sqrt( x*x + y*y + z*z ); - amrex::Real const G = - - 0.5_rt * z*z * std::atan( x*y/(z*r) ) - - 0.5_rt * y*y * std::atan( x*z/(y*r) ) - - 0.5_rt * x*x * std::atan( y*z/(x*r) ) - + y*z*std::asinh( x/std::sqrt(y*y + z*z) ) - + x*z*std::asinh( y/std::sqrt(x*x + z*z) ) - + x*y*std::asinh( z/std::sqrt(x*x + y*y) ); + amrex::Real const G = - 0.5_rt * z*z * std::atan( x*y/(z*r) ) + - 0.5_rt * y*y * std::atan( x*z/(y*r) ) + - 0.5_rt * x*x * std::atan( y*z/(x*r) ) + + y*z*std::asinh( x/std::sqrt(y*y + z*z) ) + + x*z*std::asinh( y/std::sqrt(x*x + z*z) ) + + x*y*std::asinh( z/std::sqrt(x*x + y*y) ); + return G; + } + + + /** @brief Implements equation 58 in https://doi.org/10.1016/j.jcp.2004.01.008 + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * + * @return the integrated Green function G in 2D + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + IntegratedPotential2D (amrex::Real x, amrex::Real y) + { + amrex::Real const G = 3_rt*x*y + - x*x * std::atan(y/x) + - y*y * std::atan(x/y) + - x*y * std::log(x*x + y*y); return G; } + + /** @brief add + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] z z-coordinate of given location + * @param[in] dx cell size along x + * @param[in] dy cell size along y + * @param[in] dz cell size along z + * + * @return the sum of integrated Green function G in 3D + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + SumOfIntegratedPotential3D (amrex::Real x, amrex::Real y, amrex::Real z, amrex::Real dx, amrex::Real dy, amrex::Real dz) + { + return 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential3D( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential3D( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential3D( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + + IntegratedPotential3D( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential3D( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential3D( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential3D( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + - IntegratedPotential3D( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + ); + } + + + /** @brief add + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] dx cell size along x + * @param[in] dy cell size along y + * + * @return the sum of integrated Green function G in 2D + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + SumOfIntegratedPotential2D (amrex::Real x, amrex::Real y, amrex::Real dx, amrex::Real dy) + { + return 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential2D( x+0.5_rt*dx, y+0.5_rt*dy ) + - IntegratedPotential2D( x+0.5_rt*dx, y-0.5_rt*dy ) + - IntegratedPotential2D( x-0.5_rt*dx, y+0.5_rt*dy ) + + IntegratedPotential2D( x-0.5_rt*dx, y-0.5_rt*dy ) + ); + } + + /** @brief Compute the electrostatic potential using the Integrated Green Function method * as in http://dx.doi.org/10.1103/PhysRevSTAB.9.044204 * @@ -54,12 +124,14 @@ namespace ablastr::fields * @param[out] phi the electrostatic potential amrex::MultiFab * @param[in] cell_size an arreay of 3 reals dx dy dz * @param[in] ba amrex::BoxArray with the grid of a given level + * @param[in] is_igf_2d boolean to select between fully 3D Poisson solver and quasi-3D, i.e. one 2D Poisson solve on every z slice (default: false) */ void computePhiIGF (amrex::MultiFab const & rho, amrex::MultiFab & phi, std::array const & cell_size, - amrex::BoxArray const & ba); + amrex::BoxArray const & ba, + bool is_igf_2d_slices); } // namespace ablastr::fields diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp old mode 100644 new mode 100755 index 0767ecfb2f3..31d8136e175 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -18,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -25,172 +25,81 @@ #include #include #include +#include #include -#include - - namespace ablastr::fields { void computePhiIGF ( amrex::MultiFab const & rho, amrex::MultiFab & phi, std::array const & cell_size, - amrex::BoxArray const & ba ) + amrex::BoxArray const & ba, + bool const is_igf_2d_slices) { using namespace amrex::literals; + BL_PROFILE("ablastr::fields::computePhiIGF"); + // Define box that encompasses the full domain amrex::Box domain = ba.minimalBox(); domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal domain.grow( phi.nGrowVect() ); // include guard cells - int const nx = domain.length(0); - int const ny = domain.length(1); - int const nz = domain.length(2); - - // Allocate 2x wider arrays for the convolution of rho with the Green function - // This also defines the box arrays for the global FFT: contains only one box; - amrex::Box const realspace_box = amrex::Box( - {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, - {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, - amrex::IntVect::TheNodeVector() ); - amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); - amrex::Box const spectralspace_box = amrex::Box( - {0,0,0}, - {nx, 2*ny-1, 2*nz-1}, - amrex::IntVect::TheNodeVector() ); - amrex::BoxArray const spectralspace_ba = amrex::BoxArray( spectralspace_box ); - // Define a distribution mapping for the global FFT, with only one box - amrex::DistributionMapping dm_global_fft; - dm_global_fft.define( realspace_ba ); - // Allocate required arrays - amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_rho.setVal(0); - amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_G.setVal(0); - // Allocate corresponding arrays in Fourier space - using SpectralField = amrex::FabArray< amrex::BaseFab< amrex::GpuComplex< amrex::Real > > >; - SpectralField tmp_rho_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); - SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); - - // Copy from rho to tmp_rho - tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); - - // Compute the integrated Green function + int nprocs = amrex::ParallelDescriptor::NProcs(); { - BL_PROFILE("Initialize Green function"); - amrex::BoxArray const domain_ba = amrex::BoxArray( domain ); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi(domain_ba, dm_global_fft,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - amrex::Box const bx = mfi.tilebox(); - - amrex::IntVect const lo = realspace_box.smallEnd(); - amrex::IntVect const hi = realspace_box.bigEnd(); - - // Fill values of the Green function - amrex::Real const dx = cell_size[0]; - amrex::Real const dy = cell_size[1]; - amrex::Real const dz = cell_size[2]; - amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); - amrex::ParallelFor( bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept - { - int const i0 = i - lo[0]; - int const j0 = j - lo[1]; - int const k0 = k - lo[2]; - amrex::Real const x = i0*dx; - amrex::Real const y = j0*dy; - amrex::Real const z = k0*dz; - - amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) - ); - - tmp_G_arr(i,j,k) = G_value; - // Fill the rest of the array by periodicity - if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} - if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} - if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} - if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} - if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} - } - ); - } + amrex::ParmParse pp("ablastr"); + pp.queryAdd("nprocs_igf_fft", nprocs); + nprocs = std::max(1,std::min(nprocs, amrex::ParallelDescriptor::NProcs())); } - // Perform forward FFTs - auto forward_plan_rho = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - auto forward_plan_G = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - // Loop over boxes perform FFTs - for ( amrex::MFIter mfi(realspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ - - // Note: the size of the real-space box and spectral-space box - // differ when using real-to-complex FFT. When initializing - // the FFT plan, the valid dimensions are those of the real-space box. - const amrex::IntVect fft_size = realspace_ba[mfi].length(); - - // FFT of rho - forward_plan_rho[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_rho[mfi].dataPtr(), - reinterpret_cast(tmp_rho_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(forward_plan_rho[mfi]); - - // FFT of G - forward_plan_G[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[mfi].dataPtr(), - reinterpret_cast(tmp_G_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(forward_plan_G[mfi]); - + static std::unique_ptr> obc_solver; + if (!obc_solver) { + amrex::ExecOnFinalize([&] () { obc_solver.reset(); }); + } + if (!obc_solver || obc_solver->Domain() != domain) { + amrex::FFT::Info info{}; + if (is_igf_2d_slices) { info.setTwoDMode(true); } // do 2D FFTs + info.setNumProcs(nprocs); + obc_solver = std::make_unique>(domain, info); } - // Multiply tmp_G_fft and tmp_rho_fft in spectral space - // Store the result in-place in Gtmp_G_fft, to save memory - amrex::Multiply( tmp_G_fft, tmp_rho_fft, 0, 0, 1, 0); - - // Perform inverse FFT - auto backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - // Loop over boxes perform FFTs - for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ - - // Note: the size of the real-space box and spectral-space box - // differ when using real-to-complex FFT. When initializing - // the FFT plan, the valid dimensions are those of the real-space box. - const amrex::IntVect fft_size = realspace_ba[mfi].length(); + auto const& lo = domain.smallEnd(); + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + + if (!is_igf_2d_slices){ + // fully 3D solver + obc_solver->setGreensFunction( + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::Real + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + + return SumOfIntegratedPotential3D(x, y, z, dx, dy, dz); + }); + }else{ + // 2D sliced solver + obc_solver->setGreensFunction( + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::Real + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::ignore_unused(k); + + return SumOfIntegratedPotential2D(x, y, dx, dy); + }); - // Inverse FFT: is done in-place, in the array of G - backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[mfi].dataPtr(), - reinterpret_cast( tmp_G_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(backward_plan[mfi]); } - // Normalize, since (FFT + inverse FFT) results in a factor N - const amrex::Real normalization = 1._rt / realspace_box.numPts(); - tmp_G.mult( normalization ); - // Copy from tmp_G to phi - phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect() ); + obc_solver->solve(phi, rho); +} // computePhiIGF - // Loop to destroy FFT plans - for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ - ablastr::math::anyfft::DestroyPlan(forward_plan_G[mfi]); - ablastr::math::anyfft::DestroyPlan(forward_plan_rho[mfi]); - ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); - } -} } // namespace ablastr::fields diff --git a/Source/ablastr/fields/Interpolate.H b/Source/ablastr/fields/Interpolate.H index a9f0a7fc75f..dc4ad47df94 100644 --- a/Source/ablastr/fields/Interpolate.H +++ b/Source/ablastr/fields/Interpolate.H @@ -11,12 +11,9 @@ #include #include -#include - #include #include - namespace ablastr::fields::details { /** Local interpolation from phi_cp to phi[lev+1] @@ -46,9 +43,9 @@ namespace ablastr::fields::details { 0, m_refratio); } - amrex::Array4 const m_phi_fp_arr; - amrex::Array4 const m_phi_cp_arr; - amrex::IntVect const m_refratio; + amrex::Array4 m_phi_fp_arr; + amrex::Array4 m_phi_cp_arr; + amrex::IntVect m_refratio; }; } // namespace ablastr::fields::details diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package index 01392991559..7441a6a1238 100644 --- a/Source/ablastr/fields/Make.package +++ b/Source/ablastr/fields/Make.package @@ -1,3 +1,6 @@ + +CEXE_sources += MultiFabRegister.cpp + ifeq ($(USE_FFT),TRUE) ifeq ($(DIM),3) CEXE_sources += IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H new file mode 100644 index 00000000000..11cf932c12c --- /dev/null +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -0,0 +1,840 @@ +/* Copyright 2024 The ABLASTR Community + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl + */ +#ifndef ABLASTR_FIELDS_MF_REGISTER_H +#define ABLASTR_FIELDS_MF_REGISTER_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace +{ + // type trait helpers in lieu of an amrex::is_amrex_enum + template > + struct is_castable_to_string : std::false_type {}; + + template + struct is_castable_to_string(std::declval()))>> : std::true_type {}; + + /** helper to either cast a string/char array to string to query an AMREX_ENUM */ + template + std::string getExtractedName (T name) + { + if constexpr(is_castable_to_string()) + { + // already a unique string key + return std::string(name); + } else + { + // user-defined AMREX_ENUM or compile error + return amrex::getEnumNameString(name); + } + } +} + +namespace ablastr::fields +{ + /** Components (base vector directions) of vector/tensor fields. + * + * Because of different staggering, the components of vector/tensor fields are stored + * in separate (i)MultiFab. + * + * @todo: synchronize with AMReX "enum class Direction" + */ + struct Direction + { + int dir = 0; + + bool operator<(const Direction& other) const + { + return other.dir < this->dir; + } + + /* TODO: just temporary int compatibility */ + operator int() const { return dir; } + }; + + /** A scalar field (a MultiFab) + * + * Note: might still have components, e.g., for copies at different times. + */ + using ScalarField = amrex::MultiFab*; + + /** A read-only scalar field (a MultiFab) + * + * Note: might still have components, e.g., for copies at different times. + */ + using ConstScalarField = amrex::MultiFab const *; + + /** A vector field of three MultiFab + */ + //using VectorField = ablastr::utils::ConstMap; + using VectorField = std::array; + + /** A read-only vector field of three MultiFab + */ + //using VectorField = ablastr::utils::ConstMap; + using ConstVectorField = std::array; + + /** A multi-level scalar field + */ + using MultiLevelScalarField = amrex::Vector; + + /** A read-only multi-level scalar field + */ + using ConstMultiLevelScalarField = amrex::Vector; + + /** A multi-level vector field + */ + using MultiLevelVectorField = amrex::Vector; + + /** A read-only multi-level vector field + */ + using ConstMultiLevelVectorField = amrex::Vector; + + /** A class to control the lifetime and properties of a MultiFab (field). + * + * This class is used to own the lifetime of an amrex::MultiFab and to store + * associated information around it regarding unique naming, scalar/vector/tensor + * properties, aliasing, load balancing, etc. + */ + struct MultiFabOwner + { + // TODO: also add iMultiFab via std::variant + + /** owned (i)MultiFab */ + amrex::MultiFab m_mf; + + /** Components (base vector directions) of this MultiFab */ + std::optional m_dir = std::nullopt; + + /** the MR level of this (i)MultiFab */ + int m_level = 0; + + /** remake distribution map on load balance, @see amrex::AmrCore::RemakeLevel */ + bool m_remake = true; + + /** redistribute on @see amrex::AmrCore::RemakeLevel */ + bool m_redistribute_on_remake = true; + + /** if m_mf is a non-owning alias, this string tracks the name of the owner */ + std::string m_owner; + + /** Is this part of a vector/tensor? */ + AMREX_INLINE + bool + is_vector () const { return m_dir.has_value(); } + + /** Is this an alias MultiFab? + * + * If yes, that means we do not own the memory. + */ + AMREX_INLINE + bool + is_alias () const { return !m_owner.empty(); } + }; + + /** This is a register of fields aka amrex::MultiFabs. + * + * This is owned by a simulation instance. All used fields should be registered here. + * Internally, this contains @see MultiFabOwner values. + */ + struct MultiFabRegister + { + // Avoid accidental copies when passing to member functions + MultiFabRegister() = default; + MultiFabRegister(MultiFabRegister const &) = delete; + MultiFabRegister(MultiFabRegister&&) = delete; + MultiFabRegister& operator=(MultiFabRegister const &) = delete; + MultiFabRegister& operator=(MultiFabRegister&&) = delete; + ~MultiFabRegister() = default; + + /** Allocate and optionally initialize a MultiFab (field) + * + * This registers a new MultiFab under a unique name, allocates it and + * optionally assigns it an initial value. + * + * @param name a unique name for this field + * @param level the MR level to represent + * @param ba the list of boxes to cover the field + * @param dm the distribution mapping for load balancing with MPI + * @param ncomp the number of components of the field (all with the same staggering) + * @param ngrow the number of guard (ghost, halo) cells + * @param initial_value the optional initial value + * @param remake follow the default domain decomposition of the simulation + * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel + * @return pointer to newly allocated MultiFab + */ + template + amrex::MultiFab* + alloc_init ( + T name, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ) + { + return internal_alloc_init( + getExtractedName(name), + level, + ba, + dm, + ncomp, + ngrow, + initial_value, + remake, + redistribute_on_remake + ); + } + + /** Allocate and optionally initialize a MultiFab (field) + * + * This registers a new MultiFab under a unique name, allocates it and + * optionally assigns it an initial value. + * + * @param name a unique name for this field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to represent + * @param ba the list of boxes to cover the field + * @param dm the distribution mapping for load balancing with MPI + * @param ncomp the number of components of the field (all with the same staggering) + * @param ngrow the number of guard (ghost, halo) cells + * @param initial_value the optional initial value + * @param remake follow the default domain decomposition of the simulation + * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel + * @return pointer to newly allocated MultiFab + */ + template + amrex::MultiFab* + alloc_init ( + T name, + Direction dir, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ) + { + return internal_alloc_init( + getExtractedName(name), + dir, + level, + ba, + dm, + ncomp, + ngrow, + initial_value, + remake, + redistribute_on_remake + ); + } + + /** Create an alias of a MultiFab (field) + * + * Registers a new name for an existing MultiFab (field) and optionally assigning + * a value. + * + * @param new_name new name + * @param alias_name owner name to alias + * @param level the MR level to represent + * @param initial_value the optional value to assign + * @return the newly aliased MultiFab + */ + template + amrex::MultiFab* + alias_init ( + N new_name, + A alias_name, + int level, + std::optional initial_value = std::nullopt + ) + { + return internal_alias_init( + getExtractedName(new_name), + getExtractedName(alias_name), + level, + initial_value + ); + } + + /** Create an alias of a MultiFab (field) + * + * Registers a new name for an existing MultiFab (field) and optionally assigning + * a value. + * + * @param new_name new name + * @param alias_name owner name to alias + * @param dir the field component for vector fields ("direction" of the unit vector) both in the alias and aliased + * @param level the MR level to represent + * @param initial_value the optional value to assign + * @return the newly aliased MultiFab + */ + template + amrex::MultiFab* + alias_init ( + N new_name, + A alias_name, + Direction dir, + int level, + std::optional initial_value = std::nullopt + ) + { + return internal_alias_init( + getExtractedName(new_name), + getExtractedName(alias_name), + dir, + level, + initial_value + ); + } + + /** Check if a scalar MultiFab (field) is registered. + * + * @param name the name to check if registered + * @param level the MR level to check + * @return true if contained, otherwise false + */ + template + [[nodiscard]] bool + has ( + T name, + int level + ) const + { + return internal_has( + getExtractedName(name), + level + ); + } + + /** Check if a MultiFab that is part of a vector/tensor field is registered. + * + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check + * @return true if contained, otherwise false + */ + template + [[nodiscard]] bool + has ( + T name, + Direction dir, + int level + ) const + { + return internal_has( + getExtractedName(name), + dir, + level + ); + } + + /** Check if a MultiFab vector field is registered. + * + * @param name the name to check if registered + * @param level the MR level to check + * @return true if contained, otherwise false + */ + template + [[nodiscard]] bool + has_vector ( + T name, + int level + ) const + { + return internal_has_vector( + getExtractedName(name), + level + ); + } + + /** Return a scalar MultiFab (field). + * + * This throws a runtime error if the requested field is not present. + * + * @param name the name of the field + * @param level the MR level + * @return a non-owning pointer to the MultiFab (field) + */ + template + [[nodiscard]] amrex::MultiFab* + get ( + T name, + int level + ) + { + return internal_get( + getExtractedName(name), + level + ); + } + + /** Return a MultiFab that is part of a vector/tensor field. + * + * This throws a runtime error if the requested field is not present. + * + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level + * @return a non-owning pointer to the MultiFab (field) + */ + template + [[nodiscard]] amrex::MultiFab* + get ( + T name, + Direction dir, + int level + ) + { + return internal_get( + getExtractedName(name), + dir, + level + ); + } + + /** Return a scalar MultiFab (field). + * + * This throws a runtime error if the requested field is not present. + * + * @param name the name of the field + * @param level the MR level + * @return a non-owning pointer to the MultiFab (field) + */ + template + [[nodiscard]] amrex::MultiFab const * + get ( + T name, + int level + ) const + { + return internal_get( + getExtractedName(name), + level + ); + } + + /** Return a MultiFab that is part of a vector/tensor field. + * + * This throws a runtime error if the requested field is not present. + * + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level + * @return a non-owning pointer to the MultiFab (field) + */ + template + [[nodiscard]] amrex::MultiFab const * + get ( + T name, + Direction dir, + int level + ) const + { + return internal_get( + getExtractedName(name), + dir, + level + ); + } + + /** Return the MultiFab of a scalar field on all MR levels. + * + * This throws a runtime error if the requested field is not present. + * + * @param name the name of the field + * @param finest_level the highest MR level to return + * @param skip_level_0 return a nullptr for level 0 + * @return non-owning pointers to the MultiFab (field) on all levels + */ + //@{ + template + [[nodiscard]] MultiLevelScalarField + get_mr_levels ( + T name, + int finest_level, + bool skip_level_0=false + ) + { + return internal_get_mr_levels( + getExtractedName(name), + finest_level, + skip_level_0 + ); + } + template + [[nodiscard]] ConstMultiLevelScalarField + get_mr_levels ( + T name, + int finest_level, + bool skip_level_0=false + ) const + { + return internal_get_mr_levels( + getExtractedName(name), + finest_level, + skip_level_0 + ); + } + //@} + + /** title + * + * Same as get above, but returns all levels for a name. + * + * @param name the name of the field + * @param level the MR level + * @return non-owning pointers to all components of a vector field + */ + //@{ + template + [[nodiscard]] VectorField + get_alldirs ( + T name, + int level + ) + { + return internal_get_alldirs( + getExtractedName(name), + level + ); + } + template + [[nodiscard]] ConstVectorField + get_alldirs ( + T name, + int level + ) const + { + return internal_get_alldirs( + getExtractedName(name), + level + ); + } + //@} + + /** Return a vector field on all MR levels. + * + * Out loop: MR levels. + * Inner loop: directions (components). + * + * @param name the name of the field + * @param finest_level the highest MR level to return + * @param skip_level_0 return a nullptr for level 0 + * @return non-owning pointers to all components of a vector field on all MR levels + */ + //@{ + template + [[nodiscard]] MultiLevelVectorField + get_mr_levels_alldirs ( + T name, + int finest_level, + bool skip_level_0=false + ) + { + return internal_get_mr_levels_alldirs( + getExtractedName(name), + finest_level, + skip_level_0 + ); + } + template + [[nodiscard]] ConstMultiLevelVectorField + get_mr_levels_alldirs ( + T name, + int finest_level, + bool skip_level_0=false + ) const + { + return internal_get_mr_levels_alldirs( + getExtractedName(name), + finest_level, + skip_level_0 + ); + } + //@} + + /** List the internal names of all registered fields. + * + * @return all currently allocated and registered fields + */ + [[nodiscard]] std::vector + list () const; + + /** Deallocate and remove a scalar field. + * + * @param name the name of the field + * @param level the MR level + */ + template + void + erase ( + T name, + int level + ) + { + internal_erase(getExtractedName(name), level); + } + + /** Deallocate and remove a vector field component. + * + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level + */ + template + void + erase ( + T name, + Direction dir, + int level + ) + { + internal_erase(getExtractedName(name), dir, level); + } + + /** Erase all MultiFabs on a specific MR level. + * + * Calls @see erase for all MultiFabs on a specific level. + * + * @param level the MR level to erase all MultiFabs from + */ + void + clear_level ( + int level + ); + + /** Remake all (i)MultiFab with a new distribution mapping. + * + * If redistribute is true, we also copy from the old data into the new. + * + * @param level the MR level to erase all MultiFabs from + * @param new_dm new distribution mapping + */ + void + remake_level ( + int other_level, + amrex::DistributionMapping const & new_dm + ); + + /** Create the register name of scalar field and MR level + * + * @param name the name of the field + * @param level the MR level + * @return internal name of the field in the register + */ + [[nodiscard]] std::string + mf_name ( + std::string name, + int level + ) const; + + /** Create the register name of vector field, component direction and MR level + * + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level + * @return internal name of the field in the register + */ + [[nodiscard]] std::string + mf_name ( + std::string name, + Direction dir, + int level + ) const; + + /** Temporary test function for legacy Python bindings */ + [[nodiscard]] bool + internal_has ( + std::string const & internal_name + ); + [[nodiscard]] amrex::MultiFab * + internal_get ( + std::string const & internal_name + ); + + private: + + [[nodiscard]] amrex::MultiFab const * + internal_get ( + std::string const & internal_name + ) const; + + amrex::MultiFab* + internal_alloc_init ( + std::string const & name, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ); + amrex::MultiFab* + internal_alloc_init ( + std::string const & name, + Direction dir, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ); + + amrex::MultiFab* + internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + int level, + std::optional initial_value = std::nullopt + ); + amrex::MultiFab* + internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + Direction dir, + int level, + std::optional initial_value = std::nullopt + ); + + [[nodiscard]] bool + internal_has ( + std::string const & name, + int level + ) const; + [[nodiscard]] bool + internal_has ( + std::string const & name, + Direction dir, + int level + ) const; + [[nodiscard]] bool + internal_has_vector ( + std::string const & name, + int level + ) const; + + [[nodiscard]] amrex::MultiFab * + internal_get ( + std::string const & name, + int level + ); + [[nodiscard]] amrex::MultiFab const * + internal_get ( + std::string const & name, + int level + ) const; + [[nodiscard]] amrex::MultiFab * + internal_get ( + std::string const & name, + Direction dir, + int level + ); + [[nodiscard]] amrex::MultiFab const * + internal_get ( + std::string const & name, + Direction dir, + int level + ) const; + [[nodiscard]] MultiLevelScalarField + internal_get_mr_levels ( + std::string const & name, + int finest_level, + bool skip_level_0 + ); + [[nodiscard]] ConstMultiLevelScalarField + internal_get_mr_levels ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) const; + [[nodiscard]] VectorField + internal_get_alldirs ( + std::string const & name, + int level + ); + [[nodiscard]] ConstVectorField + internal_get_alldirs ( + std::string const & name, + int level + ) const; + [[nodiscard]] MultiLevelVectorField + internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level, + bool skip_level_0 + ); + [[nodiscard]] ConstMultiLevelVectorField + internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) const; + + void + internal_erase ( + std::string const & name, + int level + ); + void + internal_erase ( + std::string const & name, + Direction dir, + int level + ); + + /** data storage: ownership and lifetime control */ + std::map< + std::string, + MultiFabOwner + > m_mf_register; + + /** the three directions of a vector field */ + std::vector m_all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + }; + + /** Little temporary helper function to pass temporary MultiFabs as VectorField. + * + * @return pointers to externally managed vector field components (3 MultiFab) + */ + VectorField + a2m ( + std::array< std::unique_ptr, 3 > const & old_vectorfield + ); + +} // namespace ablastr::fields + +#endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp new file mode 100644 index 00000000000..a1266deeab0 --- /dev/null +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -0,0 +1,662 @@ +/* Copyright 2024 The ABLASTR Community + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl + */ +#include "MultiFabRegister.H" + +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace ablastr::fields +{ + amrex::MultiFab* + MultiFabRegister::internal_alloc_init ( + std::string const & name, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value, + bool remake, + bool redistribute_on_remake + ) + { + // checks + if (has(name, level)) { + throw std::runtime_error("MultiFabRegister::alloc_init failed because " + name + " already exists."); + } + + // fully qualified name + std::string const internal_name = mf_name(name, level); + + // allocate + const auto tag = amrex::MFInfo().SetTag(internal_name); + auto [it, success] = m_mf_register.emplace( + internal_name, + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + std::nullopt, // scalar: no direction + level, + remake, + redistribute_on_remake, + "" // we own the memory + } + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + internal_name); + } + + // a shorthand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + amrex::MultiFab* + MultiFabRegister::internal_alloc_init ( + std::string const & name, + Direction dir, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value, + bool remake, + bool redistribute_on_remake + ) + { + // checks + if (has(name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alloc_init failed because " + + mf_name(name, dir, level) + + " already exists." + ); + } + + // fully qualified name + std::string const internal_name = mf_name(name, dir, level); + + // allocate + const auto tag = amrex::MFInfo().SetTag(internal_name); + auto [it, success] = m_mf_register.emplace( + internal_name, + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + dir, + level, + remake, + redistribute_on_remake, + "" // we own the memory + } + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + internal_name); + } + + // a shorthand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + amrex::MultiFab* + MultiFabRegister::internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + int level, + std::optional initial_value + ) + { + // checks + if (has(new_name, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(new_name, level) + + " already exists." + ); + } + if (!has(alias_name, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(alias_name, level) + + " does not exist." + ); + } + + // fully qualified name + std::string const internal_new_name = mf_name(new_name, level); + std::string const internal_alias_name = mf_name(alias_name, level); + + MultiFabOwner & alias = m_mf_register[internal_alias_name]; + amrex::MultiFab & mf_alias = alias.m_mf; + + // allocate + auto [it, success] = m_mf_register.emplace( + internal_new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + std::nullopt, // scalar: no direction + level, + alias.m_remake, + alias.m_redistribute_on_remake, + internal_alias_name + } + + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alias_init failed for " + internal_new_name); + } + + // a shorthand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + amrex::MultiFab* + MultiFabRegister::internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + Direction dir, + int level, + std::optional initial_value + ) + { + // checks + if (has(new_name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(new_name, dir, level) + + " already exists." + ); + } + if (!has(alias_name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(alias_name, dir, level) + + " does not exist." + ); + } + + // fully qualified name + std::string const internal_new_name = mf_name(new_name, dir, level); + std::string const internal_alias_name = mf_name(alias_name, dir, level); + + MultiFabOwner & alias = m_mf_register[internal_alias_name]; + amrex::MultiFab & mf_alias = alias.m_mf; + + // allocate + auto [it, success] = m_mf_register.emplace( + internal_new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + dir, + level, + alias.m_remake, + alias.m_redistribute_on_remake, + internal_alias_name + } + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alias_init failed for " + internal_new_name); + } + + // a short-hand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + void + MultiFabRegister::remake_level ( + int level, + amrex::DistributionMapping const & new_dm + ) + { + // Owning MultiFabs + for (auto & element : m_mf_register ) + { + MultiFabOwner & mf_owner = element.second; + + // keep distribution map as it is? + if (!mf_owner.m_remake) { + continue; + } + + // remake MultiFab with new distribution map + if (mf_owner.m_level == level && !mf_owner.is_alias()) { + const amrex::MultiFab & mf = mf_owner.m_mf; + amrex::IntVect const & ng = mf.nGrowVect(); + const auto tag = amrex::MFInfo().SetTag(mf.tags()[0]); + amrex::MultiFab new_mf(mf.boxArray(), new_dm, mf.nComp(), ng, tag); + + // copy data to new MultiFab: Only done for persistent data like E and B field, not for + // temporary things like currents, etc. + if (mf_owner.m_redistribute_on_remake) { + new_mf.Redistribute(mf, 0, 0, mf.nComp(), ng); + } + + // replace old MultiFab with new one, deallocate old one + mf_owner.m_mf = std::move(new_mf); + } + } + + // Aliases + for (auto & element : m_mf_register ) + { + MultiFabOwner & mf_owner = element.second; + + // keep distribution map as it is? + if (!mf_owner.m_remake) { + continue; + } + + if (mf_owner.m_level == level && mf_owner.is_alias()) { + const amrex::MultiFab & mf = m_mf_register[mf_owner.m_owner].m_mf; + amrex::MultiFab new_mf(mf, amrex::make_alias, 0, mf.nComp()); + + // no copy via Redistribute: the owner was already redistributed + + // replace old MultiFab with new one, deallocate old one + mf_owner.m_mf = std::move(new_mf); + } + } + } + + bool + MultiFabRegister::internal_has ( + std::string const & name, + int level + ) const + { + std::string const internal_name = mf_name(name, level); + + return m_mf_register.count(internal_name) > 0; + } + + bool + MultiFabRegister::internal_has ( + std::string const & name, + Direction dir, + int level + ) const + { + std::string const internal_name = mf_name(name, dir, level); + + return m_mf_register.count(internal_name) > 0; + } + + bool + MultiFabRegister::internal_has_vector ( + std::string const & name, + int level + ) const + { + unsigned long count = 0; + for (Direction const & dir : m_all_dirs) + { + std::string const internal_name = mf_name(name, dir, level); + count += m_mf_register.count(internal_name); + } + + return count == 3; + } + + bool + MultiFabRegister::internal_has ( + std::string const & internal_name + ) + { + return m_mf_register.count(internal_name) > 0; + } + + amrex::MultiFab* + MultiFabRegister::internal_get ( + std::string const & internal_name + ) + { + if (m_mf_register.count(internal_name) == 0) { + throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + internal_name); + } + amrex::MultiFab & mf = m_mf_register.at(internal_name).m_mf; + + return &mf; + } + + amrex::MultiFab const * + MultiFabRegister::internal_get ( + std::string const & internal_name + ) const + { + if (m_mf_register.count(internal_name) == 0) { + throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + internal_name); + } + amrex::MultiFab const & mf = m_mf_register.at(internal_name).m_mf; + + return &mf; + } + + amrex::MultiFab* + MultiFabRegister::internal_get ( + std::string const & name, + int level + ) + { + std::string const internal_name = mf_name(name, level); + return internal_get(internal_name); + } + + amrex::MultiFab* + MultiFabRegister::internal_get ( + std::string const & name, + Direction dir, + int level + ) + { + std::string const internal_name = mf_name(name, dir, level); + return internal_get(internal_name); + } + + amrex::MultiFab const * + MultiFabRegister::internal_get ( + std::string const & name, + int level + ) const + { + std::string const internal_name = mf_name(name, level); + return internal_get(internal_name); + } + + amrex::MultiFab const * + MultiFabRegister::internal_get ( + std::string const & name, + Direction dir, + int level + ) const + { + std::string const internal_name = mf_name(name, dir, level); + return internal_get(internal_name); + } + + MultiLevelScalarField + MultiFabRegister::internal_get_mr_levels ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) + { + MultiLevelScalarField field_on_level; + field_on_level.reserve(finest_level+1); + for (int lvl = 0; lvl <= finest_level; lvl++) + { + if (lvl == 0 && skip_level_0) + { + field_on_level.push_back(nullptr); + } + else + { + field_on_level.push_back(internal_get(name, lvl)); + } + } + return field_on_level; + } + + ConstMultiLevelScalarField + MultiFabRegister::internal_get_mr_levels ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) const + { + ConstMultiLevelScalarField field_on_level; + field_on_level.reserve(finest_level+1); + for (int lvl = 0; lvl <= finest_level; lvl++) + { + if (lvl == 0 && skip_level_0) + { + field_on_level.push_back(nullptr); + } + else + { + field_on_level.push_back(internal_get(name, lvl)); + } + } + return field_on_level; + } + + VectorField + MultiFabRegister::internal_get_alldirs ( + std::string const & name, + int level + ) + { + // insert a new level + VectorField vectorField; + + // insert components + for (Direction const & dir : m_all_dirs) + { + vectorField[dir] = internal_get(name, dir, level); + } + return vectorField; + } + + ConstVectorField + MultiFabRegister::internal_get_alldirs ( + std::string const & name, + int level + ) const + { + // insert a new level + ConstVectorField vectorField; + + // insert components + for (Direction const & dir : m_all_dirs) + { + vectorField[dir] = internal_get(name, dir, level); + } + return vectorField; + } + + MultiLevelVectorField + MultiFabRegister::internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) + { + MultiLevelVectorField field_on_level; + field_on_level.reserve(finest_level+1); + + for (int lvl = 0; lvl <= finest_level; lvl++) + { + // insert a new level + field_on_level.push_back(VectorField{}); + + // insert components + for (Direction const & dir : m_all_dirs) + { + if (lvl == 0 && skip_level_0) + { + field_on_level[lvl][dir] = nullptr; + } + else + { + field_on_level[lvl][dir] = internal_get(name, dir, lvl); + } + } + } + return field_on_level; + } + + ConstMultiLevelVectorField + MultiFabRegister::internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level, + bool skip_level_0 + ) const + { + ConstMultiLevelVectorField field_on_level; + field_on_level.reserve(finest_level+1); + + for (int lvl = 0; lvl <= finest_level; lvl++) + { + // insert a new level + field_on_level.push_back(ConstVectorField{}); + + // insert components + for (Direction const & dir : m_all_dirs) + { + if (lvl == 0 && skip_level_0) + { + field_on_level[lvl][dir] = nullptr; + } + else + { + field_on_level[lvl][dir] = internal_get(name, dir, lvl); + } + } + } + return field_on_level; + } + + std::vector + MultiFabRegister::list () const + { + std::vector names; + names.reserve(m_mf_register.size()); + for (auto const & str : m_mf_register) { names.push_back(str.first); } + + return names; + } + + void + MultiFabRegister::internal_erase ( + std::string const & name, + int level + ) + { + std::string const internal_name = mf_name(name, level); + + if (m_mf_register.count(internal_name) != 1) { + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + internal_name); + } + m_mf_register.erase(internal_name); + } + + void + MultiFabRegister::internal_erase ( + std::string const & name, + Direction dir, + int level + ) + { + std::string const internal_name = mf_name(name, dir, level); + + if (m_mf_register.count(internal_name) != 1) { + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + internal_name); + } + m_mf_register.erase(internal_name); + } + + void + MultiFabRegister::clear_level ( + int level + ) + { + // C++20: Replace with std::erase_if + for (auto first = m_mf_register.begin(), last = m_mf_register.end(); first != last;) + { + if (first->second.m_level == level) { + first = m_mf_register.erase(first); + } else { + ++first; + } + } + } + + std::string + MultiFabRegister::mf_name ( + std::string name, + int level + ) const + { + // Add the suffix "[level=level]" + return name.append("[level=") + .append(std::to_string(level)) + .append("]"); + } + + std::string + MultiFabRegister::mf_name ( + std::string name, + Direction dir, + int level + ) const + { + // Add the suffix for the direction [x] or [y] or [z] + // note: since Cartesian is not correct for all our supported geometries, + // in the future we might want to break this to "[dir=0/1/2]". + // This will be a breaking change for (Python) users that rely on that string. + constexpr int x_in_ascii = 120; + std::string const component_name{char(x_in_ascii + dir.dir)}; + return mf_name( + name + .append("[") + .append(component_name) + .append("]"), + level + ); + } + + VectorField + a2m ( + std::array< std::unique_ptr, 3 > const & old_vectorfield + ) + { + std::vector const all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + VectorField field_on_level; + + // insert components + for (auto const dir : {0, 1, 2}) + { + field_on_level[Direction{dir}] = old_vectorfield[dir].get(); + } + return field_on_level; + } +} // namespace ablastr::fields diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H old mode 100644 new mode 100755 index 26a4c72208d..c79736e0d1b --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -14,6 +14,7 @@ #include #include #include +#include #include #if defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D) @@ -46,19 +47,113 @@ #include #include #include -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) -# include -#endif +#include #ifdef AMREX_USE_EB # include #endif #include #include +#include namespace ablastr::fields { +/** Compute the L-infinity norm of the charge density `rho` across all MR levels + * to determine if `rho` is zero everywhere + * + * \param[in] rho The charge density a given species + * \param[in] finest_level The most refined mesh refinement level + * \param[in] absolute_tolerance The absolute convergence threshold for the MLMG solver + * \param[out] max_norm_rho The maximum L-infinity norm of `rho` across all levels + */ +inline amrex::Real getMaxNormRho ( + ablastr::fields::ConstMultiLevelScalarField const& rho, + int finest_level, + amrex::Real & absolute_tolerance) +{ + amrex::Real max_norm_rho = 0.0; + for (int lev=0; lev<=finest_level; lev++) { + max_norm_rho = amrex::max(max_norm_rho, rho[lev]->norm0()); + } + amrex::ParallelDescriptor::ReduceRealMax(max_norm_rho); + + if (max_norm_rho == 0) { + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } + ablastr::warn_manager::WMRecordWarning( + "ElectrostaticSolver", + "Max norm of rho is 0", + ablastr::warn_manager::WarnPriority::low + ); + } + return max_norm_rho; +} + +/** Interpolate the potential `phi` from level `lev` to `lev+1` + * + * Needed to solve Poisson equation on `lev+1` + * The coarser level `lev` provides both + * the boundary values and initial guess for `phi` + * on the finer level `lev+1` + * + * \param[in] phi_lev The potential on a given mesh refinement level `lev` + * \param[in] phi_lev_plus_one The potential on the next level `lev+1` + * \param[in] geom_lev The geometry of level `lev` + * \param[in] do_single_precision_comms perform communications in single precision + * \param[in] refratio mesh refinement ratio between level `lev` and `lev+1` + * \param[in] ncomp Number of components of the multifab (1) + * \param[in] ng Number of ghost cells (1 if collocated, 0 otherwise) + */ +inline void interpolatePhiBetweenLevels ( + amrex::MultiFab const* phi_lev, + amrex::MultiFab* phi_lev_plus_one, + amrex::Geometry const & geom_lev, + bool do_single_precision_comms, + const amrex::IntVect& refratio, + const int ncomp, + const int ng) +{ + using namespace amrex::literals; + + // Allocate phi_cp for lev+1 + amrex::BoxArray ba = phi_lev_plus_one->boxArray(); + ba.coarsen(refratio); + amrex::MultiFab phi_cp(ba, phi_lev_plus_one->DistributionMap(), ncomp, ng); + if (ng > 0) { + // Set all values outside the domain to zero + phi_cp.setDomainBndry(0.0_rt, geom_lev); + } + + // Copy from phi[lev] to phi_cp (in parallel) + const amrex::Periodicity& crse_period = geom_lev.periodicity(); + + ablastr::utils::communication::ParallelCopy( + phi_cp, + *phi_lev, + 0, + 0, + 1, + amrex::IntVect(0), + amrex::IntVect(ng), + do_single_precision_comms, + crse_period + ); + + // Local interpolation from phi_cp to phi[lev+1] +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*phi_lev_plus_one, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + amrex::Array4 const phi_fp_arr = phi_lev_plus_one->array(mfi); + amrex::Array4 const phi_cp_arr = phi_cp.array(mfi); + + details::PoissonInterpCPtoFP const interp(phi_fp_arr, phi_cp_arr, refratio); + + amrex::Box const& b = mfi.growntilebox(ng); + amrex::ParallelFor(b, interp); + } +} + /** Compute the potential `phi` by solving the Poisson equation * * Uses `rho` as a source, assuming that the source moves at a @@ -69,8 +164,8 @@ namespace ablastr::fields { * \vec{\nabla}^2 r \phi - (\vec{\beta}\cdot\vec{\nabla})^2 r \phi = -\frac{r \rho}{\epsilon_0} * \f] * - * \tparam T_BoundaryHandler handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \tparam T_PostPhiCalculationFunctor a calculation per level directly after phi was calculated + * \tparam T_BoundaryHandler handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler (EB ONLY) * \tparam T_FArrayBoxFactory usually nothing or an amrex::EBFArrayBoxFactory (EB ONLY) * \param[in] rho The charge density a given species * \param[out] phi The potential to be computed by this function @@ -83,190 +178,229 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] grid_type Integer that corresponds to the type of grid used in the simulation (collocated, staggered, hybrid) - * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels + * \param[in] is_igf_2d boolean to select between fully 3D Poisson solver and quasi-3D, i.e. one 2D Poisson solve on every z slice (default: false) + * \param[in] eb_enabled solve with embedded boundaries * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) + * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \param[in] current_time the current time; required for embedded boundaries (default: none) * \param[in] eb_farray_box_factory a factory for field data, @see amrex::EBFArrayBoxFactory; required for embedded boundaries (default: none) */ template< - typename T_BoundaryHandler, typename T_PostPhiCalculationFunctor = std::nullopt_t, + typename T_BoundaryHandler = std::nullopt_t, typename T_FArrayBoxFactory = void > void -computePhi (amrex::Vector const & rho, - amrex::Vector & phi, - std::array const beta, - amrex::Real const relative_tolerance, - amrex::Real absolute_tolerance, - int const max_iters, - int const verbosity, - amrex::Vector const& geom, - amrex::Vector const& dmap, - amrex::Vector const& grids, - utils::enums::GridType grid_type, - T_BoundaryHandler const boundary_handler, - bool is_solver_igf_on_lev0, - bool const do_single_precision_comms = false, - std::optional > rel_ref_ratio = std::nullopt, - [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, - [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB - [[maybe_unused]] std::optional > eb_farray_box_factory = std::nullopt // only used for EB +computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + std::array const beta, + amrex::Real relative_tolerance, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, + utils::enums::GridType grid_type, + bool is_solver_igf_on_lev0, + [[maybe_unused]] bool const is_igf_2d, + bool eb_enabled = false, + bool do_single_precision_comms = false, + std::optional > rel_ref_ratio = std::nullopt, + [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, + [[maybe_unused]] T_BoundaryHandler const boundary_handler = std::nullopt, + [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB + [[maybe_unused]] std::optional > eb_farray_box_factory = std::nullopt // only used for EB ) { using namespace amrex::literals; ABLASTR_PROFILE("computePhi"); + auto const finest_level = static_cast(rho.size() - 1); + if (!rel_ref_ratio.has_value()) { - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0u, "rel_ref_ratio must be set if mesh-refinement is used"); rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; } - auto const finest_level = static_cast(rho.size() - 1); - - // determine if rho is zero everywhere - amrex::Real max_norm_b = 0.0; - for (int lev=0; lev<=finest_level; lev++) { - max_norm_b = amrex::max(max_norm_b, rho[lev]->norm0()); - } - amrex::ParallelDescriptor::ReduceRealMax(max_norm_b); +#if !defined(AMREX_USE_EB) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, + "Embedded boundary solve requested but not compiled in"); +#endif - const bool always_use_bnorm = (max_norm_b > 0); - if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } - ablastr::warn_manager::WMRecordWarning( - "ElectrostaticSolver", - "Max norm of rho is 0", - ablastr::warn_manager::WarnPriority::low - ); - } +#if !defined(ABLASTR_USE_FFT) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "Must compile with FFT support to use the IGF solver!"); +#endif -#if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) - amrex::LPInfo info; -#else - const amrex::LPInfo info; +#if !defined(WARPX_DIM_3D) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "The FFT Poisson solver is currently only implemented for 3D!"); #endif - for (int lev=0; lev<=finest_level; lev++) { - // Set the value of beta - amrex::Array beta_solver = + // Set the value of beta + amrex::Array beta_solver = #if defined(WARPX_DIM_1D_Z) - {{ beta[2] }}; // beta_x and beta_z + {{ beta[2] }}; // beta_x and beta_z #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - {{ beta[0], beta[2] }}; // beta_x and beta_z + {{ beta[0], beta[2] }}; // beta_x and beta_z #else - {{ beta[0], beta[1], beta[2] }}; + {{ beta[0], beta[1], beta[2] }}; #endif -#if !defined(ABLASTR_USE_FFT) - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, - "Must compile with FFT support to use the IGF solver!"); -#endif + // determine if rho is zero everywhere + const amrex::Real max_norm_b = getMaxNormRho( + amrex::GetVecOfConstPtrs(rho), finest_level, absolute_tolerance); -#if !defined(WARPX_DIM_3D) - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, - "The FFT Poisson solver is currently only implemented for 3D!"); -#endif + amrex::LPInfo info; -#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) + for (int lev=0; lev<=finest_level; lev++) { + amrex::Array const dx_scaled + {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), + geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), + geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; + +#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(is_solver_igf_on_lev0 && lev==0){ - amrex::Array const dx_igf - {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), - geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), - geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; if ( max_norm_b == 0 ) { phi[lev]->setVal(0); } else { - computePhiIGF( *rho[lev], *phi[lev], dx_igf, grids[lev] ); + computePhiIGF( *rho[lev], *phi[lev], dx_scaled, grids[lev], is_igf_2d); } continue; } #endif - // Use the Multigrid (MLMG) solver if selected or on refined patches // but first scale rho appropriately - using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + rho[lev]->mult(-1._rt / ablastr::constant::SI::ep0); -#if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) - // Determine whether to use semi-coarsening - amrex::Array dx_scaled - {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), - geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), - geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; - int max_semicoarsening_level = 0; - int semicoarsening_direction = -1; - const auto min_dir = static_cast(std::distance(dx_scaled.begin(), - std::min_element(dx_scaled.begin(),dx_scaled.end()))); - const auto max_dir = static_cast(std::distance(dx_scaled.begin(), - std::max_element(dx_scaled.begin(),dx_scaled.end()))); - if (dx_scaled[max_dir] > dx_scaled[min_dir]) { - semicoarsening_direction = max_dir; - max_semicoarsening_level = static_cast - (std::log2(dx_scaled[max_dir]/dx_scaled[min_dir])); - } - if (max_semicoarsening_level > 0) { - info.setSemicoarsening(true); - info.setMaxSemicoarseningLevel(max_semicoarsening_level); - info.setSemicoarseningDirection(semicoarsening_direction); - } +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; #endif -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) - // In the presence of EB or RZ: the solver assumes that the beam is - // propagating along one of the axes of the grid, i.e. that only *one* - // of the components of `beta` is non-negligible. - amrex::MLEBNodeFDLaplacian linop( {geom[lev]}, {grids[lev]}, {dmap[lev]}, info + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx_scaled.begin(), + std::min_element(dx_scaled.begin(), dx_scaled.end()))); + const auto max_dir = static_cast(std::distance(dx_scaled.begin(), + std::max_element(dx_scaled.begin(), dx_scaled.end()))); + if (dx_scaled[max_dir] > dx_scaled[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx_scaled[max_dir] / dx_scaled[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } + } + + std::unique_ptr linop; + if (eb_enabled || is_rz) { + // In the presence of EB or RZ: the solver assumes that the beam is + // propagating along one of the axes of the grid, i.e. that only *one* + // of the components of `beta` is non-negligible. + auto linop_nodelap = std::make_unique(); + if (eb_enabled) { #if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} + if constexpr(std::is_same_v) { + throw std::runtime_error("EB requested by eb_farray_box_factory not provided!"); + } else { + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info, + amrex::Vector{eb_farray_box_factory.value()[lev]} + ); + } #endif - ); + } + else { + // TODO: rather use MLNodeTensorLaplacian (for RZ w/o EB) here? Semi-Coarsening would be nice here + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + } - // Note: this assumes that the beam is propagating along - // one of the axes of the grid, i.e. that only *one* of the - // components of `beta` is non-negligible. // we use this + // Note: this assumes that the beam is propagating along + // one of the axes of the grid, i.e. that only *one* of the + // components of `beta` is non-negligible. // we use this #if defined(WARPX_DIM_RZ) - linop.setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); + linop_nodelap->setRZ(true); + linop_nodelap->setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); #else - linop.setSigma({AMREX_D_DECL( - 1._rt-beta_solver[0]*beta_solver[0], - 1._rt-beta_solver[1]*beta_solver[1], - 1._rt-beta_solver[2]*beta_solver[2])}); + linop_nodelap->setSigma({AMREX_D_DECL( + 1._rt-beta_solver[0]*beta_solver[0], + 1._rt-beta_solver[1]*beta_solver[1], + 1._rt-beta_solver[2]*beta_solver[2])}); #endif - #if defined(AMREX_USE_EB) - // if the EB potential only depends on time, the potential can be passed - // as a float instead of a callable - if (boundary_handler.phi_EB_only_t) { - linop.setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); - } - else - linop.setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); -#endif -#else - // In the absence of EB and RZ: use a more generic solver - // that can handle beams propagating in any direction - amrex::MLNodeTensorLaplacian linop( {geom[lev]}, {grids[lev]}, - {dmap[lev]}, info ); - linop.setBeta( beta_solver ); // for the non-axis-aligned solver + if (eb_enabled) { + if constexpr (!std::is_same_v) { + // if the EB potential only depends on time, the potential can be passed + // as a float instead of a callable + if (boundary_handler.phi_EB_only_t) { + linop_nodelap->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); + } else { + linop_nodelap->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); + } + } else + { + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "EB Poisson solver enabled but no 'boundary_handler' passed!"); + } + } #endif + linop = std::move(linop_nodelap); + } else { + // In the absence of EB and RZ: use a more generic solver + // that can handle beams propagating in any direction + auto linop_tenslap = std::make_unique( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + linop_tenslap->setBeta(beta_solver); // for the non-axis-aligned solver + linop = std::move(linop_tenslap); + } + + // Level 0 domain boundary + if constexpr (std::is_same_v) { + amrex::Array const lobc = {AMREX_D_DECL( + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet, + amrex::LinOpBCType::Dirichlet + )}; + amrex::Array const hibc = lobc; + linop->setDomainBC(lobc, hibc); + } else { + linop->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + } // Solve the Poisson equation - linop.setDomainBC( boundary_handler.lobc, boundary_handler.hibc ); -#ifdef WARPX_DIM_RZ - linop.setRZ(true); -#endif - amrex::MLMG mlmg(linop); // actual solver defined here + amrex::MLMG mlmg(*linop); // actual solver defined here mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); - mlmg.setAlwaysUseBNorm(always_use_bnorm); - if (grid_type == utils::enums::GridType::Collocated) { + mlmg.setAlwaysUseBNorm((max_norm_b > 0)); + + const int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells + if (ng) { // In this case, computeE needs to use ghost nodes data. So we // ask MLMG to fill BC for us after it solves the problem. mlmg.setFinalFillBC(true); @@ -276,54 +410,22 @@ computePhi (amrex::Vector const & rho, mlmg.solve( {phi[lev]}, {rho[lev]}, relative_tolerance, absolute_tolerance ); + const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; + const int ncomp = linop->getNComp(); + // needed for solving the levels by levels: // - coarser level is initial guess for finer level // - coarser level provides boundary values for finer level patch // Interpolation from phi[lev] to phi[lev+1] // (This provides both the boundary conditions and initial guess for phi[lev+1]) if (lev < finest_level) { - - // Allocate phi_cp for lev+1 - amrex::BoxArray ba = phi[lev+1]->boxArray(); - const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; - ba.coarsen(refratio); - const int ncomp = linop.getNComp(); - const int ng = (grid_type == utils::enums::GridType::Collocated) ? 1 : 0; - amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); - if (ng > 0) { - // Set all values outside the domain to zero - phi_cp.setDomainBndry(0.0_rt, geom[lev]); - } - - // Copy from phi[lev] to phi_cp (in parallel) - const amrex::Periodicity& crse_period = geom[lev].periodicity(); - - ablastr::utils::communication::ParallelCopy( - phi_cp, - *phi[lev], - 0, - 0, - 1, - amrex::IntVect(0), - amrex::IntVect(ng), - do_single_precision_comms, - crse_period - ); - - // Local interpolation from phi_cp to phi[lev+1] -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi(*phi[lev + 1], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Array4 const phi_fp_arr = phi[lev + 1]->array(mfi); - amrex::Array4 const phi_cp_arr = phi_cp.array(mfi); - - details::PoissonInterpCPtoFP const interp(phi_fp_arr, phi_cp_arr, refratio); - - amrex::Box const& b = mfi.growntilebox(ng); - amrex::ParallelFor(b, interp); - } - + interpolatePhiBetweenLevels(phi[lev], + phi[lev+1], + geom[lev], + do_single_precision_comms, + refratio, + ncomp, + ng); } // Run additional operations, such as calculation of the E field for embedded boundaries @@ -332,10 +434,10 @@ computePhi (amrex::Vector const & rho, post_phi_calculation.value()(mlmg, lev); } } + rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again } // loop over lev(els) -} - +} // computePhi } // namespace ablastr::fields #endif // ABLASTR_POISSON_SOLVER_H diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 3ef96c30c84..16863320c1e 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -1,4 +1,4 @@ -/* Copyright 2022 S. Eric Clark, LLNL +/* Copyright 2022-2024 S. Eric Clark (Helion Energy, formerly LLNL) * * This file is part of WarpX. * @@ -71,6 +71,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see MagnetostaticSolver::VectorPoissonBoundaryHandler + * \param[in] eb_enabled solve with embedded boundaries * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_A_calculation perform a calculation per level directly after A was calculated; required for embedded boundaries (default: none) @@ -85,15 +86,16 @@ template< void computeVectorPotential ( amrex::Vector > const & curr, amrex::Vector > & A, - amrex::Real const relative_tolerance, + amrex::Real relative_tolerance, amrex::Real absolute_tolerance, - int const max_iters, - int const verbosity, + int max_iters, + int verbosity, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, - bool const do_single_precision_comms = false, + bool eb_enabled = false, + bool do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostACalculationFunctor post_A_calculation = std::nullopt, [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB @@ -130,28 +132,53 @@ computeVectorPotential ( amrex::Vector > co ); } - const amrex::LPInfo& info = amrex::LPInfo(); - // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { - amrex::MLEBNodeFDLaplacian linopx( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} -#endif - ); - amrex::MLEBNodeFDLaplacian linopy( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} + amrex::LPInfo info; + +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; #endif - ); - amrex::MLEBNodeFDLaplacian linopz( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} + + amrex::Array const dx + {AMREX_D_DECL(geom[lev].CellSize(0), + geom[lev].CellSize(1), + geom[lev].CellSize(2))}; + + + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx.begin(), + std::min_element(dx.begin(), dx.end()))); + const auto max_dir = static_cast(std::distance(dx.begin(), + std::max_element(dx.begin(), dx.end()))); + if (dx[max_dir] > dx[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx[max_dir] / dx[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } + } + + amrex::MLEBNodeFDLaplacian linopx, linopy, linopz; + if (eb_enabled) { +#ifdef AMREX_USE_EB + linopx.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); + linopy.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); + linopz.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); #endif - ); + } else { + linopx.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + linopy.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + linopz.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + } amrex::Array linop = {&linopx,&linopy,&linopz}; amrex::Array,3> mlmg; @@ -163,9 +190,9 @@ computeVectorPotential ( amrex::Vector > co // Note: this assumes that beta is zero linop[adim]->setSigma({AMREX_D_DECL(1._rt, 1._rt, 1._rt)}); -#if defined(AMREX_USE_EB) // Set Homogeneous Dirichlet Boundary on EB - linop[adim]->setEBDirichlet(0_rt); +#if defined(AMREX_USE_EB) + if (eb_enabled) { linop[adim]->setEBDirichlet(0_rt); } #endif #ifdef WARPX_DIM_RZ diff --git a/Source/ablastr/math/CMakeLists.txt b/Source/ablastr/math/CMakeLists.txt index 9093da83ae1..0ad3fe80b87 100644 --- a/Source/ablastr/math/CMakeLists.txt +++ b/Source/ablastr/math/CMakeLists.txt @@ -1 +1,9 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(ablastr_${SD} + PRIVATE + FiniteDifference.cpp + ) +endforeach() + add_subdirectory(fft) diff --git a/Source/ablastr/math/FiniteDifference.H b/Source/ablastr/math/FiniteDifference.H new file mode 100644 index 00000000000..8761318eb81 --- /dev/null +++ b/Source/ablastr/math/FiniteDifference.H @@ -0,0 +1,44 @@ +/* Copyright 2021-2025 Edoardo Zoni, Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef ABLASTR_MATH_FINITE_DIFFERENCE_H_ +#define ABLASTR_MATH_FINITE_DIFFERENCE_H_ + +#include "ablastr/utils/Enums.H" + +#include +#include + +namespace ablastr::math +{ + /** + * \brief Returns an array of coefficients (Fornberg coefficients), corresponding + * to the weight of each point in a finite-difference approximation of a derivative + * (up to order \c n_order). + * + * \param[in] n_order order of the finite-difference approximation + * \param[in] a_grid_type type of grid (collocated or not) + */ + [[nodiscard]] amrex::Vector + getFornbergStencilCoefficients ( + int n_order, ablastr::utils::enums::GridType a_grid_type); + + /** + * \brief Re-orders the Fornberg coefficients so that they can be used more conveniently for + * finite-order centering operations. For example, for finite-order centering of order 6, + * the Fornberg coefficients \c (c_0,c_1,c_2) are re-ordered as \c (c_2,c_1,c_0,c_0,c_1,c_2). + * + * \param[in,out] ordered_coeffs host vector where the re-ordered Fornberg coefficients will be stored + * \param[in] unordered_coeffs host vector storing the original sequence of Fornberg coefficients + * \param[in] order order of the finite-order centering along a given direction + */ + void + ReorderFornbergCoefficients ( + amrex::Vector& ordered_coeffs, + const amrex::Vector& unordered_coeffs, int order); +} + +#endif //ABLASTR_MATH_FINITE_DIFFERENCE_H_ diff --git a/Source/ablastr/math/FiniteDifference.cpp b/Source/ablastr/math/FiniteDifference.cpp new file mode 100644 index 00000000000..85d0b332131 --- /dev/null +++ b/Source/ablastr/math/FiniteDifference.cpp @@ -0,0 +1,77 @@ +/* Copyright 2021-2025 Edoardo Zoni, Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "FiniteDifference.H" + +#include "ablastr/utils/TextMsg.H" + +using namespace ablastr::utils::enums; +using namespace amrex; + +namespace ablastr::math +{ + + amrex::Vector + getFornbergStencilCoefficients (const int n_order, GridType a_grid_type) + { + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(n_order % 2 == 0, "n_order must be even"); + + const int m = n_order / 2; + amrex::Vector coeffs; + coeffs.resize(m); + + // There are closed-form formula for these coefficients, but they result in + // an overflow when evaluated numerically. One way to avoid the overflow is + // to calculate the coefficients by recurrence. + + // Coefficients for collocated (nodal) finite-difference approximation + if (a_grid_type == GridType::Collocated) + { + // First coefficient + coeffs.at(0) = m * 2._rt / (m+1); + // Other coefficients by recurrence + for (int n = 1; n < m; n++) + { + coeffs.at(n) = - (m-n) * 1._rt / (m+n+1) * coeffs.at(n-1); + } + } + // Coefficients for staggered finite-difference approximation + else + { + amrex::Real prod = 1.; + for (int k = 1; k < m+1; k++) + { + prod *= (m + k) / (4._rt * k); + } + // First coefficient + coeffs.at(0) = 4_rt * m * prod * prod; + // Other coefficients by recurrence + for (int n = 1; n < m; n++) + { + coeffs.at(n) = - ((2_rt*n-1) * (m-n)) * 1._rt / ((2_rt*n+1) * (m+n)) * coeffs.at(n-1); + } + } + + return coeffs; + } + + void + ReorderFornbergCoefficients ( + amrex::Vector& ordered_coeffs, + const amrex::Vector& unordered_coeffs, + const int order) + { + const int n = order / 2; + for (int i = 0; i < n; i++) { + ordered_coeffs[i] = unordered_coeffs[n-1-i]; + } + for (int i = n; i < order; i++) { + ordered_coeffs[i] = unordered_coeffs[i-n]; + } + } + +} diff --git a/Source/ablastr/math/Make.package b/Source/ablastr/math/Make.package index a0e95b11225..5e3fd22dc81 100644 --- a/Source/ablastr/math/Make.package +++ b/Source/ablastr/math/Make.package @@ -1,3 +1,5 @@ -include $(WARPX_HOME)/Source/ablastr/math/fft/Make.package +CEXE_sources += FiniteDifference.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr/math -VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr +include $(WARPX_HOME)/Source/ablastr/math/fft/Make.package diff --git a/Source/ablastr/math/fft/AnyFFT.H b/Source/ablastr/math/fft/AnyFFT.H index 8a4c11c7654..6b049c97ff9 100644 --- a/Source/ablastr/math/fft/AnyFFT.H +++ b/Source/ablastr/math/fft/AnyFFT.H @@ -10,16 +10,21 @@ #ifdef ABLASTR_USE_FFT # include +# include # include # if defined(AMREX_USE_CUDA) # include +# include # elif defined(AMREX_USE_HIP) # if __has_include() // ROCm 5.3+ # include # else # include # endif +# include +# elif defined(AMREX_USE_SYCL) +# include # else # include # endif @@ -62,6 +67,8 @@ namespace ablastr::math::anyfft # else using Complex = double2; # endif +# elif defined(AMREX_USE_SYCL) + using Complex = amrex::GpuComplex; # else # ifdef AMREX_USE_FLOAT using Complex = fftwf_complex; @@ -70,6 +77,28 @@ namespace ablastr::math::anyfft # endif # endif + /* Library-dependent multiply helpers */ +# if defined(AMREX_USE_CUDA) +# ifdef AMREX_USE_FLOAT + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = cuCmulf(a, b); } +# else + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = cuCmul(a, b); } +# endif +# elif defined(AMREX_USE_HIP) + # ifdef AMREX_USE_FLOAT + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = hipCmulf(a, b); } +# else + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = hipCmul(a, b); } +# endif +# elif defined(AMREX_USE_SYCL) + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { c = a * b; } +# else + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void multiply (Complex & c, Complex const & a, Complex const & b) { + c[0] = a[0] * b[0] - a[1] * b[1]; + c[1] = a[0] * b[1] + a[1] * b[0]; + } +# endif + /** Library-dependent FFT plans type, which holds one fft plan per box * (plans are only initialized for the boxes that are owned by the local MPI rank). */ @@ -77,6 +106,15 @@ namespace ablastr::math::anyfft using VendorFFTPlan = cufftHandle; # elif defined(AMREX_USE_HIP) using VendorFFTPlan = rocfft_plan; +# elif defined(AMREX_USE_SYCL) + using VendorFFTPlan = oneapi::mkl::dft::descriptor< +# ifdef AMREX_USE_FLOAT + oneapi::mkl::dft::precision::SINGLE, +# else + oneapi::mkl::dft::precision::DOUBLE, +# endif + oneapi::mkl::dft::domain::REAL> *; + // dft::descriptor has no default ctor, so we have to use ptr. # else # ifdef AMREX_USE_FLOAT using VendorFFTPlan = fftwf_plan; @@ -99,6 +137,9 @@ namespace ablastr::math::anyfft VendorFFTPlan m_plan; /**< Vendor FFT plan */ direction m_dir; /**< direction (C2R or R2C) */ int m_dim; /**< Dimensionality of the FFT plan */ +#ifdef AMREX_USE_SYCL + amrex::gpuStream_t m_stream; +#endif }; /** Collection of FFT plans, one FFTplan per box */ diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index 913a912e1ee..b0ebc3c8050 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -5,6 +5,8 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) target_sources(ablastr_${SD} PRIVATE WrapRocFFT.cpp) + elseif(WarpX_COMPUTE STREQUAL SYCL) + target_sources(ablastr_${SD} PRIVATE WrapMklFFT.cpp) else() target_sources(ablastr_${SD} PRIVATE WrapFFTW.cpp) endif() diff --git a/Source/ablastr/math/fft/Make.package b/Source/ablastr/math/fft/Make.package index 63786cdc006..6e7dc2b3dd4 100644 --- a/Source/ablastr/math/fft/Make.package +++ b/Source/ablastr/math/fft/Make.package @@ -3,6 +3,8 @@ ifeq ($(USE_FFT),TRUE) CEXE_sources += WrapCuFFT.cpp else ifeq ($(USE_HIP),TRUE) CEXE_sources += WrapRocFFT.cpp + else ifeq ($(USE_SYCL),TRUE) + CEXE_sources += WrapMklFFT.cpp else CEXE_sources += WrapFFTW.cpp endif diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 9ceb91457c9..73f3cdfc395 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -42,8 +42,11 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorR2C); + } else if (dim == 1) { + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorR2C, 1); } else { - ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); + ABLASTR_ABORT_WITH_MESSAGE("only dim=1 and dim=2 and dim=3 have been implemented"); } } else { if (dim == 3) { @@ -52,6 +55,9 @@ namespace ablastr::math::anyfft } else if (dim == 2) { result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorC2R); + } else if (dim == 1) { + result = cufftPlan1d( + &(fft_plan.m_plan), real_size[0], VendorC2R, 1); } else { ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); } diff --git a/Source/ablastr/math/fft/WrapFFTW.cpp b/Source/ablastr/math/fft/WrapFFTW.cpp index 6711bbface9..16f0355cc5d 100644 --- a/Source/ablastr/math/fft/WrapFFTW.cpp +++ b/Source/ablastr/math/fft/WrapFFTW.cpp @@ -25,11 +25,15 @@ namespace ablastr::math::anyfft const auto VendorCreatePlanC2R3D = fftwf_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftwf_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftwf_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftwf_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftwf_plan_dft_c2r_1d; #else const auto VendorCreatePlanR2C3D = fftw_plan_dft_r2c_3d; const auto VendorCreatePlanC2R3D = fftw_plan_dft_c2r_3d; const auto VendorCreatePlanR2C2D = fftw_plan_dft_r2c_2d; const auto VendorCreatePlanC2R2D = fftw_plan_dft_c2r_2d; + const auto VendorCreatePlanR2C1D = fftw_plan_dft_r2c_1d; + const auto VendorCreatePlanC2R1D = fftw_plan_dft_c2r_1d; #endif FFTplan CreatePlan(const amrex::IntVect& real_size, amrex::Real * const real_array, @@ -56,9 +60,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanR2C2D( real_size[1], real_size[0], real_array, complex_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanR2C1D( + real_size[0], real_array, complex_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented"); + "only dim=1 and dim=2 and dim=3 have been implemented"); } } else if (dir == direction::C2R){ if (dim == 3) { @@ -67,9 +74,12 @@ namespace ablastr::math::anyfft } else if (dim == 2) { fft_plan.m_plan = VendorCreatePlanC2R2D( real_size[1], real_size[0], complex_array, real_array, FFTW_ESTIMATE); + } else if (dim == 1) { + fft_plan.m_plan = VendorCreatePlanC2R1D( + real_size[0], complex_array, real_array, FFTW_ESTIMATE); } else { ABLASTR_ABORT_WITH_MESSAGE( - "only dim=2 and dim=3 have been implemented. Should be easy to add dim=1."); + "only dim=1 and dim=2 and dim=3 have been implemented."); } } diff --git a/Source/ablastr/math/fft/WrapMklFFT.cpp b/Source/ablastr/math/fft/WrapMklFFT.cpp new file mode 100644 index 00000000000..ef2cb6c42fa --- /dev/null +++ b/Source/ablastr/math/fft/WrapMklFFT.cpp @@ -0,0 +1,96 @@ +/* Copyright 2019-2023 + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ + +#include "AnyFFT.H" + +#include "ablastr/utils/TextMsg.H" +#include "ablastr/profiler/ProfilerWrapper.H" + +#include + +namespace ablastr::math::anyfft +{ + + void setup () {/*nothing to do*/} + + void cleanup () {/*nothing to do*/} + + FFTplan CreatePlan (const amrex::IntVect& real_size, amrex::Real * const real_array, + Complex * const complex_array, const direction dir, const int dim) + { + FFTplan fft_plan; + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan"); + + // Initialize fft_plan.m_plan with the vendor fft plan. + std::vector strides(dim+1); + if (dim == 3) { + fft_plan.m_plan = new std::remove_pointer_t( + {std::int64_t(real_size[2]), + std::int64_t(real_size[1]), + std::int64_t(real_size[0])}); + strides[0] = 0; + strides[1] = real_size[0] * real_size[1]; + strides[2] = real_size[0]; + strides[3] = 1; + } else if (dim == 2) { + fft_plan.m_plan = new std::remove_pointer_t( + {std::int64_t(real_size[1]), + std::int64_t(real_size[0])}); + strides[0] = 0; + strides[1] = real_size[0]; + strides[2] = 1; + } else if (dim == 1) { + strides[0] = 0; + strides[1] = 1; + fft_plan.m_plan = new std::remove_pointer_t( + std::int64_t(real_size[0])); + } else { + ABLASTR_ABORT_WITH_MESSAGE("only dim2 =1, dim=2 and dim=3 have been implemented"); + } + + fft_plan.m_plan->set_value(oneapi::mkl::dft::config_param::PLACEMENT, + DFTI_NOT_INPLACE); + fft_plan.m_plan->set_value(oneapi::mkl::dft::config_param::FWD_STRIDES, + strides.data()); + fft_plan.m_plan->commit(amrex::Gpu::Device::streamQueue()); + + // Store meta-data in fft_plan + fft_plan.m_real_array = real_array; + fft_plan.m_complex_array = complex_array; + fft_plan.m_dir = dir; + fft_plan.m_dim = dim; + fft_plan.m_stream = amrex::Gpu::gpuStream(); + + return fft_plan; + } + + void DestroyPlan (FFTplan& fft_plan) + { + delete fft_plan.m_plan; + } + + void Execute (FFTplan& fft_plan) + { + if (!(fft_plan.m_stream == amrex::Gpu::gpuStream())) { + amrex::Gpu::streamSynchronize(); + } + + sycl::event r; + if (fft_plan.m_dir == direction::R2C) { + r = oneapi::mkl::dft::compute_forward( + *fft_plan.m_plan, + fft_plan.m_real_array, + reinterpret_cast*>(fft_plan.m_complex_array)); + } else { + r = oneapi::mkl::dft::compute_backward( + *fft_plan.m_plan, + reinterpret_cast*>(fft_plan.m_complex_array), + fft_plan.m_real_array); + } + r.wait(); + } +} diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 75e3bca170d..2eac9eb951b 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -46,7 +46,7 @@ namespace ablastr::particles * \param nc number of components to deposit */ template< typename T_PC > -static void +void deposit_charge (typename T_PC::ParIterType& pti, typename T_PC::RealVector const& wp, amrex::Real const charge, diff --git a/Source/ablastr/utils/Enums.H b/Source/ablastr/utils/Enums.H index 1f89bede9e4..7c7129cae77 100644 --- a/Source/ablastr/utils/Enums.H +++ b/Source/ablastr/utils/Enums.H @@ -8,17 +8,19 @@ #ifndef ABLASTR_UTILS_ENUMS_H_ #define ABLASTR_UTILS_ENUMS_H_ +#include + namespace ablastr::utils::enums { /** Type of grids used in a simulation: * * Collocated at the same location (AMReX: all "NODAL"), staggered (Yee-style), or hybrid. */ - enum struct GridType { - Collocated = 0, - Staggered = 1, - Hybrid = 2 - }; + AMREX_ENUM(GridType, + Collocated, + Staggered, + Hybrid, + Default = Staggered); /** Mesh-refinement patch * diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 5eeaeec259f..bf4874b4536 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -37,7 +37,7 @@ SignalHandling::parseSignalNameToNumber (const std::string &str) #if defined(__linux__) || defined(__APPLE__) const struct { const char* abbrev; - const int value; + int value; } signals_to_parse[] = { {"ABRT", SIGABRT}, {"ALRM", SIGALRM}, diff --git a/Source/ablastr/utils/UsedInputsFile.cpp b/Source/ablastr/utils/UsedInputsFile.cpp index dfdc4bfa192..a7777556242 100644 --- a/Source/ablastr/utils/UsedInputsFile.cpp +++ b/Source/ablastr/utils/UsedInputsFile.cpp @@ -23,7 +23,7 @@ ablastr::utils::write_used_inputs_file (std::string const & filename) if (amrex::ParallelDescriptor::IOProcessor()) { std::ofstream jobInfoFile; jobInfoFile.open(filename.c_str(), std::ios::out); - amrex::ParmParse::dumpTable(jobInfoFile, true); + amrex::ParmParse::prettyPrintTable(jobInfoFile); jobInfoFile.close(); } } diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H index 401432f5dda..088a613bc87 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.H +++ b/Source/ablastr/utils/msg_logger/MsgLogger.H @@ -79,10 +79,10 @@ namespace ablastr::utils::msg_logger * \brief Same as static Msg deserialize(std::vector::const_iterator& it) * but accepting an rvalue as an argument * - * @param[in] it iterator of a byte array + * @param[in] rit iterator of a byte array * @return a Msg struct */ - static Msg deserialize(std::vector::const_iterator&& it); + static Msg deserialize(std::vector::const_iterator&& rit); }; /** @@ -115,10 +115,10 @@ namespace ablastr::utils::msg_logger * \brief Same as static Msg MsgWithCounter(std::vector::const_iterator& it) * but accepting an rvalue as an argument * - * @param[in] it iterator of a byte array + * @param[in] rit iterator of a byte array * @return a MsgWithCounter struct */ - static MsgWithCounter deserialize(std::vector::const_iterator&& it); + static MsgWithCounter deserialize(std::vector::const_iterator&& rit); }; /** @@ -154,10 +154,10 @@ namespace ablastr::utils::msg_logger * \brief Same as static Msg MsgWithCounterAndRanks(std::vector::const_iterator& it) * but accepting an rvalue as an argument * - * @param[in] it iterator of a byte array + * @param[in] rit iterator of a byte array * @return a MsgWithCounterAndRanks struct */ - static MsgWithCounterAndRanks deserialize(std::vector::const_iterator&& it); + static MsgWithCounterAndRanks deserialize(std::vector::const_iterator&& rit); }; /** @@ -280,9 +280,9 @@ namespace ablastr::utils::msg_logger #endif - const int m_rank /*! MPI rank of the current process*/; - const int m_num_procs /*! Number of MPI ranks*/; - const int m_io_rank /*! Rank of the I/O process*/; + int m_rank /*! MPI rank of the current process*/; + int m_num_procs /*! Number of MPI ranks*/; + int m_io_rank /*! Rank of the I/O process*/; std::map m_messages /*! This stores a map to associate warning messages with the corresponding counters*/; }; diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index 6537a8f61e5..6597588d085 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -147,9 +147,10 @@ Msg Msg::deserialize (std::vector::const_iterator& it) return msg; } -Msg Msg::deserialize (std::vector::const_iterator&& it) +Msg Msg::deserialize (std::vector::const_iterator&& rit) { - return Msg::deserialize(it); + auto lit = std::vector::const_iterator{std::move(rit)}; + return Msg::deserialize(lit); } std::vector MsgWithCounter::serialize() const @@ -174,9 +175,10 @@ MsgWithCounter MsgWithCounter::deserialize (std::vector::const_iterator& i return msg_with_counter; } -MsgWithCounter MsgWithCounter::deserialize (std::vector::const_iterator&& it) +MsgWithCounter MsgWithCounter::deserialize (std::vector::const_iterator&& rit) { - return MsgWithCounter::deserialize(it); + auto lit = std::vector::const_iterator{std::move(rit)}; + return MsgWithCounter::deserialize(lit); } std::vector MsgWithCounterAndRanks::serialize() const @@ -205,9 +207,10 @@ MsgWithCounterAndRanks::deserialize (std::vector::const_iterator& it) } MsgWithCounterAndRanks -MsgWithCounterAndRanks::deserialize (std::vector::const_iterator&& it) +MsgWithCounterAndRanks::deserialize (std::vector::const_iterator&& rit) { - return MsgWithCounterAndRanks::deserialize(it); + auto lit = std::vector::const_iterator{std::move(rit)}; + return MsgWithCounterAndRanks::deserialize(lit); } Logger::Logger() : diff --git a/Tools/Algorithms/psatd.ipynb b/Tools/Algorithms/psatd.ipynb index c2f326e4110..6153b904968 100644 --- a/Tools/Algorithms/psatd.ipynb +++ b/Tools/Algorithms/psatd.ipynb @@ -3,13 +3,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "import sympy as sp\n", - "from sympy import *\n", + "from sympy import * # noqa\n", "\n", "sp.init_session()\n", "sp.init_printing()" @@ -39,8 +37,8 @@ "source": [ "divE_cleaning = True\n", "divB_cleaning = True\n", - "J_in_time = 'constant'\n", - "rho_in_time = 'constant'" + "J_in_time = \"constant\"\n", + "rho_in_time = \"constant\"" ] }, { @@ -63,10 +61,13 @@ " Wd = P * D * invP\n", " for i in range(Wd.shape[0]):\n", " for j in range(Wd.shape[1]):\n", - " Wd[i,j] = Wd[i,j].expand().simplify()\n", - " diff = W[i,j] - Wd[i,j]\n", + " Wd[i, j] = Wd[i, j].expand().simplify()\n", + " diff = W[i, j] - Wd[i, j]\n", " diff = diff.expand().simplify()\n", - " assert (diff == 0), f'Diagonalization failed: W[{i},{j}] - Wd[{i},{j}] = {diff} is not zero'\n", + " assert diff == 0, (\n", + " f\"Diagonalization failed: W[{i},{j}] - Wd[{i},{j}] = {diff} is not zero\"\n", + " )\n", + "\n", "\n", "def simple_mat(W):\n", " \"\"\"\n", @@ -74,7 +75,7 @@ " \"\"\"\n", " for i in range(W.shape[0]):\n", " for j in range(W.shape[1]):\n", - " W[i,j] = W[i,j].expand().simplify()" + " W[i, j] = W[i, j].expand().simplify()" ] }, { @@ -98,41 +99,41 @@ " dim += 1\n", "\n", "# Define symbols for physical constants\n", - "c = sp.symbols(r'c', real=True, positive=True)\n", - "mu0 = sp.symbols(r'\\mu_0', real=True, positive=True)\n", + "c = sp.symbols(r\"c\", real=True, positive=True)\n", + "mu0 = sp.symbols(r\"\\mu_0\", real=True, positive=True)\n", "\n", "# Define symbols for time variables\n", "# (s is auxiliary variable used in integral over time)\n", - "s = sp.symbols(r's', real=True, positive=True)\n", - "t = sp.symbols(r't', real=True, positive=True)\n", - "tn = sp.symbols(r't_n', real=True, positive=True)\n", - "dt = sp.symbols(r'\\Delta{t}', real=True, positive=True)\n", + "s = sp.symbols(r\"s\", real=True, positive=True)\n", + "t = sp.symbols(r\"t\", real=True, positive=True)\n", + "tn = sp.symbols(r\"t_n\", real=True, positive=True)\n", + "dt = sp.symbols(r\"\\Delta{t}\", real=True, positive=True)\n", "\n", "# The assumption that kx, ky and kz are positive is general enough\n", "# and makes it easier for SymPy to perform some of the calculations\n", - "kx = sp.symbols(r'k_x', real=True, positive=True)\n", - "ky = sp.symbols(r'k_y', real=True, positive=True)\n", - "kz = sp.symbols(r'k_z', real=True, positive=True)\n", + "kx = sp.symbols(r\"k_x\", real=True, positive=True)\n", + "ky = sp.symbols(r\"k_y\", real=True, positive=True)\n", + "kz = sp.symbols(r\"k_z\", real=True, positive=True)\n", "\n", "# Define symbols for the Cartesian components of the electric field\n", - "Ex = sp.symbols(r'E^x')\n", - "Ey = sp.symbols(r'E^y')\n", - "Ez = sp.symbols(r'E^z')\n", + "Ex = sp.symbols(r\"E^x\")\n", + "Ey = sp.symbols(r\"E^y\")\n", + "Ez = sp.symbols(r\"E^z\")\n", "E = Matrix([[Ex], [Ey], [Ez]])\n", "\n", "# Define symbols for the Cartesian components of the magnetic field\n", - "Bx = sp.symbols(r'B^x')\n", - "By = sp.symbols(r'B^y')\n", - "Bz = sp.symbols(r'B^z')\n", + "Bx = sp.symbols(r\"B^x\")\n", + "By = sp.symbols(r\"B^y\")\n", + "Bz = sp.symbols(r\"B^z\")\n", "B = Matrix([[Bx], [By], [Bz]])\n", "\n", "# Define symbol for the scalar field F used with div(E) cleaning\n", "if divE_cleaning:\n", - " F = sp.symbols(r'F')\n", + " F = sp.symbols(r\"F\")\n", "\n", "# Define symbol for the scalar field G used with div(B) cleaning\n", "if divB_cleaning:\n", - " G = sp.symbols(r'G')" + " G = sp.symbols(r\"G\")" ] }, { @@ -149,30 +150,30 @@ "outputs": [], "source": [ "# Define first-order time derivatives of the electric field\n", - "dEx_dt = I*c**2*(ky*Bz-kz*By) \n", - "dEy_dt = I*c**2*(kz*Bx-kx*Bz)\n", - "dEz_dt = I*c**2*(kx*By-ky*Bx)\n", + "dEx_dt = I * c**2 * (ky * Bz - kz * By)\n", + "dEy_dt = I * c**2 * (kz * Bx - kx * Bz)\n", + "dEz_dt = I * c**2 * (kx * By - ky * Bx)\n", "\n", "# Define first-order time derivatives of the magnetic field\n", - "dBx_dt = -I*(ky*Ez-kz*Ey)\n", - "dBy_dt = -I*(kz*Ex-kx*Ez)\n", - "dBz_dt = -I*(kx*Ey-ky*Ex)\n", + "dBx_dt = -I * (ky * Ez - kz * Ey)\n", + "dBy_dt = -I * (kz * Ex - kx * Ez)\n", + "dBz_dt = -I * (kx * Ey - ky * Ex)\n", "\n", "# Define first-order time derivative of the scalar field F used with div(E) cleaning,\n", "# and related additional terms in the first-order time derivative of the electric field\n", "if divE_cleaning:\n", - " dEx_dt += I*c**2*F*kx \n", - " dEy_dt += I*c**2*F*ky\n", - " dEz_dt += I*c**2*F*kz\n", - " dF_dt = I*(kx*Ex+ky*Ey+kz*Ez)\n", + " dEx_dt += I * c**2 * F * kx\n", + " dEy_dt += I * c**2 * F * ky\n", + " dEz_dt += I * c**2 * F * kz\n", + " dF_dt = I * (kx * Ex + ky * Ey + kz * Ez)\n", "\n", "# Define first-order time derivative of the scalar field G used with div(B) cleaning,\n", "# and related additional terms in the first-order time derivative of the magnetic field\n", "if divB_cleaning:\n", - " dBx_dt += I*c**2*G*kx\n", - " dBy_dt += I*c**2*G*ky\n", - " dBz_dt += I*c**2*G*kz\n", - " dG_dt = I*(kx*Bx+ky*By+kz*Bz)\n", + " dBx_dt += I * c**2 * G * kx\n", + " dBy_dt += I * c**2 * G * ky\n", + " dBz_dt += I * c**2 * G * kz\n", + " dG_dt = I * (kx * Bx + ky * By + kz * Bz)\n", "\n", "# Define array of first-order time derivatives of the electric and magnetic fields\n", "dE_dt = Matrix([[dEx_dt], [dEy_dt], [dEz_dt]])\n", @@ -242,8 +243,8 @@ "M = zeros(dim)\n", "for i in range(M.shape[0]):\n", " for j in range(M.shape[1]):\n", - " M[i,j] = dEBFG_dt[i].coeff(EBFG[j], 1)\n", - "print(r'M = ')\n", + " M[i, j] = dEBFG_dt[i].coeff(EBFG[j], 1)\n", + "print(r\"M = \")\n", "display(M)" ] }, @@ -308,12 +309,12 @@ "\n", "# Compute matrices of eigenvectors and eigenvalues for diagonalization of M\n", "P, D = M.diagonalize()\n", - "invP = P**(-1)\n", + "invP = P ** (-1)\n", "expD = exp(D)\n", "check_diag(M, D, P, invP)\n", - "print('P = ')\n", + "print(\"P = \")\n", "display(P)\n", - "print('D = ')\n", + "print(\"D = \")\n", "display(D)" ] }, @@ -334,7 +335,7 @@ "\n", "# Compute matrices of eigenvectors and eigenvalues for diagonalization of W1\n", "P1 = P\n", - "D1 = D * (t-tn)\n", + "D1 = D * (t - tn)\n", "invP1 = invP\n", "expD1 = exp(D1)" ] @@ -418,8 +419,8 @@ "expW1 = P1 * expD1 * invP1\n", "\n", "# Compute general solution at time t = tn+dt\n", - "EBFG_h = expW1 * EBFG \n", - "EBFG_h_new = EBFG_h.subs(t, tn+dt)" + "EBFG_h = expW1 * EBFG\n", + "EBFG_h_new = EBFG_h.subs(t, tn + dt)" ] }, { @@ -436,41 +437,41 @@ "outputs": [], "source": [ "# Define J\n", - "Jx_c0 = sp.symbols(r'\\gamma_{J_x}', real=True)\n", - "Jy_c0 = sp.symbols(r'\\gamma_{J_y}', real=True)\n", - "Jz_c0 = sp.symbols(r'\\gamma_{J_z}', real=True)\n", + "Jx_c0 = sp.symbols(r\"\\gamma_{J_x}\", real=True)\n", + "Jy_c0 = sp.symbols(r\"\\gamma_{J_y}\", real=True)\n", + "Jz_c0 = sp.symbols(r\"\\gamma_{J_z}\", real=True)\n", "Jx = Jx_c0\n", "Jy = Jy_c0\n", "Jz = Jz_c0\n", - "if J_in_time == 'linear':\n", - " Jx_c1 = sp.symbols(r'\\beta_{J_x}', real=True)\n", - " Jy_c1 = sp.symbols(r'\\beta_{J_y}', real=True)\n", - " Jz_c1 = sp.symbols(r'\\beta_{J_z}', real=True)\n", - " Jx += Jx_c1*(s-tn)\n", - " Jy += Jy_c1*(s-tn)\n", - " Jz += Jz_c1*(s-tn)\n", - "if J_in_time == 'quadratic':\n", - " Jx_c1 = sp.symbols(r'\\beta_{J_x}', real=True)\n", - " Jy_c1 = sp.symbols(r'\\beta_{J_y}', real=True)\n", - " Jz_c1 = sp.symbols(r'\\beta_{J_z}', real=True)\n", - " Jx_c2 = sp.symbols(r'\\alpha_{J_x}', real=True)\n", - " Jy_c2 = sp.symbols(r'\\alpha_{J_y}', real=True)\n", - " Jz_c2 = sp.symbols(r'\\alpha_{J_z}', real=True)\n", - " Jx += Jx_c1*(s-tn) + Jx_c2*(s-tn)**2\n", - " Jy += Jy_c1*(s-tn) + Jy_c2*(s-tn)**2\n", - " Jz += Jz_c1*(s-tn) + Jz_c2*(s-tn)**2\n", + "if J_in_time == \"linear\":\n", + " Jx_c1 = sp.symbols(r\"\\beta_{J_x}\", real=True)\n", + " Jy_c1 = sp.symbols(r\"\\beta_{J_y}\", real=True)\n", + " Jz_c1 = sp.symbols(r\"\\beta_{J_z}\", real=True)\n", + " Jx += Jx_c1 * (s - tn)\n", + " Jy += Jy_c1 * (s - tn)\n", + " Jz += Jz_c1 * (s - tn)\n", + "if J_in_time == \"quadratic\":\n", + " Jx_c1 = sp.symbols(r\"\\beta_{J_x}\", real=True)\n", + " Jy_c1 = sp.symbols(r\"\\beta_{J_y}\", real=True)\n", + " Jz_c1 = sp.symbols(r\"\\beta_{J_z}\", real=True)\n", + " Jx_c2 = sp.symbols(r\"\\alpha_{J_x}\", real=True)\n", + " Jy_c2 = sp.symbols(r\"\\alpha_{J_y}\", real=True)\n", + " Jz_c2 = sp.symbols(r\"\\alpha_{J_z}\", real=True)\n", + " Jx += Jx_c1 * (s - tn) + Jx_c2 * (s - tn) ** 2\n", + " Jy += Jy_c1 * (s - tn) + Jy_c2 * (s - tn) ** 2\n", + " Jz += Jz_c1 * (s - tn) + Jz_c2 * (s - tn) ** 2\n", "\n", "# Define rho\n", "if divE_cleaning:\n", - " rho_c0 = sp.symbols(r'\\gamma_{\\rho}', real=True)\n", + " rho_c0 = sp.symbols(r\"\\gamma_{\\rho}\", real=True)\n", " rho = rho_c0\n", - " if rho_in_time == 'linear':\n", - " rho_c1 = sp.symbols(r'\\beta_{\\rho}', real=True)\n", - " rho += rho_c1*(s-tn)\n", - " if rho_in_time == 'quadratic':\n", - " rho_c1 = sp.symbols(r'\\beta_{\\rho}', real=True)\n", - " rho_c2 = sp.symbols(r'\\alpha_{\\rho}', real=True)\n", - " rho += rho_c1*(s-tn) + rho_c2*(s-tn)**2" + " if rho_in_time == \"linear\":\n", + " rho_c1 = sp.symbols(r\"\\beta_{\\rho}\", real=True)\n", + " rho += rho_c1 * (s - tn)\n", + " if rho_in_time == \"quadratic\":\n", + " rho_c1 = sp.symbols(r\"\\beta_{\\rho}\", real=True)\n", + " rho_c2 = sp.symbols(r\"\\alpha_{\\rho}\", real=True)\n", + " rho += rho_c1 * (s - tn) + rho_c2 * (s - tn) ** 2" ] }, { @@ -500,9 +501,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "%%time \n", @@ -515,7 +514,7 @@ " fields_list.append(0)\n", "S = zeros(dim, 1)\n", "for i in range(S.shape[0]):\n", - " S[i] = -mu0*c**2 * fields_list[i]\n", + " S[i] = -mu0 * c**2 * fields_list[i]\n", "\n", "# Compute integral of exp(W3)*S over s (assuming |k| is not zero)\n", "integral = zeros(dim, 1)\n", @@ -524,12 +523,12 @@ "for i in range(dim):\n", " r = integrate(tmp[i], (s, tn, t))\n", " integral[i] = r\n", - " \n", + "\n", "# Compute particular solution at time t = tn+dt\n", "tmp = invP2 * P3\n", "simple_mat(tmp)\n", "EBFG_nh = P2 * expD2 * tmp * integral\n", - "EBFG_nh_new = EBFG_nh.subs(t, tn+dt)" + "EBFG_nh_new = EBFG_nh.subs(t, tn + dt)" ] }, { @@ -549,13 +548,13 @@ "\n", "for i in range(EBFG.shape[0]):\n", " lhs = dEBFG_dt[i] + S[i]\n", - " lhs = lhs.subs(s, tn) # sources were written as functions of s\n", + " lhs = lhs.subs(s, tn) # sources were written as functions of s\n", " rhs = (EBFG_h[i] + EBFG_nh[i]).diff(t)\n", - " rhs = rhs.subs(t, tn) # results were written as functions of t\n", + " rhs = rhs.subs(t, tn) # results were written as functions of t\n", " rhs = rhs.simplify()\n", " diff = lhs - rhs\n", " diff = diff.simplify()\n", - " assert (diff == 0), f'Integration of linear system of ODEs failed'" + " assert diff == 0, \"Integration of linear system of ODEs failed\"" ] }, { @@ -575,22 +574,30 @@ "source": [ "%%time\n", "\n", - "L = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz']\n", - "R = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz']\n", + "L = [\"Ex\", \"Ey\", \"Ez\", \"Bx\", \"By\", \"Bz\"]\n", + "R = [\"Ex\", \"Ey\", \"Ez\", \"Bx\", \"By\", \"Bz\"]\n", "if divE_cleaning:\n", - " L.append('F')\n", - " R.append('F')\n", + " L.append(\"F\")\n", + " R.append(\"F\")\n", "if divB_cleaning:\n", - " L.append('G')\n", - " R.append('G')\n", + " L.append(\"G\")\n", + " R.append(\"G\")\n", "\n", "# Compute individual coefficients in the update equations\n", "coeff_h = dict()\n", "for i in range(dim):\n", " for j in range(dim):\n", " key = (L[i], R[j])\n", - " coeff_h[key] = EBFG_h_new[i].coeff(EBFG[j], 1).expand().simplify().rewrite(cos).trigsimp().simplify()\n", - " print(f'Coefficient of {L[i]} with respect to {R[j]}:')\n", + " coeff_h[key] = (\n", + " EBFG_h_new[i]\n", + " .coeff(EBFG[j], 1)\n", + " .expand()\n", + " .simplify()\n", + " .rewrite(cos)\n", + " .trigsimp()\n", + " .simplify()\n", + " )\n", + " print(f\"Coefficient of {L[i]} with respect to {R[j]}:\")\n", " display(coeff_h[key])" ] }, @@ -611,36 +618,36 @@ "source": [ "%%time\n", "\n", - "L = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz']\n", - "R = ['Jx_c0', 'Jy_c0', 'Jz_c0']\n", - "if J_in_time == 'linear':\n", - " R.append('Jx_c1')\n", - " R.append('Jy_c1')\n", - " R.append('Jz_c1')\n", - "if J_in_time == 'quadratic':\n", - " R.append('Jx_c1')\n", - " R.append('Jy_c1')\n", - " R.append('Jz_c1')\n", - " R.append('Jx_c2')\n", - " R.append('Jy_c2')\n", - " R.append('Jz_c2')\n", + "L = [\"Ex\", \"Ey\", \"Ez\", \"Bx\", \"By\", \"Bz\"]\n", + "R = [\"Jx_c0\", \"Jy_c0\", \"Jz_c0\"]\n", + "if J_in_time == \"linear\":\n", + " R.append(\"Jx_c1\")\n", + " R.append(\"Jy_c1\")\n", + " R.append(\"Jz_c1\")\n", + "if J_in_time == \"quadratic\":\n", + " R.append(\"Jx_c1\")\n", + " R.append(\"Jy_c1\")\n", + " R.append(\"Jz_c1\")\n", + " R.append(\"Jx_c2\")\n", + " R.append(\"Jy_c2\")\n", + " R.append(\"Jz_c2\")\n", "if divE_cleaning:\n", - " L.append('F')\n", - " R.append('rho_c0')\n", - " if rho_in_time == 'linear':\n", - " R.append('rho_c1')\n", - " if rho_in_time == 'quadratic':\n", - " R.append('rho_c1')\n", - " R.append('rho_c2')\n", + " L.append(\"F\")\n", + " R.append(\"rho_c0\")\n", + " if rho_in_time == \"linear\":\n", + " R.append(\"rho_c1\")\n", + " if rho_in_time == \"quadratic\":\n", + " R.append(\"rho_c1\")\n", + " R.append(\"rho_c2\")\n", "if divB_cleaning:\n", - " L.append('G')\n", + " L.append(\"G\")\n", "\n", "cs = [Jx_c0, Jy_c0, Jz_c0]\n", - "if J_in_time == 'linear':\n", + "if J_in_time == \"linear\":\n", " cs.append(Jx_c1)\n", " cs.append(Jy_c1)\n", " cs.append(Jz_c1)\n", - "if J_in_time == 'quadratic':\n", + "if J_in_time == \"quadratic\":\n", " cs.append(Jx_c1)\n", " cs.append(Jy_c1)\n", " cs.append(Jz_c1)\n", @@ -649,19 +656,28 @@ " cs.append(Jz_c2)\n", "if divE_cleaning:\n", " cs.append(rho_c0)\n", - " if rho_in_time == 'linear':\n", + " if rho_in_time == \"linear\":\n", " cs.append(rho_c1)\n", - " if rho_in_time == 'quadratic':\n", + " if rho_in_time == \"quadratic\":\n", " cs.append(rho_c1)\n", " cs.append(rho_c2)\n", - " \n", + "\n", "# Compute individual coefficients in the update equation\n", "coeff_nh = dict()\n", "for i in range(len(L)):\n", " for j in range(len(R)):\n", " key = (L[i], R[j])\n", - " coeff_nh[key] = EBFG_nh_new[i].expand().coeff(cs[j], 1).expand().simplify().rewrite(cos).trigsimp().simplify()\n", - " print(f'Coefficient of {L[i]} with respect to {R[j]}:')\n", + " coeff_nh[key] = (\n", + " EBFG_nh_new[i]\n", + " .expand()\n", + " .coeff(cs[j], 1)\n", + " .expand()\n", + " .simplify()\n", + " .rewrite(cos)\n", + " .trigsimp()\n", + " .simplify()\n", + " )\n", + " print(f\"Coefficient of {L[i]} with respect to {R[j]}:\")\n", " display(coeff_nh[key])" ] }, @@ -677,29 +693,25 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ - "coeff_h[('Ex', 'By')]" + "coeff_h[(\"Ex\", \"By\")]" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ - "coeff_nh[('Ex', 'Jx_c0')]" + "coeff_nh[(\"Ex\", \"Jx_c0\")]" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -713,7 +725,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/Tools/Algorithms/psatd_pml.ipynb b/Tools/Algorithms/psatd_pml.ipynb index 897ffa70b9e..66ae219747b 100644 --- a/Tools/Algorithms/psatd_pml.ipynb +++ b/Tools/Algorithms/psatd_pml.ipynb @@ -8,9 +8,8 @@ }, "outputs": [], "source": [ - "import inspect\n", "import sympy as sp\n", - "from sympy import *\n", + "from sympy import * # noqa\n", "from sympy.solvers.solveset import linsolve\n", "\n", "sp.init_session()\n", @@ -35,16 +34,16 @@ "DD = 12\n", "\n", "# Speed of light, time, time step\n", - "c = sp.symbols(r'c', real = True, positive = True)\n", - "t = sp.symbols(r't', real = True)\n", - "tn = sp.symbols(r't_n', real = True)\n", - "dt = sp.symbols(r'\\Delta{t}', real = True, positive = True)\n", + "c = sp.symbols(r\"c\", real=True, positive=True)\n", + "t = sp.symbols(r\"t\", real=True)\n", + "tn = sp.symbols(r\"t_n\", real=True)\n", + "dt = sp.symbols(r\"\\Delta{t}\", real=True, positive=True)\n", "\n", "# Components of k vector, omega, norm of k vector\n", - "kx = sp.symbols(r'k_x', real = True)\n", - "ky = sp.symbols(r'k_y', real = True)\n", - "kz = sp.symbols(r'k_z', real = True)\n", - "om = sp.symbols(r'omega', real = True, positive = True)\n", + "kx = sp.symbols(r\"k_x\", real=True)\n", + "ky = sp.symbols(r\"k_y\", real=True)\n", + "kz = sp.symbols(r\"k_z\", real=True)\n", + "om = sp.symbols(r\"omega\", real=True, positive=True)\n", "knorm = sp.sqrt(kx**2 + ky**2 + kz**2)" ] }, @@ -98,10 +97,12 @@ "outputs": [], "source": [ "def C(omega, t):\n", - " return sp.cos(omega * (t-tn))\n", + " return sp.cos(omega * (t - tn))\n", + "\n", "\n", "def S(omega, t):\n", - " return (t-tn) if omega == 0. else sp.sin(omega * (t-tn)) / omega\n", + " return (t - tn) if omega == 0.0 else sp.sin(omega * (t - tn)) / omega\n", + "\n", "\n", "def Xt(eigenpairs, a, b, t):\n", " \"\"\"\n", @@ -114,7 +115,7 @@ " # Loop over matrix eigenpairs\n", " for ep in eigenpairs:\n", " # ep[0] is an eigenvalue and om = sp.sqrt(-ep[0])\n", - " omega = 0. if ep[0] == 0. else om\n", + " omega = 0.0 if ep[0] == 0.0 else om\n", " # am is the algebraic multiplicity of the eigenvalue\n", " am = ep[1]\n", " # vF is the list of all eigenvectors corresponding to the eigenvalue\n", @@ -125,6 +126,7 @@ " i += 1\n", " return XX\n", "\n", + "\n", "def evolve(X, dX_dt, d2X_dt2):\n", " \"\"\"\n", " Solve ordinary differential equation X'' = M*X.\n", @@ -133,15 +135,15 @@ " MX = zeros(DD)\n", " for i in range(DD):\n", " for j in range(DD):\n", - " MX[i,j] = d2X_dt2[i].coeff(X[j], 1)\n", - " #MX /= c**2\n", + " MX[i, j] = d2X_dt2[i].coeff(X[j], 1)\n", + " # MX /= c**2\n", "\n", " print()\n", - " print(r'Matrix:')\n", + " print(r\"Matrix:\")\n", " display(MX)\n", "\n", " # Characteristic matrix\n", - " lamda = sp.symbols(r'lamda')\n", + " lamda = sp.symbols(r\"lamda\")\n", " Id = eye(DD)\n", " MX_charmat = MX - lamda * Id\n", "\n", @@ -149,9 +151,9 @@ " MX_charpoly = MX_charmat.det()\n", " MX_charpoly = factor(MX_charpoly.as_expr())\n", "\n", - " print(r'Characteristic polynomial:')\n", + " print(r\"Characteristic polynomial:\")\n", " display(MX_charpoly)\n", - " \n", + "\n", " MX_eigenvals = sp.solve(MX_charpoly, lamda)\n", "\n", " # List of eigenvectors\n", @@ -159,59 +161,58 @@ "\n", " # List of eigenpairs\n", " MX_eigenpairs = []\n", - " \n", - " # Compute eigenvectors as null spaces\n", - " for l in MX_eigenvals:\n", "\n", + " # Compute eigenvectors as null spaces\n", + " for ev in MX_eigenvals:\n", " # M - lamda * Id\n", - " A = MX_charmat.subs(lamda, l)\n", + " A = MX_charmat.subs(lamda, ev)\n", " A.simplify()\n", "\n", - " print(r'Eigenvalue:')\n", - " display(l)\n", + " print(r\"Eigenvalue:\")\n", + " display(ev)\n", "\n", - " print(r'Characteristic matrix:')\n", + " print(r\"Characteristic matrix:\")\n", " display(A)\n", "\n", " # Perform Gaussian elimination (necessary for lamda != 0)\n", - " if (l != 0.):\n", - " print(r'Gaussian elimination:')\n", - " print(r'A[0,:] += A[1,:]')\n", - " A[0,:] += A[1,:]\n", - " print(r'A[0,:] += A[2,:]')\n", - " A[0,:] += A[2,:]\n", - " print(r'Swap A[0,:] and A[11,:]')\n", - " row = A[11,:]\n", - " A[11,:] = A[0,:]\n", - " A[0,:] = row\n", - " print(r'A[3,:] += A[4,:]')\n", - " A[3,:] += A[4,:]\n", - " print(r'A[3,:] += A[5,:]')\n", - " A[3,:] += A[5,:]\n", - " print(r'Swap A[3,:] and A[10,:]')\n", - " row = A[10,:]\n", - " A[10,:] = A[3,:]\n", - " A[3,:] = row\n", - " print(r'A[0,:] += A[3,:]')\n", - " A[0,:] += A[3,:]\n", - " print(r'A[0,:] += A[9,:]')\n", - " A[0,:] += A[9,:]\n", - " print(r'Swap A[0,:] and A[9,:]')\n", - " row = A[9,:]\n", - " A[9,:] = A[0,:]\n", - " A[0,:] = row\n", - " print(r'A[6,:] += A[8,:]')\n", - " A[6,:] += A[8,:]\n", - " print(r'A[6,:] += A[7,:]')\n", - " A[6,:] += A[7,:]\n", - " print(r'Swap A[6,:] and A[8,:]')\n", - " row = A[8,:]\n", - " A[8,:] = A[6,:]\n", - " A[6,:] = row\n", - " print(r'A[6,:] += A[7,:]')\n", - " A[6,:] += A[7,:]\n", - " print(r'A[4,:] += A[5,:]')\n", - " A[4,:] += A[5,:]\n", + " if ev != 0.0:\n", + " print(r\"Gaussian elimination:\")\n", + " print(r\"A[0,:] += A[1,:]\")\n", + " A[0, :] += A[1, :]\n", + " print(r\"A[0,:] += A[2,:]\")\n", + " A[0, :] += A[2, :]\n", + " print(r\"Swap A[0,:] and A[11,:]\")\n", + " row = A[11, :]\n", + " A[11, :] = A[0, :]\n", + " A[0, :] = row\n", + " print(r\"A[3,:] += A[4,:]\")\n", + " A[3, :] += A[4, :]\n", + " print(r\"A[3,:] += A[5,:]\")\n", + " A[3, :] += A[5, :]\n", + " print(r\"Swap A[3,:] and A[10,:]\")\n", + " row = A[10, :]\n", + " A[10, :] = A[3, :]\n", + " A[3, :] = row\n", + " print(r\"A[0,:] += A[3,:]\")\n", + " A[0, :] += A[3, :]\n", + " print(r\"A[0,:] += A[9,:]\")\n", + " A[0, :] += A[9, :]\n", + " print(r\"Swap A[0,:] and A[9,:]\")\n", + " row = A[9, :]\n", + " A[9, :] = A[0, :]\n", + " A[0, :] = row\n", + " print(r\"A[6,:] += A[8,:]\")\n", + " A[6, :] += A[8, :]\n", + " print(r\"A[6,:] += A[7,:]\")\n", + " A[6, :] += A[7, :]\n", + " print(r\"Swap A[6,:] and A[8,:]\")\n", + " row = A[8, :]\n", + " A[8, :] = A[6, :]\n", + " A[6, :] = row\n", + " print(r\"A[6,:] += A[7,:]\")\n", + " A[6, :] += A[7, :]\n", + " print(r\"A[4,:] += A[5,:]\")\n", + " A[4, :] += A[5, :]\n", " A.simplify()\n", " display(A)\n", "\n", @@ -219,36 +220,36 @@ " v = A.nullspace()\n", " MX_eigenvects.append(v)\n", "\n", - " print(r'Eigenvectors:')\n", + " print(r\"Eigenvectors:\")\n", " display(v)\n", - " \n", + "\n", " # Store eigenpairs (eigenvalue, algebraic multiplicity, eigenvectors)\n", - " MX_eigenpairs.append((l, len(v), v))\n", - " \n", - " #print(r'Eigenpairs:')\n", - " #display(MX_eigenpairs)\n", + " MX_eigenpairs.append((ev, len(v), v))\n", + "\n", + " # print(r'Eigenpairs:')\n", + " # display(MX_eigenpairs)\n", "\n", " # Verify that the eigenpairs satisfy the characteristic equations\n", " for ep in MX_eigenpairs:\n", " for j in range(ep[1]):\n", " diff = MX * ep[2][j] - ep[0] * ep[2][j]\n", " diff.simplify()\n", - " if diff != zeros(DD,1):\n", - " print('The charcteristic equation is not verified for some eigenpairs')\n", + " if diff != zeros(DD, 1):\n", + " print(\"The charcteristic equation is not verified for some eigenpairs\")\n", " display(diff)\n", "\n", " # Define integration constants\n", " a = []\n", " b = []\n", " for i in range(DD):\n", - " an = r'a_{:d}'.format(i+1)\n", - " bn = r'b_{:d}'.format(i+1) \n", + " an = r\"a_{:d}\".format(i + 1)\n", + " bn = r\"b_{:d}\".format(i + 1)\n", " a.append(sp.symbols(an))\n", " b.append(sp.symbols(bn))\n", "\n", " # Set equations corresponding to initial conditions\n", " lhs_a = Xt(MX_eigenpairs, a, b, tn) - X\n", - " lhs_b = Xt(MX_eigenpairs, a, b, t ).diff(t).subs(t, tn) - dX_dt\n", + " lhs_b = Xt(MX_eigenpairs, a, b, t).diff(t).subs(t, tn) - dX_dt\n", "\n", " # Compute integration constants from initial conditions\n", " # (convert list of tuples to list using list comprehension)\n", @@ -258,7 +259,7 @@ " b = [item for el in b for item in el]\n", "\n", " # Evaluate solution at t = tn + dt\n", - " X_new = Xt(MX_eigenpairs, a, b, tn+dt).expand()\n", + " X_new = Xt(MX_eigenpairs, a, b, tn + dt).expand()\n", " for d in range(DD):\n", " for Eij in E:\n", " X_new[d] = X_new[d].collect(Eij)\n", @@ -272,12 +273,12 @@ " # Check correctness by taking *second* derivative\n", " # and comparing with initial right-hand side at time tn\n", " X_t = Xt(MX_eigenpairs, a, b, t)\n", - " diff = X_t.diff(t).diff(t).subs(t, tn).subs(om, c*knorm).expand() - d2X_dt2\n", + " diff = X_t.diff(t).diff(t).subs(t, tn).subs(om, c * knorm).expand() - d2X_dt2\n", " diff.simplify()\n", - " if diff != zeros(DD,1):\n", - " print('Integration in time failed')\n", + " if diff != zeros(DD, 1):\n", + " print(\"Integration in time failed\")\n", " display(diff)\n", - " \n", + "\n", " return X_t, X_new" ] }, @@ -334,121 +335,167 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "# indices 0 1 2 3 4 5 6 7 8\n", - "labels = ['xx', 'xy', 'xz', 'yx', 'yy', 'yz', 'zx', 'zy', 'zz']\n", + "labels = [\"xx\", \"xy\", \"xz\", \"yx\", \"yy\", \"yz\", \"zx\", \"zy\", \"zz\"]\n", "\n", "# E fields\n", - "Exx = sp.symbols(r'E_{xx}')\n", - "Exy = sp.symbols(r'E_{xy}')\n", - "Exz = sp.symbols(r'E_{xz}')\n", - "Eyx = sp.symbols(r'E_{yx}')\n", - "Eyy = sp.symbols(r'E_{yy}')\n", - "Eyz = sp.symbols(r'E_{yz}')\n", - "Ezx = sp.symbols(r'E_{zx}')\n", - "Ezy = sp.symbols(r'E_{zy}')\n", - "Ezz = sp.symbols(r'E_{zz}')\n", - "E = Matrix([[Exx],[Exy],[Exz],[Eyx],[Eyy],[Eyz],[Ezx],[Ezy],[Ezz]])\n", + "Exx = sp.symbols(r\"E_{xx}\")\n", + "Exy = sp.symbols(r\"E_{xy}\")\n", + "Exz = sp.symbols(r\"E_{xz}\")\n", + "Eyx = sp.symbols(r\"E_{yx}\")\n", + "Eyy = sp.symbols(r\"E_{yy}\")\n", + "Eyz = sp.symbols(r\"E_{yz}\")\n", + "Ezx = sp.symbols(r\"E_{zx}\")\n", + "Ezy = sp.symbols(r\"E_{zy}\")\n", + "Ezz = sp.symbols(r\"E_{zz}\")\n", + "E = Matrix([[Exx], [Exy], [Exz], [Eyx], [Eyy], [Eyz], [Ezx], [Ezy], [Ezz]])\n", "\n", "# B fields\n", - "Bxx = sp.symbols(r'B_{xx}')\n", - "Bxy = sp.symbols(r'B_{xy}')\n", - "Bxz = sp.symbols(r'B_{xz}')\n", - "Byx = sp.symbols(r'B_{yx}')\n", - "Byy = sp.symbols(r'B_{yy}')\n", - "Byz = sp.symbols(r'B_{yz}')\n", - "Bzx = sp.symbols(r'B_{zx}')\n", - "Bzy = sp.symbols(r'B_{zy}')\n", - "Bzz = sp.symbols(r'B_{zz}')\n", - "B = Matrix([[Bxx],[Bxy],[Bxz],[Byx],[Byy],[Byz],[Bzx],[Bzy],[Bzz]])\n", + "Bxx = sp.symbols(r\"B_{xx}\")\n", + "Bxy = sp.symbols(r\"B_{xy}\")\n", + "Bxz = sp.symbols(r\"B_{xz}\")\n", + "Byx = sp.symbols(r\"B_{yx}\")\n", + "Byy = sp.symbols(r\"B_{yy}\")\n", + "Byz = sp.symbols(r\"B_{yz}\")\n", + "Bzx = sp.symbols(r\"B_{zx}\")\n", + "Bzy = sp.symbols(r\"B_{zy}\")\n", + "Bzz = sp.symbols(r\"B_{zz}\")\n", + "B = Matrix([[Bxx], [Bxy], [Bxz], [Byx], [Byy], [Byz], [Bzx], [Bzy], [Bzz]])\n", "\n", "# F fields\n", - "Fx = sp.symbols(r'F_{x}')\n", - "Fy = sp.symbols(r'F_{y}')\n", - "Fz = sp.symbols(r'F_{z}')\n", - "F = Matrix([[Fx],[Fy],[Fz]])\n", + "Fx = sp.symbols(r\"F_{x}\")\n", + "Fy = sp.symbols(r\"F_{y}\")\n", + "Fz = sp.symbols(r\"F_{z}\")\n", + "F = Matrix([[Fx], [Fy], [Fz]])\n", "\n", "# G fields\n", - "Gx = sp.symbols(r'G_{x}')\n", - "Gy = sp.symbols(r'G_{y}')\n", - "Gz = sp.symbols(r'G_{z}')\n", - "G = Matrix([[Gx],[Gy],[Gz]])\n", + "Gx = sp.symbols(r\"G_{x}\")\n", + "Gy = sp.symbols(r\"G_{y}\")\n", + "Gz = sp.symbols(r\"G_{z}\")\n", + "G = Matrix([[Gx], [Gy], [Gz]])\n", "\n", "# dE/dt\n", - "dExx_dt = c**2 * I * kx * (Fx + Fy + Fz)\n", - "dExy_dt = c**2 * I * ky * (Bzx + Bzy + Bzz)\n", - "dExz_dt = - c**2 * I * kz * (Byx + Byy + Byz)\n", - "dEyx_dt = - c**2 * I * kx * (Bzx + Bzy + Bzz)\n", - "dEyy_dt = c**2 * I * ky * (Fx + Fy + Fz)\n", - "dEyz_dt = c**2 * I * kz * (Bxx + Bxy + Bxz)\n", - "dEzx_dt = c**2 * I * kx * (Byx + Byy + Byz)\n", - "dEzy_dt = - c**2 * I * ky * (Bxx + Bxy + Bxz)\n", - "dEzz_dt = c**2 * I * kz * (Fx + Fy + Fz)\n", - "dE_dt = Matrix([[dExx_dt],[dExy_dt],[dExz_dt],[dEyx_dt],[dEyy_dt],[dEyz_dt],[dEzx_dt],[dEzy_dt],[dEzz_dt]])\n", + "dExx_dt = c**2 * I * kx * (Fx + Fy + Fz)\n", + "dExy_dt = c**2 * I * ky * (Bzx + Bzy + Bzz)\n", + "dExz_dt = -(c**2) * I * kz * (Byx + Byy + Byz)\n", + "dEyx_dt = -(c**2) * I * kx * (Bzx + Bzy + Bzz)\n", + "dEyy_dt = c**2 * I * ky * (Fx + Fy + Fz)\n", + "dEyz_dt = c**2 * I * kz * (Bxx + Bxy + Bxz)\n", + "dEzx_dt = c**2 * I * kx * (Byx + Byy + Byz)\n", + "dEzy_dt = -(c**2) * I * ky * (Bxx + Bxy + Bxz)\n", + "dEzz_dt = c**2 * I * kz * (Fx + Fy + Fz)\n", + "dE_dt = Matrix(\n", + " [\n", + " [dExx_dt],\n", + " [dExy_dt],\n", + " [dExz_dt],\n", + " [dEyx_dt],\n", + " [dEyy_dt],\n", + " [dEyz_dt],\n", + " [dEzx_dt],\n", + " [dEzy_dt],\n", + " [dEzz_dt],\n", + " ]\n", + ")\n", "\n", "# dB/dt\n", - "dBxx_dt = I * kx * (Gx + Gy + Gz)\n", - "dBxy_dt = - I * ky * (Ezx + Ezy + Ezz)\n", - "dBxz_dt = I * kz * (Eyx + Eyy + Eyz)\n", - "dByx_dt = I * kx * (Ezx + Ezy + Ezz)\n", - "dByy_dt = I * ky * (Gx + Gy + Gz)\n", - "dByz_dt = - I * kz * (Exx + Exy + Exz)\n", - "dBzx_dt = - I * kx * (Eyx + Eyy + Eyz)\n", - "dBzy_dt = I * ky * (Exx + Exy + Exz)\n", - "dBzz_dt = I * kz * (Gx + Gy + Gz)\n", - "dB_dt = Matrix([[dBxx_dt],[dBxy_dt],[dBxz_dt],[dByx_dt],[dByy_dt],[dByz_dt],[dBzx_dt],[dBzy_dt],[dBzz_dt]])\n", + "dBxx_dt = I * kx * (Gx + Gy + Gz)\n", + "dBxy_dt = -I * ky * (Ezx + Ezy + Ezz)\n", + "dBxz_dt = I * kz * (Eyx + Eyy + Eyz)\n", + "dByx_dt = I * kx * (Ezx + Ezy + Ezz)\n", + "dByy_dt = I * ky * (Gx + Gy + Gz)\n", + "dByz_dt = -I * kz * (Exx + Exy + Exz)\n", + "dBzx_dt = -I * kx * (Eyx + Eyy + Eyz)\n", + "dBzy_dt = I * ky * (Exx + Exy + Exz)\n", + "dBzz_dt = I * kz * (Gx + Gy + Gz)\n", + "dB_dt = Matrix(\n", + " [\n", + " [dBxx_dt],\n", + " [dBxy_dt],\n", + " [dBxz_dt],\n", + " [dByx_dt],\n", + " [dByy_dt],\n", + " [dByz_dt],\n", + " [dBzx_dt],\n", + " [dBzy_dt],\n", + " [dBzz_dt],\n", + " ]\n", + ")\n", "\n", "# dF/dt\n", "dFx_dt = I * kx * (Exx + Exy + Exz)\n", "dFy_dt = I * ky * (Eyx + Eyy + Eyz)\n", "dFz_dt = I * kz * (Ezx + Ezy + Ezz)\n", - "dF_dt = Matrix([[dFx_dt],[dFy_dt],[dFz_dt]])\n", + "dF_dt = Matrix([[dFx_dt], [dFy_dt], [dFz_dt]])\n", "\n", "# dG/dt\n", "dGx_dt = c**2 * I * kx * (Bxx + Bxy + Bxz)\n", "dGy_dt = c**2 * I * ky * (Byx + Byy + Byz)\n", "dGz_dt = c**2 * I * kz * (Bzx + Bzy + Bzz)\n", - "dG_dt = Matrix([[dGx_dt],[dGy_dt],[dGz_dt]])\n", + "dG_dt = Matrix([[dGx_dt], [dGy_dt], [dGz_dt]])\n", "\n", "# d2E/dt2\n", - "d2Exx_dt2 = c**2 * I * kx * (dFx_dt + dFy_dt + dFz_dt)\n", - "d2Exy_dt2 = c**2 * I * ky * (dBzx_dt + dBzy_dt + dBzz_dt)\n", - "d2Exz_dt2 = - c**2 * I * kz * (dByx_dt + dByy_dt + dByz_dt)\n", - "d2Eyx_dt2 = - c**2 * I * kx * (dBzx_dt + dBzy_dt + dBzz_dt)\n", - "d2Eyy_dt2 = c**2 * I * ky * (dFx_dt + dFy_dt + dFz_dt)\n", - "d2Eyz_dt2 = c**2 * I * kz * (dBxx_dt + dBxy_dt + dBxz_dt)\n", - "d2Ezx_dt2 = c**2 * I * kx * (dByx_dt + dByy_dt + dByz_dt)\n", - "d2Ezy_dt2 = - c**2 * I * ky * (dBxx_dt + dBxy_dt + dBxz_dt)\n", - "d2Ezz_dt2 = c**2 * I * kz * (dFx_dt + dFy_dt + dFz_dt)\n", - "d2E_dt2 = Matrix([[d2Exx_dt2],[d2Exy_dt2],[d2Exz_dt2],[d2Eyx_dt2],[d2Eyy_dt2],[d2Eyz_dt2],[d2Ezx_dt2],[d2Ezy_dt2],[d2Ezz_dt2]])\n", + "d2Exx_dt2 = c**2 * I * kx * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2Exy_dt2 = c**2 * I * ky * (dBzx_dt + dBzy_dt + dBzz_dt)\n", + "d2Exz_dt2 = -(c**2) * I * kz * (dByx_dt + dByy_dt + dByz_dt)\n", + "d2Eyx_dt2 = -(c**2) * I * kx * (dBzx_dt + dBzy_dt + dBzz_dt)\n", + "d2Eyy_dt2 = c**2 * I * ky * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2Eyz_dt2 = c**2 * I * kz * (dBxx_dt + dBxy_dt + dBxz_dt)\n", + "d2Ezx_dt2 = c**2 * I * kx * (dByx_dt + dByy_dt + dByz_dt)\n", + "d2Ezy_dt2 = -(c**2) * I * ky * (dBxx_dt + dBxy_dt + dBxz_dt)\n", + "d2Ezz_dt2 = c**2 * I * kz * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2E_dt2 = Matrix(\n", + " [\n", + " [d2Exx_dt2],\n", + " [d2Exy_dt2],\n", + " [d2Exz_dt2],\n", + " [d2Eyx_dt2],\n", + " [d2Eyy_dt2],\n", + " [d2Eyz_dt2],\n", + " [d2Ezx_dt2],\n", + " [d2Ezy_dt2],\n", + " [d2Ezz_dt2],\n", + " ]\n", + ")\n", "\n", "# d2B/dt2\n", - "d2Bxx_dt2 = I * kx * (dGx_dt + dGy_dt + dGz_dt)\n", - "d2Bxy_dt2 = - I * ky * (dEzx_dt + dEzy_dt + dEzz_dt)\n", - "d2Bxz_dt2 = I * kz * (dEyx_dt + dEyy_dt + dEyz_dt)\n", - "d2Byx_dt2 = I * kx * (dEzx_dt + dEzy_dt + dEzz_dt)\n", - "d2Byy_dt2 = I * ky * (dGx_dt + dGy_dt + dGz_dt)\n", - "d2Byz_dt2 = - I * kz * (dExx_dt + dExy_dt + dExz_dt)\n", - "d2Bzx_dt2 = - I * kx * (dEyx_dt + dEyy_dt + dEyz_dt)\n", - "d2Bzy_dt2 = I * ky * (dExx_dt + dExy_dt + dExz_dt)\n", - "d2Bzz_dt2 = I * kz * (dGx_dt + dGy_dt + dGz_dt)\n", - "d2B_dt2 = Matrix([[d2Bxx_dt2],[d2Bxy_dt2],[d2Bxz_dt2],[d2Byx_dt2],[d2Byy_dt2],[d2Byz_dt2],[d2Bzx_dt2],[d2Bzy_dt2],[d2Bzz_dt2]])\n", + "d2Bxx_dt2 = I * kx * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2Bxy_dt2 = -I * ky * (dEzx_dt + dEzy_dt + dEzz_dt)\n", + "d2Bxz_dt2 = I * kz * (dEyx_dt + dEyy_dt + dEyz_dt)\n", + "d2Byx_dt2 = I * kx * (dEzx_dt + dEzy_dt + dEzz_dt)\n", + "d2Byy_dt2 = I * ky * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2Byz_dt2 = -I * kz * (dExx_dt + dExy_dt + dExz_dt)\n", + "d2Bzx_dt2 = -I * kx * (dEyx_dt + dEyy_dt + dEyz_dt)\n", + "d2Bzy_dt2 = I * ky * (dExx_dt + dExy_dt + dExz_dt)\n", + "d2Bzz_dt2 = I * kz * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2B_dt2 = Matrix(\n", + " [\n", + " [d2Bxx_dt2],\n", + " [d2Bxy_dt2],\n", + " [d2Bxz_dt2],\n", + " [d2Byx_dt2],\n", + " [d2Byy_dt2],\n", + " [d2Byz_dt2],\n", + " [d2Bzx_dt2],\n", + " [d2Bzy_dt2],\n", + " [d2Bzz_dt2],\n", + " ]\n", + ")\n", "\n", "# d2F/dt2\n", "d2Fx_dt2 = I * kx * (dExx_dt + dExy_dt + dExz_dt)\n", "d2Fy_dt2 = I * ky * (dEyx_dt + dEyy_dt + dEyz_dt)\n", "d2Fz_dt2 = I * kz * (dEzx_dt + dEzy_dt + dEzz_dt)\n", - "d2F_dt2 = Matrix([[d2Fx_dt2],[d2Fy_dt2],[d2Fz_dt2]])\n", + "d2F_dt2 = Matrix([[d2Fx_dt2], [d2Fy_dt2], [d2Fz_dt2]])\n", "\n", "# d2G/dt2\n", "d2Gx_dt2 = c**2 * I * kx * (dBxx_dt + dBxy_dt + dBxz_dt)\n", "d2Gy_dt2 = c**2 * I * ky * (dByx_dt + dByy_dt + dByz_dt)\n", "d2Gz_dt2 = c**2 * I * kz * (dBzx_dt + dBzy_dt + dBzz_dt)\n", - "d2G_dt2 = Matrix([[d2Gx_dt2],[d2Gy_dt2],[d2Gz_dt2]])\n", + "d2G_dt2 = Matrix([[d2Gx_dt2], [d2Gy_dt2], [d2Gz_dt2]])\n", "\n", "for i in range(dd):\n", " d2E_dt2[i] = sp.expand(d2E_dt2[i])\n", @@ -458,51 +505,51 @@ "\n", "for i in range(3):\n", " d2F_dt2[i] = sp.expand(d2F_dt2[i])\n", - " \n", + "\n", "for i in range(3):\n", " d2G_dt2[i] = sp.expand(d2G_dt2[i])\n", - " \n", + "\n", "# Extended array for E and G\n", - "EG = zeros(DD,1)\n", + "EG = zeros(DD, 1)\n", "for i in range(dd):\n", " EG[i] = E[i]\n", - "for i in range(dd,DD):\n", - " EG[i] = G[i-dd]\n", + "for i in range(dd, DD):\n", + " EG[i] = G[i - dd]\n", "\n", "# dEG/dt\n", - "dEG_dt = zeros(DD,1)\n", + "dEG_dt = zeros(DD, 1)\n", "for i in range(dd):\n", " dEG_dt[i] = dE_dt[i]\n", - "for i in range(dd,DD):\n", - " dEG_dt[i] = dG_dt[i-dd]\n", - " \n", + "for i in range(dd, DD):\n", + " dEG_dt[i] = dG_dt[i - dd]\n", + "\n", "# d2EG/dt2\n", - "d2EG_dt2 = zeros(DD,1)\n", + "d2EG_dt2 = zeros(DD, 1)\n", "for i in range(dd):\n", " d2EG_dt2[i] = d2E_dt2[i]\n", - "for i in range(dd,DD):\n", - " d2EG_dt2[i] = d2G_dt2[i-dd]\n", - " \n", + "for i in range(dd, DD):\n", + " d2EG_dt2[i] = d2G_dt2[i - dd]\n", + "\n", "# Extended array for B and F\n", - "BF = zeros(DD,1)\n", + "BF = zeros(DD, 1)\n", "for i in range(dd):\n", " BF[i] = B[i]\n", - "for i in range(dd,DD):\n", - " BF[i] = F[i-dd]\n", + "for i in range(dd, DD):\n", + " BF[i] = F[i - dd]\n", "\n", "# dBF/dt\n", - "dBF_dt = zeros(DD,1)\n", + "dBF_dt = zeros(DD, 1)\n", "for i in range(dd):\n", " dBF_dt[i] = dB_dt[i]\n", - "for i in range(dd,DD):\n", - " dBF_dt[i] = dF_dt[i-dd]\n", + "for i in range(dd, DD):\n", + " dBF_dt[i] = dF_dt[i - dd]\n", "\n", "# d2BF/dt2\n", - "d2BF_dt2 = zeros(DD,1)\n", + "d2BF_dt2 = zeros(DD, 1)\n", "for i in range(dd):\n", " d2BF_dt2[i] = d2B_dt2[i]\n", - "for i in range(dd,DD):\n", - " d2BF_dt2[i] = d2F_dt2[i-dd]" + "for i in range(dd, DD):\n", + " d2BF_dt2[i] = d2F_dt2[i - dd]" ] }, { @@ -515,30 +562,28 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ - "print(r'Solve equations for E and G:')\n", + "print(r\"Solve equations for E and G:\")\n", "EG_t, EG_new = evolve(EG, dEG_dt, d2EG_dt2)\n", "\n", - "print(r'Solve equations for B and F:')\n", + "print(r\"Solve equations for B and F:\")\n", "BF_t, BF_new = evolve(BF, dBF_dt, d2BF_dt2)\n", "\n", "# Check correctness by taking *first* derivative\n", "# and comparing with initial right-hand side at time tn\n", "# E,G\n", - "diff = EG_t.diff(t).subs(t, tn).subs(om, c*knorm).expand() - dEG_dt\n", + "diff = EG_t.diff(t).subs(t, tn).subs(om, c * knorm).expand() - dEG_dt\n", "diff.simplify()\n", - "if diff != zeros(DD,1):\n", - " print('Integration in time failed')\n", + "if diff != zeros(DD, 1):\n", + " print(\"Integration in time failed\")\n", " display(diff)\n", "# B,F\n", - "diff = BF_t.diff(t).subs(t, tn).subs(om, c*knorm).expand() - dBF_dt\n", + "diff = BF_t.diff(t).subs(t, tn).subs(om, c * knorm).expand() - dBF_dt\n", "diff.simplify()\n", - "if diff != zeros(DD,1):\n", - " print('Integration in time failed')\n", + "if diff != zeros(DD, 1):\n", + " print(\"Integration in time failed\")\n", " display(diff)" ] }, @@ -552,13 +597,10 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "# Code generation\n", - "from sympy.codegen.ast import Assignment\n", "\n", "# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11\n", "# EG: Exx, Exy, Exz, Eyx, Eyy, Eyz, Ezx, Ezy, Ezz, Gx, Gy, Gz\n", @@ -570,22 +612,22 @@ "# Extract individual terms (right hand side)\n", "for i in range(DD):\n", " X = EG[i]\n", - " C = X_new.coeff(X, 1).simplify()\n", - " print(r'Coefficient multiplying ' + str(X))\n", - " display(C)\n", - " #print(ccode(Assignment(sp.symbols(r'LHS'), C)))\n", + " C1 = X_new.coeff(X, 1).simplify()\n", + " print(r\"Coefficient multiplying \" + str(X))\n", + " display(C1)\n", + " # print(ccode(Assignment(sp.symbols(r'LHS'), C1)))\n", "for i in range(DD):\n", " X = BF[i]\n", - " C = X_new.coeff(X, 1).simplify()\n", - " print(r'Coefficient multiplying ' + str(X))\n", - " display(C)\n", - " #print(ccode(Assignment(sp.symbols(r'LHS'), C)))" + " C2 = X_new.coeff(X, 1).simplify()\n", + " print(r\"Coefficient multiplying \" + str(X))\n", + " display(C2)\n", + " # print(ccode(Assignment(sp.symbols(r'LHS'), C2)))" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -599,7 +641,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/Tools/Algorithms/stencil.py b/Tools/Algorithms/stencil.py index dde7398daaa..1fafd3837a7 100644 --- a/Tools/Algorithms/stencil.py +++ b/Tools/Algorithms/stencil.py @@ -15,18 +15,19 @@ import os import sys -sys.path.append('../Parser/') +sys.path.append("../Parser/") import matplotlib.pyplot as plt import numpy as np from input_file_parser import parse_input_file from scipy.constants import c -plt.style.use('tableau-colorblind10') -plt.rcParams.update({'font.size': 14}) +plt.style.use("tableau-colorblind10") +plt.rcParams.update({"font.size": 14}) sp = np.finfo(np.float32).eps dp = np.finfo(np.float64).eps + def get_Fornberg_coeffs(order, staggered): """ Compute the centered or staggered Fornberg coefficients at finite order. @@ -43,24 +44,29 @@ def get_Fornberg_coeffs(order, staggered): coeffs : numpy.ndarray Array of centered or staggered Fornberg coefficients. """ - m = order//2 - coeffs = np.zeros(m+1) + m = order // 2 + coeffs = np.zeros(m + 1) # Compute Fornberg coefficients by recurrence if staggered: - prod = 1. - for k in range(1, m+1): - prod = prod*(m+k)/(4*k) - coeffs[0] = 4*m*prod**2 - for n in range(1, m+1): - coeffs[n] = -(((2*n-3)*(m+1-n))/((2*n-1)*(m-1+n))*coeffs[n-1]) + prod = 1.0 + for k in range(1, m + 1): + prod = prod * (m + k) / (4 * k) + coeffs[0] = 4 * m * prod**2 + for n in range(1, m + 1): + coeffs[n] = -( + ((2 * n - 3) * (m + 1 - n)) + / ((2 * n - 1) * (m - 1 + n)) + * coeffs[n - 1] + ) else: - coeffs[0] = -2. - for n in range(1, m+1): - coeffs[n] = -(m+1-n)/(m+n)*coeffs[n-1] + coeffs[0] = -2.0 + for n in range(1, m + 1): + coeffs[n] = -(m + 1 - n) / (m + n) * coeffs[n - 1] return coeffs + def modified_k(kx, dx, order, staggered): """ Compute the centered or staggered modified wave vector at finite order. @@ -81,24 +87,29 @@ def modified_k(kx, dx, order, staggered): k_mod : numpy.ndarray Centered or staggered modified wave vector. """ - m = order//2 + m = order // 2 coeffs = get_Fornberg_coeffs(order, staggered) # Array of values for n: from 1 to m - n = np.arange(1, m+1) + n = np.arange(1, m + 1) # Array of values of sin # (first axis corresponds to k and second axis to n) if staggered: - sin_kn = (np.sin(kx[:,np.newaxis]*(n[np.newaxis,:]-0.5)*dx)/((n[np.newaxis,:]-0.5)*dx)) + sin_kn = np.sin(kx[:, np.newaxis] * (n[np.newaxis, :] - 0.5) * dx) / ( + (n[np.newaxis, :] - 0.5) * dx + ) else: - sin_kn = (np.sin(kx[:,np.newaxis]*n[np.newaxis,:]*dx)/(n[np.newaxis,:]*dx)) + sin_kn = np.sin(kx[:, np.newaxis] * n[np.newaxis, :] * dx) / ( + n[np.newaxis, :] * dx + ) # Modified k - k_mod = np.tensordot(sin_kn, coeffs[1:], axes=(-1,-1)) + k_mod = np.tensordot(sin_kn, coeffs[1:], axes=(-1, -1)) return k_mod + def func_cosine(om, w_c, dt): """ Compute the leading spectral coefficient of the general PSATD equations: @@ -120,10 +131,11 @@ def func_cosine(om, w_c, dt): coeff : numpy.ndarray Leading spectral coefficient of the general PSATD equations. """ - theta_c = np.exp(1.j*w_c*dt*0.5) - coeff = theta_c**2*np.cos(om*dt) + theta_c = np.exp(1.0j * w_c * dt * 0.5) + coeff = theta_c**2 * np.cos(om * dt) return coeff + def compute_stencils(coeff_nodal, coeff_stagg, axis): """ Compute nodal and staggered stencils along a given direction. @@ -159,17 +171,20 @@ def compute_stencils(coeff_nodal, coeff_stagg, axis): # Average over the other two directions i1 = (axis + 1) % 3 i2 = (axis + 2) % 3 - stencil_avg_nodal = (stencil_nodal.sum(axis=(i1,i2)) / - (stencil_nodal.shape[i1]*stencil_nodal.shape[i2])) - stencil_avg_stagg = (stencil_stagg.sum(axis=(i1,i2)) / - (stencil_stagg.shape[i1]*stencil_stagg.shape[i2])) + stencil_avg_nodal = stencil_nodal.sum(axis=(i1, i2)) / ( + stencil_nodal.shape[i1] * stencil_nodal.shape[i2] + ) + stencil_avg_stagg = stencil_stagg.sum(axis=(i1, i2)) / ( + stencil_stagg.shape[i1] * stencil_stagg.shape[i2] + ) stencils = dict() - stencils['nodal'] = abs(stencil_avg_nodal) - stencils['stagg'] = abs(stencil_avg_stagg) + stencils["nodal"] = abs(stencil_avg_nodal) + stencils["stagg"] = abs(stencil_avg_stagg) return stencils + def compute_all(dx_boosted, dt, psatd_order, v_gal, nx=None): """ Compute nodal and staggered stencils along all directions. @@ -200,12 +215,12 @@ def compute_all(dx_boosted, dt, psatd_order, v_gal, nx=None): nx = np.full(shape=dims, fill_value=256) # k vectors and modified k vectors - k_arr = [] + k_arr = [] k_arr_c = [] k_arr_s = [] for i in range(dims): - k_arr.append(2*np.pi*np.fft.fftfreq(nx[i], dx_boosted[i])) - if psatd_order[i] != 'inf': + k_arr.append(2 * np.pi * np.fft.fftfreq(nx[i], dx_boosted[i])) + if psatd_order[i] != "inf": k_arr_c.append(modified_k(k_arr[i], dx_boosted[i], psatd_order[i], False)) k_arr_s.append(modified_k(k_arr[i], dx_boosted[i], psatd_order[i], True)) else: @@ -219,9 +234,9 @@ def compute_all(dx_boosted, dt, psatd_order, v_gal, nx=None): kk_s = np.sqrt(sum(k**2 for k in k_s)) # Frequencies - om_c = c*kk_c - om_s = c*kk_s - w_c = v_gal*k_c[-1] + om_c = c * kk_c + om_s = c * kk_s + w_c = v_gal * k_c[-1] # Spectral coefficient coeff_nodal = func_cosine(om_c, w_c, dt) @@ -234,6 +249,7 @@ def compute_all(dx_boosted, dt, psatd_order, v_gal, nx=None): return stencils + def compute_guard_cells(errmin, errmax, stencil): """ Compute the minimum number of guard cells for a given error threshold @@ -254,15 +270,16 @@ def compute_guard_cells(errmin, errmax, stencil): """ diff = stencil - errmin v = next(d for d in diff if d < 0) - gcmin = np.argwhere(diff == v)[0,0] + gcmin = np.argwhere(diff == v)[0, 0] diff = stencil - errmax try: v = next(d for d in diff if d < 0) - gcmax = np.argwhere(diff == v)[0,0] - 1 + gcmax = np.argwhere(diff == v)[0, 0] - 1 except StopIteration: - gcmin, gcmax = compute_guard_cells(errmin, errmax*10, stencil) + gcmin, gcmax = compute_guard_cells(errmin, errmax * 10, stencil) return (gcmin, gcmax) + def plot_stencil(cells, stencil_nodal, stencil_stagg, label, path, name): """ Plot stencil extent for nodal and staggered/hybrid solver, @@ -281,34 +298,37 @@ def plot_stencil(cells, stencil_nodal, stencil_stagg, label, path, name): name : str Label for figure name. """ - fig = plt.figure(figsize=[10,6]) + fig = plt.figure(figsize=[10, 6]) ax = fig.add_subplot(111) - ax.plot(cells, stencil_nodal, linestyle='-', label='nodal') - ax.plot(cells, stencil_stagg, linestyle='-', label='staggered or hybrid') + ax.plot(cells, stencil_nodal, linestyle="-", label="nodal") + ax.plot(cells, stencil_stagg, linestyle="-", label="staggered or hybrid") # Plot single and double precision machine epsilons - ax.axhline(y=sp, c='grey', ls='dashed', label='machine epsilon (single precision)') - ax.axhline(y=dp, c='grey', ls='dotted', label='machine epsilon (double precision)') + ax.axhline(y=sp, c="grey", ls="dashed", label="machine epsilon (single precision)") + ax.axhline(y=dp, c="grey", ls="dotted", label="machine epsilon (double precision)") # Shade regions between single and double precision machine epsilons xmin, xmax = compute_guard_cells(sp, dp, stencil_nodal) - ax.fill_between(cells[xmin:xmax+1], stencil_nodal[xmin:xmax+1], alpha=0.5) + ax.fill_between(cells[xmin : xmax + 1], stencil_nodal[xmin : xmax + 1], alpha=0.5) xmin, xmax = compute_guard_cells(sp, dp, stencil_stagg) - ax.fill_between(cells[xmin:xmax+1], stencil_stagg[xmin:xmax+1], alpha=0.5) + ax.fill_between(cells[xmin : xmax + 1], stencil_stagg[xmin : xmax + 1], alpha=0.5) # - ax.set_yscale('log') + ax.set_yscale("log") ax.set_xticks(cells, minor=True) - ax.grid(which='minor', linewidth=0.2) - ax.grid(which='major', linewidth=0.4) + ax.grid(which="minor", linewidth=0.2) + ax.grid(which="major", linewidth=0.4) ax.legend() - ax.set_xlabel('number of cells') - ax.set_ylabel('signal to be truncated') - ax.set_title(r'Stencil extent along ${:s}$'.format(label)) + ax.set_xlabel("number of cells") + ax.set_ylabel("signal to be truncated") + ax.set_title(r"Stencil extent along ${:s}$".format(label)) fig.tight_layout() - fig_name = os.path.join(path, 'figure_stencil_' + label) + fig_name = os.path.join(path, "figure_stencil_" + label) if name: - fig_name += '_' + name - fig.savefig(fig_name + '.png', dpi=150) + fig_name += "_" + name + fig.savefig(fig_name + ".png", dpi=150) + -def run_main(dims, dx_boosted, dt, psatd_order, gamma=1., galilean=False, path='.', name=''): +def run_main( + dims, dx_boosted, dt, psatd_order, gamma=1.0, galilean=False, path=".", name="" +): """ Main function. @@ -333,44 +353,44 @@ def run_main(dims, dx_boosted, dt, psatd_order, gamma=1., galilean=False, path=' """ # Galilean velocity (default = 0.) - v_gal = 0. + v_gal = 0.0 if galilean: - v_gal = -np.sqrt(1.-1./gamma**2)*c + v_gal = -np.sqrt(1.0 - 1.0 / gamma**2) * c # Display some output - print('\nCell size:') - print(f'- dx = {dx_boosted}') + print("\nCell size:") + print(f"- dx = {dx_boosted}") if dims > 1: - print(f'- dx[1:]/dx[0] = {dx_boosted[1:]/dx_boosted[0]}') - print('\nTime step:') - print(f'- dt = {dt}') - print(f'- c*dt/dx = {c*dt/dx_boosted}') - print('\nSpectral order:') - print(f'- order = {psatd_order}') - print('\nLorentz boost, Galilean velocity:') - print(f'- gamma = {gamma}') - print(f'- v_gal = {v_gal}') + print(f"- dx[1:]/dx[0] = {dx_boosted[1:] / dx_boosted[0]}") + print("\nTime step:") + print(f"- dt = {dt}") + print(f"- c*dt/dx = {c * dt / dx_boosted}") + print("\nSpectral order:") + print(f"- order = {psatd_order}") + print("\nLorentz boost, Galilean velocity:") + print(f"- gamma = {gamma}") + print(f"- v_gal = {v_gal}") stencils = compute_all(dx_boosted, dt, psatd_order, v_gal) # Maximum number of cells - nc = dims*[65] + nc = dims * [65] # Arrays of stencils for i, s in enumerate(stencils): - s['nodal'] = s['nodal'][:nc[i]] - s['stagg'] = s['stagg'][:nc[i]] + s["nodal"] = s["nodal"][: nc[i]] + s["stagg"] = s["stagg"][: nc[i]] # Axis labels - label = ['x'] + label = ["x"] if dims == 3: - label.append('y') + label.append("y") if dims > 1: - label.append('z') + label.append("z") # Plot stencils for i, s in enumerate(stencils): - plot_stencil(np.arange(nc[i]), s['nodal'], s['stagg'], label[i], path, name) + plot_stencil(np.arange(nc[i]), s["nodal"], s["stagg"], label[i], path, name) # Compute min and max numbers of guard cells gcmin_nodal = [] @@ -378,38 +398,46 @@ def run_main(dims, dx_boosted, dt, psatd_order, gamma=1., galilean=False, path=' gcmin_stagg = [] gcmax_stagg = [] for s in stencils: - gcmin, gcmax = compute_guard_cells(sp, dp, s['nodal']) + gcmin, gcmax = compute_guard_cells(sp, dp, s["nodal"]) gcmin_nodal.append(gcmin) gcmax_nodal.append(gcmax) - gcmin, gcmax = compute_guard_cells(sp, dp, s['stagg']) + gcmin, gcmax = compute_guard_cells(sp, dp, s["stagg"]) gcmin_stagg.append(gcmin) gcmax_stagg.append(gcmax) fig_path = os.path.abspath(path) - print(f'\nFigures saved in {fig_path}/.') - print('\nThe plots show the extent of the signal to be truncated (y-axis)' - + '\nby choosing a given number of cells (x-axis) for the ghost regions' - + '\nof each simulation grid, along x, y, and z.') - print('\nIt is recommended to choose a number of ghost cells that corresponds to' - + '\na truncation of the signal between single and double machine precision.' - + '\nThe more ghost cells, the more accurate, yet expensive, results.' - + '\nFor each stencil the region of accuracy between single and double precision' - + '\nis shaded to help you identify a suitable number of ghost cells.') - print('\nFor a nodal simulation, choose:') + print(f"\nFigures saved in {fig_path}/.") + print( + "\nThe plots show the extent of the signal to be truncated (y-axis)" + + "\nby choosing a given number of cells (x-axis) for the ghost regions" + + "\nof each simulation grid, along x, y, and z." + ) + print( + "\nIt is recommended to choose a number of ghost cells that corresponds to" + + "\na truncation of the signal between single and double machine precision." + + "\nThe more ghost cells, the more accurate, yet expensive, results." + + "\nFor each stencil the region of accuracy between single and double precision" + + "\nis shaded to help you identify a suitable number of ghost cells." + ) + print("\nFor a nodal simulation, choose:") for i in range(dims): - print(f'- between {gcmin_nodal[i]} and {gcmax_nodal[i]} ghost cells along {label[i]}') - print('\nFor a staggered or hybrid simulation, choose:') + print( + f"- between {gcmin_nodal[i]} and {gcmax_nodal[i]} ghost cells along {label[i]}" + ) + print("\nFor a staggered or hybrid simulation, choose:") for i in range(dims): - print(f'- between {gcmin_stagg[i]} and {gcmax_stagg[i]} ghost cells along {label[i]}') + print( + f"- between {gcmin_stagg[i]} and {gcmax_stagg[i]} ghost cells along {label[i]}" + ) print() return -if __name__ == '__main__': +if __name__ == "__main__": # Parse path to input file from command line parser = argparse.ArgumentParser() - parser.add_argument('--input_file', help='path to input file to be parsed') + parser.add_argument("--input_file", help="path to input file to be parsed") args = parser.parse_args() input_file = args.input_file @@ -417,45 +445,45 @@ def run_main(dims, dx_boosted, dt, psatd_order, gamma=1., galilean=False, path=' input_dict = parse_input_file(input_file) # TODO Handle RZ - dims = int(input_dict['geometry.dims'][0]) + dims = int(input_dict["geometry.dims"][0]) # Notation considering x as vector of coordinates (x,y,z) - nx = np.array([int(w) for w in input_dict['amr.n_cell']]) - xmin = np.array([float(w) for w in input_dict['geometry.prob_lo']]) - xmax = np.array([float(w) for w in input_dict['geometry.prob_hi']]) + nx = np.array([int(w) for w in input_dict["amr.n_cell"]]) + xmin = np.array([float(w) for w in input_dict["geometry.prob_lo"]]) + xmax = np.array([float(w) for w in input_dict["geometry.prob_hi"]]) # Cell size in the lab frame and boosted frame (boost along z) ## lab frame - dx = (xmax-xmin) / nx + dx = (xmax - xmin) / nx ## boosted frame - gamma = 1. - if 'warpx.gamma_boost' in input_dict: - gamma = float(input_dict['warpx.gamma_boost'][0]) - beta = np.sqrt(1. - 1./gamma**2) + gamma = 1.0 + if "warpx.gamma_boost" in input_dict: + gamma = float(input_dict["warpx.gamma_boost"][0]) + beta = np.sqrt(1.0 - 1.0 / gamma**2) dx_boosted = np.copy(dx) - dx_boosted[-1] = (1. + beta) * gamma * dx[-1] + dx_boosted[-1] = (1.0 + beta) * gamma * dx[-1] # Time step for pseudo-spectral scheme cfl = 0.999 - if 'warpx.cfl' in input_dict: - cfl = float(input_dict['warpx.cfl'][0]) + if "warpx.cfl" in input_dict: + cfl = float(input_dict["warpx.cfl"][0]) dt = cfl * np.min(dx_boosted) / c # Pseudo-spectral order psatd_order = np.full(shape=dims, fill_value=16) - if 'psatd.nox' in input_dict: - psatd_order[0] = int(input_dict['psatd.nox'][0]) - if 'psatd.noy' in input_dict: - psatd_order[1] = int(input_dict['psatd.noy'][0]) - if 'psatd.noz' in input_dict: - psatd_order[-1] = int(input_dict['psatd.noz'][0]) + if "psatd.nox" in input_dict: + psatd_order[0] = int(input_dict["psatd.nox"][0]) + if "psatd.noy" in input_dict: + psatd_order[1] = int(input_dict["psatd.noy"][0]) + if "psatd.noz" in input_dict: + psatd_order[-1] = int(input_dict["psatd.noz"][0]) # Galilean flag galilean = False - if 'psatd.use_default_v_galilean' in input_dict: - galilean = bool(input_dict['psatd.use_default_v_galilean'][0]) - if 'psatd.v_galilean' in input_dict: - galilean = bool(input_dict['psatd.v_galilean'][-1]) + if "psatd.use_default_v_galilean" in input_dict: + galilean = bool(input_dict["psatd.use_default_v_galilean"][0]) + if "psatd.v_galilean" in input_dict: + galilean = bool(input_dict["psatd.v_galilean"][-1]) # Run main function (some arguments are optional, # see definition of run_main function for help) diff --git a/Tools/Ascent/ascent_replay_warpx.ipynb b/Tools/Ascent/ascent_replay_warpx.ipynb index 636e9721c74..e9361d298de 100644 --- a/Tools/Ascent/ascent_replay_warpx.ipynb +++ b/Tools/Ascent/ascent_replay_warpx.ipynb @@ -14,14 +14,12 @@ "metadata": {}, "outputs": [], "source": [ + "import glob\n", + "\n", + "import ascent\n", "import conduit\n", "import conduit.blueprint\n", - "import conduit.relay\n", - "import ascent\n", - "\n", - "from IPython.display import Image\n", - "\n", - "import glob" + "import conduit.relay" ] }, { @@ -30,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "#print(conduit.about())" + "# print(conduit.about())" ] }, { @@ -85,8 +83,10 @@ "# we publish this data now once to visualize it in multiple ways below\n", "a = ascent.Ascent()\n", "opts = conduit.Node()\n", - "opts[\"actions_file\"] = \"\" # work-around: ascent_actions.yaml file must not overwrite our replay action\n", - "#opts[\"exceptions\"] = \"forward\"\n", + "opts[\"actions_file\"] = (\n", + " \"\" # work-around: ascent_actions.yaml file must not overwrite our replay action\n", + ")\n", + "# opts[\"exceptions\"] = \"forward\"\n", "a.open(opts)\n", "a.publish(data)" ] diff --git a/Tools/DevUtils/compare_wx_w_3d.ipynb b/Tools/DevUtils/compare_wx_w_3d.ipynb index d4b0520dd36..7d59554ed23 100644 --- a/Tools/DevUtils/compare_wx_w_3d.ipynb +++ b/Tools/DevUtils/compare_wx_w_3d.ipynb @@ -23,16 +23,14 @@ "outputs": [], "source": [ "# Import statements\n", - "import sys\n", - "from tqdm import tqdm\n", - "import yt, glob\n", + "import glob\n", + "\n", + "import yt\n", + "\n", "yt.funcs.mylog.setLevel(50)\n", - "from IPython.display import clear_output\n", - "import numpy as np\n", - "from ipywidgets import interact, RadioButtons, IntSlider\n", - "from openpmd_viewer import OpenPMDTimeSeries\n", - "from yt.units import volt\n", "import matplotlib.pyplot as plt\n", + "from ipywidgets import IntSlider, RadioButtons, interact\n", + "\n", "%matplotlib" ] }, @@ -45,9 +43,11 @@ "outputs": [], "source": [ "# Define path way for WarpX results and find iterations\n", - "path_warpx = '/Users/mthevenet/warpX_directory/res_warpx/valid_pwfa/'\n", - "file_list_warpx = glob.glob(path_warpx + 'plt?????')\n", - "iterations_warpx = [ int(file_name[len(file_name)-5:]) for file_name in file_list_warpx ]" + "path_warpx = \"/Users/mthevenet/warpX_directory/res_warpx/valid_pwfa/\"\n", + "file_list_warpx = glob.glob(path_warpx + \"plt?????\")\n", + "iterations_warpx = [\n", + " int(file_name[len(file_name) - 5 :]) for file_name in file_list_warpx\n", + "]" ] }, { @@ -59,9 +59,12 @@ "outputs": [], "source": [ "# Define path way for Warp results and find iterations\n", - "path_warp = '/Users/mthevenet/warp_results/valid_pwfa/'\n", - "file_list_warp = glob.glob(path_warp + 'diags/hdf5/data????????.h5')\n", - "iterations_warp = [ int(file_name[len(file_name)-11:len(file_name)-3]) for file_name in file_list_warp ]" + "path_warp = \"/Users/mthevenet/warp_results/valid_pwfa/\"\n", + "file_list_warp = glob.glob(path_warp + \"diags/hdf5/data????????.h5\")\n", + "iterations_warp = [\n", + " int(file_name[len(file_name) - 11 : len(file_name) - 3])\n", + " for file_name in file_list_warp\n", + "]" ] }, { @@ -81,89 +84,104 @@ }, "outputs": [], "source": [ - "def plot_field( iteration, field, slicing_direction='y'):\n", - " \n", + "def plot_field(iteration, field, slicing_direction=\"y\"):\n", " # First dataset\n", - " ds = yt.load( path_warpx + '/plt%05d/' %iteration )\n", - " all_data_level_0 = ds.covering_grid(level=0, \n", - " left_edge=ds.domain_left_edge, dims=ds.domain_dimensions)\n", - " \n", + " ds = yt.load(path_warpx + \"/plt%05d/\" % iteration)\n", + " all_data_level_0 = ds.covering_grid(\n", + " level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions\n", + " )\n", + "\n", " # Second dataset\n", - "# ts = OpenPMDTimeSeries(path_warp + 'diags/hdf5/') \n", - " ds2 = yt.load( path_warp + 'diags/hdf5/data%08d.h5' %iteration )\n", - " all_data_level_02 = ds2.covering_grid(level=0, \n", - " left_edge=ds2.domain_left_edge, dims=ds2.domain_dimensions)\n", + " # ts = OpenPMDTimeSeries(path_warp + 'diags/hdf5/')\n", + " ds2 = yt.load(path_warp + \"diags/hdf5/data%08d.h5\" % iteration)\n", + " all_data_level_02 = ds2.covering_grid(\n", + " level=0, left_edge=ds2.domain_left_edge, dims=ds2.domain_dimensions\n", + " )\n", "\n", " # first dataset loaded via yt\n", - " left_edge = ds.domain_left_edge.convert_to_mks()*1e6\n", - " right_edge = ds.domain_right_edge.convert_to_mks()*1e6\n", - " if slicing_direction == 'x':\n", - " n = int( ds.domain_dimensions[0]//2 )\n", + " left_edge = ds.domain_left_edge.convert_to_mks() * 1e6\n", + " right_edge = ds.domain_right_edge.convert_to_mks() * 1e6\n", + " if slicing_direction == \"x\":\n", + " n = int(ds.domain_dimensions[0] // 2)\n", " data2d = all_data_level_0[field][n, :, :]\n", - " extent = [ left_edge[2], right_edge[2], left_edge[1], right_edge[1] ]\n", - " elif slicing_direction == 'y':\n", - " n = int( ds.domain_dimensions[1]//2 )\n", + " extent = [left_edge[2], right_edge[2], left_edge[1], right_edge[1]]\n", + " elif slicing_direction == \"y\":\n", + " n = int(ds.domain_dimensions[1] // 2)\n", " data2d = all_data_level_0[field][:, n, :]\n", - " extent = [ left_edge[2], right_edge[2], left_edge[0], right_edge[0] ]\n", - " elif slicing_direction == 'z':\n", - " n = int( ds.domain_dimensions[2]//2 )\n", + " extent = [left_edge[2], right_edge[2], left_edge[0], right_edge[0]]\n", + " elif slicing_direction == \"z\":\n", + " n = int(ds.domain_dimensions[2] // 2)\n", " data2d = all_data_level_0[field][:, :, n]\n", - " extent = [ left_edge[1], right_edge[1], left_edge[0], right_edge[0] ]\n", + " extent = [left_edge[1], right_edge[1], left_edge[0], right_edge[0]]\n", "\n", " # second dataset loaded via yt\n", - " left_edge = ds2.domain_left_edge.convert_to_mks()*1e6\n", - " right_edge = ds2.domain_right_edge.convert_to_mks()*1e6\n", - " field_reformat = field[0].upper() + '_' + field[1]\n", - " if slicing_direction == 'x':\n", - " n = int( ds2.domain_dimensions[0]//2 )\n", + " left_edge = ds2.domain_left_edge.convert_to_mks() * 1e6\n", + " right_edge = ds2.domain_right_edge.convert_to_mks() * 1e6\n", + " field_reformat = field[0].upper() + \"_\" + field[1]\n", + " if slicing_direction == \"x\":\n", + " n = int(ds2.domain_dimensions[0] // 2)\n", " data2d2 = all_data_level_02[field_reformat][n, :, :]\n", - " extent2 = [ left_edge[2], right_edge[2], left_edge[1], right_edge[1] ]\n", - " elif slicing_direction == 'y':\n", - " n = int( ds2.domain_dimensions[1]//2 )\n", + " extent2 = [left_edge[2], right_edge[2], left_edge[1], right_edge[1]]\n", + " elif slicing_direction == \"y\":\n", + " n = int(ds2.domain_dimensions[1] // 2)\n", " data2d2 = all_data_level_02[field_reformat][:, n, :]\n", - " extent2 = [ left_edge[2], right_edge[2], left_edge[0], right_edge[0] ]\n", - " elif slicing_direction == 'z':\n", - " n = int( ds2.domain_dimensions[2]//2 )\n", + " extent2 = [left_edge[2], right_edge[2], left_edge[0], right_edge[0]]\n", + " elif slicing_direction == \"z\":\n", + " n = int(ds2.domain_dimensions[2] // 2)\n", " data2d2 = all_data_level_02[field_reformat][:, :, n]\n", - " extent2 = [ left_edge[1], right_edge[1], left_edge[0], right_edge[0] ]\n", + " extent2 = [left_edge[1], right_edge[1], left_edge[0], right_edge[0]]\n", "\n", - " if slicing_direction == 'x':\n", - " yaxis_label = 'y'\n", - " if slicing_direction == 'y':\n", - " yaxis_label = 'x'\n", - " if slicing_direction == 'z':\n", - " yaxis_label = 'y'\n", + " if slicing_direction == \"x\":\n", + " yaxis_label = \"y\"\n", + " if slicing_direction == \"y\":\n", + " yaxis_label = \"x\"\n", + " if slicing_direction == \"z\":\n", + " yaxis_label = \"y\"\n", "\n", - "# xlim = [5,25]\n", - "# ylim = [-20,20]\n", + " xlim = [extent[0], extent[1]]\n", + " ylim = [extent[2], extent[3]]\n", "\n", " plt.clf()\n", " # first dataset\n", - " plt.subplot(2,1,1)\n", - " plt.title(\"WarpX %s at iteration %d\" %(field, iteration) )\n", - " plt.imshow( data2d, interpolation='nearest', cmap='viridis',\n", - " origin='lower', extent=extent, aspect='auto')\n", - " plt.ylabel(yaxis_label + ' (um)')\n", + " plt.subplot(2, 1, 1)\n", + " plt.title(\"WarpX %s at iteration %d\" % (field, iteration))\n", + " plt.imshow(\n", + " data2d,\n", + " interpolation=\"nearest\",\n", + " cmap=\"viridis\",\n", + " origin=\"lower\",\n", + " extent=extent,\n", + " aspect=\"auto\",\n", + " )\n", + " plt.ylabel(yaxis_label + \" (um)\")\n", " vmin, vmax = plt.gci().get_clim()\n", " plt.colorbar()\n", - " if 'xlim' in locals():\n", + " if \"xlim\" in locals():\n", " plt.xlim(xlim)\n", - " if 'ylim' in locals():\n", + " if \"ylim\" in locals():\n", " plt.ylim(ylim)\n", " plt.xticks([])\n", - " \n", + "\n", " # second dataset\n", - " plt.subplot(2,1,2)\n", - " plt.title(\"Warp %s at iteration %d\" %(field, iteration) )\n", - " plt.imshow( data2d2, interpolation='nearest', cmap='viridis',\n", - " origin='lower', extent=extent2, aspect='auto',vmin=vmin,vmax=vmax)\n", + " plt.subplot(2, 1, 2)\n", + " plt.title(\"Warp %s at iteration %d\" % (field, iteration))\n", + " plt.imshow(\n", + " data2d2,\n", + " interpolation=\"nearest\",\n", + " cmap=\"viridis\",\n", + " origin=\"lower\",\n", + " extent=extent2,\n", + " aspect=\"auto\",\n", + " vmin=vmin,\n", + " vmax=vmax,\n", + " )\n", " plt.colorbar()\n", - " if 'xlim' in locals():\n", + " if \"xlim\" in locals():\n", " plt.xlim(xlim)\n", - " if 'ylim' in locals():\n", + " if \"ylim\" in locals():\n", " plt.ylim(ylim)\n", - " plt.xlabel('z (um)')\n", - " plt.ylabel(yaxis_label + ' (um)')" + " plt.xlabel(\"z (um)\")\n", + " plt.ylabel(yaxis_label + \" (um)\")" ] }, { @@ -182,10 +200,14 @@ "outputs": [], "source": [ "iterations = iterations_warp\n", - "interact(plot_field, \n", - " iteration = IntSlider(min=min(iterations), max=max(iterations), step=iterations[1]-iterations[0]),\n", - " field = RadioButtons( options=['jx', 'jy', 'jz', 'Ex', 'Ey', 'Ez'], value='jx'),\n", - " slicing_direction = RadioButtons( options=[ 'x', 'y', 'z'], value='x'))" + "interact(\n", + " plot_field,\n", + " iteration=IntSlider(\n", + " min=min(iterations), max=max(iterations), step=iterations[1] - iterations[0]\n", + " ),\n", + " field=RadioButtons(options=[\"jx\", \"jy\", \"jz\", \"Ex\", \"Ey\", \"Ez\"], value=\"jx\"),\n", + " slicing_direction=RadioButtons(options=[\"x\", \"y\", \"z\"], value=\"x\"),\n", + ")" ] }, { diff --git a/Tools/DevUtils/compute_domain.py b/Tools/DevUtils/compute_domain.py index b54412639bd..d54ef2abad4 100644 --- a/Tools/DevUtils/compute_domain.py +++ b/Tools/DevUtils/compute_domain.py @@ -6,7 +6,7 @@ import numpy as np -''' +""" This Python script helps a user to parallelize a WarpX simulation. The user specifies the minimal size of the physical domain and the resolution @@ -22,96 +22,106 @@ Note that the script has no notion of blocking_factor. It is assumed that blocking_factor = max_grid_size, and that all boxes have the same size. -''' +""" # Update the lines below for your simulation # ------------------------------------------ # 2 elements for 2D, 3 elements for 3D # Lower corner of the box -box_lo0 = np.array([-25.e-6, -25.e-6, -15.e-6]) +box_lo0 = np.array([-25.0e-6, -25.0e-6, -15.0e-6]) # Upper corner of the box -box_hi0 = np.array([ 25.e-6, 25.e-6, 60.e-6]) +box_hi0 = np.array([25.0e-6, 25.0e-6, 60.0e-6]) # Cell size -dx = 1.e-6 +dx = 1.0e-6 dz = dx cell_size = np.array([dx, dx, dz]) # Use this for simulations in a boosted frame if you # want to enforce dz < dx / dx_over_dz_boosted_frame compute_dz_boosted_frame = True -gamma_boost = 30. -dx_over_dz_boosted_frame = 1.1 # >1. is usually more stable +gamma_boost = 30.0 +dx_over_dz_boosted_frame = 1.1 # >1. is usually more stable # ------------------------------------------ + # similar to numpy.ceil, except the output data type is int def intceil(num): return np.ceil(num).astype(int) + # Enlarge simulation boundaries to satisfy three conditions: # - The resolution must be exactly the one provided by the user # - The physical domain must cover the domain specified by box_lo0, box_hi0 # - The number of cells must be a multiple of mgs (max_grid_size). def adjust_bounds(box_lo0, box_hi0, box_ncell0, mgs): - cell_size = (box_hi0-box_lo0) / box_ncell0 - box_ncell = intceil(box_ncell0/mgs)*mgs + cell_size = (box_hi0 - box_lo0) / box_ncell0 + box_ncell = intceil(box_ncell0 / mgs) * mgs box_lo = box_ncell * cell_size * box_lo0 / (box_hi0 - box_lo0) box_hi = box_ncell * cell_size * box_hi0 / (box_hi0 - box_lo0) return box_lo, box_hi, box_ncell + # Calculate parallelization for the simulation, given numerical parameters # (number of cells, max_grid_size, number of threads per node etc.) -def nb_nodes_mpi(box_ncell,mgs,threadspernode,ompnumthreads,ngridpernode, ndim): - nmpipernode = threadspernode/ompnumthreads - ngridpermpi = ngridpernode/nmpipernode - box_ngrids = box_ncell/mgs +def nb_nodes_mpi(box_ncell, mgs, threadspernode, ompnumthreads, ngridpernode, ndim): + nmpipernode = threadspernode / ompnumthreads + ngridpermpi = ngridpernode / nmpipernode + box_ngrids = box_ncell / mgs if ndim == 2: ngrids = box_ngrids[0] * box_ngrids[1] elif ndim == 3: ngrids = np.prod(box_ngrids) - n_mpi = intceil( ngrids/ngridpermpi ) - n_node = intceil( n_mpi/nmpipernode ) + n_mpi = intceil(ngrids / ngridpermpi) + n_node = intceil(n_mpi / nmpipernode) return n_node, n_mpi + # Get number of dimensions (2 or 3) ndim = box_lo0.size if compute_dz_boosted_frame: # Adjust dz so that dx/dz = dx_over_dz_boosted_frame in simulation frame - cell_size[-1] = cell_size[0] / dx_over_dz_boosted_frame / 2. / gamma_boost + cell_size[-1] = cell_size[0] / dx_over_dz_boosted_frame / 2.0 / gamma_boost # Given the resolution, compute number of cells a priori -box_ncell0 = ( box_hi0 - box_lo0 ) / cell_size +box_ncell0 = (box_hi0 - box_lo0) / cell_size if ndim == 2: # Set of parameters suitable for a 2D simulation on Cori KNL - ngridpernode = 16. - ompnumthreads = 8. - mgs = 1024. - threadspernode = 64. # HyperThreading level = 1: no hyperthreading - distance_between_threads = int(68*4/threadspernode) - c_option = int( ompnumthreads*distance_between_threads ) + ngridpernode = 16.0 + ompnumthreads = 8.0 + mgs = 1024.0 + threadspernode = 64.0 # HyperThreading level = 1: no hyperthreading + distance_between_threads = int(68 * 4 / threadspernode) + c_option = int(ompnumthreads * distance_between_threads) elif ndim == 3: # Set of parameters suitable for a 3D simulation on Cori KNL - ngridpernode = 8. - ompnumthreads = 8. - mgs = 64. - threadspernode = 64. # HyperThreading level = 1: no hyperthreading - distance_between_threads = int(68*4/threadspernode) - c_option = int( ompnumthreads*distance_between_threads ) + ngridpernode = 8.0 + ompnumthreads = 8.0 + mgs = 64.0 + threadspernode = 64.0 # HyperThreading level = 1: no hyperthreading + distance_between_threads = int(68 * 4 / threadspernode) + c_option = int(ompnumthreads * distance_between_threads) # Adjust simulation bounds box_lo, box_hi, box_ncell = adjust_bounds(box_lo0, box_hi0, box_ncell0, mgs) # Calculate parallelization -n_node,n_mpi = nb_nodes_mpi(box_ncell, mgs, threadspernode, ompnumthreads, ngridpernode, ndim) +n_node, n_mpi = nb_nodes_mpi( + box_ncell, mgs, threadspernode, ompnumthreads, ngridpernode, ndim +) # Print results -string_output = ' ### Parameters used ### \n' -string_output += 'ngridpernode = ' + str(ngridpernode) + '\n' -string_output += 'ompnumthreads = ' + str(ompnumthreads) + '\n' -string_output += 'mgs (max_grid_size) = ' + str(mgs) + '\n' -string_output += 'threadspernode ( = # MPI ranks per node * OMP_NUM_THREADS) = ' + str(threadspernode) + '\n' -string_output += 'ndim = ' + str(ndim) + '\n\n' -string_output += 'box_lo = ' + str(box_lo) + '\n' -string_output += 'box_hi = ' + str(box_hi) + '\n' -string_output += 'box_ncell = ' + str(box_ncell) + '\n' -string_output += 'n_node = ' + str(n_node) + '\n' -string_output += 'n_mpi = ' + str(n_mpi) + '\n' +string_output = " ### Parameters used ### \n" +string_output += "ngridpernode = " + str(ngridpernode) + "\n" +string_output += "ompnumthreads = " + str(ompnumthreads) + "\n" +string_output += "mgs (max_grid_size) = " + str(mgs) + "\n" +string_output += ( + "threadspernode ( = # MPI ranks per node * OMP_NUM_THREADS) = " + + str(threadspernode) + + "\n" +) +string_output += "ndim = " + str(ndim) + "\n\n" +string_output += "box_lo = " + str(box_lo) + "\n" +string_output += "box_hi = " + str(box_hi) + "\n" +string_output += "box_ncell = " + str(box_ncell) + "\n" +string_output += "n_node = " + str(n_node) + "\n" +string_output += "n_mpi = " + str(n_mpi) + "\n" print(string_output) diff --git a/Tools/DevUtils/update_benchmarks_from_azure_output.py b/Tools/DevUtils/update_benchmarks_from_azure_output.py index ec7b17d1050..bcff995b21a 100644 --- a/Tools/DevUtils/update_benchmarks_from_azure_output.py +++ b/Tools/DevUtils/update_benchmarks_from_azure_output.py @@ -1,4 +1,4 @@ -# Copyright 2023 Neil Zaim +# Copyright 2023 Neil Zaim, Edoardo Zoni # # This file is part of WarpX. # @@ -8,58 +8,46 @@ import re import sys -''' -This Python script updates the Azure benchmarks automatically using a raw Azure output textfile -that is given as the first and only argument of the script. +""" +This Python script updates the Azure benchmarks automatically using a raw +Azure output text file that is passed as command line argument of the script. +""" -In the Azure output, we read the lines contained between -"New file for Test_Name:" -and the next occurrence of -"'----------------'" -And use these lines to update the benchmarks -''' +# read path to Azure output text file +azure_output = sys.argv[1] -azure_output_filename = sys.argv[1] +# string to identify failing tests that require a checksums reset +new_checksums = "New checksums" +failing_test = "" -pattern_test_name = 'New file for (?P[\w\-]*)' -closing_string = '----------------' +# path of all checksums benchmark files benchmark_path = "../../Regression/Checksum/benchmarks_json/" -benchmark_suffix = ".json" -first_line_read = False -current_test = "" - -with open(azure_output_filename, "r") as f: +with open(azure_output, "r") as f: + # find length of Azure prefix to be removed from each line, + # first line of Azure output starts with "##[section]Starting:" + first_line = f.readline() + prefix_length = first_line.find("#") + # loop over lines for line in f: - - if current_test == "": - # Here we search lines that read, for example, - # "New file for LaserAcceleration_BTD" - # and we set current_test = "LaserAcceleration_BTD" - match_test_name = re.search(pattern_test_name, line) - if match_test_name: - current_test = match_test_name.group('testname') - new_file_string = "" - + # remove Azure prefix from line + line = line[prefix_length:] + if failing_test == "": + # no failing test found yet + if re.search(new_checksums, line): + # failing test found, set failing test name + failing_test = line[line.find("test_") : line.find(".json")] + json_file_string = "" else: - # We add each line to the new file string until we find the line containing - # "----------------" - # which indicates that we have read the new file entirely - - if not closing_string in line: - if not first_line_read: - # Raw Azure output comes with a prefix at the beginning of each line that we do - # not need here. The first line that we will read is the prefix followed by the - # "{" character, so we determine how long the prefix is by finding the last - # occurrence of the "{" character in this line. - azure_indent = line.rfind('{') - first_line_read = True - new_file_string += line[azure_indent:] - - else: - # We have read the new file entirely. Dump it in the json file. - new_file_json = json.loads(new_file_string) - json_filepath = benchmark_path+current_test+benchmark_suffix - with open(json_filepath, "w") as f_json: - json.dump(new_file_json, f_json, indent=2) - current_test = "" + # extract and dump new checksums of failing test + json_file_string += line + if line.startswith("}"): # end of new checksums + json_file = json.loads(json_file_string) + json_filename = failing_test + ".json" + json_filepath = benchmark_path + json_filename + print(f"\nDumping new checksums file {json_filename}:") + print(json_file_string) + with open(json_filepath, "w") as json_f: + json.dump(json_file, json_f, indent=2) + # reset to empty string to continue search of failing tests + failing_test = "" diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh index 046c72d7b27..4c1948cf372 100755 --- a/Tools/Linter/runClangTidy.sh +++ b/Tools/Linter/runClangTidy.sh @@ -55,13 +55,13 @@ ${CTIDY} --version echo echo "This can be overridden by setting the environment" echo "variables CLANG, CLANGXX, and CLANGTIDY e.g.: " -echo "$ export CLANG=clang-15" -echo "$ export CLANGXX=clang++-15" -echo "$ export CTIDCLANGTIDYY=clang-tidy-15" +echo "$ export CLANG=clang-17" +echo "$ export CLANGXX=clang++-17" +echo "$ export CTIDCLANGTIDYY=clang-tidy-17" echo "$ ./Tools/Linter/runClangTidy.sh" echo echo "******************************************************" -echo "* Warning: clang v15 is currently used in CI tests. *" +echo "* Warning: clang v17 is currently used in CI tests. *" echo "* It is therefore recommended to use this version. *" echo "* Otherwise, a newer version may find issues not *" echo "* currently covered by CI tests while older versions *" diff --git a/Tools/Parser/input_file_parser.py b/Tools/Parser/input_file_parser.py index 0ab134f6222..9aeba6bbacf 100644 --- a/Tools/Parser/input_file_parser.py +++ b/Tools/Parser/input_file_parser.py @@ -16,16 +16,18 @@ def parse_input_file(input_file): input_dict = dict() with open(input_file) as ff: for line in ff: - sline = line.split('=') + sline = line.split("=") # skip lines that are commented out, blank, or continuation of previous parameters - skip_line = sline[0].startswith('#') or sline[0].startswith('\n') or len(sline) == 1 + skip_line = ( + sline[0].startswith("#") or sline[0].startswith("\n") or len(sline) == 1 + ) if not skip_line: key = sline[0].strip() val = sline[1].split() # The value corresponding to a given key of input_dict is a list # of strings, from which we remove any leftover comments for i in range(len(val)): - if val[i].startswith('#'): + if val[i].startswith("#"): val = val[:i] break input_dict[key] = val diff --git a/Tools/PerformanceTests/automated_test_1_uniform_rest_32ppc b/Tools/PerformanceTests/automated_test_1_uniform_rest_32ppc deleted file mode 100644 index e5e722aa761..00000000000 --- a/Tools/PerformanceTests/automated_test_1_uniform_rest_32ppc +++ /dev/null @@ -1,58 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 20.e-6 - -# Boundaries -boundary.field_lo = pec pec periodic -boundary.field_hi = pec pec periodic -boundary.particle_lo = absorbing absorbing periodic -boundary.particle_hi = absorbing absorbing periodic - -# Verbosity -warpx.verbose = 1 - -algo.particle_shape = 3 - -# CFL -warpx.cfl = 1.0 - -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 2 2 4 -electrons.profile = constant -electrons.density = 1.e20 # number of electrons per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.01 -electrons.uy_th = 0.01 -electrons.uz_th = 0.01 -electrons.ux_m = 0. -electrons.uy_m = 0. -electrons.uz_m = 0. - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = "NUniformPerCell" -ions.num_particles_per_cell_each_dim = 2 2 4 -ions.profile = constant -ions.density = 1.e20 # number of electrons per m^3 -ions.momentum_distribution_type = "gaussian" -ions.ux_th = 0.01 -ions.uy_th = 0.01 -ions.uz_th = 0.01 -ions.ux_m = 0. -ions.uy_m = 0. -ions.uz_m = 0. diff --git a/Tools/PerformanceTests/automated_test_2_uniform_rest_1ppc b/Tools/PerformanceTests/automated_test_2_uniform_rest_1ppc deleted file mode 100644 index 610990ac140..00000000000 --- a/Tools/PerformanceTests/automated_test_2_uniform_rest_1ppc +++ /dev/null @@ -1,44 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 32 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 20.e-6 - -# Boundaries -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -boundary.particle_lo = absorbing absorbing absorbing -boundary.particle_hi = absorbing absorbing absorbing - -# Verbosity -warpx.verbose = 1 - -algo.particle_shape = 3 - -# CFL -warpx.cfl = 1.0 - -particles.species_names = electrons - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 1 1 1 -electrons.profile = constant -electrons.density = 1.e20 # number of electrons per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.01 -electrons.uy_th = 0.01 -electrons.uz_th = 0.01 -electrons.ux_m = 0. -electrons.uy_m = 0. -electrons.uz_m = 0. diff --git a/Tools/PerformanceTests/automated_test_3_uniform_drift_4ppc b/Tools/PerformanceTests/automated_test_3_uniform_drift_4ppc deleted file mode 100644 index 5a834cb4117..00000000000 --- a/Tools/PerformanceTests/automated_test_3_uniform_drift_4ppc +++ /dev/null @@ -1,59 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 20.e-6 - -# Boundaries -boundary.field_lo = pec pec periodic -boundary.field_hi = pec pec periodic -boundary.particle_lo = absorbing absorbing periodic -boundary.particle_hi = absorbing absorbing periodic - -# Verbosity -warpx.verbose = 1 - -# Algorithms -algo.particle_shape = 3 - -# CFL -warpx.cfl = 1.0 - -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 2 2 4 -electrons.profile = constant -electrons.density = 1.e20 # number of electrons per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.01 -electrons.uy_th = 0.01 -electrons.uz_th = 0.01 -electrons.ux_m = 0. -electrons.uy_m = 0. -electrons.uz_m = 100. - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = "NUniformPerCell" -ions.num_particles_per_cell_each_dim = 2 2 4 -ions.profile = constant -ions.density = 1.e20 # number of electrons per m^3 -ions.momentum_distribution_type = "gaussian" -ions.ux_th = 0.01 -ions.uy_th = 0.01 -ions.uz_th = 0.01 -ions.ux_m = 0. -ions.uy_m = 0. -ions.uz_m = 100. diff --git a/Tools/PerformanceTests/automated_test_4_labdiags_2ppc b/Tools/PerformanceTests/automated_test_4_labdiags_2ppc deleted file mode 100644 index f49d92acf26..00000000000 --- a/Tools/PerformanceTests/automated_test_4_labdiags_2ppc +++ /dev/null @@ -1,79 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -150.e-6 -150.e-6 -80.e-6 # physical domain -geometry.prob_hi = 150.e-6 150.e-6 0. - -# Boundaries -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -boundary.particle_lo = absorbing absorbing absorbing -boundary.particle_hi = absorbing absorbing absorbing - -# Verbosity -warpx.verbose = 1 - -# Numerics -algo.particle_shape = 3 -warpx.use_filter = 1 -warpx.cfl = 1.0 - -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light - -# Boosted frame -warpx.gamma_boost = 15. -warpx.boost_direction = z - -# Species -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.xmin = -150.e-6 -electrons.xmax = 150.e-6 -electrons.ymin = -150.e-6 -electrons.ymax = 150.e-6 -electrons.zmin = 0.e-6 -electrons.num_particles_per_cell_each_dim = 1 1 1 -electrons.profile = constant -electrons.density = 1. -electrons.momentum_distribution_type = "at_rest" -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = "NUniformPerCell" -ions.xmin = -150.e-6 -ions.xmax = 150.e-6 -ions.ymin = -150.e-6 -ions.ymax = 150.e-6 -ions.zmin = 0.e-6 -ions.num_particles_per_cell_each_dim = 1 1 1 -ions.profile = constant -ions.density = 1. -ions.momentum_distribution_type = "at_rest" -ions.do_continuous_injection = 1 - -# Laser -lasers.names = laser -laser.profile = Gaussian -laser.position = 0. 0. -1.e-6 # This point is on the laser plane -laser.direction = 0. 0. 1. # The plane normal direction -laser.polarization = 1. 0. 0. # The main polarization vector -laser.e_max = 8.e12 # Maximum amplitude of the laser field (in V/m) -laser.profile_waist = 5.e-5 # The waist of the laser (in meters) -laser.profile_duration = 16.7e-15 # The duration of the laser (in seconds) -laser.profile_t_peak = 33.4e-15 # The time at which the laser reaches its peak (in seconds) -laser.profile_focal_distance = 0.e-6 # Focal distance from the antenna (in meters) -laser.wavelength = 0.8e-6 # The wavelength of the laser (in meters) diff --git a/Tools/PerformanceTests/automated_test_5_loadimbalance b/Tools/PerformanceTests/automated_test_5_loadimbalance deleted file mode 100644 index 76b1a53efdb..00000000000 --- a/Tools/PerformanceTests/automated_test_5_loadimbalance +++ /dev/null @@ -1,59 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 32 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 20.e-6 - -# Boundaries -boundary.field_lo = pec pec periodic -boundary.field_hi = pec pec periodic -boundary.particle_lo = absorbing absorbing periodic -boundary.particle_hi = absorbing absorbing periodic - -warpx.verbose = 1 -algo.load_balance_intervals = -5 -algo.particle_shape = 3 - -# CFL -warpx.cfl = 1.0 - -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 2 2 4 -electrons.profile = constant -electrons.density = 1.e20 # number of electrons per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.zmax = 0. -electrons.ux_th = 0.01 -electrons.uy_th = 0.01 -electrons.uz_th = 0.01 -electrons.ux_m = 0. -electrons.uy_m = 0. -electrons.uz_m = 0. - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = "NUniformPerCell" -ions.num_particles_per_cell_each_dim = 2 2 4 -ions.profile = constant -ions.density = 1.e20 # number of electrons per m^3 -ions.momentum_distribution_type = "gaussian" -ions.zmax = 0. -ions.ux_th = 0.01 -ions.uy_th = 0.01 -ions.uz_th = 0.01 -ions.ux_m = 0. -ions.uy_m = 0. -ions.uz_m = 0. diff --git a/Tools/PerformanceTests/automated_test_6_output_2ppc b/Tools/PerformanceTests/automated_test_6_output_2ppc deleted file mode 100644 index afd4a6df336..00000000000 --- a/Tools/PerformanceTests/automated_test_6_output_2ppc +++ /dev/null @@ -1,64 +0,0 @@ -# Maximum number of time steps: command-line argument -# number of grid points: command-line argument - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -amr.max_grid_size = 64 - -# Maximum level in hierarchy (for now must be 0, i.e., one level in total) -amr.max_level = 0 - -# Geometry -geometry.dims = 3 -geometry.prob_lo = -20.e-6 -20.e-6 -20.e-6 # physical domain -geometry.prob_hi = 20.e-6 20.e-6 20.e-6 - -# Boundaries -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec -boundary.particle_lo = absorbing absorbing absorbing -boundary.particle_hi = absorbing absorbing absorbing - -# Verbosity -warpx.verbose = 1 - -algo.particle_shape = 3 - -# CFL -warpx.cfl = 1.0 - -particles.species_names = electrons ions - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = "NUniformPerCell" -electrons.num_particles_per_cell_each_dim = 1 1 1 -electrons.profile = constant -electrons.density = 1.e20 # number of electrons per m^3 -electrons.momentum_distribution_type = "gaussian" -electrons.ux_th = 0.01 -electrons.uy_th = 0.01 -electrons.uz_th = 0.01 -electrons.ux_m = 0. -electrons.uy_m = 0. -electrons.uz_m = 0. - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = "NUniformPerCell" -ions.num_particles_per_cell_each_dim = 1 1 1 -ions.profile = constant -ions.density = 1.e20 # number of electrons per m^3 -ions.momentum_distribution_type = "gaussian" -ions.ux_th = 0.01 -ions.uy_th = 0.01 -ions.uz_th = 0.01 -ions.ux_m = 0. -ions.uy_m = 0. -ions.uz_m = 0. - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.file_prefix = "./diags/plt" -diag1.diag_type = Full diff --git a/Tools/PerformanceTests/cori.py b/Tools/PerformanceTests/cori.py deleted file mode 100644 index 046767713f0..00000000000 --- a/Tools/PerformanceTests/cori.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2019 Axel Huebl, Luca Fedeli, Maxence Thevenet -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import copy -import os - -from functions_perftest import test_element - -module_name = {'cpu': 'haswell.', 'knl': 'mic-knl.', 'gpu':'.'} - -def executable_name(compiler, architecture): - return 'perf_tests3d.' + compiler + \ - '.' + module_name[architecture] + 'TPROF.MTMPI.OMP.QED.ex' - -def get_config_command(compiler, architecture): - config_command = '' - config_command += 'module unload darshan;' - if architecture == 'knl': - if compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-haswell;' - config_command += 'module load craype-mic-knl;' - elif architecture == 'cpu': - if compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-mic-knl;' - config_command += 'module load craype-haswell;' - return config_command - -# This function runs a batch script with -# dependencies to perform the analysis -# after all performance tests are done. -def process_analysis(automated, cwd, compiler, architecture, n_node_list, start_date, path_source, path_results): - dependencies = '' - f_log = open(cwd + 'log_jobids_tmp.txt' ,'r') - for line in f_log.readlines(): - dependencies += line.split()[3] + ':' - - batch_string = '''#!/bin/bash -#SBATCH --job-name=warpx_1node_read -#SBATCH --time=00:07:00 -#SBATCH -C knl -#SBATCH -N 1 -#SBATCH -S 4 -#SBATCH -q regular -#SBATCH -e read_error.txt -#SBATCH -o read_output.txt -#SBATCH --mail-type=end -#SBATCH --account=m2852 -module load h5py-parallel -''' - batch_string += 'python run_automated.py --compiler=' + \ - compiler + ' --architecture=' + architecture + \ - ' --mode=read' + \ - ' --n_node_list=' + '"' + n_node_list + '"' + \ - ' --start_date=' + start_date + \ - ' --path_source=' + path_source + \ - ' --path_results=' + path_results - if automated == True: - batch_string += ' --automated' - batch_string += '\n' - batch_file = 'slurm_perfread' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + batch_file) - print( 'process_analysis line: ' + 'sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) - os.system('sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) - -# Calculate simulation time. Take 5 min + 5 min / simulation -def time_min(nb_simulations): - return 5. + nb_simulations*5. - -def get_submit_job_command(): - return ' sbatch ' - -def get_batch_string(test_list, job_time_min, Cname, n_node): - - job_time_str = str(int(job_time_min/60)) + ':' + str(int(job_time_min%60)) + ':00' - - batch_string = '' - batch_string += '#!/bin/bash\n' - batch_string += '#SBATCH --job-name=' + test_list[0].input_file + '\n' - batch_string += '#SBATCH --time=' + job_time_str + '\n' - batch_string += '#SBATCH -C ' + Cname + '\n' - batch_string += '#SBATCH -N ' + str(n_node) + '\n' - batch_string += '#SBATCH -q regular\n' - batch_string += '#SBATCH -e error.txt\n' - batch_string += '#SBATCH --account=m2852\n' - batch_string += 'module unload PrgEnv-gnu\n' - batch_string += 'module load PrgEnv-intel\n' - return batch_string - -def get_run_string(current_test, architecture, n_node, count, bin_name, runtime_param_string): - srun_string = '' - srun_string += 'export OMP_NUM_THREADS=' + str(current_test.n_omp) + '\n' - # number of logical cores per MPI process - if architecture == 'cpu': - cflag_value = max(1, int(32/current_test.n_mpi_per_node) * 2) # Follow NERSC directives - elif architecture == 'knl': - cflag_value = max(1, int(64/current_test.n_mpi_per_node) * 4) # Follow NERSC directives - output_filename = 'out_' + '_'.join([current_test.input_file, str(n_node), str(current_test.n_mpi_per_node), str(current_test.n_omp), str(count)]) + '.txt' - srun_string += 'srun --cpu_bind=cores '+ \ - ' -n ' + str(n_node*current_test.n_mpi_per_node) + \ - ' -c ' + str(cflag_value) + \ - ' ./' + bin_name + \ - ' ' + current_test.input_file + \ - runtime_param_string + \ - ' > ' + output_filename + '\n' - return srun_string - -def get_test_list(n_repeat): - test_list_unq = [] - # n_node is kept to None and passed in functions as an external argument - # That way, several test_element_instance run with the same n_node on the same batch job - test_list_unq.append( test_element(input_file='automated_test_1_uniform_rest_32ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], - max_grid_size=64, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_2_uniform_rest_1ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[256, 256, 512], - max_grid_size=64, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_3_uniform_drift_4ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], - max_grid_size=64, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_4_labdiags_2ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[64, 64, 128], - max_grid_size=64, - blocking_factor=32, - n_step=50) ) - test_list_unq.append( test_element(input_file='automated_test_5_loadimbalance', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], - max_grid_size=64, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_6_output_2ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 256, 256], - max_grid_size=64, - blocking_factor=32, - n_step=1) ) - test_list = [copy.deepcopy(item) for item in test_list_unq for _ in range(n_repeat) ] - return test_list diff --git a/Tools/PerformanceTests/functions_perftest.py b/Tools/PerformanceTests/functions_perftest.py deleted file mode 100644 index 8d7f4e29246..00000000000 --- a/Tools/PerformanceTests/functions_perftest.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright 2018-2019 Axel Huebl, Luca Fedeli, Maxence Thevenet -# Remi Lehe -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import copy -import os -import re -import shutil - -import git -import numpy as np -import pandas as pd - -# import cori -# import summit - -# Each instance of this class contains information for a single test. -class test_element(): - def __init__(self, input_file=None, n_node=None, n_mpi_per_node=None, - n_omp=None, n_cell=None, n_step=None, max_grid_size=None, - blocking_factor=None): - self.input_file = input_file - self.n_node = n_node - self.n_mpi_per_node = n_mpi_per_node - self.n_omp = n_omp - self.n_cell = n_cell - self.n_step = n_step - self.max_grid_size = max_grid_size - self.blocking_factor = blocking_factor - - def scale_n_cell(self, n_node=0): - n_cell_scaled = copy.deepcopy(self.n_cell) - index_dim = 0 - while n_node > 1: - n_cell_scaled[index_dim] *= 2 - n_node /= 2 - index_dim = (index_dim+1) % 3 - self.n_cell = n_cell_scaled - -def scale_n_cell(ncell, n_node): - ncell_scaled = ncell[:] - index_dim = 0 - while n_node > 1: - ncell_scaled[index_dim] *= 2 - n_node /= 2 - index_dim = (index_dim+1) % 3 - return ncell_scaled - -def store_git_hash(repo_path=None, filename=None, name=None): - repo = git.Repo(path=repo_path) - sha = repo.head.object.hexsha - file_handler = open( filename, 'a+' ) - file_handler.write( name + ':' + sha + ' ') - file_handler.close() - -def get_file_content(filename=None): - file_handler = open( filename, 'r' ) - file_content = file_handler.read() - file_handler.close() - return file_content - -def run_batch(run_name, res_dir, bin_name, config_command, architecture='knl',\ - Cname='knl', n_node=1, n_mpi=1, n_omp=1): - # Clean res_dir - if os.path.exists(res_dir): - shutil.rmtree(res_dir) - os.makedirs(res_dir) - # Copy files to res_dir - cwd = os.environ['WARPX'] + '/Tools/PerformanceTests/' - bin_dir = cwd + 'Bin/' - shutil.copy(bin_dir + bin_name, res_dir) - shutil.copyfile(cwd + run_name, res_dir + 'inputs') - os.chdir(res_dir) - batch_string = '' - batch_string += '#!/bin/bash\n' - batch_string += '#SBATCH --job-name=' + run_name + str(n_node) + str(n_mpi) + str(n_omp) + '\n' - batch_string += '#SBATCH --time=00:23:00\n' - batch_string += '#SBATCH -C ' + Cname + '\n' - batch_string += '#SBATCH -N ' + str(n_node) + '\n' - batch_string += '#SBATCH -q regular\n' - batch_string += '#SBATCH -e error.txt\n' - batch_string += '#SBATCH --account=m2852\n' - batch_string += 'export OMP_NUM_THREADS=' + str(n_omp) + '\n' - if architecture == 'cpu': - cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives - batch_string += 'srun --cpu_bind=cores '+ \ - ' -n ' + str(n_node*n_mpi) + \ - ' -c ' + str(cflag_value) + \ - ' ./' + bin_name + ' inputs > perf_output.txt' - elif architecture == 'knl': - # number of logical cores per MPI process - cflag_value = max(1, int(64/n_mpi) * 4) # Follow NERSC directives - batch_string += 'srun --cpu_bind=cores ' + \ - ' -n ' + str(n_node*n_mpi) + \ - ' -c ' + str(cflag_value) + \ - ' ./' + bin_name + ' inputs > perf_output.txt\n' - batch_file = 'slurm' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + bin_name) - os.system(config_command + 'sbatch ' + batch_file + ' >> ' + cwd + 'log_jobids_tmp.txt') - return 0 - -def run_batch_nnode(test_list, res_dir, cwd, bin_name, config_command, batch_string, submit_job_command): - # Clean res_dir - if os.path.exists(res_dir): - shutil.rmtree(res_dir, ignore_errors=True) - os.makedirs(res_dir) - # Copy files to res_dir - bin_dir = cwd + 'Bin/' - shutil.copy(bin_dir + bin_name, res_dir) - os.chdir(res_dir) - - for count, current_test in enumerate(test_list): - shutil.copy(cwd + current_test.input_file, res_dir) - batch_file = 'batch_script.sh' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + bin_name) - os.system(config_command + submit_job_command + batch_file +\ - ' >> ' + cwd + 'log_jobids_tmp.txt') - -# Read output file and return init time and 1-step time -def read_run_perf(filename, n_steps): - timing_list = [] - # Search inclusive time to get simulation step time - partition_limit = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' - with open(filename) as file_handler: - output_text = file_handler.read() - # Get total simulation time - line_match_totaltime = re.search('TinyProfiler total time across processes.*', output_text) - total_time = float(line_match_totaltime.group(0).split()[8]) - search_area = output_text.partition(partition_limit)[2] - line_match_looptime = re.search('\nWarpX::Evolve().*', search_area) - time_wo_initialization = float(line_match_looptime.group(0).split()[3]) - timing_list += [str(total_time - time_wo_initialization)] - timing_list += [str(time_wo_initialization/n_steps)] - partition_limit1 = 'NCalls Excl. Min Excl. Avg Excl. Max Max %' - partition_limit2 = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' - file_handler.close() - with open(filename) as file_handler: - output_text = file_handler.read() - # Search EXCLISUSIVE routine timings - search_area = output_text.partition(partition_limit1)[2].partition(partition_limit2)[0] - pattern_list = ['\nParticleContainer::Redistribute().*',\ - '\nFabArray::FillBoundary().*',\ - '\nFabArray::ParallelCopy().*',\ - '\nPPC::CurrentDeposition.*',\ - '\nPPC::FieldGather.*',\ - '\nPPC::ParticlePush.*',\ - '\nPPC::Evolve::Copy.*',\ - '\nWarpX::Evolve().*',\ - 'Checkpoint().*',\ - 'WriteParticles().*',\ - '\nVisMF::Write(FabArray).*',\ - '\nWriteMultiLevelPlotfile().*',\ - '\nParticleContainer::RedistributeMPI().*'] - for pattern in pattern_list: - timing = '0' - line_match = re.search(pattern, search_area) - if line_match is not None: - timing = [str(float(line_match.group(0).split()[3])/n_steps)] - timing_list += timing - return timing_list - -# Write time into logfile -def write_perf_logfile(log_file, log_line): - f_log = open(log_file, 'a') - f_log.write(log_line) - f_log.close() - return 0 - -def get_nsteps(run_name): - with open(run_name) as file_handler: - run_name_text = file_handler.read() - line_match_nsteps = re.search('\nmax_step.*', run_name_text) - nsteps = float(line_match_nsteps.group(0).split()[2]) - return nsteps - -def extract_dataframe(filename, n_steps): - # Get init time and total time through Inclusive time - partition_limit_start = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' - print(filename) - with open(filename) as file_handler: - output_text = file_handler.read() - # get total simulation time - line_match_totaltime = re.search('TinyProfiler total time across processes.*', output_text) - total_time = float(line_match_totaltime.group(0).split()[8]) - # get time performing steps as Inclusive WarpX::Evolve() time - search_area = output_text.partition(partition_limit_start)[2] - line_match_looptime = re.search('\nWarpX::Evolve().*', search_area) - time_wo_initialization = float(line_match_looptime.group(0).split()[3]) - # New, might break something - line_match_WritePlotFile = re.search('\nDiagnostics::FilterComputePackFlush().*', search_area) - if line_match_WritePlotFile is not None: - time_WritePlotFile = float(line_match_WritePlotFile.group(0).split()[3]) - else: - time_WritePlotFile = 0. - # Get timers for all routines - # Where to start and stop in the output_file - partition_limit_start = 'NCalls Excl. Min Excl. Avg Excl. Max Max %' - partition_limit_end = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' - # Put file content in a string - with open(filename) as file_handler: - output_text = file_handler.read() - # Keep only profiling data - search_area = output_text.partition(partition_limit_start)[2]\ - .partition(partition_limit_end)[0] - list_string = search_area.split('\n')[2:-4] - time_array = np.zeros(len(list_string)) - column_list= [] - for i in np.arange(len(list_string)): - column_list.append(list_string[i].split()[0]) - time_array[i] = float(list_string[i].split()[3]) - df = pd.DataFrame(columns=column_list) - df.loc[0] = time_array - df['time_initialization'] = total_time - time_wo_initialization - df['time_running'] = time_wo_initialization - df['time_WritePlotFile'] = time_WritePlotFile - # df['string_output'] = partition_limit_start + '\n' + search_area - return df - -# Run a performance test in an interactive allocation -# def run_interactive(run_name, res_dir, n_node=1, n_mpi=1, n_omp=1): -# # Clean res_dir # -# if os.path.exists(res_dir): -# shutil.rmtree(res_dir) -# os.makedirs(res_dir) -# # Copy files to res_dir # -# shutil.copyfile(bin_dir + bin_name, res_dir + bin_name) -# shutil.copyfile(cwd + run_name, res_dir + 'inputs') -# os.chdir(res_dir) -# if args.architecture == 'cpu': -# cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives # -# exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ -# 'srun --cpu_bind=cores ' + \ -# ' -n ' + str(n_node*n_mpi) + \ -# ' -c ' + str(cflag_value) + \ -# ' ./' + bin_name + ' inputs > perf_output.txt' -# elif args.architecture == 'knl': -# # number of logical cores per MPI process # -# cflag_value = max(1,int(68/n_mpi) * 4) # Follow NERSC directives # -# exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ -# 'srun --cpu_bind=cores ' + \ -# ' -n ' + str(n_node*n_mpi) + \ -# ' -c ' + str(cflag_value) + \ -# ' ./' + bin_name + ' inputs > perf_output.txt' -# os.system('chmod 700 ' + bin_name) -# os.system(config_command + exec_command) -# return 0 diff --git a/Tools/PerformanceTests/performance_log.txt b/Tools/PerformanceTests/performance_log.txt deleted file mode 100644 index 72fece34939..00000000000 --- a/Tools/PerformanceTests/performance_log.txt +++ /dev/null @@ -1,81 +0,0 @@ -## year month day run_name compiler architecture n_node n_mpi n_omp time_initialization time_one_iteration Redistribute FillBoundary ParallelCopy CurrentDeposition FieldGather ParthiclePush Copy Evolve Checkpoint WriteParticles Write_FabArray WriteMultiLevelPlotfile(unit: second) RedistributeMPI -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.21 0.3997 0.1717 0.01706 0.0162 0.07026 0.03655 0.01928 0.01999 0.003687 0 0 0 0 0.006799 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.89 0.4779 0.04441 0.1143 0.09117 0.1072 0.01254 0.003702 0.004217 0.01247 0 0 0 0 0.003441 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.58 0.4626 0.04424 0.1048 0.0851 0.1073 0.01259 0.003767 0.004282 0.01311 0 0 0 0 0.002798 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.63 0.4616 0.04441 0.1033 0.08398 0.1079 0.01312 0.003802 0.004224 0.01278 0 0 0 0 0.003188 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.72 0.461 0.04419 0.1038 0.08424 0.1074 0.01257 0.003799 0.0043 0.01318 0 0 0 0 0.002816 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.32 0.3986 0.1712 0.01804 0.01697 0.06999 0.03615 0.01842 0.01896 0.003445 0 0 0 0 0.00738 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.17 0.3974 0.1711 0.01722 0.01587 0.07016 0.03642 0.01844 0.01902 0.003431 0 0 0 0 0.007332 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.88 0.3946 0.1709 0.01686 0.01562 0.06972 0.03595 0.01848 0.01916 0.003269 0 0 0 0 0.006887 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.95 0.4094 0.1708 0.01761 0.01632 0.07001 0.03651 0.01863 0.01906 0.003314 0 0 0 0 0.01898 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.3 0.4787 0.04447 0.1139 0.09124 0.108 0.01287 0.003811 0.004205 0.01249 0 0 0 0 0.003045 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 3.16 0.4578 0.04412 0.1015 0.08339 0.1078 0.01301 0.003919 0.004182 0.0125 0 0 0 0 0.002701 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.78 0.4679 0.04418 0.1035 0.08456 0.1079 0.01303 0.003902 0.004214 0.0127 0 0 0 0 0.009118 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.12 0.4613 0.04425 0.1043 0.08517 0.1073 0.01242 0.003797 0.004221 0.01239 0 0 0 0 0.003665 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.48 0.1237 0.03056 0.01622 0.01468 0.02039 0.005016 0.003737 0.002632 0.00326 0 0 0 0 0.006871 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.79 0.1287 0.0308 0.01706 0.01715 0.02042 0.005452 0.003636 0.002797 0.003143 0 0 0 0 0.007324 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1296 0.03084 0.01711 0.01731 0.02053 0.005379 0.003641 0.002843 0.003137 0 0 0 0 0.008151 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1323 0.03081 0.01703 0.01736 0.02065 0.005339 0.003638 0.002751 0.004008 0 0 0 0 0.01015 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.85 0.2896 0.03832 0.06449 0.07493 0.003507 0.002987 0.0001515 0.0001762 0.007921 0.0371 0.001537 0 0.0004387 0.03832 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.12 0.2895 0.03845 0.06423 0.07481 0.003489 0.002994 0.000152 0.0001779 0.00834 0.0357 0.001545 0 0.0005249 0.03845 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.76 0.3243 0.03804 0.0646 0.07462 0.003483 0.002991 0.0001508 0.0001769 0.008051 0.05983 0.001565 0 0.005392 0.03804 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.74 0.3143 0.03941 0.06478 0.07547 0.003486 0.003007 0.0001518 0.0001808 0.007845 0.05079 0.001543 0 0.0007033 0.03941 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.2 0.3845 0.08558 0.1042 0.1332 0 0 0 0 0.01226 0 0 0 0 0.08558 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.19 0.3864 0.085 0.1051 0.134 0 0 0 0 0.01202 0 0 0 0 0.085 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 8.98 0.3912 0.08665 0.1061 0.1356 0 0 0 0 0.01193 0 0 0 0 0.08665 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.03 0.3826 0.08484 0.1031 0.1329 0 0 0 0 0.01205 0 0 0 0 0.08484 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.6 1.086 0.0898 0.1311 0.09441 0.1345 0.027 0.008783 0.009792 0.02151 0.08454 0.04962 0 0.0008218 0.005303 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.7 1.136 0.09059 0.1437 0.09535 0.1358 0.02915 0.009238 0.01002 0.02315 0.09088 0.05006 0 0.01081 0.005381 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.0 1.132 0.09145 0.1377 0.09592 0.1365 0.02817 0.009353 0.0103 0.02447 0.066 0.05309 0 0.02047 0.009196 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.8 1.135 0.09088 0.1308 0.09623 0.135 0.02762 0.008839 0.009758 0.02561 0.1144 0.04874 0 0.0008693 0.008112 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.87 0.4053 0.1754 0.01914 0.01871 0.0691 0.03648 0.01879 0.0193 0.003268 0 0 0 0 0.007445 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 4.38 0.405 0.1741 0.01901 0.01839 0.07034 0.03718 0.01894 0.0195 0.003845 0 0 0 0 0.007187 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.79 0.3999 0.1739 0.01859 0.01631 0.06918 0.0367 0.01906 0.01952 0.003278 0 0 0 0 0.006658 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.93 0.4044 0.1746 0.01854 0.01695 0.06975 0.03721 0.0191 0.01941 0.003979 0 0 0 0 0.007381 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.83 0.4773 0.04582 0.1089 0.08772 0.1072 0.01304 0.003335 0.004231 0.01385 0 0 0 0 0.002991 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.51 0.4654 0.04556 0.1027 0.08351 0.1068 0.01292 0.003114 0.004249 0.01356 0 0 0 0 0.002748 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.62 0.4755 0.0457 0.1082 0.08761 0.1069 0.0131 0.003205 0.00431 0.01388 0 0 0 0 0.002738 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.44 0.4798 0.04561 0.1133 0.08962 0.1064 0.01246 0.003076 0.004241 0.01318 0 0 0 0 0.003164 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.92 0.1282 0.03185 0.01747 0.01557 0.01956 0.005103 0.003455 0.00274 0.00346 0 0 0 0 0.007196 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.97 0.1301 0.03157 0.01788 0.01732 0.01957 0.00508 0.003335 0.002803 0.003454 0 0 0 0 0.007446 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.91 0.1289 0.03137 0.01765 0.0155 0.02026 0.005636 0.003513 0.002716 0.003381 0 0 0 0 0.007087 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.64 0.1308 0.03142 0.0181 0.01777 0.01953 0.005204 0.003371 0.002782 0.003057 0 0 0 0 0.007769 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 3.19 0.3005 0.0383 0.06396 0.07274 0.003502 0.003005 0.0001628 0.0001839 0.008869 0.04427 0.001522 0 0.0005522 0.0383 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.41 0.2945 0.0389 0.06251 0.0723 0.003508 0.003009 0.000164 0.0001825 0.009131 0.04042 0.001538 0 0.0005936 0.0389 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.32 0.3066 0.0387 0.06558 0.07547 0.003463 0.003017 0.0001631 0.0001837 0.008431 0.04955 0.001555 0 0.000454 0.0387 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.71 0.3391 0.03987 0.06534 0.07626 0.003475 0.003004 0.0001643 0.0001821 0.008152 0.06677 0.001534 0 0.01029 0.03987 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.68 0.3956 0.08701 0.1051 0.1352 0 0 0 0 0.01387 0 0 0 0 0.08701 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.65 0.3987 0.0866 0.1051 0.1332 0 0 0 0 0.0191 0 0 0 0 0.0866 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.11 0.4013 0.08782 0.1087 0.1359 0 0 0 0 0.01379 0 0 0 0 0.08782 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.94 0.39 0.08702 0.1028 0.132 0 0 0 0 0.0142 0 0 0 0 0.08702 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.292 0.2639 0.01424 0.03424 0.01742 0.01893 0.003449 0.001364 0.001712 0.009362 0.04053 0.01765 0 0.002558 0.001185 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.779 0.3155 0.01125 0.03605 0.01628 0.02431 0.009672 0.002843 0.001334 0.008876 0.05925 0.02047 0 0.001897 0.0006917 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.635 0.2568 0.01083 0.03443 0.01592 0.01963 0.003027 0.001439 0.001286 0.009288 0.03879 0.01815 0 0.001509 0.0007743 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.371 0.2648 0.01401 0.03376 0.01593 0.01936 0.003443 0.001351 0.00169 0.01161 0.03936 0.01785 0 0.002107 0.001171 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.7 0.4573 0.01159 0.02139 0.02206 0.06934 0.03845 0.0192 0.02062 0.003496 0 0 0 0 0.01159 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.45 0.4603 0.01356 0.02085 0.02488 0.06946 0.03777 0.01908 0.02031 0.003356 0 0 0 0 0.01356 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.72 0.4552 0.01245 0.02003 0.02013 0.06874 0.03766 0.01907 0.0203 0.003667 0 0 0 0 0.01245 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.94 0.4557 0.01381 0.01979 0.02053 0.0687 0.03694 0.01886 0.02012 0.006396 0 0 0 0 0.01381 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.33 0.4937 0.005316 0.1103 0.09802 0.1071 0.01258 0.00326 0.004435 0.01347 0 0 0 0 0.005316 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.27 0.5063 0.004948 0.1213 0.1019 0.1067 0.01183 0.003056 0.004479 0.01327 0 0 0 0 0.004948 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.2 0.4983 0.005787 0.1141 0.1002 0.1067 0.0121 0.00307 0.00445 0.01343 0 0 0 0 0.005787 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.39 0.5018 0.005339 0.1152 0.1007 0.1073 0.01249 0.003196 0.004484 0.01348 0 0 0 0 0.005339 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.98 0.1342 0.007843 0.01855 0.01798 0.01936 0.005198 0.003471 0.002626 0.003161 0 0 0 0 0.007843 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.63 0.1367 0.008055 0.01917 0.01818 0.01992 0.006097 0.003388 0.002639 0.003079 0 0 0 0 0.008055 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.66 0.1365 0.008017 0.0196 0.01819 0.01979 0.005769 0.00331 0.002668 0.003111 0 0 0 0 0.008017 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.89 0.1367 0.008249 0.01947 0.01818 0.01956 0.005585 0.003341 0.002697 0.003217 0 0 0 0 0.008249 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.14 0.3087 0.04174 0.0637 0.0734 0.00345 0.002967 0.0001664 0.0001849 0.008714 0.05156 0.001539 0 0.0004984 0.04174 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.21 0.3407 0.07513 0.07261 0.0713 0.003428 0.002994 0.0001638 0.0001848 0.009408 0.003442 0.00173 0 0.0005256 0.07513 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.73 0.347 0.04077 0.06476 0.07148 0.00345 0.002998 0.0001637 0.0001829 0.009379 0.03947 0.001574 0 0.04989 0.04077 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.52 0.3469 0.04088 0.06365 0.07183 0.003493 0.002957 0.0001659 0.0001827 0.009064 0.04694 0.001959 0 0.04099 0.04088 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.92 0.4206 0.08811 0.1186 0.1402 0 0 0 0 0.01443 0 0 0 0 0.08811 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.12 0.3884 0.08626 0.1027 0.1305 0 0 0 0 0.01368 0 0 0 0 0.08626 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.91 0.4097 0.08598 0.1119 0.1381 0 0 0 0 0.01414 0 0 0 0 0.08598 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.63 0.4257 0.0876 0.1213 0.1441 0 0 0 0 0.01422 0 0 0 0 0.0876 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.23 0.274 0.003227 0.03782 0.01724 0.01945 0.003219 0.001468 0.0014 0.01094 0.03943 0.0175 0 0.00509 0.001122 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 2.076 0.3023 0.002995 0.035 0.01619 0.02462 0.01126 0.006984 0.001548 0.01009 0.04604 0.01734 0 0.08398 0.001151 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.378 0.273 0.004545 0.03721 0.01754 0.02039 0.003415 0.00145 0.001561 0.01058 0.04009 0.01763 0 0.002519 0.001187 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.61 0.2911 0.004065 0.03726 0.01782 0.02439 0.01289 0.003463 0.001689 0.008778 0.03975 0.01723 0 0.00247 0.00129 diff --git a/Tools/PerformanceTests/run_alltests.py b/Tools/PerformanceTests/run_alltests.py deleted file mode 100644 index b1083fc6f45..00000000000 --- a/Tools/PerformanceTests/run_alltests.py +++ /dev/null @@ -1,353 +0,0 @@ -# Copyright 2017-2020 Luca Fedeli, Maxence Thevenet, Remi Lehe -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import argparse -import datetime -import os -import re -import shutil -import time - -from functions_perftest import * - -# This script runs automated performance tests for WarpX. -# It runs tests in list test_list defined below, and write -# results in file performance_log.txt in warpx/Tools/PerformanceTests/ - -# ---- User's manual ---- -# Before running performance tests, make sure you have the latest version -# of performance_log.txt -# A typical execution reads: -# > python run_alltests.py --no-recompile --compiler=gnu --architecture=cpu --mode=run --log_file='my_performance_log.txt' -# These are default values, and will give the same result as -# > python run_alltests.py -# To add a new test item, extent the test_list with a line like -# test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*3) -# - my_input_file must be in warpx/Tools/PerformanceTests -# - the test will run 3 times, to have some statistics -# - the test must take <1h or it will timeout - -# ---- Developer's manual ---- -# This script can run in two modes: -# - 'run' mode: for each test item, a batch job is executed. -# create folder '$SCRATCH/performance_warpx/' -# recompile the code if option --recompile is used -# loop over test_list and submit one batch script per item -# Submit a batch job that executes the script in read mode -# This last job runs once all others are completed -# - 'read' mode: Get performance data from all test items -# create performance log file if does not exist -# loop over test_file -# read initialization time and step time -# write data into the performance log file -# push file performance_log.txt on the repo - -# Define the list of tests to run -# ------------------------------- -# each element of test_list contains -# [str runname, int n_node, int n_mpi PER NODE, int n_omp] -test_list = [] -n_repeat = 3 - -test_list.extend([['ompscaling_32ppc' , 1, 1, 1]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 2]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 4]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 8]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 16]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 32]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1, 64]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1,128]]*n_repeat) -test_list.extend([['ompscaling_32ppc' , 1, 1,256]]*n_repeat) - -#test_list.extend([['mil_weak1_0ppc_1' , 1, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_8' , 8, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_64' , 64, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_512' , 512, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_1024' , 1024, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_2048' , 2048, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_0ppc_4096' , 4096, 8, 8]]*n_repeat) - -#test_list.extend([['mil_weak1_32ppc_1' , 1, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_8' , 8, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_64' , 64, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_512' , 512, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_1024' , 1024, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_2048' , 2048, 8, 8]]*n_repeat) -#test_list.extend([['mil_weak1_32ppc_4096' , 4096, 8, 8]]*n_repeat) - -#test_list.extend([['strong_32ppc1' , 1, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc1' , 8, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc1' , 64, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc1' , 128, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 128, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 256, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 512, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 1024, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 2048, 8, 8]]*n_repeat) -#test_list.extend([['strong_32ppc128' , 4096, 8, 8]]*n_repeat) - -#test_list.extend([['strong1_0ppc_1' , 1, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_1' , 8, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_1' , 64, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_1' , 128, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_1' , 256, 8, 8]]*n_repeat) - -#test_list.extend([['strong1_0ppc_128' , 128, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_128' , 256, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_128' , 512, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_128' , 1024, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_128' , 2048, 8, 8]]*n_repeat) -#test_list.extend([['strong1_0ppc_128' , 4096, 8, 8]]*n_repeat) - -n_tests = len(test_list) - -# Read command-line arguments -# --------------------------- -# Create parser and read arguments -parser = argparse.ArgumentParser( - description='Run performance tests and write results in files') -parser.add_argument('--recompile', dest='recompile', action='store_true', default=False) -parser.add_argument('--no-recompile', dest='recompile', action='store_false', default=False) -parser.add_argument('--commit', dest='commit', action='store_true', default=False) -parser.add_argument( '--compiler', choices=['gnu', 'intel'], default='gnu', - help='which compiler to use') -parser.add_argument( '--architecture', choices=['cpu', 'knl'], default='cpu', - help='which architecture to cross-compile for NERSC machines') -parser.add_argument( '--mode', choices=['run', 'read'], default='run', - help='whether to run perftests or read their perf output. run calls read') -parser.add_argument( '--log_file', dest = 'log_file', default='my_performance_log.txt', - help='name of log file where data will be written. ignored if option --commit is used') -parser.add_argument( '--n_steps', dest = 'n_steps', default=None, - help='Number of time steps in the simulation. Should be read automatically from the input file') - -args = parser.parse_args() - -log_file = args.log_file -if args.commit == True: - log_file = 'performance_log.txt' - -# Dictionaries -# compiler names. Used for WarpX executable name -compiler_name = {'intel': 'intel', 'gnu': 'gcc'} -# architecture. Used for WarpX executable name -module_name = {'cpu': 'haswell', 'knl': 'mic-knl'} -# architecture. Used in batch scripts -module_Cname = {'cpu': 'haswell', 'knl': 'knl,quad,cache'} -# Define environment variables -cwd = os.getcwd() + '/' -res_dir_base = os.environ['SCRATCH'] + '/performance_warpx/' -bin_dir = cwd + 'Bin/' -bin_name = 'perf_tests3d.' + args.compiler + '.' + module_name[args.architecture] + '.TPROF.MTMPI.OMP.QED.ex' -log_dir = cwd - -perf_database_file = cwd + 'perf_database_warpx.h5' -do_rename = False -store_test = False - -day = time.strftime('%d') -month = time.strftime('%m') -year = time.strftime('%Y') - -# Initialize tests -# ---------------- -if args.mode == 'run': -# Set default options for compilation and execution - config_command = '' - config_command += 'module unload darshan;' - config_command += 'module load craype-hugepages4M;' - if args.architecture == 'knl': - if args.compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif args.compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-haswell;' - config_command += 'module load craype-mic-knl;' - elif args.architecture == 'cpu': - if args.compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif args.compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-mic-knl;' - config_command += 'module load craype-haswell;' - # Create main result directory if does not exist - if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) - -# Recompile if requested -if args.recompile == True: - with open(cwd + 'GNUmakefile_perftest') as makefile_handler: - makefile_text = makefile_handler.read() - makefile_text = re.sub('\nCOMP.*', '\nCOMP=%s' %compiler_name[args.compiler], makefile_text) - with open(cwd + 'GNUmakefile_perftest', 'w') as makefile_handler: - makefile_handler.write( makefile_text ) - os.system(config_command + " make -f GNUmakefile_perftest realclean ; " + " rm -r tmp_build_dir *.mod; make -j 8 -f GNUmakefile_perftest") - -# Define functions to run a test and analyse results -# -------------------------------------------------- -def process_analysis(): - dependencies = '' - f_log = open(cwd + 'log_jobids_tmp.txt','r') - for count, current_run in enumerate(test_list): - line = f_log.readline() - print(line) - dependencies += line.split()[3] + ':' - batch_string = '' - batch_string += '#!/bin/bash\n' - batch_string += '#SBATCH --job-name=warpx_read\n' - batch_string += '#SBATCH --time=00:05:00\n' - batch_string += '#SBATCH -C ' + module_Cname[args.architecture] + '\n' - batch_string += '#SBATCH -N 1\n' - batch_string += '#SBATCH -S 4\n' - batch_string += '#SBATCH -q regular\n' - batch_string += '#SBATCH -e read_error.txt\n' - batch_string += '#SBATCH -o read_output.txt\n' - batch_string += '#SBATCH --mail-type=end\n' - batch_string += '#SBATCH --account=m2852\n' - batch_string += 'python ' + __file__ + ' --no-recompile --compiler=' + \ - args.compiler + ' --architecture=' + args.architecture + \ - ' --mode=read' + ' --log_file=' + log_file - if args.commit == True: - batch_string += ' --commit' - batch_string += '\n' - batch_file = 'slurm_perfread' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + batch_file) - os.system('sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) - return 0 - -# Loop over the tests and return run time + details -# ------------------------------------------------- -if args.mode == 'run': - # Remove file log_jobids_tmp.txt if exists. - # This file contains the jobid of every perf test - # It is used to manage the analysis script dependencies - if os.path.isfile(cwd + 'log_jobids_tmp.txt'): - os.remove(cwd + 'log_jobids_tmp.txt') - for count, current_run in enumerate(test_list): - # Results folder - print('run ' + str(current_run)) - run_name = current_run[0] - n_node = current_run[1] - n_mpi = current_run[2] - n_omp = current_run[3] - n_steps = get_nsteps(cwd + run_name) - res_dir = res_dir_base - res_dir += '_'.join([run_name, args.compiler,\ - args.architecture, str(n_node), str(n_mpi),\ - str(n_omp), str(count)]) + '/' - # Run the simulation. - # If you are currently in an interactive session and want to run interactive, - # just replace run_batch with run_interactive - run_batch(run_name, res_dir, bin_name, config_command, architecture=args.architecture, \ - Cname=module_Cname[args.architecture], n_node=n_node, n_mpi=n_mpi, n_omp=n_omp) - os.chdir(cwd) - process_analysis() - -if args.mode == 'read': - # Create log_file for performance tests if does not exist - if not os.path.isfile(log_dir + log_file): - log_line = '## year month day run_name compiler architecture n_node n_mpi ' +\ - 'n_omp time_initialization time_one_iteration Redistribute '+\ - 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParticlePush Copy Evolve Checkpoint '+\ - 'WriteParticles Write_FabArray '+\ - 'WriteMultiLevelPlotfile '+\ - 'RedistributeMPI(unit: second)\n' - f_log = open(log_dir + log_file, 'a') - f_log.write(log_line) - f_log.close() - for count, current_run in enumerate(test_list): - # Results folder - print('read ' + str(current_run)) - run_name = current_run[0] - n_node = current_run[1] - n_mpi = current_run[2] - n_omp = current_run[3] - if args.n_steps is None: - n_steps = get_nsteps(cwd + run_name) - else: - n_steps = int(args.n_steps) - res_dir = res_dir_base - res_dir += '_'.join([run_name, args.compiler,\ - args.architecture, str(n_node), str(n_mpi),\ - str(n_omp), str(count)]) + '/' - # Read to store in text file - # -------------------------- - output_filename = 'perf_output.txt' - timing_list = read_run_perf(res_dir + output_filename, n_steps) - # Write performance data to the performance log file - log_line = ' '.join([year, month, day, run_name, args.compiler,\ - args.architecture, str(n_node), str(n_mpi),\ - str(n_omp)] + timing_list + ['\n']) - write_perf_logfile(log_dir + log_file, log_line) - - # Read data for all test to put in hdf5 a database - # ------------------------------------------------ - # This is an hdf5 file containing ALL the simulation parameters and results. Might be too large for a repo - df_newline = extract_dataframe(res_dir + 'perf_output.txt', n_steps) - # Add all simulation parameters to the dataframe - df_newline['run_name'] = run_name - df_newline['n_node'] = n_node - df_newline['n_mpi'] = n_mpi - df_newline['n_omp'] = n_omp - df_newline['n_steps'] = n_steps - df_newline['rep'] = count - df_newline['date'] = datetime.datetime.now() - input_file = open(cwd + run_name, 'r') - input_file_content = input_file.read() - input_file.close() - df_newline['inputs_content'] = input_file_content - if os.path.exists(perf_database_file): - df_base = pd.read_hdf(perf_database_file, 'all_data') - updated_df = df_base.append(df_newline, ignore_index=True) - else: - updated_df = df_newline - updated_df.to_hdf(perf_database_file, key='all_data', mode='w') - - # Store test parameters for record if requested - if store_test == True: - dir_record_base = './perf_warpx_record/' - if not os.path.exists(dir_record_base): - os.mkdir(dir_record_base) - count = 0 - dir_record = dir_record_base + '_'.join([year, month, day]) + '_0' - while os.path.exists(dir_record): - dir_record = dir_record[:-1] + str(count) - os.mkdir(dir_record) - shutil.copy(__file__, dir_record) - shutil.copy(log_dir + log_file, dir_record) - for count, current_run in enumerate(test_list): - shutil.copy(current_run[0], dir_record) - - if do_rename == True: - # Rename files if requested - for count, current_run in enumerate(test_list): - run_name = current_run[0] - n_node = current_run[1] - n_mpi = current_run[2] - n_omp = current_run[3] - res_dir = res_dir_base - res_dir += '_'.join([run_name, args.compiler,\ - args.architecture, str(n_node), str(n_mpi),\ - str(n_omp), str(count)]) + '/' - res_dir_arch = res_dir_base - res_dir_arch += '_'.join([year, month, day, run_name, args.compiler,\ - args.architecture, str(n_node), str(n_mpi), \ - str(n_omp), str(count)]) + '/' - os.rename(res_dir, res_dir_arch) - - # Commit results to the Repo - if args.commit == True: - os.system('git add ' + log_dir + log_file + ';'\ - 'git commit -m "performance tests";'\ - 'git push -u origin development') diff --git a/Tools/PerformanceTests/run_alltests_1node.py b/Tools/PerformanceTests/run_alltests_1node.py deleted file mode 100644 index f112552b36e..00000000000 --- a/Tools/PerformanceTests/run_alltests_1node.py +++ /dev/null @@ -1,362 +0,0 @@ -# Copyright 2018-2020 Luca Fedeli, Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import argparse -import datetime -import os -import re -import shutil -import time - -from functions_perftest import * - -# This script runs automated performance tests for WarpX. -# It runs tests in list test_list defined below, and write -# results in file performance_log.txt in warpx/Tools/PerformanceTests/ - -# ---- User's manual ---- -# Before running performance tests, make sure you have the latest version -# of performance_log.txt - -# ---- Running a custom set of performance tests ---- -# > python run_alltests_1node.py --no-recompile --compiler=intel -# > --architecture=knl --mode=run --input_file=uniform_plasma -# > --n_node=1 --log_file='my_performance_log.txt' - -# ---- Running the pre-defined automated tests ---- -# Compile and run: -# > python run_alltests_1node.py --automated --recompile -# Just run: -# > python run_alltests_1node.py --automated - -# To add a new test item, extent the test_list with a line like -# test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*n_repeat) -# - my_input_file must be in warpx/Tools/PerformanceTests - -# ---- Developer's manual ---- -# This script can run in two modes: -# - 'run' mode: for each test item, a batch job is executed. -# create folder '$SCRATCH/performance_warpx/' -# recompile the code if option --recompile is used -# loop over test_list and submit one batch script per item -# Submit a batch job that executes the script in read mode -# This last job runs once all others are completed -# - 'read' mode: Get performance data from all test items -# create performance log file if does not exist -# loop over test_file -# read initialization time and step time -# write data into the performance log file -# push file performance_log.txt on the repo - -# Read command-line arguments -# --------------------------- -# Create parser and read arguments -parser = argparse.ArgumentParser( - description='Run performance tests and write results in files') -parser.add_argument('--recompile', dest='recompile', action='store_true', default=False) -parser.add_argument('--no-recompile', dest='recompile', action='store_false', default=False) -parser.add_argument('--commit', dest='commit', action='store_true', default=False) -parser.add_argument( '--compiler', choices=['gnu', 'intel'], default='intel', - help='which compiler to use') -parser.add_argument( '--architecture', choices=['cpu', 'knl'], default='knl', - help='which architecture to cross-compile for NERSC machines') -parser.add_argument( '--mode', choices=['run', 'read'], default='run', - help='whether to run perftests or read their perf output. run calls read') -parser.add_argument( '--log_file', dest = 'log_file', default='my_performance_log.txt', - help='name of log file where data will be written. ignored if option --commit is used') -parser.add_argument('--n_node', dest='n_node', default=1, help='nomber of nodes for the runs') -parser.add_argument('--input_file', dest='input_file', default='input_file.pixr', - type=str, help='input file to run') -parser.add_argument('--automated', dest='automated', action='store_true', default=False, - help='Use to run the automated test list') - -args = parser.parse_args() -log_file = args.log_file -do_commit = args.commit -run_name = args.input_file - -# list of tests to run and analyse. -# Note: This is overwritten if option --automated is used -# each element of test_list contains -# [str input_file, int n_node, int n_mpi PER NODE, int n_omp] -test_list = [] -n_repeat = 2 -filename1 = args.input_file -test_list.extend([[filename1, 1, 128, 1]]*n_repeat) -test_list.extend([[filename1, 1, 64, 2]]*n_repeat) - -# Nothing should be changed after this line -# if flag --automated is used, test_list and do_commit are -# overwritten - -if args.automated == True: - test_list = [] - n_repeat = 2 - test_list.extend([['automated_test_1_uniform_rest_32ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_2_uniform_rest_1ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_3_uniform_drift_4ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_4_labdiags_2ppc', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_5_loadimbalance', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_6_output_2ppc', 1, 16, 8]]*n_repeat) - do_commit = False - run_name = 'automated_tests' - -n_tests = len(test_list) -if do_commit == True: - log_file = 'performance_log.txt' - -# Dictionaries -# compiler names. Used for WarpX executable name -compiler_name = {'intel': 'intel', 'gnu': 'gcc'} -# architecture. Used for WarpX executable name -module_name = {'cpu': 'haswell', 'knl': 'mic-knl'} -# architecture. Used in batch scripts -module_Cname = {'cpu': 'haswell', 'knl': 'knl,quad,cache'} -# Define environment variables -cwd = os.getcwd() + '/' -res_dir_base = os.environ['SCRATCH'] + '/performance_warpx/' -bin_dir = cwd + 'Bin/' -bin_name = 'perf_tests3d.' + args.compiler + '.' + module_name[args.architecture] + '.TPROF.MTMPI.OMP.QED.ex' -log_dir = cwd - -day = time.strftime('%d') -month = time.strftime('%m') -year = time.strftime('%Y') -n_node = int(args.n_node) - -perf_database_file = cwd + 'perf_database_warpx.h5' - -# Initialize tests -# ---------------- -if args.mode == 'run': -# Set default options for compilation and execution - config_command = '' - config_command += 'module unload darshan;' - config_command += 'module load craype-hugepages4M;' - if args.architecture == 'knl': - if args.compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif args.compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-haswell;' - config_command += 'module load craype-mic-knl;' - elif args.architecture == 'cpu': - if args.compiler == 'intel': - config_command += 'module unload PrgEnv-gnu;' - config_command += 'module load PrgEnv-intel;' - elif args.compiler == 'gnu': - config_command += 'module unload PrgEnv-intel;' - config_command += 'module load PrgEnv-gnu;' - config_command += 'module unload craype-mic-knl;' - config_command += 'module load craype-haswell;' - # Create main result directory if does not exist - if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) - -# Recompile if requested -if args.recompile == True: - with open(cwd + 'GNUmakefile_perftest') as makefile_handler: - makefile_text = makefile_handler.read() - makefile_text = re.sub('\nCOMP.*', '\nCOMP=%s' %compiler_name[args.compiler], makefile_text) - with open(cwd + 'GNUmakefile_perftest', 'w') as makefile_handler: - makefile_handler.write( makefile_text ) - os.system(config_command + " make -f GNUmakefile_perftest realclean ; " + " rm -r tmp_build_dir *.mod; make -j 8 -f GNUmakefile_perftest") - -# This function runs a batch script with dependencies to perform the analysis -# when performance runs are done. -def process_analysis(): - dependencies = '' - f_log = open(cwd + 'log_jobids_tmp.txt','r') - line = f_log.readline() - print(line) - dependencies += line.split()[3] + ':' - batch_string = '' - batch_string += '#!/bin/bash\n' - batch_string += '#SBATCH --job-name=warpx_1node_read\n' - batch_string += '#SBATCH --time=00:05:00\n' - batch_string += '#SBATCH -C haswell\n' - batch_string += '#SBATCH -N 1\n' - batch_string += '#SBATCH -S 4\n' - batch_string += '#SBATCH -q regular\n' - batch_string += '#SBATCH -e read_error.txt\n' - batch_string += '#SBATCH -o read_output.txt\n' - batch_string += '#SBATCH --mail-type=end\n' - batch_string += '#SBATCH --account=m2852\n' - batch_string += 'python ' + __file__ + ' --no-recompile --compiler=' + \ - args.compiler + ' --architecture=' + args.architecture + \ - ' --mode=read' + ' --log_file=' + log_file + \ - ' --input_file=' + args.input_file - if do_commit == True: - batch_string += ' --commit' - if args.automated == True: - batch_string += ' --automated' - batch_string += '\n' - batch_file = 'slurm_perfread' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + batch_file) - os.system('sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) - return 0 - -# Loop over the tests and return run time + details -# ------------------------------------------------- -if args.mode == 'run': - # Remove file log_jobids_tmp.txt if exists. - # This file contains the jobid of every perf test - # It is used to manage the analysis script dependencies - if os.path.isfile(cwd + 'log_jobids_tmp.txt'): - os.remove(cwd + 'log_jobids_tmp.txt') - res_dir = res_dir_base - res_dir += '_'.join([run_name, args.compiler,\ - args.architecture, str(n_node)]) + '/' - # Run the simulation. - run_batch_nnode(test_list, res_dir, bin_name, config_command,\ - architecture=args.architecture, Cname=module_Cname[args.architecture], \ - n_node=n_node) - os.chdir(cwd) - process_analysis() - -if args.mode == 'read': - # Create log_file for performance tests if does not exist - if not os.path.isfile(log_dir + log_file): - log_line = '## year month day input_file compiler architecture n_node n_mpi ' +\ - 'n_omp time_initialization time_one_iteration Redistribute '+\ - 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParthiclePush Copy Evolve Checkpoint '+\ - 'WriteParticles Write_FabArray '+\ - 'WriteMultiLevelPlotfile(unit: second) '+\ - 'RedistributeMPI\n' - f_log = open(log_dir + log_file, 'a') - f_log.write(log_line) - f_log.close() - for count, current_run in enumerate(test_list): - # Results folder - print('read ' + str(current_run)) - input_file = current_run[0] - # Do not read n_node = current_run[1], it is an external parameter - n_mpi = current_run[2] - n_omp = current_run[3] - n_steps = get_nsteps(cwd + input_file) - print('n_steps = ' + str(n_steps)) - res_dir = res_dir_base - res_dir += '_'.join([run_name, args.compiler,\ - args.architecture, str(n_node)]) + '/' - # Read performance data from the output file - output_filename = 'out_' + '_'.join([input_file, str(n_node), str(n_mpi), str(n_omp), str(count)]) + '.txt' - timing_list = read_run_perf(res_dir + output_filename, n_steps) - # Write performance data to the performance log file - log_line = ' '.join([year, month, day, input_file, args.compiler,\ - args.architecture, str(n_node), str(n_mpi),\ - str(n_omp)] + timing_list + ['\n']) - write_perf_logfile(log_dir + log_file, log_line) - # Read data for all test to put in hdf5 a database - # ------------------------------------------------ - # This is an hdf5 file containing ALL the simulation parameters and results. Might be too large for a repo - df_newline = extract_dataframe(res_dir + output_filename, n_steps) - # Add all simulation parameters to the dataframe - df_newline['run_name'] = run_name - df_newline['n_node'] = n_node - df_newline['n_mpi'] = n_mpi - df_newline['n_omp'] = n_omp - df_newline['n_steps'] = n_steps - df_newline['rep'] = count - df_newline['date'] = datetime.datetime.now() - input_file_open = open(cwd + input_file, 'r') - input_file_content = input_file_open.read() - input_file_open.close() - df_newline['inputs_content'] = input_file_content - if os.path.exists(perf_database_file): - df_base = pd.read_hdf(perf_database_file, 'all_data') - updated_df = df_base.append(df_newline, ignore_index=True) - else: - updated_df = df_newline - updated_df.to_hdf(perf_database_file, key='all_data', mode='w') - - # Store test parameters fot record - dir_record_base = './perf_warpx_record/' - if not os.path.exists(dir_record_base): - os.mkdir(dir_record_base) - count = 0 - dir_record = dir_record_base + '_'.join([year, month, day]) + '_0' - while os.path.exists(dir_record): - count += 1 - dir_record = dir_record[:-1] + str(count) - os.mkdir(dir_record) - shutil.copy(__file__, dir_record) - shutil.copy(log_dir + log_file, dir_record) - for count, current_run in enumerate(test_list): - shutil.copy(current_run[0], dir_record) - - # Rename directory with precise date for archive purpose - res_dir_arch = res_dir_base - res_dir_arch += '_'.join([year, month, day, run_name, args.compiler,\ - args.architecture, str(n_node)]) + '/' - os.rename(res_dir, res_dir_arch) - - # Commit results to the Repo - if do_commit == True: - os.system('git add ' + log_dir + log_file + ';'\ - 'git commit -m "performance tests";'\ - 'git push -u origin development') - - # Plot file - import matplotlib - import numpy as np - matplotlib.use('Agg') - import matplotlib.pyplot as plt - filename0 = 'my_performance_log' - filename = filename0 + '.txt' - fontsize = 14 - matplotlib.rcParams.update({'font.size': fontsize}) - nsteps = 100. - nrepeat = 4 - legends = [ 'n_node', 'n_mpi', 'n_omp', 'time_initialization', 'time_one_iteration', \ - 'Redistribute', 'FillBoundary', 'ParallelCopy', 'CurrentDeposition', \ - 'FieldGather', 'ParthiclePush', 'Copy', 'Evolve', 'Checkpoint', \ - 'WriteParticles', 'Write_FabArray', 'WriteMultiLevelPlotfile', \ - 'RedistributeMPI'] - date = np.loadtxt( filename, usecols = np.arange(0, 3 )) - data = np.loadtxt( filename, usecols = np.arange(6, 6+len(legends)) ) - # Read run name - with open(filename) as f: - namelist_tmp = zip(*[line.split() for line in f])[3] - # Remove first line = comments - namelist = list(namelist_tmp[1:]) - selector_list = ['automated_test_1_uniform_rest_32ppc',\ - 'automated_test_2_uniform_rest_1ppc',\ - 'automated_test_3_uniform_drift_4ppc',\ - 'automated_test_4_labdiags_2ppc',\ - 'automated_test_5_loadimbalance',\ - 'automated_test_6_output_2ppc'] - selector_string = selector_list[0] - selector = [idx for idx in range(len(namelist)) if selector_string in namelist[idx]] - lin_date = date[:,0]+date[:,1]/12.+date[:,2]/366. - unique_lin_date = np.unique(lin_date) - my_xticks = unique_lin_date -# cmap = plt.get_cmap("tab20") - cycle = plt.rcParams['axes.prop_cycle'].by_key()['color'] - for selector_string in selector_list: - selector = [idx for idx in range(len(namelist)) if selector_string in namelist[idx]] - plt.figure(num=0, figsize=(8,4)) - plt.clf() - plt.title('warpx ' + selector_string) - for i in np.arange(data.shape[1]): - icolors = i-3 - if i>3 and (data[selector,i] > 5./100*data[selector,4]).any(): - plt.plot(lin_date[selector], data[selector,i],'+', ms=6, \ - mew=2, label=legends[i] ) - # plt.plot(lin_date[selector], data[selector,i],'+', ms=6, \ - # mew=2, label=legends[i], color=cmap(i) ) - plt.xlabel('date') - plt.ylabel('time/step (s)') - plt.grid() - plt.legend(loc='best') - plt.legend(bbox_to_anchor=(1.1, 1.05)) - plt.savefig( selector_string + '.pdf', bbox_inches='tight') - plt.savefig( selector_string + '.png', bbox_inches='tight') diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py deleted file mode 100644 index f03ead05376..00000000000 --- a/Tools/PerformanceTests/run_automated.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright 2018-2019 Axel Huebl, Luca Fedeli, Maxence Thevenet -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -import argparse -import copy -import datetime -import os -import shutil -import sys -import time - -import git -import pandas as pd -from functions_perftest import ( - extract_dataframe, - get_file_content, - run_batch_nnode, - store_git_hash, -) - -# Get name of supercomputer and import configuration functions from -# machine-specific file -if os.getenv("LMOD_SYSTEM_NAME") == 'summit': - machine = 'summit' - from summit import ( - executable_name, - get_batch_string, - get_config_command, - get_run_string, - get_submit_job_command, - get_test_list, - process_analysis, - time_min, - ) -if os.getenv("NERSC_HOST") == 'cori': - machine = 'cori' - from cori import ( - executable_name, - get_batch_string, - get_config_command, - get_run_string, - get_submit_job_command, - get_test_list, - process_analysis, - time_min, - ) - -# typical use: python run_automated.py --n_node_list='1,8,16,32' --automated -# Assume warpx, picsar, amrex and perf_logs repos ar in the same directory and -# environment variable AUTOMATED_PERF_TESTS contains the path to this directory - -# requirements: -# - python packages: gitpython and pandas -# - AUTOMATED_PERF_TESTS: environment variables where warpx, -# amrex and picsar are installed ($AUTOMATED_PERF_TESTS/warpx etc.) -# - SCRATCH: environment variable where performance results are written. -# This script will create folder $SCRATCH/performance_warpx/ - -if "AUTOMATED_PERF_TESTS" not in os.environ: - raise ValueError("environment variable AUTOMATED_PERF_TESTS is not defined.\n" - "It should contain the path to the directory where WarpX, " - "AMReX and PICSAR repos are.") -if "SCRATCH" not in os.environ: - raise ValueError("environment variable SCRATCH is not defined.\n" - "This script will create $SCRATCH/performance_warpx/ " - "to store performance results.") -# Handle parser -############### -parser = argparse.ArgumentParser( description='Run performance tests and write results in files' ) -parser.add_argument('--recompile', - dest='recompile', - action='store_true', - default=False) -parser.add_argument('--commit', - dest='commit', - action='store_true', - default=False) -parser.add_argument('--automated', - dest='automated', - action='store_true', - default=False, - help='Use to run the automated test list') -parser.add_argument('--n_node_list', - dest='n_node_list', - default=[], - help='list of number of nodes for the runs', type=str) -parser.add_argument('--start_date', - dest='start_date' ) -parser.add_argument('--compiler', - choices=['gnu', 'intel', 'pgi'], - default='intel', - help='which compiler to use') -parser.add_argument('--architecture', - choices=['cpu', 'knl', 'gpu'], - default='knl', - help='which architecture to cross-compile for NERSC machines') -parser.add_argument('--mode', - choices=['run', 'read', 'browse_output_files'], - default='run', - help='whether to run perftests or read their perf output. run calls read') -parser.add_argument('--path_source', - default=None, - help='path to parent folder containing amrex, picsar and warpx folders') -parser.add_argument('--path_results', - default=None, - help='path to result directory, where simulations run') - -args = parser.parse_args() -n_node_list_string = args.n_node_list.split(',') -n_node_list = [int(i) for i in n_node_list_string] -start_date = args.start_date - -# Set behavior variables -######################## -run_name = 'custom_perftest' -perf_database_file = 'my_tests_database.h5' -rename_archive = False -store_full_input = False -update_perf_log_repo = False -push_on_perf_log_repo = False -recompile = args.recompile -pull_3_repos = False -recompile = True -compiler = args.compiler -architecture = args.architecture -source_dir_base = args.path_source -res_dir_base = args.path_results - -browse_output_files = False -if args.mode == 'browse_output_files': - browse_output_file = True -if args.mode == 'read': - browse_output_files = True - -if args.automated == True: - run_name = 'automated_tests' - perf_database_file = machine + '_results.h5' - rename_archive = True - store_full_input = False - update_perf_log_repo = True - push_on_perf_log_repo = False - pull_3_repos = True - recompile = True - source_dir_base = os.environ['AUTOMATED_PERF_TESTS'] - res_dir_base = os.environ['SCRATCH'] + '/performance_warpx/' - if machine == 'summit': - compiler = 'gnu' - architecture = 'gpu' - -# List of tests to perform -# ------------------------ -# Each test runs n_repeat times -n_repeat = 2 -# test_list is machine-specific -test_list = get_test_list(n_repeat) - -# Define directories -# ------------------ -warpx_dir = source_dir_base + '/warpx/' -picsar_dir = source_dir_base + '/picsar/' -amrex_dir = source_dir_base + '/amrex/' -perf_logs_repo = source_dir_base + 'perf_logs/' - -# Define dictionaries -# ------------------- -compiler_name = {'intel': 'intel', 'gnu': 'gcc', 'pgi':'pgi'} -module_Cname = {'cpu': 'haswell', 'knl': 'knl,quad,cache', 'gpu':''} -csv_file = {'cori':'cori_knl.csv', 'summit':'summit.csv'} -# cwd = os.getcwd() + '/' -cwd = warpx_dir + 'Tools/PerformanceTests/' - -path_hdf5 = cwd -if args.automated: - path_hdf5 = perf_logs_repo + '/logs_hdf5/' - -bin_dir = cwd + 'Bin/' -bin_name = executable_name(compiler, architecture) - -log_dir = cwd -day = time.strftime('%d') -month = time.strftime('%m') -year = time.strftime('%Y') - -# Initialize tests -# ---------------- -if args.mode == 'run': - start_date = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - # Set default options for compilation and execution - config_command = get_config_command(compiler, architecture) - # Create main result directory if does not exist - if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) - - # Recompile if requested - # ---------------------- - if recompile == True: - if pull_3_repos == True: - git_repo = git.cmd.Git( picsar_dir ) - git_repo.pull() - git_repo = git.cmd.Git( amrex_dir ) - git_repo.pull() - git_repo = git.cmd.Git( warpx_dir ) - git_repo.pull() - - # Copy WarpX/GNUmakefile to current directory and recompile - # with specific options for automated performance tests. - # This way, performance test compilation does not mess with user's - # compilation - shutil.copyfile("../../GNUmakefile","./GNUmakefile") - make_realclean_command = " make realclean WARPX_HOME=../.. " \ - "AMREX_HOME=../../../amrex/ PICSAR_HOME=../../../picsar/ " \ - "EBASE=perf_tests COMP=%s" %compiler_name[compiler] + ";" - make_command = "make -j 16 WARPX_HOME=../.. " \ - "AMREX_HOME=../../../amrex/ PICSAR_HOME=../../../picsar/ " \ - "EBASE=perf_tests COMP=%s" %compiler_name[compiler] - if machine == 'summit': - make_command += ' USE_GPU=TRUE ' - os.system(config_command + make_realclean_command + \ - "rm -r tmp_build_dir *.mod; " + make_command ) - - # Store git hashes for WarpX, AMReX and PICSAR into file, so that - # they can be read when running the analysis. - if os.path.exists( cwd + 'store_git_hashes.txt' ): - os.remove( cwd + 'store_git_hashes.txt' ) - store_git_hash(repo_path=picsar_dir, filename=cwd + 'store_git_hashes.txt', name='picsar') - store_git_hash(repo_path=amrex_dir , filename=cwd + 'store_git_hashes.txt', name='amrex' ) - store_git_hash(repo_path=warpx_dir , filename=cwd + 'store_git_hashes.txt', name='warpx' ) - -# Loop over the tests and run all simulations: -# One batch job submitted per n_node. Several -# tests run within the same batch job. -# -------------------------------------------- -if args.mode == 'run': - if os.path.exists( 'log_jobids_tmp.txt' ): - os.remove( 'log_jobids_tmp.txt' ) - # loop on n_node. One batch script per n_node - for n_node in n_node_list: - res_dir = res_dir_base - res_dir += '_'.join([run_name, compiler, architecture, str(n_node)]) + '/' - runtime_param_list = [] - # Deep copy as we change the attribute n_cell of - # each instance of class test_element - test_list_n_node = copy.deepcopy(test_list) - job_time_min = time_min(len(test_list)) - batch_string = get_batch_string(test_list_n_node, job_time_min, module_Cname[architecture], n_node) - # Loop on tests - for count, current_run in enumerate(test_list_n_node): - current_run.scale_n_cell(n_node) - runtime_param_string = ' amr.n_cell=' + ' '.join(str(i) for i in current_run.n_cell) - runtime_param_string += ' amr.max_grid_size=' + str(current_run.max_grid_size) - runtime_param_string += ' amr.blocking_factor=' + str(current_run.blocking_factor) - runtime_param_string += ' max_step=' + str( current_run.n_step ) - # runtime_param_list.append( runtime_param_string ) - run_string = get_run_string(current_run, architecture, n_node, count, bin_name, runtime_param_string) - batch_string += run_string - batch_string += 'rm -rf plotfiles lab_frame_data diags\n' - - submit_job_command = get_submit_job_command() - # Run the simulations. - run_batch_nnode(test_list_n_node, res_dir, cwd, bin_name, config_command, batch_string, submit_job_command) - os.chdir(cwd) - # submit batch for analysis - if os.path.exists( 'read_error.txt' ): - os.remove( 'read_error.txt' ) - if os.path.exists( 'read_output.txt' ): - os.remove( 'read_output.txt' ) - process_analysis(args.automated, cwd, compiler, architecture, - args.n_node_list, start_date, source_dir_base, res_dir_base) - -# read the output file from each test and store timers in -# hdf5 file with pandas format -# ------------------------------------------------------- -for n_node in n_node_list: - print(n_node) - if browse_output_files: - res_dir = res_dir_base - res_dir += '_'.join([run_name, compiler,\ - architecture, str(n_node)]) + '/' - for count, current_run in enumerate(test_list): - # Read performance data from the output file - output_filename = 'out_' + '_'.join([current_run.input_file, str(n_node), str(current_run.n_mpi_per_node), str(current_run.n_omp), str(count)]) + '.txt' - # Read data for all test to put in hdf5 a database - # This is an hdf5 file containing ALL the simulation - # parameters and results. Might be too large for a repo - df_newline = extract_dataframe(res_dir + output_filename, current_run.n_step) - # Add all simulation parameters to the dataframe - df_newline['git_hashes'] = get_file_content(filename=cwd+'store_git_hashes.txt') - df_newline['start_date'] = start_date - df_newline['run_name'] = run_name - df_newline['input_file'] = current_run.input_file - df_newline['n_node'] = n_node - df_newline['n_mpi_per_node'] = current_run.n_mpi_per_node - df_newline['n_omp'] = current_run.n_omp - df_newline['n_steps'] = current_run.n_step - df_newline['rep'] = count%n_repeat - df_newline['date'] = datetime.datetime.now() - if store_full_input: - df_newline['inputs_content'] = get_file_content( filename=cwd+current_run.input_file ) - # Load file perf_database_file if exists, and - # append with results from this scan - if os.path.exists(path_hdf5 + perf_database_file): - df_base = pd.read_hdf(path_hdf5 + perf_database_file, 'all_data') - updated_df = df_base.append(df_newline, ignore_index=True) - else: - updated_df = df_newline - # Write dataframe to file perf_database_file - # (overwrite if file exists) - updated_df.to_hdf(path_hdf5 + perf_database_file, key='all_data', mode='w', format='table') - -# Extract sub-set of pandas data frame, write it to -# csv file and copy this file to perf_logs repo -# ------------------------------------------------- -if args.mode=='read' and update_perf_log_repo: - # get perf_logs repo - git_repo = git.Repo( perf_logs_repo ) - if push_on_perf_log_repo: - git_repo.git.stash('save') - git_repo.git.pull() - os.chdir( perf_logs_repo ) - sys.path.append('./') - import write_csv - git_repo.git.add('./logs_csv/' + csv_file[machine]) - git_repo.git.add('./logs_hdf5/' + perf_database_file) - index = git_repo.index - index.commit("automated tests") - -# Rename all result directories for archiving purposes: -# include date in the name, and a counter to avoid over-writing -for n_node in n_node_list: - if browse_output_files: - res_dir = res_dir_base - res_dir += '_'.join([run_name, compiler,\ - architecture, str(n_node)]) + '/' - # Rename directory with precise date+hour for archive purpose - if rename_archive == True: - loc_counter = 0 - res_dir_arch = res_dir_base - res_dir_arch += '_'.join([year, month, day, run_name, compiler,\ - architecture, str(n_node), str(loc_counter)]) + '/' - while os.path.exists( res_dir_arch ): - loc_counter += 1 - res_dir_arch = res_dir_base - res_dir_arch += '_'.join([year, month, day, run_name, compiler,\ - architecture, str(n_node), str(loc_counter)]) + '/' - print("renaming " + res_dir + " -> " + res_dir_arch) - os.rename( res_dir, res_dir_arch ) diff --git a/Tools/PerformanceTests/summit.py b/Tools/PerformanceTests/summit.py deleted file mode 100644 index c2ba6c70a2e..00000000000 --- a/Tools/PerformanceTests/summit.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2019 Axel Huebl, Luca Fedeli, Maxence Thevenet -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# requirements: -# - module load python/3.7.0-anaconda3-5.3.0 - -import copy -import os - -from functions_perftest import test_element - - -def executable_name(compiler,architecture): - return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.ex' - -def get_config_command(compiler, architecture): - config_command = '' - config_command += 'module load gcc;' - config_command += 'module load cuda;' - return config_command - -# This function runs a batch script with -# dependencies to perform the analysis -# after all performance tests are done. -def process_analysis(automated, cwd, compiler, architecture, n_node_list, start_date, path_source, path_results): - - batch_string = '''#!/bin/bash -#BSUB -P APH114 -#BSUB -W 00:10 -#BSUB -nnodes 1 -#BSUB -J perf_test -#BSUB -o read_output.txt -#BSUB -e read_error.txt -''' - f_log = open(cwd + 'log_jobids_tmp.txt' ,'r') - for line in f_log.readlines(): - dependency = line.split()[1][1:-1] - batch_string += '#BSUB -w ended(' + dependency + ')\n' - - batch_string += 'python run_automated.py --compiler=' + \ - compiler + ' --architecture=' + architecture + \ - ' --mode=read' + \ - ' --n_node_list=' + '"' + n_node_list + '"' + \ - ' --start_date=' + start_date + \ - ' --path_source=' + path_source + \ - ' --path_results=' + path_results - if automated == True: - batch_string += ' --automated' - batch_string += '\n' - batch_file = 'bsub_perfread' - f_exe = open(batch_file,'w') - f_exe.write(batch_string) - f_exe.close() - os.system('chmod 700 ' + batch_file) - print( 'process_analysis line: ' + 'bsub ' + batch_file) - os.system('bsub ' + batch_file) - -# Calculate simulation time. Take 2 min + 2 min / simulation -def time_min(nb_simulations): - return 2. + nb_simulations*2. - -def get_submit_job_command(): - return ' bsub ' - -def get_batch_string(test_list, job_time_min, Cname, n_node): - - job_time_str = str(int(job_time_min/60)) + ':' + str(int(job_time_min%60)) - - batch_string = '' - batch_string += '#!/bin/bash\n' - batch_string += '#BSUB -P APH114\n' - batch_string += '#BSUB -W ' + job_time_str + '\n' - batch_string += '#BSUB -nnodes ' + str(n_node) + '\n' - batch_string += '#BSUB -J ' + test_list[0].input_file + '\n' - batch_string += '#BSUB -e error.txt\n' - batch_string += 'module load gcc\n' - batch_string += 'module load cuda\n' - return batch_string - -def get_run_string(current_test, architecture, n_node, count, bin_name, runtime_param_string): - - output_filename = 'out_' + '_'.join([current_test.input_file, str(n_node), str(current_test.n_mpi_per_node), str(current_test.n_omp), str(count)]) + '.txt' - - ngpu = str(current_test.n_mpi_per_node) - srun_string = '' - srun_string += 'jsrun ' - srun_string += ' -n ' + str(n_node) - srun_string += ' -a ' + ngpu + ' -g ' + ngpu + ' -c ' + ngpu + ' --bind=packed:1 ' - srun_string += ' ./' + bin_name + ' ' - srun_string += current_test.input_file + ' ' - srun_string += runtime_param_string - srun_string += ' > ' + output_filename + '\n' - return srun_string - -def get_test_list(n_repeat): - test_list_unq = [] - # n_node is kept to None and passed in functions as an external argument - # That way, several test_element_instance run with the same n_node on the same batch job - test_list_unq.append( test_element(input_file='automated_test_1_uniform_rest_32ppc', - n_mpi_per_node=6, - n_omp=1, - n_cell=[128, 128, 192], - max_grid_size=256, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_2_uniform_rest_1ppc', - n_mpi_per_node=6, - n_omp=1, - n_cell=[256, 512, 768], - max_grid_size=512, - blocking_factor=256, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_3_uniform_drift_4ppc', - n_mpi_per_node=6, - n_omp=1, - n_cell=[128, 128, 384], - max_grid_size=256, - blocking_factor=64, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_4_labdiags_2ppc', - n_mpi_per_node=6, - n_omp=1, - n_cell=[384, 256, 512], - max_grid_size=256, - blocking_factor=128, - n_step=50) ) - test_list_unq.append( test_element(input_file='automated_test_5_loadimbalance', - n_mpi_per_node=6, - n_omp=1, - n_cell=[64, 64, 192], - max_grid_size=64, - blocking_factor=32, - n_step=10) ) - test_list_unq.append( test_element(input_file='automated_test_6_output_2ppc', - n_mpi_per_node=6, - n_omp=1, - n_cell=[384, 256, 512], - max_grid_size=256, - blocking_factor=64, - n_step=1) ) - test_list = [copy.deepcopy(item) for item in test_list_unq for _ in range(n_repeat) ] - return test_list diff --git a/Tools/PostProcessing/Visualization.ipynb b/Tools/PostProcessing/Visualization.ipynb index ef05b69c2c0..848beb57256 100644 --- a/Tools/PostProcessing/Visualization.ipynb +++ b/Tools/PostProcessing/Visualization.ipynb @@ -16,10 +16,11 @@ "outputs": [], "source": [ "# Import statements\n", - "import yt ; yt.funcs.mylog.setLevel(50)\n", - "import numpy as np\n", - "import scipy.constants as scc\n", + "import yt\n", + "\n", + "yt.funcs.mylog.setLevel(50)\n", "import matplotlib.pyplot as plt\n", + "\n", "%matplotlib notebook" ] }, @@ -45,12 +46,12 @@ "metadata": {}, "outputs": [], "source": [ - "diag_name = 'diag' # E.g., diagnostics.diags_names = diag\n", + "diag_name = \"diag\" # E.g., diagnostics.diags_names = diag\n", "iteration = 0\n", - "plotfile = './diags/{}{:05d}'.format(diag_name, iteration)\n", - "field = 'Ex'\n", - "species = 'electron'\n", - "ds = yt.load( plotfile ) # Load the plotfile\n", + "plotfile = \"./diags/{}{:05d}\".format(diag_name, iteration)\n", + "field = \"Ex\"\n", + "species = \"electron\"\n", + "ds = yt.load(plotfile) # Load the plotfile\n", "# ds.field_list # Print all available quantities" ] }, @@ -67,10 +68,10 @@ "metadata": {}, "outputs": [], "source": [ - "sl = yt.SlicePlot(ds, 2, field, aspect=.2) # Create a sliceplot object\n", - "sl.annotate_particles(width=(10.e-6, 'm'), p_size=2, ptype=species, col='black')\n", - "sl.annotate_grids() # Show grids\n", - "sl.show() # Show the plot" + "sl = yt.SlicePlot(ds, 2, field, aspect=0.2) # Create a sliceplot object\n", + "sl.annotate_particles(width=(10.0e-6, \"m\"), p_size=2, ptype=species, col=\"black\")\n", + "sl.annotate_grids() # Show grids\n", + "sl.show() # Show the plot" ] }, { @@ -87,21 +88,27 @@ "outputs": [], "source": [ "# Get field quantities\n", - "all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions)\n", - "Bx = all_data_level_0['boxlib', field].v.squeeze()\n", - "Dx = ds.domain_width/ds.domain_dimensions\n", - "extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1],\n", - " ds.domain_left_edge[0], ds.domain_right_edge[0] ]\n", + "all_data_level_0 = ds.covering_grid(\n", + " level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions\n", + ")\n", + "Bx = all_data_level_0[\"boxlib\", field].v.squeeze()\n", + "Dx = ds.domain_width / ds.domain_dimensions\n", + "extent = [\n", + " ds.domain_left_edge[ds.dimensionality - 1],\n", + " ds.domain_right_edge[ds.dimensionality - 1],\n", + " ds.domain_left_edge[0],\n", + " ds.domain_right_edge[0],\n", + "]\n", "\n", "# Get particle quantities\n", "ad = ds.all_data()\n", - "x = ad[species, 'particle_position_x'].v\n", - "z = ad[species, 'particle_position_z'].v\n", + "x = ad[species, \"particle_position_x\"].v\n", + "z = ad[species, \"particle_position_z\"].v\n", "\n", "# Plot image\n", "plt.figure()\n", - "plt.imshow(Bx[:,Bx.shape[1]//2,:], extent=extent, aspect='auto')\n", - "plt.scatter(z,x,s=.1,c='k')" + "plt.imshow(Bx[:, Bx.shape[1] // 2, :], extent=extent, aspect=\"auto\")\n", + "plt.scatter(z, x, s=0.1, c=\"k\")" ] } ], diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 4b0cdfd532b..07a353cdc3d 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -11,6 +11,7 @@ class SimData: """ Structure for easy access to load costs reduced diagnostics """ + def __init__(self, directory, prange, is_3D): """ Set data-containing dir, data range; load data @@ -26,7 +27,7 @@ def __call__(self, i): print("No data_fields!") return - if not i in self.keys: + if i not in self.keys: print("Index is out of range!") print("Valid keys are ", self.keys) return @@ -40,7 +41,6 @@ def __call__(self, i): # Data_fields index currently set self.idx = i - def _get_costs_reduced_diagnostics(self, directory, prange): """ Read costs reduced diagnostics @@ -58,20 +58,21 @@ def _get_costs_reduced_diagnostics(self, directory, prange): if len(data.shape) == 1: data = data.reshape(-1, data.shape[0]) - steps = data[:,0].astype(int) + steps = data[:, 0].astype(int) - times = data[:,1] - data = data[:,2:] + times = data[:, 1] + data = data[:, 2:] # Compute the number of datafields saved per box n_data_fields = 0 with open(directory) as f: h = f.readlines()[0] - unique_headers=[''.join([l for l in w if not l.isdigit()]) - for w in h.split()][2::] + unique_headers = [ + "".join([ln for ln in w if not ln.isdigit()]) for w in h.split() + ][2::] # Either 9 or 10 depending if GPU - n_data_fields = 9 if len(set(unique_headers))%9 == 0 else 10 + n_data_fields = 9 if len(set(unique_headers)) % 9 == 0 else 10 f.close() # From data header, data layout is: @@ -86,9 +87,11 @@ def _get_costs_reduced_diagnostics(self, directory, prange): # cost_box_n, proc_box_n, lev_box_n, i_low_box_n, j_low_box_n, # k_low_box_n, num_cells_n, num_macro_particles_n, # (, gpu_ID_box_n if GPU run), hostname_box_n - i, j, k = (data[0,3::n_data_fields], - data[0,4::n_data_fields], - data[0,5::n_data_fields]) + i, j, k = ( + data[0, 3::n_data_fields], + data[0, 4::n_data_fields], + data[0, 5::n_data_fields], + ) i_blocks = np.diff(np.array(sorted(i.astype(int)))) j_blocks = np.diff(np.array(sorted(j.astype(int)))) @@ -103,21 +106,23 @@ def _get_costs_reduced_diagnostics(self, directory, prange): j_blocking_factor = 1 if len(j_non_zero) == 0 else j_non_zero.min() k_blocking_factor = 1 if len(k_non_zero) == 0 else k_non_zero.min() - imax = i.astype(int).max()//i_blocking_factor - jmax = j.astype(int).max()//j_blocking_factor - kmax = k.astype(int).max()//k_blocking_factor + imax = i.astype(int).max() // i_blocking_factor + jmax = j.astype(int).max() // j_blocking_factor + kmax = k.astype(int).max() // k_blocking_factor for key in self.keys: row = np.where(key == steps)[0][0] costs = data[row, 0::n_data_fields].astype(float) ranks = data[row, 1::n_data_fields].astype(int) - icoords = i.astype(int)//i_blocking_factor - jcoords = j.astype(int)//j_blocking_factor - kcoords = k.astype(int)//k_blocking_factor + icoords = i.astype(int) // i_blocking_factor + jcoords = j.astype(int) // j_blocking_factor + kcoords = k.astype(int) // k_blocking_factor # Fill in cost array - shape = (kmax+1, jmax+1, imax+1)[:2+self.is_3D] - coords = [coord[:2+self.is_3D] for coord in zip(kcoords, jcoords, icoords)] + shape = (kmax + 1, jmax + 1, imax + 1)[1 - self.is_3D :] + coords = [ + coord[1 - self.is_3D :] for coord in zip(kcoords, jcoords, icoords) + ] cost_arr = np.full(shape, 0.0) rank_arr = np.full(shape, -1) @@ -127,43 +132,56 @@ def _get_costs_reduced_diagnostics(self, directory, prange): rank_arr[coord] = ranks[nc] # For non-uniform blocks: fill with the corresponding cost/rank - visited = np.full(shape, False) + visited = np.full(shape, False) + def dfs(corner, pos, prev): # Exit conditions - if any([pos[i]>=shape[i] for i in range(len(shape))]): return - edges = list(rank_arr[corner[0]:pos[0]+1, pos[1], pos[2]]) \ - + list(rank_arr[pos[0], corner[1]:pos[1]+1, pos[2]]) \ - + list(rank_arr[pos[0], pos[1], corner[2]:pos[2]+1]) \ - if self.is_3D else \ - list(rank_arr[corner[0]:pos[0]+1, pos[1]]) \ - + list(rank_arr[pos[0], corner[1]:pos[1]+1]) - if visited[pos] or not set(edges).issubset(set([prev, -1])): return + if any([pos[i] >= shape[i] for i in range(len(shape))]): + return + edges = ( + list(rank_arr[corner[0] : pos[0] + 1, pos[1], pos[2]]) + + list(rank_arr[pos[0], corner[1] : pos[1] + 1, pos[2]]) + + list(rank_arr[pos[0], pos[1], corner[2] : pos[2] + 1]) + if self.is_3D + else list(rank_arr[corner[0] : pos[0] + 1, pos[1]]) + + list(rank_arr[pos[0], corner[1] : pos[1] + 1]) + ) + if visited[pos] or not set(edges).issubset(set([prev, -1])): + return visited[pos] = True - if rank_arr[pos] not in [-1, prev]: prev, corner = rank_arr[pos], pos - else: rank_arr[pos] = prev + if rank_arr[pos] not in [-1, prev]: + prev, corner = rank_arr[pos], pos + else: + rank_arr[pos] = prev args = [[0, 1] for _ in range(len(shape))] - neighbors = [tuple(np.array(pos) + np.array(p)) for p in product(*args) - if not p == (0,)*len(shape)] - for n in neighbors: dfs(corner, n, prev) + neighbors = [ + tuple(np.array(pos) + np.array(p)) + for p in product(*args) + if not p == (0,) * len(shape) + ] + for n in neighbors: + dfs(corner, n, prev) - for corner in coords: dfs(corner, corner, rank_arr[corner]) + for corner in coords: + dfs(corner, corner, rank_arr[corner]) - self.data_fields[key]['cost_arr'] = cost_arr - self.data_fields[key]['rank_arr'] = rank_arr + self.data_fields[key]["cost_arr"] = cost_arr + self.data_fields[key]["rank_arr"] = rank_arr # Compute load balance efficiency - rank_to_cost_map = {r:0. for r in set(ranks)} - for c, r in zip(costs, ranks): rank_to_cost_map[r] += c + rank_to_cost_map = {r: 0.0 for r in set(ranks)} + for c, r in zip(costs, ranks): + rank_to_cost_map[r] += c efficiencies = np.array(list(rank_to_cost_map.values())) efficiencies /= efficiencies.max() - self.data_fields[key]['ranks'] = np.array(list(rank_to_cost_map.keys())) - self.data_fields[key]['lb_efficiencies'] = efficiencies - self.data_fields[key]['lb_efficiency'] = efficiencies.mean() - self.data_fields[key]['lb_efficiency_max'] = efficiencies.max() - self.data_fields[key]['lb_efficiency_min'] = efficiencies.min() - self.data_fields[key]['t'] = times[row] - self.data_fields[key]['step'] = steps[row] + self.data_fields[key]["ranks"] = np.array(list(rank_to_cost_map.keys())) + self.data_fields[key]["lb_efficiencies"] = efficiencies + self.data_fields[key]["lb_efficiency"] = efficiencies.mean() + self.data_fields[key]["lb_efficiency_max"] = efficiencies.max() + self.data_fields[key]["lb_efficiency_min"] = efficiencies.min() + self.data_fields[key]["t"] = times[row] + self.data_fields[key]["step"] = steps[row] # ... diff --git a/Tools/PostProcessing/plot_nci_growth_rate.ipynb b/Tools/PostProcessing/plot_nci_growth_rate.ipynb index 559d5250237..5413d909229 100644 --- a/Tools/PostProcessing/plot_nci_growth_rate.ipynb +++ b/Tools/PostProcessing/plot_nci_growth_rate.ipynb @@ -21,11 +21,14 @@ "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", - "from scipy.constants import c\n", "import numpy as np\n", "import scipy.constants as scc\n", - "import yt ; yt.funcs.mylog.setLevel(50)\n", + "import yt\n", + "from scipy.constants import c\n", + "\n", + "yt.funcs.mylog.setLevel(50)\n", "import glob\n", + "\n", "%matplotlib inline" ] }, @@ -44,7 +47,7 @@ }, "outputs": [], "source": [ - "path_wx = 'path to diags folder'" + "path_wx = \"path to diags folder\"" ] }, { @@ -55,8 +58,10 @@ }, "outputs": [], "source": [ - "file_list_warpx = glob.glob(path_wx + 'diag1?????')\n", - "iterations_warpx = [ int(file_name[len(file_name)-5:]) for file_name in file_list_warpx ]" + "file_list_warpx = glob.glob(path_wx + \"diag1?????\")\n", + "iterations_warpx = [\n", + " int(file_name[len(file_name) - 5 :]) for file_name in file_list_warpx\n", + "]" ] }, { @@ -68,25 +73,24 @@ "outputs": [], "source": [ "def calculate_parameters(path):\n", - " iteration=200\n", - " dsx = yt.load( path + 'diag1%05d/' %iteration )\n", - " dxx = dsx.domain_width/dsx.domain_dimensions\n", - " dx=dxx[0];\n", - " dx = 1.*dx.ndarray_view()\n", - "\n", - " dz=dxx[1];\n", - " \n", - " dz = 1.*dz.ndarray_view()\n", - " cell_volume_x = np.prod(dxx)\n", - "\n", - " ds1 = yt.load(path+'/diag100100/')\n", - " ds2 = yt.load(path+'/diag100200/')\n", - " \n", + " iteration = 200\n", + " dsx = yt.load(path + \"diag1%05d/\" % iteration)\n", + " dxx = dsx.domain_width / dsx.domain_dimensions\n", + " dx = dxx[0]\n", + " dx = 1.0 * dx.ndarray_view()\n", + "\n", + " dz = dxx[1]\n", + "\n", + " dz = 1.0 * dz.ndarray_view()\n", + "\n", + " ds1 = yt.load(path + \"/diag100100/\")\n", + " ds2 = yt.load(path + \"/diag100200/\")\n", + "\n", " cur_t1 = ds1.current_time\n", - " cur_t2 = ds2.current_time \n", + " cur_t2 = ds2.current_time\n", " cur_t2.to_ndarray\n", - " dt = (cur_t2-cur_t1)/100\n", - " dt = 1.*dt.ndarray_view();\n", + " dt = (cur_t2 - cur_t1) / 100\n", + " dt = 1.0 * dt.ndarray_view()\n", " return dx, dz, dt" ] }, @@ -97,7 +101,7 @@ "outputs": [], "source": [ "dx, dz, dt = calculate_parameters(path_wx)\n", - "print(dx,dz,dt)" + "print(dx, dz, dt)" ] }, { @@ -108,35 +112,32 @@ }, "outputs": [], "source": [ - "def get_fourier_transform_wx( path, fieldcomp, \n", - " iteration, plot=False, remove_last=True ):\n", + "def get_fourier_transform_wx(path, fieldcomp, iteration, plot=False, remove_last=True):\n", " \"\"\"\n", " Calculate the Fourier transform of the field at a given iteration\n", " \"\"\"\n", - " \n", - " ds = yt.load(path + '/diag1%05d/' %iteration )\n", + "\n", + " ds = yt.load(path + \"/diag1%05d/\" % iteration)\n", "\n", " grid = ds.index.grids[0]\n", " F = grid[fieldcomp]\n", " F = F.ndarray_view()\n", "\n", - " \n", " if remove_last:\n", - " F = F[:-1,:-1]\n", - " F = F[:,:,0]\n", + " F = F[:-1, :-1]\n", + " F = F[:, :, 0]\n", "\n", - " kxmax = np.pi/dx\n", - " kzmax = np.pi/dz\n", + " kxmax = np.pi / dx\n", + " kzmax = np.pi / dz\n", " Nx = F.shape[0]\n", " Nz = F.shape[1]\n", - " spectralF = np.fft.fftshift( np.fft.fft2(F) )[int(Nx/2):, int(Nz/2):]\n", + " spectralF = np.fft.fftshift(np.fft.fft2(F))[int(Nx / 2) :, int(Nz / 2) :]\n", "\n", " if plot:\n", - " plt.imshow( np.log(abs(spectralF)), origin='lower',\n", - " extent=[0, kxmax, 0, kzmax] )\n", + " plt.imshow(np.log(abs(spectralF)), origin=\"lower\", extent=[0, kxmax, 0, kzmax])\n", " plt.colorbar()\n", - " \n", - " return( spectralF, kxmax, kzmax )" + "\n", + " return (spectralF, kxmax, kzmax)" ] }, { @@ -147,28 +148,30 @@ }, "outputs": [], "source": [ - "def growth_rate_between_wx( path, iteration1, iteration2, \n", - " remove_last=False, threshold=-13 ):\n", + "def growth_rate_between_wx(\n", + " path, iteration1, iteration2, remove_last=False, threshold=-13\n", + "):\n", " \"\"\"\n", - " Calculate the difference in spectral amplitude between two iterations, \n", + " Calculate the difference in spectral amplitude between two iterations,\n", " return the growth rate\n", - " \n", + "\n", " \"\"\"\n", - " spec1, kxmax, kzmax = \\\n", - " get_fourier_transform_wx( path, 'Ez', iteration=iteration1, remove_last=remove_last )\n", - " spec1 = np.where( abs(spec1) > np.exp(threshold), spec1, np.exp(threshold) )\n", - " \n", + " spec1, kxmax, kzmax = get_fourier_transform_wx(\n", + " path, \"Ez\", iteration=iteration1, remove_last=remove_last\n", + " )\n", + " spec1 = np.where(abs(spec1) > np.exp(threshold), spec1, np.exp(threshold))\n", "\n", - " spec2, kxmax, kzmax = \\\n", - " get_fourier_transform_wx( path, 'Ez', iteration=iteration2, remove_last=remove_last )\n", - " \n", - " spec2 = np.where( abs(spec2) > np.exp(threshold), spec2, np.exp(threshold) )\n", - " diff_growth = np.log( abs(spec2) ) - np.log( abs(spec1) )\n", + " spec2, kxmax, kzmax = get_fourier_transform_wx(\n", + " path, \"Ez\", iteration=iteration2, remove_last=remove_last\n", + " )\n", "\n", - " diff_time = (iteration2-iteration1)*dt;\n", - " growth_rate = diff_growth/diff_time/c;\n", + " spec2 = np.where(abs(spec2) > np.exp(threshold), spec2, np.exp(threshold))\n", + " diff_growth = np.log(abs(spec2)) - np.log(abs(spec1))\n", "\n", - " return( growth_rate, [0, kxmax, 0, kzmax] )" + " diff_time = (iteration2 - iteration1) * dt\n", + " growth_rate = diff_growth / diff_time / c\n", + "\n", + " return (growth_rate, [0, kxmax, 0, kzmax])" ] }, { @@ -179,18 +182,18 @@ }, "outputs": [], "source": [ - "def energy( ts ):\n", - " Ex= ts.index.grids[0]['boxlib', 'Ex'].squeeze().v\n", - " Ey= ts.index.grids[0]['boxlib', 'Ey'].squeeze().v\n", - " Ez= ts.index.grids[0]['boxlib', 'Ez'].squeeze().v\n", - " \n", - " Bx= ts.index.grids[0]['boxlib', 'Ex'].squeeze().v\n", - " By= ts.index.grids[0]['boxlib', 'Ey'].squeeze().v\n", - " Bz= ts.index.grids[0]['boxlib', 'Ez'].squeeze().v\n", - "\n", - " energyE = scc.epsilon_0*np.sum(Ex**2+Ey**2+Ez**2)\n", - " energyB= np.sum(Bx**2+By**2+Bz**2)/scc.mu_0 \n", - " energy = energyE + energyB\n", + "def energy(ts):\n", + " Ex = ts.index.grids[0][\"boxlib\", \"Ex\"].squeeze().v\n", + " Ey = ts.index.grids[0][\"boxlib\", \"Ey\"].squeeze().v\n", + " Ez = ts.index.grids[0][\"boxlib\", \"Ez\"].squeeze().v\n", + "\n", + " Bx = ts.index.grids[0][\"boxlib\", \"Ex\"].squeeze().v\n", + " By = ts.index.grids[0][\"boxlib\", \"Ey\"].squeeze().v\n", + " Bz = ts.index.grids[0][\"boxlib\", \"Ez\"].squeeze().v\n", + "\n", + " energyE = scc.epsilon_0 * np.sum(Ex**2 + Ey**2 + Ez**2)\n", + " energyB = np.sum(Bx**2 + By**2 + Bz**2) / scc.mu_0\n", + " energy = energyE + energyB\n", " return energy" ] }, @@ -202,9 +205,9 @@ "source": [ "energy_list = []\n", "for iter in iterations_warpx:\n", - " path = path_wx + '/diag1%05d/' %iter\n", - " ds = yt.load( path) \n", - " energy_list.append( energy(ds) )" + " path = path_wx + \"/diag1%05d/\" % iter\n", + " ds = yt.load(path)\n", + " energy_list.append(energy(ds))" ] }, { @@ -214,23 +217,23 @@ "outputs": [], "source": [ "iteration_start = 1700\n", - "iteration_end= 1900\n", + "iteration_end = 1900\n", "iter_delta = iterations_warpx[2] - iterations_warpx[1]\n", "\n", "\n", - "ds_start = yt.load(path_wx + 'diag1%05d/' %iteration_start)\n", - "Ez_start= ds.index.grids[0]['boxlib', 'Ez'].squeeze().v\n", + "ds_start = yt.load(path_wx + \"diag1%05d/\" % iteration_start)\n", + "Ez_start = ds.index.grids[0][\"boxlib\", \"Ez\"].squeeze().v\n", "\n", - "ds_end = yt.load(path_wx + 'diag1%05d/' %iteration_end)\n", - "Ez_end= ds_end.index.grids[0]['boxlib', 'Ez'].squeeze().v\n", + "ds_end = yt.load(path_wx + \"diag1%05d/\" % iteration_end)\n", + "Ez_end = ds_end.index.grids[0][\"boxlib\", \"Ez\"].squeeze().v\n", "\n", - "gr_wx, extent = growth_rate_between_wx(path_wx, iteration_start, iteration_end )\n", + "gr_wx, extent = growth_rate_between_wx(path_wx, iteration_start, iteration_end)\n", "\n", "\n", "fs = 13\n", "vmax = 0.05\n", - "vmin=-vmax\n", - "cmap_special = 'bwr'" + "vmin = -vmax\n", + "cmap_special = \"bwr\"" ] }, { @@ -239,23 +242,23 @@ "metadata": {}, "outputs": [], "source": [ - "fig, (ax1, ax2) = plt.subplots( ncols=2, figsize=(10,4.) )\n", + "fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(10, 4.0))\n", "\n", - "fs=14\n", + "fs = 14\n", "cmap = \"viridis\"\n", - "aspect='auto'\n", - "\n", - "img1 = ax1.imshow(Ez_start,aspect=aspect, cmap=cmap, extent=extent)\n", - "ax1.set_title('$t_{step}=$%i' % iteration_start, size=fs)\n", - "ax1.set_xlabel(' $k_{p,r} z$ ',size=fs)\n", - "ax1.set_ylabel(' $k_{p,r} x $ ',size=fs)\n", - "fig.colorbar(img1, ax=ax1, label = '$E_z, [V/m]$')\n", - "\n", - "img2 = ax2.imshow(Ez_end,aspect=aspect, cmap=cmap, extent=extent)\n", - "ax2.set_title('$t_{step}=$%i' % iteration_end,size=fs)\n", - "ax2.set_xlabel(' $k_{p,r} z$ ',size=fs)\n", - "ax2.set_ylabel(' $k_{p,r} x $ ',size=fs)\n", - "fig.colorbar(img2, ax=ax2, label = '$E_z, [V/m]$')\n", + "aspect = \"auto\"\n", + "\n", + "img1 = ax1.imshow(Ez_start, aspect=aspect, cmap=cmap, extent=extent)\n", + "ax1.set_title(\"$t_{step}=$%i\" % iteration_start, size=fs)\n", + "ax1.set_xlabel(\" $k_{p,r} z$ \", size=fs)\n", + "ax1.set_ylabel(\" $k_{p,r} x $ \", size=fs)\n", + "fig.colorbar(img1, ax=ax1, label=\"$E_z, [V/m]$\")\n", + "\n", + "img2 = ax2.imshow(Ez_end, aspect=aspect, cmap=cmap, extent=extent)\n", + "ax2.set_title(\"$t_{step}=$%i\" % iteration_end, size=fs)\n", + "ax2.set_xlabel(\" $k_{p,r} z$ \", size=fs)\n", + "ax2.set_ylabel(\" $k_{p,r} x $ \", size=fs)\n", + "fig.colorbar(img2, ax=ax2, label=\"$E_z, [V/m]$\")\n", "plt.tight_layout()" ] }, @@ -265,23 +268,31 @@ "metadata": {}, "outputs": [], "source": [ - "fig, (ax1, ax2) = plt.subplots( ncols=2,nrows=1, figsize=(13,5.) )\n", + "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, figsize=(13, 5.0))\n", "\n", - "fs=14\n", + "fs = 14\n", "\n", - "img1 = ax1.semilogy(iterations_warpx,energy_list)\n", - "ax1.semilogy(iteration_start,energy_list[iteration_start/iter_delta],'ro')\n", - "ax1.semilogy(iteration_end,energy_list[iteration_end/iter_delta],'ro')\n", + "img1 = ax1.semilogy(iterations_warpx, energy_list)\n", + "ax1.semilogy(iteration_start, energy_list[iteration_start / iter_delta], \"ro\")\n", + "ax1.semilogy(iteration_end, energy_list[iteration_end / iter_delta], \"ro\")\n", "ax1.grid()\n", "ax1.legend()\n", - "ax1.set_xlabel('time step',size=fs)\n", - "ax1.set_ylabel('Total EM energy',size=fs)\n", - "\n", - "img2 = ax2.imshow( gr_wx, origin='lower' ,cmap='bwr', vmax=0.05, vmin=-vmax, interpolation='nearest', extent=[0, 1, 0, 1] )\n", - "ax2.set_title('NCI growth rate',size=fs)\n", - "ax2.set_xlabel('$k_{p,r} z$ ',size=fs)\n", - "ax2.set_ylabel('$k_{p,r} x $ ',size=fs)\n", - "fig.colorbar(img2, ax=ax2, label = '$Im(\\omega)/\\omega_{p,r}$')" + "ax1.set_xlabel(\"time step\", size=fs)\n", + "ax1.set_ylabel(\"Total EM energy\", size=fs)\n", + "\n", + "img2 = ax2.imshow(\n", + " gr_wx,\n", + " origin=\"lower\",\n", + " cmap=\"bwr\",\n", + " vmax=0.05,\n", + " vmin=-vmax,\n", + " interpolation=\"nearest\",\n", + " extent=[0, 1, 0, 1],\n", + ")\n", + "ax2.set_title(\"NCI growth rate\", size=fs)\n", + "ax2.set_xlabel(\"$k_{p,r} z$ \", size=fs)\n", + "ax2.set_ylabel(\"$k_{p,r} x $ \", size=fs)\n", + "fig.colorbar(img2, ax=ax2, label=\"$Im(\\omega)/\\omega_{p,r}$\")" ] } ], diff --git a/Tools/PostProcessing/plot_parallel.py b/Tools/PostProcessing/plot_parallel.py index a4309b3896e..2bdac5d0177 100644 --- a/Tools/PostProcessing/plot_parallel.py +++ b/Tools/PostProcessing/plot_parallel.py @@ -16,7 +16,7 @@ import matplotlib.pyplot as plt import numpy as np -''' +""" This script loops over all WarpX plotfiles in a directory and, for each plotfile, saves an image showing the field and particles. @@ -41,33 +41,71 @@ To get help, run > python plot_parallel --help -''' +""" # Parse command line for options. parser = argparse.ArgumentParser() -parser.add_argument('--path', default=None, - help='path to plotfiles, defaults to diags/plotfiles. Plotfiles names must be plt?????') -parser.add_argument('--image_dir', default=None, - help='path where images are placed, defaults to diags/plotfiles or path if specified.') -parser.add_argument('--plotlib', default='yt', - choices=['yt','matplotlib'], - help='Plotting library to use') -parser.add_argument('--field', default='Ez', - help='Which field to plot, e.g., Ez, By, jx or rho. The central slice in y is plotted') -parser.add_argument('--pjump', default=20, - help='When plotlib=matplotlib, we plot every pjump particle') -parser.add_argument('--vmax', type=float, default=None, - help='If specified, the colormap will have bounds [-vmax, vmax]') -parser.add_argument('--slicewidth', default=10.e-6, - help='Only particles with -slicewidth/2 1: @@ -232,21 +310,23 @@ def reduce_evolved_quantity(z, q): else: return z, q + ### Analysis ### # Get list of plotfiles -file_list = glob.glob(os.path.join(path, 'plt?????')) +file_list = glob.glob(os.path.join(path, "plt?????")) file_list.sort() nfiles = len(file_list) # Get list of particle species to plot -pslist = get_species(file_list); +pslist = get_species(file_list) rank = 0 size = 1 if not args.serial: try: from mpi4py import MPI + comm_world = MPI.COMM_WORLD rank = comm_world.Get_rank() size = comm_world.Get_size() @@ -254,9 +334,9 @@ def reduce_evolved_quantity(z, q): pass if rank == 0: - print('number of MPI ranks: %d'%size) - print('Number of plotfiles: %s'%nfiles) - print('list of species: ', pslist) + print("number of MPI ranks: %d" % size) + print("Number of plotfiles: %s" % nfiles) + print("list of species: ", pslist) if plot_evolution is not None: # Fill with a value less than any possible value @@ -271,14 +351,16 @@ def reduce_evolved_quantity(z, q): # - plot field snapshot # - store window position and field max in arrays for count, filename in enumerate(file_list): - if count%size != rank: + if count % size != rank: continue - plot_snapshot( filename ) + plot_snapshot(filename) if plot_evolution is not None: - zwin[count], quantity[count] = get_evolution_quantity( filename, plot_evolution ) + zwin[count], quantity[count] = get_evolution_quantity(filename, plot_evolution) if plot_particle_evolution is not None: - zbar[count], xstd[count] = get_particle_evolution_quantity(filename, plot_particle_evolution) + zbar[count], xstd[count] = get_particle_evolution_quantity( + filename, plot_particle_evolution + ) if plot_evolution is not None: zwin, quantity = reduce_evolved_quantity(zwin, quantity) diff --git a/Tools/PostProcessing/plot_particle_path.py b/Tools/PostProcessing/plot_particle_path.py index 9bf7f896c10..af29dfc0e11 100644 --- a/Tools/PostProcessing/plot_particle_path.py +++ b/Tools/PostProcessing/plot_particle_path.py @@ -9,7 +9,7 @@ class AMReXParticleHeader(object): - ''' + """ This class is designed to parse and store the information contained in an AMReX particle header file. @@ -22,19 +22,18 @@ class AMReXParticleHeader(object): etc... - ''' + """ def __init__(self, header_filename): - self.real_component_names = [] self.int_component_names = [] with open(header_filename, "r") as f: self.version_string = f.readline().strip() - particle_real_type = self.version_string.split('_')[-1] - if particle_real_type == 'double': + particle_real_type = self.version_string.split("_")[-1] + if particle_real_type == "double": self.real_type = np.float64 - elif particle_real_type == 'single': + elif particle_real_type == "single": self.real_type = np.float32 else: raise RuntimeError("Did not recognize particle real type.") @@ -62,7 +61,7 @@ def __init__(self, header_filename): self.num_int_extra = 0 self.num_int = 0 - self.grids_per_level = np.zeros(self.num_levels, dtype='int64') + self.grids_per_level = np.zeros(self.num_levels, dtype="int64") self.grids = [] for level_num in range(self.num_levels): self.grids_per_level[level_num] = int(f.readline().strip()) @@ -75,7 +74,7 @@ def __init__(self, header_filename): def read_particle_data(fn, ptype="particle0"): - ''' + """ This function returns the particle data stored in a particular plot file and particle type. It returns two numpy arrays, the @@ -89,7 +88,7 @@ def read_particle_data(fn, ptype="particle0"): idata, rdata = read_particle_data("plt00000", "particle0") - ''' + """ base_fn = fn + "/" + ptype header = AMReXParticleHeader(base_fn + "/Header") @@ -99,22 +98,23 @@ def read_particle_data(fn, ptype="particle0"): elif header.real_type == np.float32: fdtype = "(%d,)f4" % header.num_real - idata = np.empty((header.num_particles, header.num_int )) + idata = np.empty((header.num_particles, header.num_int)) rdata = np.empty((header.num_particles, header.num_real)) ip = 0 for lvl, level_grids in enumerate(header.grids): - for (which, count, where) in level_grids: - if count == 0: continue + for which, count, where in level_grids: + if count == 0: + continue fn = base_fn + "/Level_%d/DATA_%04d" % (lvl, which) - with open(fn, 'rb') as f: + with open(fn, "rb") as f: f.seek(where) - ints = np.fromfile(f, dtype = idtype, count=count) - floats = np.fromfile(f, dtype = fdtype, count=count) + ints = np.fromfile(f, dtype=idtype, count=count) + floats = np.fromfile(f, dtype=fdtype, count=count) - idata[ip:ip+count] = ints - rdata[ip:ip+count] = floats + idata[ip : ip + count] = ints + rdata[ip : ip + count] = floats ip += count return idata, rdata @@ -143,10 +143,10 @@ def read_particle_data(fn, ptype="particle0"): fig = plt.gcf() fig.set_size_inches(8, 8) - plt.plot(x0, y0, 'r.') - plt.plot(x1, y1, 'b.') - plt.axis((-2., 2., -2., 2.)) + plt.plot(x0, y0, "r.") + plt.plot(x1, y1, "b.") + plt.axis((-2.0, 2.0, -2.0, 2.0)) ax = plt.gca() - ax.set_xlabel(r'$x$') - ax.set_ylabel(r'$y$') - plt.savefig('particles.png') + ax.set_xlabel(r"$x$") + ax.set_ylabel(r"$y$") + plt.savefig("particles.png") diff --git a/Tools/PostProcessing/plot_timestep_duration.py b/Tools/PostProcessing/plot_timestep_duration.py index 9858eb6a422..7b893f1ad1e 100755 --- a/Tools/PostProcessing/plot_timestep_duration.py +++ b/Tools/PostProcessing/plot_timestep_duration.py @@ -8,25 +8,24 @@ def extract_data(filename): - regex_step = re.compile( - r"STEP [0-9]* ends.*\n.* Avg\. per step = ([0-9]*[.])?[0-9]+ s", re.MULTILINE) + r"STEP [0-9]* ends.*\n.* Avg\. per step = ([0-9]*[.])?[0-9]+ s", re.MULTILINE + ) string_data = [] - print("Processing " + filename + " ...", end='') + print("Processing " + filename + " ...", end="") with open(filename) as f: text = f.read() string_data = [s.group(0) for s in regex_step.finditer(text)] - regex_real = re.compile( - r" -?[\d.]+(?:e-?\d+)?", re.MULTILINE) + regex_real = re.compile(r" -?[\d.]+(?:e-?\d+)?", re.MULTILINE) time_data = np.zeros([len(string_data), 6]) for i, ss in enumerate(string_data): numbers = regex_real.findall(ss) - time_data[i,:] = np.array(numbers) + time_data[i, :] = np.array(numbers) print("...done!") return time_data @@ -34,22 +33,22 @@ def extract_data(filename): def plot_timestep_duration(time_data, name): fig_name = name + "_ts_duration.png" - print("Generating " + fig_name + "...", end='') + print("Generating " + fig_name + "...", end="") - plt.rcParams.update({'font.size': 20}) - plt.rcParams['axes.linewidth'] = 3 + plt.rcParams.update({"font.size": 20}) + plt.rcParams["axes.linewidth"] = 3 - f, ax = plt.subplots(figsize=(12,6)) + f, ax = plt.subplots(figsize=(12, 6)) ax.set_ylabel("timestep duration [s]") ax.set_xlabel("step [#]") - ax.semilogy(time_data[:,0], time_data[:,4]) + ax.semilogy(time_data[:, 0], time_data[:, 4]) - ax.spines['bottom'].set_color('gray') - ax.spines['top'].set_visible(False) - ax.spines['left'].set_color('gray') - ax.spines['right'].set_visible(False) + ax.spines["bottom"].set_color("gray") + ax.spines["top"].set_visible(False) + ax.spines["left"].set_color("gray") + ax.spines["right"].set_visible(False) plt.tight_layout() @@ -59,22 +58,22 @@ def plot_timestep_duration(time_data, name): def plot_cumulative_duration(time_data, name): fig_name = name + "_cumulative_duration.png" - print("Generating " + fig_name + "...", end='') + print("Generating " + fig_name + "...", end="") - plt.rcParams.update({'font.size': 20}) - plt.rcParams['axes.linewidth'] = 3 + plt.rcParams.update({"font.size": 20}) + plt.rcParams["axes.linewidth"] = 3 - f, ax = plt.subplots(figsize=(12,6)) + f, ax = plt.subplots(figsize=(12, 6)) ax.set_ylabel("cumulative duration [s]") ax.set_xlabel("step [#]") - ax.plot(time_data[:,0], np.cumsum(time_data[:,4])) + ax.plot(time_data[:, 0], np.cumsum(time_data[:, 4])) - ax.spines['bottom'].set_color('gray') - ax.spines['top'].set_visible(False) - ax.spines['left'].set_color('gray') - ax.spines['right'].set_visible(False) + ax.spines["bottom"].set_color("gray") + ax.spines["top"].set_visible(False) + ax.spines["left"].set_color("gray") + ax.spines["right"].set_visible(False) plt.tight_layout() @@ -83,9 +82,16 @@ def plot_cumulative_duration(time_data, name): def do_plot_timestep_duration(): - parser = argparse.ArgumentParser(description='Generates plots of timestep duration from WarpX standard output logs') - parser.add_argument('file_name', metavar='file_name', type=str, nargs=1, - help='the name of the WarpX output log to process') + parser = argparse.ArgumentParser( + description="Generates plots of timestep duration from WarpX standard output logs" + ) + parser.add_argument( + "file_name", + metavar="file_name", + type=str, + nargs=1, + help="the name of the WarpX output log to process", + ) args = parser.parse_args() log_file_name = args.file_name[0] @@ -95,5 +101,6 @@ def do_plot_timestep_duration(): plot_timestep_duration(time_data, log_file_name) plot_cumulative_duration(time_data, log_file_name) + if __name__ == "__main__": do_plot_timestep_duration() diff --git a/Tools/PostProcessing/read_raw_data.py b/Tools/PostProcessing/read_raw_data.py index c34ea11d301..bc63b43f3cf 100644 --- a/Tools/PostProcessing/read_raw_data.py +++ b/Tools/PostProcessing/read_raw_data.py @@ -10,10 +10,11 @@ import numpy as np -HeaderInfo = namedtuple('HeaderInfo', ['version', 'how', 'ncomp', 'nghost']) +HeaderInfo = namedtuple("HeaderInfo", ["version", "how", "ncomp", "nghost"]) + def read_data(plt_file): - ''' + """ This function reads the raw (i.e. not averaged to cell centers) data from a WarpX plt file. The plt file must have been written with the @@ -33,9 +34,9 @@ def read_data(plt_file): >>> data = read_data("plt00016") >>> print(data.keys()) - >>> print(data['Ex'].shape) + >>> print(data["Ex"].shape) - ''' + """ all_data = [] raw_files = sorted(glob(plt_file + "/raw_fields/Level_*/")) for raw_file in raw_files: @@ -69,33 +70,35 @@ def _line_to_numpy_arrays(line): def _read_local_Header(header_file, dim): with open(header_file, "r") as f: t_snapshot = float(f.readline()) - if dim==2: + if dim == 2: nx, nz = [int(x) for x in f.readline().split()] ny = 1 xmin, zmin = [float(x) for x in f.readline().split()] ymin = 0 xmax, zmax = [float(x) for x in f.readline().split()] ymax = 0 - if dim==3: + if dim == 3: nx, ny, nz = [int(x) for x in f.readline().split()] xmin, ymin, zmin = [float(x) for x in f.readline().split()] xmax, ymax, zmax = [float(x) for x in f.readline().split()] field_names = f.readline().split() local_info = { - 't_snapshot' : t_snapshot, - 'field_names' : field_names, - 'xmin' : xmin, - 'ymin' : ymin, - 'zmin' : zmin, - 'xmax' : xmax, - 'ymax' : ymax, - 'zmax' : zmax, - 'nx' : nx, - 'ny' : ny, - 'nz' : nz - } + "t_snapshot": t_snapshot, + "field_names": field_names, + "xmin": xmin, + "ymin": ymin, + "zmin": zmin, + "xmax": xmax, + "ymax": ymax, + "zmax": zmax, + "nx": nx, + "ny": ny, + "nz": nz, + } return local_info + + ## ------------------------------------------------------------ ## USE THIS INSTEAD OF THE PREVIOUS FUNCTION IF Header contains ## (x,y,z) min and max vectors instead of zmin and zmax @@ -115,25 +118,23 @@ def _read_local_Header(header_file, dim): def _read_global_Header(header_file): with open(header_file, "r") as f: - nshapshots = int(f.readline()) dt_between_snapshots = float(f.readline()) gamma_boost = float(f.readline()) beta_boost = float(f.readline()) global_info = { - 'nshapshots' : nshapshots, - 'dt_between_snapshots' : dt_between_snapshots, - 'gamma_boost' : gamma_boost, - 'beta_boost' : beta_boost - } + "nshapshots": nshapshots, + "dt_between_snapshots": dt_between_snapshots, + "gamma_boost": gamma_boost, + "beta_boost": beta_boost, + } return global_info def _read_header(header_file): with open(header_file, "r") as f: - version = int(f.readline()) how = int(f.readline()) ncomp = int(f.readline()) @@ -142,9 +143,11 @@ def _read_header(header_file): # If the number of ghost cells varies depending on the direction, # s is a string of the form '(9,8)\n' in 2D or '(9,8,9)\n' in 3D. s = f.readline() - s = s.replace('(', '') # remove left parenthesis '(', if any - s = s.replace(')', '') # remove right parenthesis ')', if any - nghost = np.fromstring(s, dtype = int, sep = ',') # convert from string to numpy array + s = s.replace("(", "") # remove left parenthesis '(', if any + s = s.replace(")", "") # remove right parenthesis ')', if any + nghost = np.fromstring( + s, dtype=int, sep="," + ) # convert from string to numpy array header = HeaderInfo(version, how, ncomp, nghost) @@ -155,12 +158,10 @@ def _read_header(header_file): boxes = [] for line in f: clean_line = line.strip().split() - if clean_line == [')']: + if clean_line == [")"]: break lo_corner, hi_corner, node_type = _line_to_numpy_arrays(clean_line) - boxes.append((lo_corner - nghost, - hi_corner + nghost, - node_type)) + boxes.append((lo_corner - nghost, hi_corner + nghost, node_type)) # read the file and offset position for the corresponding box file_names = [] @@ -182,7 +183,6 @@ def _combine_boxes(boxes): def _read_field(raw_file, field_name): - header_file = raw_file + field_name + "_H" boxes, file_names, offsets, header = _read_header(header_file) @@ -200,11 +200,11 @@ def _read_field(raw_file, field_name): shape = np.append(shape, header.ncomp) with open(raw_file + fn, "rb") as f: f.seek(offset) - if (header.version == 1): + if header.version == 1: f.readline() # skip the first line - arr = np.fromfile(f, 'float64', np.product(shape)) - arr = arr.reshape(shape, order='F') - box_shape = [slice(l,h+1) for l, h in zip(lo, hi)] + arr = np.fromfile(f, "float64", np.prod(shape)) + arr = arr.reshape(shape, order="F") + box_shape = [slice(low, hig + 1) for low, hig in zip(lo, hi)] if header.ncomp > 1: box_shape += [slice(None)] data[tuple(box_shape)] = arr @@ -212,9 +212,7 @@ def _read_field(raw_file, field_name): return data - def _read_buffer(snapshot, header_fn, _component_names): - boxes, file_names, offsets, header = _read_header(header_fn) dom_lo, dom_hi = _combine_boxes(boxes) @@ -227,21 +225,24 @@ def _read_buffer(snapshot, header_fn, _component_names): lo = box[0] - dom_lo hi = box[1] - dom_lo shape = hi - lo + 1 - size = np.product(shape) + size = np.prod(shape) with open(snapshot + "/Level_0/" + fn, "rb") as f: f.seek(offset) - if (header.version == 1): + if header.version == 1: f.readline() # skip the first line - arr = np.fromfile(f, 'float64', header.ncomp*size) + arr = np.fromfile(f, "float64", header.ncomp * size) for i in range(header.ncomp): - comp_data = arr[i*size:(i+1)*size].reshape(shape, order='F') + comp_data = arr[i * size : (i + 1) * size].reshape(shape, order="F") data = all_data[_component_names[i]] - data[tuple([slice(l,h+1) for l, h in zip(lo, hi)])] = comp_data + data[tuple([slice(low, hig + 1) for low, hig in zip(lo, hi)])] = ( + comp_data + ) all_data[_component_names[i]] = data return all_data -def read_reduced_diags(filename, delimiter=' '): - ''' + +def read_reduced_diags(filename, delimiter=" "): + """ Read data written by WarpX Reduced Diagnostics, and return them into Python objects input: - filename name of file to open @@ -249,54 +250,67 @@ def read_reduced_diags(filename, delimiter=' '): output: - metadata_dict dictionary where first key is the type of metadata, second is the field - data dictionary with data - ''' + """ # Read header line - unformatted_header = list( np.genfromtxt( filename, comments="@", max_rows=1, dtype="str", delimiter=delimiter) ) + unformatted_header = list( + np.genfromtxt( + filename, comments="@", max_rows=1, dtype="str", delimiter=delimiter + ) + ) # From header line, get field name, units and column number - field_names = [s[s.find("]")+1:s.find("(")] for s in unformatted_header] - field_units = [s[s.find("(")+1:s.find(")")] for s in unformatted_header] - field_column = [s[s.find("[")+1:s.find("]")] for s in unformatted_header] + field_names = [s[s.find("]") + 1 : s.find("(")] for s in unformatted_header] + field_units = [s[s.find("(") + 1 : s.find(")")] for s in unformatted_header] + field_column = [s[s.find("[") + 1 : s.find("]")] for s in unformatted_header] # Load data and re-format to a dictionary - data = np.loadtxt( filename, delimiter=delimiter ) + data = np.loadtxt(filename, delimiter=delimiter) if data.ndim == 1: data_dict = {key: np.atleast_1d(data[i]) for i, key in enumerate(field_names)} else: - data_dict = {key: data[:,i] for i, key in enumerate(field_names)} + data_dict = {key: data[:, i] for i, key in enumerate(field_names)} # Put header data into a dictionary metadata_dict = {} - metadata_dict['units'] = {key: field_units[i] for i, key in enumerate(field_names)} - metadata_dict['column'] = {key: field_column[i] for i, key in enumerate(field_names)} + metadata_dict["units"] = {key: field_units[i] for i, key in enumerate(field_names)} + metadata_dict["column"] = { + key: field_column[i] for i, key in enumerate(field_names) + } return metadata_dict, data_dict -def read_reduced_diags_histogram(filename, delimiter=' '): - ''' + +def read_reduced_diags_histogram(filename, delimiter=" "): + """ Modified based on read_reduced_diags Two extra return objects: - bin_value: the values of bins - bin_data: the histogram data values of bins - ''' + """ # Read header line - unformatted_header = list( np.genfromtxt( filename, comments="@", max_rows=1, dtype="str", delimiter=delimiter) ) + unformatted_header = list( + np.genfromtxt( + filename, comments="@", max_rows=1, dtype="str", delimiter=delimiter + ) + ) # From header line, get field name, units and column number - field_names = [s[s.find("]")+1:s.find("(")] for s in unformatted_header] - field_names[2:] = [s[s.find("b"):s.find("=")] for s in field_names[2:]] - field_units = [s[s.find("(")+1:s.find(")")] for s in unformatted_header] - field_column = [s[s.find("[")+1:s.find("]")] for s in unformatted_header] - field_bin = [s[s.find("=")+1:s.find("(")] for s in unformatted_header] + field_names = [s[s.find("]") + 1 : s.find("(")] for s in unformatted_header] + field_names[2:] = [s[s.find("b") : s.find("=")] for s in field_names[2:]] + field_units = [s[s.find("(") + 1 : s.find(")")] for s in unformatted_header] + field_column = [s[s.find("[") + 1 : s.find("]")] for s in unformatted_header] + field_bin = [s[s.find("=") + 1 : s.find("(")] for s in unformatted_header] # Load data and re-format to a dictionary - data = np.loadtxt( filename, delimiter=delimiter ) + data = np.loadtxt(filename, delimiter=delimiter) if data.ndim == 1: data_dict = {key: data[i] for i, key in enumerate(field_names)} else: - data_dict = {key: data[:,i] for i, key in enumerate(field_names)} + data_dict = {key: data[:, i] for i, key in enumerate(field_names)} # Put header data into a dictionary metadata_dict = {} - metadata_dict['units'] = {key: field_units[i] for i, key in enumerate(field_names)} - metadata_dict['column'] = {key: field_column[i] for i, key in enumerate(field_names)} + metadata_dict["units"] = {key: field_units[i] for i, key in enumerate(field_names)} + metadata_dict["column"] = { + key: field_column[i] for i, key in enumerate(field_names) + } # Save bin values - bin_value = np.asarray(field_bin[2:], dtype=np.float64, order='C') + bin_value = np.asarray(field_bin[2:], dtype=np.float64, order="C") if data.ndim == 1: - bin_data = data[2:] + bin_data = data[2:] else: - bin_data = data[:,2:] + bin_data = data[:, 2:] return metadata_dict, data_dict, bin_value, bin_data diff --git a/Tools/PostProcessing/video_yt.py b/Tools/PostProcessing/video_yt.py index 61046b3c074..90aad9f8d17 100644 --- a/Tools/PostProcessing/video_yt.py +++ b/Tools/PostProcessing/video_yt.py @@ -5,7 +5,7 @@ # # License: BSD-3-Clause-LBNL -''' +""" This script loops over 3D plotfiles plt*****, generates a 3D rendering of the data with fields and particles, and saves one image per plotfile to plt_****_img.png. It was written for a laser-wakefield acceleration @@ -18,7 +18,7 @@ > mpirun -np 4 python video_yt.py to generate the images. It can be quite slow for even moderately large plotfiles. -''' +""" import glob @@ -28,37 +28,43 @@ yt.enable_parallelism() import numpy as np -field = 'Ez' -my_max = int(5.e9) # Field maximum amplitude +field = "Ez" +my_max = int(5.0e9) # Field maximum amplitude do_particles = True -species0 = 'beam' -species1 = 'electrons' -do_patch = False # if want to plot an MR patch +species0 = "beam" +species1 = "electrons" +do_patch = False # if want to plot an MR patch resolution = (512, 512) -camera_position = np.array([15., 20., -5.])*yt.units.micrometer -file_list = glob.glob('./diags/plotfiles/plt?????') +camera_position = np.array([15.0, 20.0, -5.0]) * yt.units.micrometer +file_list = glob.glob("./diags/plotfiles/plt?????") + +clight = 299792458.0 # must be the same value as in WarpX -clight = 299792458.0 # must be the same value as in WarpX def plot_species(species, ad, radii, transparency, abs_xmax): # Color for each of these particles - colors_vect = [1., 1., 1., .05] # the last value is overwritten later - x = ad[species,'particle_position_x'].v - y = ad[species,'particle_position_y'].v - z = ad[species,'particle_position_z'].v + colors_vect = [1.0, 1.0, 1.0, 0.05] # the last value is overwritten later + x = ad[species, "particle_position_x"].v + y = ad[species, "particle_position_y"].v + z = ad[species, "particle_position_z"].v selector = np.abs(x) < abs_xmax - x = x[selector] ; y = y[selector] ; z = z[selector] - vertices = np.column_stack((x,y,z)) - colors = np.tile(colors_vect,(vertices.shape[0], 1)) - colors[:,3] = transparency - point = yt.visualization.volume_rendering.render_source.PointSource(vertices, colors=colors, radii=radii) + x = x[selector] + y = y[selector] + z = z[selector] + vertices = np.column_stack((x, y, z)) + colors = np.tile(colors_vect, (vertices.shape[0], 1)) + colors[:, 3] = transparency + point = yt.visualization.volume_rendering.render_source.PointSource( + vertices, colors=colors, radii=radii + ) return point + # Create the 3d image for 1 timestep # filename is the name of the folder (e.g. plt00000) def img_onestep(filename): # Load the data - ds = yt.load( filename ) + ds = yt.load(filename) ad = ds.all_data() # Calculate the z position of the box. @@ -66,30 +72,48 @@ def img_onestep(filename): # was used in the simulation, the rendering shows some jitter. # This is because a cell is added in z at some iterations but not all. # These lines calculate this jitter z_shift and remove it from the camera position and focus - iteration=int(filename[-5:]) - dt = 1./clight * 1./np.sqrt((1./ad['dx'][-1]**2 + 1./ad['dy'][-1]**2 + 1./ad['dz'][-1]**2)) + iteration = int(filename[-5:]) + dt = ( + 1.0 + / clight + * 1.0 + / np.sqrt( + ( + 1.0 / ad["dx"][-1] ** 2 + + 1.0 / ad["dy"][-1] ** 2 + + 1.0 / ad["dz"][-1] ** 2 + ) + ) + ) z_front = dt * float(iteration) * clight - z_shift = z_front-ds.domain_right_edge[2] + z_shift = z_front - ds.domain_right_edge[2] # Create a yt source object for the level1 patch if do_patch: box_patch = yt.visualization.volume_rendering.render_source.BoxSource( - left_edge=ds.index.grids[1].LeftEdge+np.array([0., 0., z_shift])*yt.units.meter, - right_edge=ds.index.grids[1].RightEdge+np.array([0., 0., z_shift])*yt.units.meter, - color=[1.,0.1,0.1,.01]) + left_edge=ds.index.grids[1].LeftEdge + + np.array([0.0, 0.0, z_shift]) * yt.units.meter, + right_edge=ds.index.grids[1].RightEdge + + np.array([0.0, 0.0, z_shift]) * yt.units.meter, + color=[1.0, 0.1, 0.1, 0.01], + ) # Handle 2 populations of particles: beam and plasma electrons if do_particles: - point0 = plot_species(species0, ad, 2, .01, 1.) - point1 = plot_species(species1, ad, 1, .002, 20.e-6) + point0 = plot_species(species0, ad, 2, 0.01, 1.0) + point1 = plot_species(species1, ad, 1, 0.002, 20.0e-6) sc = yt.create_scene(ds, field=field) # Set camera properties cam = sc.camera dom_length = ds.domain_width[2].v cam.set_width(ds.quan(dom_length, yt.units.meter)) - cam.position = ds.domain_center + camera_position + np.array([0., 0., z_shift])*yt.units.meter - cam.focus = ds.domain_center + np.array([0., 0., z_shift])*yt.units.meter + cam.position = ( + ds.domain_center + + camera_position + + np.array([0.0, 0.0, z_shift]) * yt.units.meter + ) + cam.focus = ds.domain_center + np.array([0.0, 0.0, z_shift]) * yt.units.meter cam.resolution = resolution # Field rendering properties source = sc[0] @@ -98,17 +122,17 @@ def img_onestep(filename): source.use_ghost_zones = True bounds = (-my_max, my_max) tf = yt.ColorTransferFunction(bounds) - w = (.01*my_max)**2 + w = (0.01 * my_max) ** 2 # Define the transfer function for 3d rendering # 3 isocontours for negative field values # The sharpness of the contour is controlled by argument width - tf.add_gaussian(-.04 *my_max, width=8*w, height=[0.1, 0.1, 1.0, 0.02]) - tf.add_gaussian(-.2 *my_max, width=5*w, height=[0.1, 0.1, 1.0, 0.05]) - tf.add_gaussian(-.6 *my_max, width=w, height=[0.0, 0.0, 1.0, 0.3]) + tf.add_gaussian(-0.04 * my_max, width=8 * w, height=[0.1, 0.1, 1.0, 0.02]) + tf.add_gaussian(-0.2 * my_max, width=5 * w, height=[0.1, 0.1, 1.0, 0.05]) + tf.add_gaussian(-0.6 * my_max, width=w, height=[0.0, 0.0, 1.0, 0.3]) # 3 isocontours for positive field values - tf.add_gaussian(.04 *my_max, width=8*w, height=[1.0, 1.0, 0.2, 0.02]) - tf.add_gaussian(.2 *my_max, width=5*w, height=[1.0, 1.0, 0.2, 0.05]) - tf.add_gaussian(.6 *my_max, width=w, height=[1.0, 1.0, 0.0, 0.3]) + tf.add_gaussian(0.04 * my_max, width=8 * w, height=[1.0, 1.0, 0.2, 0.02]) + tf.add_gaussian(0.2 * my_max, width=5 * w, height=[1.0, 1.0, 0.2, 0.05]) + tf.add_gaussian(0.6 * my_max, width=w, height=[1.0, 1.0, 0.0, 0.3]) source.tfh.tf = tf source.tfh.bounds = bounds source.tfh.set_log(False) @@ -118,7 +142,8 @@ def img_onestep(filename): sc.add_source(point1) if do_patch: sc.add_source(box_patch) - sc.save('./img_' + filename[-8:] + '.png', sigma_clip=1.) + sc.save("./img_" + filename[-8:] + ".png", sigma_clip=1.0) + # Get plt folders in current folder and loop over them. file_list.sort() diff --git a/Tools/PostProcessing/yt3d_mpi.py b/Tools/PostProcessing/yt3d_mpi.py index 655327aff3d..10734494280 100644 --- a/Tools/PostProcessing/yt3d_mpi.py +++ b/Tools/PostProcessing/yt3d_mpi.py @@ -4,7 +4,7 @@ # # License: BSD-3-Clause-LBNL -''' +""" This script loops over 3D plotfiles plt*****, generates a 3D rendering of the data with fields and particles, and saves one image per plotfile to img_*****.png. It was written for a beam-driven wakefield acceleration @@ -15,7 +15,7 @@ > mpirun -np 12 python yt3d_mpi.py to generate the images. It can be quite slow for even moderately large plotfiles. -''' +""" import glob @@ -27,60 +27,79 @@ yt.funcs.mylog.setLevel(50) # my_max = 1.e11 # for smooth rendering -my_max = 5.e10 # for layered rendering -species_to_plot = ['plasma_e', 'beam', 'driver'] +my_max = 5.0e10 # for layered rendering +species_to_plot = ["plasma_e", "beam", "driver"] # For each species, provide [red, green, blue, alpha] between 0. and 1. -species_colors = { 'plasma_e': [1., 1., 1., .15], - 'beam' : [1., 1., 1., .2 ], - 'driver' : [1., 1., 1., .2 ] } +species_colors = { + "plasma_e": [1.0, 1.0, 1.0, 0.15], + "beam": [1.0, 1.0, 1.0, 0.2], + "driver": [1.0, 1.0, 1.0, 0.2], +} # provide these to avoid jitter when using a moving window use_moving_window = True plot_mr_patch = False -rendering_type = 'layers' # 'layers' or 'smooth' -maxwell_solver = 'ckc' # 'ckc' or 'yee' +rendering_type = "layers" # 'layers' or 'smooth' +maxwell_solver = "ckc" # 'ckc' or 'yee' cfl = 0.99 -file_list = glob.glob('plotfiles/plt?????') +file_list = glob.glob("plotfiles/plt?????") + +bounds = (-my_max, my_max) +z_shift = 0.0 +w = (0.01 * my_max) ** 2 -bounds = ( -my_max, my_max ) -z_shift = 0. -w = (.01*my_max)**2 def jitter_shift(ds, ad, cfl, iteration): - if maxwell_solver == 'yee': - dt = 1./scc.c * 1./np.sqrt((1./ad['dx'][-1]**2 + 1./ad['dy'][-1]**2 + 1./ad['dz'][-1]**2)) - elif maxwell_solver == 'ckc': - dt = cfl * min( [ ad['dx'][-1], ad['dy'][-1], ad['dz'][-1] ] ) / scc.c - z_front = dt * float(iteration) * scc.c + 7.5e-6*yt.units.meter - z_shift = z_front-ds.domain_right_edge[2] + if maxwell_solver == "yee": + dt = ( + 1.0 + / scc.c + * 1.0 + / np.sqrt( + ( + 1.0 / ad["dx"][-1] ** 2 + + 1.0 / ad["dy"][-1] ** 2 + + 1.0 / ad["dz"][-1] ** 2 + ) + ) + ) + elif maxwell_solver == "ckc": + dt = cfl * min([ad["dx"][-1], ad["dy"][-1], ad["dz"][-1]]) / scc.c + z_front = dt * float(iteration) * scc.c + 7.5e-6 * yt.units.meter + z_shift = z_front - ds.domain_right_edge[2] return z_shift + def get_species_ytpoints(ad, species, color_vec): - xp = ad[species,'particle_position_x'].v - yp = ad[species,'particle_position_y'].v - zp = ad[species,'particle_position_z'].v - if species == 'plasma_e': - selection = np.abs(xp)<2.e-6 + xp = ad[species, "particle_position_x"].v + yp = ad[species, "particle_position_y"].v + zp = ad[species, "particle_position_z"].v + if species == "plasma_e": + selection = np.abs(xp) < 2.0e-6 zp = zp[selection] yp = yp[selection] xp = xp[selection] - vertices = np.column_stack((xp,yp,zp)) - colors = np.tile(color_vec,(vertices.shape[0], 1)) - points = yt.visualization.volume_rendering.render_source.PointSource(vertices, colors=colors, radii=1) + vertices = np.column_stack((xp, yp, zp)) + colors = np.tile(color_vec, (vertices.shape[0], 1)) + points = yt.visualization.volume_rendering.render_source.PointSource( + vertices, colors=colors, radii=1 + ) return points + def img_onestep(filename): - ds = yt.load( filename ) + ds = yt.load(filename) ad = ds.all_data() - iteration=int(filename[-5:]) - sc = yt.create_scene(ds, field='Ez') + iteration = int(filename[-5:]) + sc = yt.create_scene(ds, field="Ez") if use_moving_window: - z_shift = jitter_shift( ds, ad, cfl, iteration ) - array_shift = z_shift * np.array([0., 0., 1.]) + z_shift = jitter_shift(ds, ad, cfl, iteration) + array_shift = z_shift * np.array([0.0, 0.0, 1.0]) if plot_mr_patch: box_patch = yt.visualization.volume_rendering.render_source.BoxSource( - left_edge =ds.index.grids[1].LeftEdge +array_shift, - right_edge=ds.index.grids[1].RightEdge+array_shift, - color=[1.,0.1,0.1,.01] ) + left_edge=ds.index.grids[1].LeftEdge + array_shift, + right_edge=ds.index.grids[1].RightEdge + array_shift, + color=[1.0, 0.1, 0.1, 0.01], + ) sc.add_source(box_patch) ######################## ### volume rendering ### @@ -90,27 +109,28 @@ def img_onestep(filename): source.grey_opacity = True source.set_log(False) tf = yt.ColorTransferFunction(bounds) - if rendering_type == 'smooth': - tf.add_gaussian(-my_max/4, width=15**2*w, height=[0.0, 0.0, 1.0, 1]) - tf.add_gaussian( my_max/4, width=15**2*w, height=[1.0, 0.0, 0.0, 1]) - if rendering_type == 'layers': + if rendering_type == "smooth": + tf.add_gaussian(-my_max / 4, width=15**2 * w, height=[0.0, 0.0, 1.0, 1]) + tf.add_gaussian(my_max / 4, width=15**2 * w, height=[1.0, 0.0, 0.0, 1]) + if rendering_type == "layers": # NEGATIVE - tf.add_gaussian(-.04 *my_max, width=8*w, height=[0.1, 0.1, 1.0, 0.2]) - tf.add_gaussian(-.2 *my_max, width=5*w, height=[0.1, 0.1, 1.0, 0.5]) - tf.add_gaussian(-.6 *my_max, width=w, height=[0.0, 0.0, 1.0, 1.]) + tf.add_gaussian(-0.04 * my_max, width=8 * w, height=[0.1, 0.1, 1.0, 0.2]) + tf.add_gaussian(-0.2 * my_max, width=5 * w, height=[0.1, 0.1, 1.0, 0.5]) + tf.add_gaussian(-0.6 * my_max, width=w, height=[0.0, 0.0, 1.0, 1.0]) # POSITIVE - tf.add_gaussian(.04 *my_max, width=8*w, height=[1.0, 1.0, 0.2, 0.2]) - tf.add_gaussian(.2 *my_max, width=5*w, height=[1.0, 1.0, 0.2, 0.5]) - tf.add_gaussian(.6 *my_max, width=w, height=[1.0, 1.0, 0.0, 1.]) + tf.add_gaussian(0.04 * my_max, width=8 * w, height=[1.0, 1.0, 0.2, 0.2]) + tf.add_gaussian(0.2 * my_max, width=5 * w, height=[1.0, 1.0, 0.2, 0.5]) + tf.add_gaussian(0.6 * my_max, width=w, height=[1.0, 1.0, 0.0, 1.0]) ###################### ### plot particles ### ###################### species_points = {} for species in species_to_plot: - species_points[ species ] = get_species_ytpoints(ad, - species, species_colors[species]) - sc.add_source( species_points[ species ] ) + species_points[species] = get_species_ytpoints( + ad, species, species_colors[species] + ) + sc.add_source(species_points[species]) source.tfh.tf = tf source.tfh.bounds = bounds ######################### @@ -118,20 +138,23 @@ def img_onestep(filename): ######################### cam = sc.camera cam.resolution = (2048, 2048) - cam.width = .00018*yt.units.meter - cam.focus = ds.domain_center + \ - np.array([0., 0., 10.e-6 ])*yt.units.meter + \ - array_shift - cam.position = ds.domain_center + \ - np.array([15., 15., -5. ])*yt.units.micrometer + \ - array_shift - cam.normal_vector = [-0.3, -0.3, -.2] + cam.width = 0.00018 * yt.units.meter + cam.focus = ( + ds.domain_center + np.array([0.0, 0.0, 10.0e-6]) * yt.units.meter + array_shift + ) + cam.position = ( + ds.domain_center + + np.array([15.0, 15.0, -5.0]) * yt.units.micrometer + + array_shift + ) + cam.normal_vector = [-0.3, -0.3, -0.2] cam.switch_orientation() # save image - if rendering_type == 'smooth': - sc.save('img_' + str(my_number_list[count]).zfill(5), sigma_clip=5.) - if rendering_type == 'layers': - sc.save('img_' + str(my_number_list[count]).zfill(5), sigma_clip=2.) + if rendering_type == "smooth": + sc.save("img_" + str(my_number_list[count]).zfill(5), sigma_clip=5.0) + if rendering_type == "layers": + sc.save("img_" + str(my_number_list[count]).zfill(5), sigma_clip=2.0) + file_list.sort() # Total number of files @@ -142,9 +165,9 @@ def img_onestep(filename): me = comm_world.Get_rank() nrank = comm_world.Get_size() # List of files to process for current proc -my_list = file_list[ (me*nfiles)/nrank : ((me+1)*nfiles)/nrank ] +my_list = file_list[(me * nfiles) / nrank : ((me + 1) * nfiles) / nrank] # List if file numbers for current proc -my_number_list = number_list[ (me*nfiles)/nrank : ((me+1)*nfiles)/nrank ] +my_number_list = number_list[(me * nfiles) / nrank : ((me + 1) * nfiles) / nrank] for count, filename in enumerate(my_list): - print('processing ' + filename) + print("processing " + filename) img_onestep(filename) diff --git a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp index 41d27477dcf..a92f191037d 100644 --- a/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp +++ b/Tools/QedTablesUtils/Source/ArgParser/QedTablesArgParser.cpp @@ -85,7 +85,7 @@ ArgParser::ParseArgs (const std::vector& keys, const int argc, char const* void ArgParser::PrintHelp (const vector& cmd_list) { - cout << "Command line options: " << endl; + cout << "Command line options:\n"; for (const auto& el : cmd_list){ const auto type = get<1>(el); @@ -102,7 +102,7 @@ ArgParser::PrintHelp (const vector& cmd_list) cout << get<0>(el) << " " << stype << " " << get<2>(el) - << endl; + << "\n"; } } diff --git a/Tools/QedTablesUtils/Source/QedTableCommons.H b/Tools/QedTablesUtils/Source/QedTableCommons.H index 40551b9e13c..903ba4623a8 100644 --- a/Tools/QedTablesUtils/Source/QedTableCommons.H +++ b/Tools/QedTablesUtils/Source/QedTableCommons.H @@ -12,14 +12,14 @@ bool Contains (const ContainerType& container, const ElementType& el) void AbortWithMessage(const std::string& msg) { - std::cout << "### ABORT : " << msg << std::endl; - std::cout << "___________________________" << std::endl; + std::cerr << "### ABORT : " << msg << "\n"; + std::cerr << "___________________________\n"; exit(1); } void SuccessExit() { - std::cout << "___________________________" << std::endl; + std::cout << "___________________________\n"; exit(0); } diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp index 1ea62b5c6ed..7bfa5787ec8 100644 --- a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -49,7 +49,7 @@ void GenerateTableQS (const ParsedArgs& args, const string& outfile_name); int main (int argc, char** argv) { - cout << "### QED Table Generator ###" << endl; + cout << "### QED Table Generator ###\n"; const auto args_map = ParseArgs(line_commands, argc, argv); if (args_map.empty() || Contains(args_map, "-h")){ diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp index ba9d58775f2..27b284f34c2 100644 --- a/Tools/QedTablesUtils/Source/QedTableReader.cpp +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -57,7 +57,7 @@ class qs_photon_emission_table_wrapper : int main (int argc, char** argv) { - cout << "### QED Table Reader ###" << endl; + cout << "### QED Table Reader ###\n"; const auto args_map = ParseArgs(line_commands, argc, argv); if (args_map.empty() || Contains(args_map, "-h")){ diff --git a/Tools/Release/newVersion.sh b/Tools/Release/newVersion.sh index b1d2a6aad27..9491401b120 100755 --- a/Tools/Release/newVersion.sh +++ b/Tools/Release/newVersion.sh @@ -104,25 +104,25 @@ sed -i -E "s/"\ # setup.py: version = '21.02', sed -i -E "s/"\ -"([[:blank:]]*version[[:blank:]]*=[[:blank:]]*')(.*)('.+)/"\ +"([[:blank:]]*version[[:blank:]]*=[[:blank:]]*\")(.*)(\".+)/"\ "\1${VERSION_STR}\3/g" \ ${REPO_DIR}/setup.py # Python/setup.py: version = '21.02', sed -i -E "s/"\ -"([[:blank:]]*version[[:blank:]]*=[[:blank:]]*')(.*)('.+)/"\ +"([[:blank:]]*version[[:blank:]]*=[[:blank:]]*\")(.*)(\".+)/"\ "\1${VERSION_STR}\3/g" \ ${REPO_DIR}/Python/setup.py # sphinx / RTD # docs/source/conf.py sed -i "s/"\ -"[[:blank:]]*version[[:blank:]]*=[[:blank:]]*u.*/"\ -"version = u'${VERSION_STR_NOSUFFIX}'/g" \ +"[[:blank:]]*version[[:blank:]]*=[[:blank:]]*.*/"\ +"version = \"${VERSION_STR_NOSUFFIX}\"/g" \ ${REPO_DIR}/Docs/source/conf.py sed -i "s/"\ -"[[:blank:]]*release[[:blank:]]*=[[:blank:]]*u.*/"\ -"release = u'${VERSION_STR}'/g" \ +"[[:blank:]]*release[[:blank:]]*=[[:blank:]]*.*/"\ +"release = \"${VERSION_STR}\"/g" \ ${REPO_DIR}/Docs/source/conf.py diff --git a/Tools/Release/releasePR.py b/Tools/Release/releasePR.py new file mode 100755 index 00000000000..47a380901b1 --- /dev/null +++ b/Tools/Release/releasePR.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl +# + +# This file is a maintainer tool to open a release PR for WarpX. +# It is highly automated and does a few assumptions, e.g., that you +# are releasing for the current month. +# +# You also need to have git and the GitHub CLI tool "gh" installed and properly +# configured for it to work: +# https://cli.github.com/ +# +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +# Maintainer Inputs ########################################################### + +print("""Hi there, this is a WarpX maintainer tool to ...\n. +For it to work, you need write access on the source directory and +you should be working in a clean git branch without ongoing +rebase/merge/conflict resolves and without unstaged changes.""") + +# check source dir +REPO_DIR = Path(__file__).parent.parent.parent.absolute() +print(f"\nYour current source directory is: {REPO_DIR}") + +REPLY = input("Are you sure you want to continue? [y/N] ") +print() +if REPLY not in ["Y", "y"]: + print("You did not confirm with 'y', aborting.") + sys.exit(1) + +release_repo = input("What is the name of your git remote? (e.g., ax3l) ") +commit_sign = input("How to sign the commit? (e.g., -sS) ") + + +# Helpers ##################################################################### + + +def concat_answers(answers): + return "\n".join(answers) + "\n" + + +# Stash current work ########################################################## + +subprocess.run(["git", "stash"], capture_output=True, text=True) + + +# Git Branch ################################################################## + +WarpX_version_yr = f"{datetime.now().strftime('%y')}" +WarpX_version_mn = f"{datetime.now().strftime('%m')}" +WarpX_version = f"{WarpX_version_yr}.{WarpX_version_mn}" +release_branch = f"release-{WarpX_version}" +subprocess.run(["git", "checkout", "development"], capture_output=True, text=True) +subprocess.run(["git", "fetch"], capture_output=True, text=True) +subprocess.run(["git", "pull", "--ff-only"], capture_output=True, text=True) +subprocess.run(["git", "branch", "-D", release_branch], capture_output=True, text=True) +subprocess.run( + ["git", "checkout", "-b", release_branch], capture_output=True, text=True +) + + +# AMReX New Version ########################################################### + +AMReX_version = f"{datetime.now().strftime('%y')}.{datetime.now().strftime('%m')}" +answers = concat_answers(["y", AMReX_version, AMReX_version, "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updateAMReX.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +subprocess.run( + ["git", "commit", commit_sign, "-m", f"AMReX: {AMReX_version}"], text=True +) + + +# PICSAR New Version ########################################################## + +PICSAR_version = "25.01" +answers = concat_answers(["y", PICSAR_version, PICSAR_version, "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updatePICSAR.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +subprocess.run( + ["git", "commit", commit_sign, "-m", f"PICSAR: {PICSAR_version}"], text=True +) + + +# pyAMReX New Version ######################################################### + +pyAMReX_version = f"{datetime.now().strftime('%y')}.{datetime.now().strftime('%m')}" +answers = concat_answers(["y", pyAMReX_version, pyAMReX_version, "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updatepyAMReX.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +subprocess.run( + ["git", "commit", commit_sign, "-m", f"pyAMReX: {pyAMReX_version}"], text=True +) + + +# WarpX New Version ########################################################### + +answers = concat_answers(["y", WarpX_version_yr, WarpX_version_mn, "", "", "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/newVersion.sh")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +subprocess.run( + ["git", "commit", commit_sign, "-m", f"WarpX: {WarpX_version}"], text=True +) + + +# GitHub PR ################################################################### + +subprocess.run(["git", "push", "-u", release_repo, release_branch], text=True) + +subprocess.run( + [ + "gh", + "pr", + "create", + "--title", + f"Release {WarpX_version}", + "--body", + f"""Prepare the {datetime.now().strftime("%B")} release of WarpX: +```bash +# update dependencies +./Tools/Release/updateAMReX.py +./Tools/Release/updatePICSAR.py # no changes, still {PICSAR_version} +./Tools/Release/updatepyAMReX.py +# bump version number +./Tools/Release/newVersion.sh +``` + +Following this workflow: https://warpx.readthedocs.io/en/latest/maintenance/release.html +""", + "--label", + "component: documentation", + "--label", + "component: third party", + "--web", + ], + text=True, +) + + +# Epilogue #################################################################### + +print("""Done. Please check your source, e.g. via + git diff +now and commit the changes if no errors occurred.""") diff --git a/Tools/Release/updateAMReX.py b/Tools/Release/updateAMReX.py index 9dfa7fbeb41..99bd4899826 100755 --- a/Tools/Release/updateAMReX.py +++ b/Tools/Release/updateAMReX.py @@ -15,15 +15,6 @@ import requests -try: - from configupdater import ConfigUpdater -except ImportError: - print("Warning: Cannot update .ini files without 'configupdater'") - print("Consider running 'python -m pip install configupdater'") - ConfigUpdater = None - sys.exit(1) - - # Maintainer Inputs ########################################################### print("""Hi there, this is a WarpX maintainer tool to update the source @@ -38,7 +29,7 @@ REPLY = input("Are you sure you want to continue? [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) @@ -46,7 +37,9 @@ # Current Versions ############################################################ # AMReX development HEAD -amrex_gh = requests.get('https://api.github.com/repos/AMReX-Codes/amrex/commits/development') +amrex_gh = requests.get( + "https://api.github.com/repos/AMReX-Codes/amrex/commits/development" +) amrex_HEAD = amrex_gh.json()["sha"] # WarpX references to AMReX: cmake/dependencies/AMReX.cmake @@ -54,18 +47,20 @@ # branch/commit/tag (git fetcher) version # set(WarpX_amrex_branch "development" ... amrex_branch = f"unknown (format issue in {amrex_cmake_path})" -with open(amrex_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*set\(WarpX_amrex_branch\s+"(.+)"\s+.*', - f.read(), re.MULTILINE) +with open(amrex_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r'.*set\(WarpX_amrex_branch\s+"(.+)"\s+.*', f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: amrex_branch = r_minimal[0] # minimal (external) version # find_package(AMReX YY.MM CONFIG ... amrex_minimal = f"unknown (format issue in {amrex_cmake_path})" -with open(amrex_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*find_package\(AMReX\s+(.+)\s+CONFIG\s+.*', - f.read(), re.MULTILINE) +with open(amrex_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r".*find_package\(AMReX\s+(.+)\s+CONFIG\s+.*", f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: amrex_minimal = r_minimal[0] @@ -78,13 +73,15 @@ print(f"Currently, WarpX builds against this AMReX commit/branch/sha: {amrex_branch}") print(f"AMReX HEAD commit (development branch): {amrex_HEAD}") -amrex_new_branch = input(f"Update AMReX commit/branch/sha: ").strip() +amrex_new_branch = input("Update AMReX commit/branch/sha: ").strip() if not amrex_new_branch: - amrex_new_branch = amrex_branch - print(f"--> Nothing entered, will keep: {amrex_branch}") + amrex_new_branch = amrex_HEAD + print(f"--> Nothing entered, use: {amrex_HEAD}") print() -print(f"Currently, a pre-installed AMReX is required at least at version: {amrex_minimal}") +print( + f"Currently, a pre-installed AMReX is required at least at version: {amrex_minimal}" +) today = datetime.date.today().strftime("%y.%m") amrex_new_minimal = input(f"New minimal AMReX version (e.g. {today})? ").strip() if not amrex_new_minimal: @@ -97,77 +94,52 @@ REPLY = input("Is this information correct? Will now start updating! [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) # Updates ##################################################################### -# run_test.sh (used also for Azure Pipelines) -run_test_path = str(REPO_DIR.joinpath("run_test.sh")) -with open(run_test_path, encoding='utf-8') as f: - run_test_content = f.read() - # branch/commit/tag (git fetcher) version - # cd amrex && git checkout COMMIT_TAG_OR_BRANCH && cd - - run_test_content = re.sub( - r'(.*cd\s+amrex.+git checkout\s+--detach\s+)(.+)(\s+&&\s.*)', - r'\g<1>{}\g<3>'.format(amrex_new_branch), - run_test_content, flags = re.MULTILINE) - -with open(run_test_path, "w", encoding='utf-8') as f: - f.write(run_test_content) - # CI: legacy build check in .github/workflows/cuda.yml ci_gnumake_path = str(REPO_DIR.joinpath(".github/workflows/cuda.yml")) -with open(ci_gnumake_path, encoding='utf-8') as f: +with open(ci_gnumake_path, encoding="utf-8") as f: ci_gnumake_content = f.read() # branch/commit/tag (git fetcher) version # cd ../amrex && git checkout COMMIT_TAG_OR_BRANCH && cd - ci_gnumake_content = re.sub( - r'(.*cd\s+\.\./amrex.+git checkout\s+--detach\s+)(.+)(\s+&&\s.*)', - r'\g<1>{}\g<3>'.format(amrex_new_branch), - ci_gnumake_content, flags = re.MULTILINE) + r"(.*cd\s+\.\./amrex.+git checkout\s+--detach\s+)(.+)(\s+&&\s.*)", + r"\g<1>{}\g<3>".format(amrex_new_branch), + ci_gnumake_content, + flags=re.MULTILINE, + ) -with open(ci_gnumake_path, "w", encoding='utf-8') as f: +with open(ci_gnumake_path, "w", encoding="utf-8") as f: f.write(ci_gnumake_content) -if ConfigUpdater is not None: - # WarpX-tests.ini - tests_ini_path = str(REPO_DIR.joinpath("Regression/WarpX-tests.ini")) - cp = ConfigUpdater() - cp.optionxform = str - cp.read(tests_ini_path) - cp['AMReX']['branch'].value = amrex_new_branch - cp.update_file() - - # WarpX-GPU-tests.ini - tests_gpu_ini_path = str(REPO_DIR.joinpath("Regression/WarpX-GPU-tests.ini")) - cp = ConfigUpdater() - cp.optionxform = str - cp.read(tests_gpu_ini_path) - cp['AMReX']['branch'].value = amrex_new_branch - cp.update_file() - # WarpX references to AMReX: cmake/dependencies/AMReX.cmake -with open(amrex_cmake_path, encoding='utf-8') as f: +with open(amrex_cmake_path, encoding="utf-8") as f: amrex_cmake_content = f.read() # branch/commit/tag (git fetcher) version # set(WarpX_amrex_branch "development" ... amrex_cmake_content = re.sub( r'(.*set\(WarpX_amrex_branch\s+")(.+)("\s+.*)', - r'\g<1>{}\g<3>'.format(amrex_new_branch), - amrex_cmake_content, flags = re.MULTILINE) + r"\g<1>{}\g<3>".format(amrex_new_branch), + amrex_cmake_content, + flags=re.MULTILINE, + ) # minimal (external) version # find_package(AMReX YY.MM CONFIG ... amrex_cmake_content = re.sub( - r'(.*find_package\(AMReX\s+)(.+)(\s+CONFIG\s+.*)', - r'\g<1>{}\g<3>'.format(amrex_new_minimal), - amrex_cmake_content, flags = re.MULTILINE) + r"(.*find_package\(AMReX\s+)(.+)(\s+CONFIG\s+.*)", + r"\g<1>{}\g<3>".format(amrex_new_minimal), + amrex_cmake_content, + flags=re.MULTILINE, + ) -with open(amrex_cmake_path, "w", encoding='utf-8') as f: +with open(amrex_cmake_path, "w", encoding="utf-8") as f: f.write(amrex_cmake_content) diff --git a/Tools/Release/updatePICSAR.py b/Tools/Release/updatePICSAR.py index 7e61679d371..90c928472a5 100755 --- a/Tools/Release/updatePICSAR.py +++ b/Tools/Release/updatePICSAR.py @@ -29,7 +29,7 @@ REPLY = input("Are you sure you want to continue? [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) @@ -37,7 +37,9 @@ # Current Versions ############################################################ # PICSAR development HEAD -PICSAR_gh = requests.get('https://api.github.com/repos/ECP-WarpX/picsar/commits/development') +PICSAR_gh = requests.get( + "https://api.github.com/repos/ECP-WarpX/picsar/commits/development" +) PICSAR_HEAD = PICSAR_gh.json()["sha"] # WarpX references to PICSAR: cmake/dependencies/PICSAR.cmake @@ -45,18 +47,20 @@ # branch/commit/tag (git fetcher) version # set(WarpX_picsar_branch "development" ... PICSAR_branch = f"unknown (format issue in {PICSAR_cmake_path})" -with open(PICSAR_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*set\(WarpX_picsar_branch\s+"(.+)"\s+.*', - f.read(), re.MULTILINE) +with open(PICSAR_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r'.*set\(WarpX_picsar_branch\s+"(.+)"\s+.*', f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: PICSAR_branch = r_minimal[0] # minimal (external) version # find_package(PICSAR YY.MM CONFIG ... PICSAR_minimal = f"unknown (format issue in {PICSAR_cmake_path})" -with open(PICSAR_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*find_package\(PICSAR\s+(.+)\s+CONFIG\s+.*', - f.read(), re.MULTILINE) +with open(PICSAR_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r".*find_package\(PICSAR\s+(.+)\s+CONFIG\s+.*", f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: PICSAR_minimal = r_minimal[0] @@ -69,13 +73,15 @@ print(f"Currently, WarpX builds against this PICSAR commit/branch/sha: {PICSAR_branch}") print(f"PICSAR HEAD commit (development branch): {PICSAR_HEAD}") -PICSAR_new_branch = input(f"Update PICSAR commit/branch/sha: ").strip() +PICSAR_new_branch = input("Update PICSAR commit/branch/sha: ").strip() if not PICSAR_new_branch: - PICSAR_new_branch = PICSAR_branch - print(f"--> Nothing entered, will keep: {PICSAR_branch}") + PICSAR_new_branch = PICSAR_HEAD + print(f"--> Nothing entered, will use: {PICSAR_HEAD}") print() -print(f"Currently, a pre-installed PICSAR is required at least at version: {PICSAR_minimal}") +print( + f"Currently, a pre-installed PICSAR is required at least at version: {PICSAR_minimal}" +) today = datetime.date.today().strftime("%y.%m") PICSAR_new_minimal = input(f"New minimal PICSAR version (e.g. {today})? ").strip() if not PICSAR_new_minimal: @@ -88,7 +94,7 @@ REPLY = input("Is this information correct? Will now start updating! [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) @@ -96,24 +102,28 @@ # Updates ##################################################################### # WarpX references to PICSAR: cmake/dependencies/PICSAR.cmake -with open(PICSAR_cmake_path, encoding='utf-8') as f: +with open(PICSAR_cmake_path, encoding="utf-8") as f: PICSAR_cmake_content = f.read() # branch/commit/tag (git fetcher) version # set(WarpX_picsar_branch "development" ... PICSAR_cmake_content = re.sub( r'(.*set\(WarpX_picsar_branch\s+")(.+)("\s+.*)', - r'\g<1>{}\g<3>'.format(PICSAR_new_branch), - PICSAR_cmake_content, flags = re.MULTILINE) + r"\g<1>{}\g<3>".format(PICSAR_new_branch), + PICSAR_cmake_content, + flags=re.MULTILINE, + ) # minimal (external) version # find_package(PICSAR YY.MM CONFIG ... PICSAR_cmake_content = re.sub( - r'(.*find_package\(PICSAR\s+)(.+)(\s+CONFIG\s+.*)', - r'\g<1>{}\g<3>'.format(PICSAR_new_minimal), - PICSAR_cmake_content, flags = re.MULTILINE) + r"(.*find_package\(PICSAR\s+)(.+)(\s+CONFIG\s+.*)", + r"\g<1>{}\g<3>".format(PICSAR_new_minimal), + PICSAR_cmake_content, + flags=re.MULTILINE, + ) -with open(PICSAR_cmake_path, "w", encoding='utf-8') as f: +with open(PICSAR_cmake_path, "w", encoding="utf-8") as f: f.write(PICSAR_cmake_content) diff --git a/Tools/Release/updatepyAMReX.py b/Tools/Release/updatepyAMReX.py index 500781e0880..13c044b3bdd 100755 --- a/Tools/Release/updatepyAMReX.py +++ b/Tools/Release/updatepyAMReX.py @@ -29,7 +29,7 @@ REPLY = input("Are you sure you want to continue? [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) @@ -37,7 +37,9 @@ # Current Versions ############################################################ # pyAMReX development HEAD -pyamrex_gh = requests.get('https://api.github.com/repos/AMReX-Codes/pyamrex/commits/development') +pyamrex_gh = requests.get( + "https://api.github.com/repos/AMReX-Codes/pyamrex/commits/development" +) pyamrex_HEAD = pyamrex_gh.json()["sha"] # WarpX references to pyAMReX: cmake/dependencies/pyAMReX.cmake @@ -45,18 +47,20 @@ # branch/commit/tag (git fetcher) version # set(WarpX_pyamrex_branch "development" ... pyamrex_branch = f"unknown (format issue in {pyamrex_cmake_path})" -with open(pyamrex_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*set\(WarpX_pyamrex_branch\s+"(.+)"\s+.*', - f.read(), re.MULTILINE) +with open(pyamrex_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r'.*set\(WarpX_pyamrex_branch\s+"(.+)"\s+.*', f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: pyamrex_branch = r_minimal[0] # minimal (external) version # find_package(AMReX YY.MM CONFIG ... pyamrex_minimal = f"unknown (format issue in {pyamrex_cmake_path})" -with open(pyamrex_cmake_path, encoding='utf-8') as f: - r_minimal = re.findall(r'.*find_package\(pyAMReX\s+(.+)\s+CONFIG\s+.*', - f.read(), re.MULTILINE) +with open(pyamrex_cmake_path, encoding="utf-8") as f: + r_minimal = re.findall( + r".*find_package\(pyAMReX\s+(.+)\s+CONFIG\s+.*", f.read(), re.MULTILINE + ) if len(r_minimal) >= 1: pyamrex_minimal = r_minimal[0] @@ -67,15 +71,19 @@ Please answer the following questions about the version number you want to require from pyAMReX:\n""") -print(f"Currently, WarpX builds against this pyAMReX commit/branch/sha: {pyamrex_branch}") +print( + f"Currently, WarpX builds against this pyAMReX commit/branch/sha: {pyamrex_branch}" +) print(f"pyAMReX HEAD commit (development branch): {pyamrex_HEAD}") -pyamrex_new_branch = input(f"Update pyAMReX commit/branch/sha: ").strip() +pyamrex_new_branch = input("Update pyAMReX commit/branch/sha: ").strip() if not pyamrex_new_branch: - pyamrex_new_branch = pyamrex_branch - print(f"--> Nothing entered, will keep: {pyamrex_branch}") + pyamrex_new_branch = pyamrex_HEAD + print(f"--> Nothing entered, will use: {pyamrex_HEAD}") print() -print(f"Currently, a pre-installed pyAMReX is required at least at version: {pyamrex_minimal}") +print( + f"Currently, a pre-installed pyAMReX is required at least at version: {pyamrex_minimal}" +) today = datetime.date.today().strftime("%y.%m") pyamrex_new_minimal = input(f"New minimal pyAMReX version (e.g. {today})? ").strip() if not pyamrex_new_minimal: @@ -88,7 +96,7 @@ REPLY = input("Is this information correct? Will now start updating! [y/N] ") print() -if not REPLY in ["Y", "y"]: +if REPLY not in ["Y", "y"]: print("You did not confirm with 'y', aborting.") sys.exit(1) @@ -96,24 +104,28 @@ # Updates ##################################################################### # WarpX references to pyAMReX: cmake/dependencies/pyAMReX.cmake -with open(pyamrex_cmake_path, encoding='utf-8') as f: +with open(pyamrex_cmake_path, encoding="utf-8") as f: pyAMReX_cmake_content = f.read() # branch/commit/tag (git fetcher) version # set(WarpX_pyamrex_branch "development" ... pyAMReX_cmake_content = re.sub( r'(.*set\(WarpX_pyamrex_branch\s+")(.+)("\s+.*)', - r'\g<1>{}\g<3>'.format(pyamrex_new_branch), - pyAMReX_cmake_content, flags = re.MULTILINE) + r"\g<1>{}\g<3>".format(pyamrex_new_branch), + pyAMReX_cmake_content, + flags=re.MULTILINE, + ) # minimal (external) version # find_package(AMReX YY.MM CONFIG ... pyAMReX_cmake_content = re.sub( - r'(.*find_package\(pyAMReX\s+)(.+)(\s+CONFIG\s+.*)', - r'\g<1>{}\g<3>'.format(pyamrex_new_minimal), - pyAMReX_cmake_content, flags = re.MULTILINE) + r"(.*find_package\(pyAMReX\s+)(.+)(\s+CONFIG\s+.*)", + r"\g<1>{}\g<3>".format(pyamrex_new_minimal), + pyAMReX_cmake_content, + flags=re.MULTILINE, + ) -with open(pyamrex_cmake_path, "w", encoding='utf-8') as f: +with open(pyamrex_cmake_path, "w", encoding="utf-8") as f: f.write(pyAMReX_cmake_content) diff --git a/Tools/Release/weeklyUpdate.py b/Tools/Release/weeklyUpdate.py new file mode 100755 index 00000000000..6c32993f79e --- /dev/null +++ b/Tools/Release/weeklyUpdate.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl +# + +# This file is a maintainer tool to open a weekly dependency update PR for WarpX. +# +# You also need to have git and the GitHub CLI tool "gh" installed and properly +# configured for it to work: +# https://cli.github.com/ +# +import subprocess +import sys +from pathlib import Path + +# Maintainer Inputs ########################################################### + +print("""Hi there, this is a WarpX maintainer tool to ...\n. +For it to work, you need write access on the source directory and +you should be working in a clean git branch without ongoing +rebase/merge/conflict resolves and without unstaged changes.""") + +# check source dir +REPO_DIR = Path(__file__).parent.parent.parent.absolute() +print(f"\nYour current source directory is: {REPO_DIR}") + +REPLY = input("Are you sure you want to continue? [y/N] ") +print() +if REPLY not in ["Y", "y"]: + print("You did not confirm with 'y', aborting.") + sys.exit(1) + +update_repo = input("What is the name of your git remote? (e.g., ax3l) ") +commit_sign = input("How to sign the commit? (e.g., -sS) ") + + +# Helpers ##################################################################### + + +def concat_answers(answers): + return "\n".join(answers) + "\n" + + +# Stash current work ########################################################## + +subprocess.run(["git", "stash"], capture_output=True, text=True) + + +# Git Branch ################################################################## + +update_branch = "topic-amrexWeekly" +subprocess.run(["git", "checkout", "development"], capture_output=True, text=True) +subprocess.run(["git", "fetch"], capture_output=True, text=True) +subprocess.run(["git", "pull", "--ff-only"], capture_output=True, text=True) +subprocess.run(["git", "branch", "-D", update_branch], capture_output=True, text=True) +subprocess.run(["git", "checkout", "-b", update_branch], capture_output=True, text=True) + + +# AMReX New Version ########################################################### + +answers = concat_answers(["y", "", "", "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updateAMReX.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +amrex_diff = subprocess.run(["git", "diff", "--cached"], capture_output=True, text=True) +print("AMReX Commit...") +subprocess.run( + ["git", "commit", commit_sign, "-m", "AMReX: Weekly Update"], + capture_output=True, + text=True, +) + + +# PICSAR New Version ########################################################## + +PICSAR_version = "25.01" +answers = concat_answers(["y", PICSAR_version, PICSAR_version, "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updatePICSAR.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +picsar_diff = subprocess.run( + ["git", "diff", "--cached"], capture_output=True, text=True +) +print("PICSAR Commit...") +subprocess.run( + ["git", "commit", commit_sign, "-m", "PICSAR: Weekly Update"], + capture_output=True, + text=True, +) + + +# pyAMReX New Version ######################################################### + +answers = concat_answers(["y", "", "", "y"]) + +process = subprocess.Popen( + [Path(REPO_DIR).joinpath("Tools/Release/updatepyAMReX.py")], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +process.communicate(answers) +del process + +# commit +subprocess.run(["git", "add", "-u"], capture_output=True, text=True) +pyamrex_diff = subprocess.run( + ["git", "diff", "--cached"], capture_output=True, text=True +) +print("pyAMReX Commit...") +subprocess.run( + ["git", "commit", commit_sign, "-m", "pyAMReX: Weekly Update"], + capture_output=True, + text=True, +) + +# GitHub PR ################################################################### + +subprocess.run(["git", "push", "-f", "-u", update_repo, update_branch], text=True) + +amrex_changes = " (no changes)" if amrex_diff.stdout == "" else "" +picsar_changes = " (no changes)" if picsar_diff.stdout == "" else "" +pyamrex_changes = " (no changes)" if pyamrex_diff.stdout == "" else "" + +subprocess.run( + [ + "gh", + "pr", + "create", + "--title", + "AMReX/pyAMReX/PICSAR: Weekly Update", + "--body", + f"""Weekly update to latest AMReX{amrex_changes}. +Weekly update to latest pyAMReX{pyamrex_changes}. +Weekly update to latest PICSAR{picsar_changes}. + +```console +./Tools/Release/updateAMReX.py +./Tools/Release/updatepyAMReX.py +./Tools/Release/updatePICSAR.py +``` +""", + "--label", + "component: documentation", + "--label", + "component: third party", + "--web", + ], + text=True, +) + + +# Epilogue #################################################################### + +print("""Done. Please check your source, e.g. via + git diff +now and commit the changes if no errors occurred.""") diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 3cba4346421..8aaff6e4450 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -8,7 +8,9 @@ module load cpe/23.12 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray module load CCE-GPU-3.0.0 -module load amd-mixed/5.2.3 +module load amd-mixed/5.7.1 +module load develop +module load cmake/3.27.9 # optional: for PSATD in RZ geometry support export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH diff --git a/Tools/machines/cori-nersc/cori_gpu.sbatch b/Tools/machines/cori-nersc/cori_gpu.sbatch deleted file mode 100644 index 7ccfcbf277f..00000000000 --- a/Tools/machines/cori-nersc/cori_gpu.sbatch +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -l - -# Copyright 2021 Axel Huebl -# This file is part of WarpX. -# License: BSD-3-Clause-LBNL -# -# Ref: -# - https://docs-dev.nersc.gov/cgpu/hardware/ -# - https://docs-dev.nersc.gov/cgpu/access/ -# - https://docs-dev.nersc.gov/cgpu/usage/#controlling-task-and-gpu-binding - -# Just increase this number of you need more nodes. -#SBATCH -N 2 -#SBATCH -t 03:00:00 -#SBATCH -J -#SBATCH -A m1759 -#SBATCH -q regular -#SBATCH -C gpu -# 8 V100 GPUs (16 GB) per node -#SBATCH --gres=gpu:8 -#SBATCH --exclusive -# one MPI rank per GPU (a quarter-socket) -#SBATCH --tasks-per-node=8 -# request all logical (virtual) cores per quarter-socket -#SBATCH --cpus-per-task=10 -#SBATCH -e WarpX.e%j -#SBATCH -o WarpX.o%j - - -# each Cori GPU node has 2 sockets of Intel Xeon Gold 6148 ('Skylake') @ 2.40 GHz -export WARPX_NMPI_PER_NODE=8 - -# each MPI rank per half-socket has 10 physical cores -# or 20 logical (virtual) cores -# we split half-sockets again by 2 to have one MPI rank per GPU -# over-subscribing each physical core with 2x -# hyperthreading leads to often to slight speedup on Intel -# the settings below make sure threads are close to the -# controlling MPI rank (process) per half socket and -# distribute equally over close-by physical cores and, -# for N>20, also equally over close-by logical cores -export OMP_PROC_BIND=spread -export OMP_PLACES=threads -export OMP_NUM_THREADS=10 - -# for async_io support: (optional) -export MPICH_MAX_THREAD_SAFETY=multiple - -EXE="" - -srun --cpu_bind=cores --gpus-per-task=1 --gpu-bind=map_gpu:0,1,2,3,4,5,6,7 \ - -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) \ - ${EXE} \ - > output.txt diff --git a/Tools/machines/cori-nersc/cori_haswell.sbatch b/Tools/machines/cori-nersc/cori_haswell.sbatch deleted file mode 100644 index bce5bfcaa3a..00000000000 --- a/Tools/machines/cori-nersc/cori_haswell.sbatch +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -l - -# Just increase this number of you need more nodes. -#SBATCH -N 1 -#SBATCH -t 03:00:00 -#SBATCH -q regular -#SBATCH -C haswell -#SBATCH -J -#SBATCH -A -#SBATCH -e error.txt -#SBATCH -o output.txt -# one MPI rank per half-socket (see below) -#SBATCH --tasks-per-node=4 -# request all logical (virtual) cores per half-socket -#SBATCH --cpus-per-task=16 - - -# each Cori Haswell node has 2 sockets of Intel Xeon E5-2698 v3 -# each Xeon CPU is divided into 2 bus rings that each have direct L3 access -export WARPX_NMPI_PER_NODE=4 - -# each MPI rank per half-socket has 8 physical cores -# or 16 logical (virtual) cores -# over-subscribing each physical core with 2x -# hyperthreading leads to a slight (3.5%) speedup -# the settings below make sure threads are close to the -# controlling MPI rank (process) per half socket and -# distribute equally over close-by physical cores and, -# for N>8, also equally over close-by logical cores -export OMP_PROC_BIND=spread -export OMP_PLACES=threads -export OMP_NUM_THREADS=16 - -# for async_io support: (optional) -export MPICH_MAX_THREAD_SAFETY=multiple - -EXE="" - -srun --cpu_bind=cores -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) \ - ${EXE} \ - > output.txt diff --git a/Tools/machines/cori-nersc/cori_knl.sbatch b/Tools/machines/cori-nersc/cori_knl.sbatch deleted file mode 100644 index 3c969827599..00000000000 --- a/Tools/machines/cori-nersc/cori_knl.sbatch +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -l - -# Copyright 2019 Maxence Thevenet -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - - -#SBATCH -N 2 -#SBATCH -t 01:00:00 -#SBATCH -q regular -#SBATCH -C knl -#SBATCH -S 4 -#SBATCH -J -#SBATCH -A -#SBATCH -e WarpX.e%j -#SBATCH -o WarpX.o%j - -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# KNLs have 4 hyperthreads max -export CORI_MAX_HYPETHREAD_LEVEL=4 -# We use 64 cores out of the 68 available on Cori KNL, -# and leave 4 to the system (see "#SBATCH -S 4" above). -export CORI_NCORES_PER_NODE=64 - -# Typically use 8 MPI ranks per node without hyperthreading, -# i.e., OMP_NUM_THREADS=8 -export WARPX_NMPI_PER_NODE=8 -export WARPX_HYPERTHREAD_LEVEL=1 - -# Compute OMP_NUM_THREADS and the thread count (-c option) -export CORI_NHYPERTHREADS_MAX=$(( ${CORI_MAX_HYPETHREAD_LEVEL} * ${CORI_NCORES_PER_NODE} )) -export WARPX_NTHREADS_PER_NODE=$(( ${WARPX_HYPERTHREAD_LEVEL} * ${CORI_NCORES_PER_NODE} )) -export OMP_NUM_THREADS=$(( ${WARPX_NTHREADS_PER_NODE} / ${WARPX_NMPI_PER_NODE} )) -export WARPX_THREAD_COUNT=$(( ${CORI_NHYPERTHREADS_MAX} / ${WARPX_NMPI_PER_NODE} )) - -# for async_io support: (optional) -export MPICH_MAX_THREAD_SAFETY=multiple - -srun --cpu_bind=cores -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) -c ${WARPX_THREAD_COUNT} \ - \ - > output.txt diff --git a/Tools/machines/cori-nersc/gpu_warpx.profile.example b/Tools/machines/cori-nersc/gpu_warpx.profile.example deleted file mode 100644 index 0d9d364cb1c..00000000000 --- a/Tools/machines/cori-nersc/gpu_warpx.profile.example +++ /dev/null @@ -1,32 +0,0 @@ -export proj="m1759" - -module purge -module load modules -module load cgpu -module load esslurm -module load gcc/8.3.0 cuda/11.4.0 cmake/3.22.1 -module load openmpi - -export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/cori_gpu/adios2-2.7.1-install:$CMAKE_PREFIX_PATH - -if [ -d "$HOME/sw/cori_gpu/venvs/cori_gpu_warpx" ] -then - source $HOME/sw/cori_gpu/venvs/cori_gpu_warpx/bin/activate -fi - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=$(which g++) - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 - -# allocate a GPU, e.g. to compile on -# 10 logical cores (5 physical), 1 GPU -function getNode() { - salloc -C gpu -N 1 -t 30 -c 10 --gres=gpu:1 -A $proj -} diff --git a/Tools/machines/cori-nersc/haswell_warpx.profile.example b/Tools/machines/cori-nersc/haswell_warpx.profile.example deleted file mode 100644 index f636b1ab555..00000000000 --- a/Tools/machines/cori-nersc/haswell_warpx.profile.example +++ /dev/null @@ -1,17 +0,0 @@ -module swap PrgEnv-intel PrgEnv-gnu -module load cmake/3.22.1 -module switch cray-libsci cray-libsci/20.09.1 -module load cray-hdf5-parallel/1.10.5.2 -module load cray-fftw/3.3.8.10 -module load cray-python/3.9.7.1 - -export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH - -if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ] -then - source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate -fi diff --git a/Tools/machines/cori-nersc/knl_warpx.profile.example b/Tools/machines/cori-nersc/knl_warpx.profile.example deleted file mode 100644 index fdd7e14d594..00000000000 --- a/Tools/machines/cori-nersc/knl_warpx.profile.example +++ /dev/null @@ -1,21 +0,0 @@ -module swap craype-haswell craype-mic-knl -module swap PrgEnv-intel PrgEnv-gnu -module load cmake/3.22.1 -module switch cray-libsci cray-libsci/20.09.1 -module load cray-hdf5-parallel/1.10.5.2 -module load cray-fftw/3.3.8.10 -module load cray-python/3.9.7.1 - -export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH - -if [ -d "$HOME/sw/knl/venvs/knl_warpx" ] -then - source $HOME/sw/knl/venvs/knl_warpx/bin/activate -fi - -export CXXFLAGS="-march=knl" -export CFLAGS="-march=knl" diff --git a/Tools/machines/dane-llnl/dane.sbatch b/Tools/machines/dane-llnl/dane.sbatch new file mode 100644 index 00000000000..b2a114b3f1b --- /dev/null +++ b/Tools/machines/dane-llnl/dane.sbatch @@ -0,0 +1,39 @@ +#!/bin/bash -l + +# Just increase this number of you need more nodes. +#SBATCH -N 2 +#SBATCH -t 24:00:00 +#SBATCH -A + +#SBATCH -J WarpX +#SBATCH -q pbatch +#SBATCH --qos=normal +#SBATCH --license=lustre1,lustre2 +#SBATCH --export=ALL +#SBATCH -e error.txt +#SBATCH -o output.txt +# one MPI rank per half-socket (see below) +#SBATCH --tasks-per-node=2 +# request all logical (virtual) cores per half-socket +#SBATCH --cpus-per-task=112 + + +# each Dane node has 2 sockets of Intel Sapphire Rapids with 56 cores each +export WARPX_NMPI_PER_NODE=2 + +# each MPI rank per half-socket has 56 physical cores +# or 112 logical (virtual) cores +# over-subscribing each physical core with 2x +# hyperthreading led to a slight (3.5%) speedup on Cori's Intel Xeon E5-2698 v3, +# so we do the same here +# the settings below make sure threads are close to the +# controlling MPI rank (process) per half socket and +# distribute equally over close-by physical cores and, +# for N>9, also equally over close-by logical cores +export OMP_PROC_BIND=spread +export OMP_PLACES=threads +export OMP_NUM_THREADS=112 + +EXE="" # e.g. ./warpx + +srun --cpu_bind=cores -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) ${EXE} diff --git a/Tools/machines/dane-llnl/dane_warpx.profile.example b/Tools/machines/dane-llnl/dane_warpx.profile.example new file mode 100644 index 00000000000..dcb895509cc --- /dev/null +++ b/Tools/machines/dane-llnl/dane_warpx.profile.example @@ -0,0 +1,54 @@ +# please set your project account +#export proj="" # edit this and comment in + +# required dependencies +module load cmake/3.26.3 +module load clang/14.0.6-magic +module load mvapich2/2.3.7 + +# optional: for PSATD support +module load fftw/3.3.10 + +# optional: for QED lookup table generation support +module load boost/1.80.0 + +# optional: for openPMD support +module load hdf5-parallel/1.14.0 + +SW_DIR="/usr/workspace/${USER}/dane" +export CMAKE_PREFIX_PATH=${SW_DIR}/install/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/install/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${SW_DIR}/install/adios2-2.8.3/bin:${PATH} + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/install/blaspp-2024.10.26:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/install/lapackpp-2024.10.26:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/install/blaspp-2024.10.26/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/install/lapackpp-2024.10.26/lib64:$LD_LIBRARY_PATH + +# optional: for Python bindings +module load python/3.12.2 + +if [ -d "${SW_DIR}/venvs/warpx-dane" ] +then + source ${SW_DIR}/venvs/warpx-dane/bin/activate +fi + +# optional: an alias to request an interactive node for two hours +alias getNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=56 -p pdebug --pty bash" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=56 -p pdebug" + +# fix system defaults: do not escape $ with a \ on tab completion +shopt -s direxpand + +# optimize CPU microarchitecture for Intel Sapphire Rapids +# note: the cc/CC/ftn wrappers below add those +export CXXFLAGS="-march=sapphirerapids" +export CFLAGS="-march=sapphirerapids" + +# compiler environment hints +export CC=$(which clang) +export CXX=$(which clang++) +export FC=$(which gfortran) diff --git a/Tools/machines/dane-llnl/install_dependencies.sh b/Tools/machines/dane-llnl/install_dependencies.sh new file mode 100755 index 00000000000..06bee0cead8 --- /dev/null +++ b/Tools/machines/dane-llnl/install_dependencies.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, David Grote +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was dane_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your dane_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# The src directory should have already been created when cloning WarpX ####### +# +SW_DIR="/usr/workspace/${USER}/dane" +rm -rf ${SW_DIR}/install +mkdir -p ${SW_DIR}/install + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with ${HOME} and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d ${SW_DIR}/src/c-blosc ] +then + cd ${SW_DIR}/src/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SW_DIR}/src/c-blosc +fi +cmake -S ${SW_DIR}/src/c-blosc -B ${build_dir}/c-blosc-dane-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/install/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-dane-build --target install --parallel 6 + +# ADIOS2 +if [ -d ${SW_DIR}/src/adios2 ] +then + cd ${SW_DIR}/src/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SW_DIR}/src/adios2 +fi +cmake -S ${SW_DIR}/src/adios2 -B ${build_dir}/adios2-dane-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/install/adios2-2.8.3 +cmake --build ${build_dir}/adios2-dane-build --target install -j 6 + +# BLAS++ (for PSATD+RZ) +if [ -d ${SW_DIR}/src/blaspp ] +then + cd ${SW_DIR}/src/blaspp + git fetch --prune + git checkout v2024.10.26 + cd - +else + git clone -b v2024.10.26 https://github.com/icl-utk-edu/blaspp.git ${SW_DIR}/src/blaspp +fi +cmake -S ${SW_DIR}/src/blaspp -B ${build_dir}/blaspp-dane-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/install/blaspp-2024.10.26 +cmake --build ${build_dir}/blaspp-dane-build --target install --parallel 6 + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SW_DIR}/src/lapackpp ] +then + cd ${SW_DIR}/src/lapackpp + git fetch --prune + git checkout v2024.10.26 + cd - +else + git clone -b v2024.10.26 https://github.com/icl-utk-edu/lapackpp.git ${SW_DIR}/src/lapackpp +fi +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SW_DIR}/src/lapackpp -B ${build_dir}/lapackpp-dane-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/install/lapackpp-2024.10.26 +cmake --build ${build_dir}/lapackpp-dane-build --target install --parallel 6 + + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/warpx-dane +python3 -m venv ${SW_DIR}/venvs/warpx-dane +source ${SW_DIR}/venvs/warpx-dane/bin/activate +python3 -m pip install --upgrade pip +#python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SW_DIR}/src/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch + + +# remove build temporary directory ############################################ +# +rm -rf ${build_dir} diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 3ea78625b78..820cf7069fd 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,7 +23,6 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 19b9ae12e24..08d0c95ee4b 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,7 +25,6 @@ spack: - cuda - fftw - hdf5 - - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index 1eb7d4074a7..b658f1e009d 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,7 +22,6 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 7eee1baa13c..45c9b0f776e 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,7 +21,6 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 - - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index f59f2d3d058..b51946ce832 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -6,11 +6,15 @@ export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi # required dependencies -module load cmake/3.23.2 +module load cmake/3.27.9 module load craype-accel-amd-gfx90a module load rocm/5.7.1 module load cray-mpich/8.1.28 module load cce/17.0.0 # must be loaded after rocm +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#compatible-compiler-rocm-toolchain-versions + +# Fix for OpenMP Runtime (OLCFHELP-21543) +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ROCM_PATH}/llvm/lib # optional: faster builds module load ccache @@ -26,11 +30,11 @@ export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRA export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.79.0 +module load boost/1.85.0 # optional: for openPMD support -module load adios2/2.8.3-mpi -module load hdf5/1.12.1-mpi +module load adios2/2.10.0-mpi +module load hdf5/1.14.3-mpi # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index fd1d28e76b5..8e8565788bc 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -87,11 +87,20 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# cupy and h5py need an older Cython +# cupy needs an older Cython # https://github.com/cupy/cupy/issues/4610 -# https://github.com/h5py/h5py/issues/2268 python3 -m pip install --upgrade "cython<3.0" +# cupy for ROCm +# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source +# https://github.com/cupy/cupy/issues/7830 +CC=cc CXX=CC \ +CUPY_INSTALL_USE_HIP=1 \ +ROCM_HOME=${ROCM_PATH} \ +HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ + python3 -m pip install -v cupy +python3 -m pip install --upgrade "cython>=3.0" # for latest mpi4py and everything else python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade h5py python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy MPICC="cc -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py @@ -100,14 +109,6 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# cupy for ROCm -# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source -# https://github.com/cupy/cupy/issues/7830 -CC=cc CXX=CC \ -CUPY_INSTALL_USE_HIP=1 \ -ROCM_HOME=${ROCM_PATH} \ -HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ - python3 -m pip install -v cupy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 #python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch index 0353c08456f..4814c439dd9 100644 --- a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch +++ b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch @@ -26,8 +26,7 @@ INPUTS=inputs # per node are 2x 2.4 GHz Intel Xeon Gold 6148 # note: the system seems to only expose cores (20 per socket), # not hyperthreads (40 per socket) -export SRUN_CPUS_PER_TASK=20 -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example index 017613f9d60..970dc980347 100644 --- a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example +++ b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example @@ -6,7 +6,7 @@ export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi # required dependencies -module load cmake/3.22.1 +module load cmake/3.30.2 module load gcc/11.2.0 module load cuda/11.7.1 module load openmpi/4.1.2/gcc.11.2.0 diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index b585c2702b6..c4c31dd4066 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -118,6 +118,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade pipx python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index cba455f3d29..33d8462b55f 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.29.0 +python -m pip install --user --upgrade picmistandard==0.33.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch index 6171ff03abc..ccb4f3dc2c3 100644 --- a/Tools/machines/karolina-it4i/karolina_gpu.sbatch +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -25,13 +25,12 @@ #SBATCH -o stdout_%j #SBATCH -e stderr_%j -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 -export SRUN_CPUS_PER_TASK=16 - # set user rights to u=rwx;g=r-x;o=--- umask 0027 +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + # executable & inputs file or python interpreter & PICMI script here EXE=./warpx.rz INPUTS=./inputs_rz diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml index 1cb6a4ac209..ead49f06fab 100644 --- a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -20,7 +20,7 @@ spack: - py-cython - py-mpi4py - py-numpy@1.24.2 - - openpmd-api@0.15.2 +python + - openpmd-api@0.16.1 +python - py-periodictable@1.5.0 - py-h5py # optional diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 1b14159cd22..86f330060f6 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -114,7 +114,7 @@ rm -rf ${SW_DIR}/venvs/warpx-lassen-toss3 python3 -m venv ${SW_DIR}/venvs/warpx-lassen-toss3 source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate python3 -m pip install --upgrade pip -python3 -m pip cache purge +# python3 -m pip cache purge # error: pip cache commands can not function since cache is disabled python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index 98e8d6410b3..99e61a2fbf6 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -2,7 +2,7 @@ #export proj="" # edit this and comment in # required dependencies -module load cmake/3.23.1 +module load cmake/3.29.2 module load gcc/11.2.1 module load cuda/12.0.0 diff --git a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example index 8db2b44b8a7..62f80433233 100644 --- a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example +++ b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example @@ -2,19 +2,16 @@ #export proj="" # change me, e.g., ac_blast # required dependencies -module load cmake/3.24.1 -module load cuda/11.4 -module load gcc/7.4.0 -module load openmpi/4.0.1-gcc +module load cmake/3.27.7 +module load gcc/11.4.0 +module load cuda/12.2.1 +module load openmpi/4.1.6 # optional: for QED support with detailed tables -module load boost/1.70.0-gcc +module load boost/1.83.0 # optional: for openPMD and PSATD+RZ support -module load hdf5/1.10.5-gcc-p -module load lapack/3.8.0-gcc -# CPU only: -#module load fftw/3.3.8-gcc +module load hdf5/1.14.3 export CMAKE_PREFIX_PATH=$HOME/sw/v100/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/adios2-2.8.3:$CMAKE_PREFIX_PATH @@ -27,7 +24,7 @@ export PATH=$HOME/sw/v100/adios2-2.8.3/bin:$PATH #module load ccache # missing # optional: for Python bindings or libEnsemble -module load python/3.8.8 +module load python/3.11.6-gcc-11.4.0 if [ -d "$HOME/sw/v100/venvs/warpx" ] then diff --git a/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh new file mode 100755 index 00000000000..fd3a2d3f756 --- /dev/null +++ b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was lonestar6_warpx_a100.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lonestar6_warpx_a100.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SW_DIR="${WORK}/sw/lonestar6/sw/lonestar6/a100" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d $HOME/src/c-blosc ] +then + cd $HOME/src/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc +fi +rm -rf $HOME/src/c-blosc-a100-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-a100-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-a100-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-a100-build + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-a100-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-a100-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-a100-build --target install -j 16 +rm -rf ${build_dir}/adios2-a100-build + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-a100-build +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-a100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-a100-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-a100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-a100-build +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-a100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-a100-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-a100-build + +# Python ###################################################################### +# +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +python3 -m pip cache purge +rm -rf ${SW_DIR}/venvs/warpx-a100 +python3 -m venv ${SW_DIR}/venvs/warpx-a100 +source ${SW_DIR}/venvs/warpx-a100/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +#python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +#python3 -m pip install --upgrade torch # CUDA 12 compatible wheel +#python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch b/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch new file mode 100644 index 00000000000..933f21093a2 --- /dev/null +++ b/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch @@ -0,0 +1,42 @@ +#!/bin/bash -l + +# Copyright 2021-2022 Axel Huebl, Kevin Gott +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +#SBATCH -t 00:10:00 +#SBATCH -N 2 +#SBATCH -J WarpX +# note: must end on _g +#SBATCH -A +#SBATCH -q regular +#SBATCH -C gpu +#SBATCH --exclusive +#SBATCH --cpus-per-task=32 +#SBATCH --gpu-bind=none +#SBATCH --gpus-per-node=4 +#SBATCH -o WarpX.o%j +#SBATCH -e WarpX.e%j + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=inputs_small + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +# threads for OpenMP and threaded compressors per MPI rank +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# depends on https://github.com/ECP-WarpX/WarpX/issues/2009 +#GPU_AWARE_MPI="amrex.the_arena_is_managed=0 amrex.use_gpu_aware_mpi=1" +GPU_AWARE_MPI="" + +# CUDA visible devices are ordered inverse to local task IDs +# Reference: nvidia-smi topo -m +srun --cpu-bind=cores bash -c " + export CUDA_VISIBLE_DEVICES=\$((3-SLURM_LOCALID)); + ${EXE} ${INPUTS} ${GPU_AWARE_MPI}" \ + > output.txt diff --git a/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example new file mode 100644 index 00000000000..57c98da9b4a --- /dev/null +++ b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example @@ -0,0 +1,57 @@ +# please set your project account +#export proj="" # change me + +# required dependencies +module purge +module load TACC +module load gcc/11.2.0 +module load cuda/12.2 +module load cmake +module load mvapich2 + +# optional: for QED support with detailed tables +module load boost/1.84 + +# optional: for openPMD and PSATD+RZ support +module load phdf5/1.10.4 + +SW_DIR="${WORK}/sw/lonestar6/sw/lonestar6/a100" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH} + +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} + +# optional: CCache +#module load ccache # TODO: request from support + +# optional: for Python bindings or libEnsemble +module load python3/3.9.7 + +if [ -d "$WORK/sw/lonestar6/a100/venvs/warpx-a100" ] +then + source $WORK/sw/lonestar6/a100/venvs/warpx-a100/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 -t 1:00:00 -p gpu-100 --gpu-bind=single:1 -c 32 -G 2 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 -t 0:30:00 -p gpu-100 --gpu-bind=single:1 -c 32 -G 2 -A $proj" + +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH=8.0 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 13fb6b1d81e..915f976f4ab 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -2,9 +2,9 @@ #export proj="project_..." # required dependencies -module load LUMI/23.09 partition/G -module load rocm/5.2.3 # waiting for 5.5 for next bump -module load buildtools/23.09 +module load LUMI/24.03 partition/G +module load rocm/6.0.3 +module load buildtools/24.03 # optional: just an additional text editor module load nano @@ -27,7 +27,7 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.10.10 +module load cray-python/3.11.7 if [ -d "${SW_DIR}/venvs/warpx-lumi" ] then diff --git a/Tools/machines/lxplus-cern/lxplus_warpx.profile.example b/Tools/machines/lxplus-cern/lxplus_warpx.profile.example index 3f3770fac3f..e461e4503f0 100644 --- a/Tools/machines/lxplus-cern/lxplus_warpx.profile.example +++ b/Tools/machines/lxplus-cern/lxplus_warpx.profile.example @@ -22,7 +22,7 @@ then # create and activate the spack environment export SPACK_STACK_USE_PYTHON=1 export SPACK_STACK_USE_CUDA=1 - spack env create warpx-lxplus-cuda-py $WORK/WarpX/Tools/machines/lxplus-cern/spack.yaml + spack env create warpx-lxplus-cuda-py $WORK/warpx/Tools/machines/lxplus-cern/spack.yaml spack env activate warpx-lxplus-cuda-py spack install else diff --git a/Tools/machines/ookami-sbu/ookami_warpx.profile.example b/Tools/machines/ookami-sbu/ookami_warpx.profile.example index 321e3ce1d59..dc8c7ac6639 100644 --- a/Tools/machines/ookami-sbu/ookami_warpx.profile.example +++ b/Tools/machines/ookami-sbu/ookami_warpx.profile.example @@ -2,7 +2,7 @@ #export proj= # required dependencies -module load cmake/3.19.0 +module load cmake/3.19.0 # please check for a 3.24+ module and report back module load gcc/10.3.0 module load openmpi/gcc10/4.1.0 diff --git a/Tools/machines/perlmutter-nersc/Containerfile b/Tools/machines/perlmutter-nersc/Containerfile new file mode 100644 index 00000000000..5a8553c2619 --- /dev/null +++ b/Tools/machines/perlmutter-nersc/Containerfile @@ -0,0 +1,167 @@ +# Build this container from its current directory: +# podman build --build-arg NJOBS=6 -t warpx-perlmutter-dev . +# Adjust NJOBS to the number of processes to use for parallel compilation. +# +# Run from the via WarpX source directory +# podman run -v ${PWD}:/opt/warpx -it warpx-perlmutter-dev +# then +# cd /opt/warpx +# and compile. + +# Base System and Essential Tools Installation +FROM nvidia/cuda:12.6.0-devel-ubuntu22.04 AS base + +# parallel builds +ARG NJOBS +ENV NJOBS=$NJOBS + +# Set up environment variables +ENV DEBIAN_FRONTEND=noninteractive \ + SW_DIR=/opt/software \ + FORCE_UNSAFE_CONFIGURE=1 + +# Perlmutter A100 compilation optimization +ENV AMREX_CUDA_ARCH=8.0 \ + CUDAARCHS=80 \ + CXXFLAGS="-march=znver3" \ + CFLAGS="-march=znver3" + +# Install essential system dependencies including MPI libraries +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + autoconf \ + build-essential \ + ca-certificates \ + coreutils \ + curl \ + environment-modules \ + gfortran \ + git \ + openssh-server \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + unzip \ + vim \ + libmpich-dev \ + cmake \ + libblas-dev \ + liblapack-dev \ + g++ \ + pkg-config \ + libbz2-dev \ + zlib1g-dev \ + libpng-dev \ + libzstd-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install c-blosc from source +FROM base AS c-blosc + +RUN git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git /tmp/c-blosc && \ + cmake \ + -S /tmp/c-blosc \ + -B /tmp/c-blosc-build \ + -DCMAKE_INSTALL_PREFIX=/usr .. && \ + cmake --build /tmp/c-blosc-build \ + --target install \ + -j${NJOBS} && \ + rm -rf /tmp/c-blosc* + +# Install ADIOS2 from source +FROM base AS adios2 + +# Ensure c-blosc is installed before ADIOS2 +COPY --from=c-blosc /usr /usr + +# Verify the location of Blosc library +RUN find /usr -name 'libblosc*' + +# Ensure Blosc library paths are correctly configured +ENV BLOSC_LIBRARY=/usr/lib/libblosc.so.1.21.1 +ENV BLOSC_INCLUDE_DIR=/usr/include + +# Install ADIOS2 +RUN git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git /tmp/adios2 && \ + cd /tmp/adios2 && \ + cmake -S . -B build \ + -DADIOS2_USE_Blosc=ON \ + -DBLOSC_LIBRARY=${BLOSC_LIBRARY} \ + -DBLOSC_INCLUDE_DIR=${BLOSC_INCLUDE_DIR} \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DADIOS2_USE_BZip2=ON \ + -DADIOS2_USE_ZFP=OFF \ + -DADIOS2_USE_SZ=OFF \ + -DADIOS2_USE_MGARD=OFF \ + -DADIOS2_USE_PNG=ON \ + -DCMAKE_INSTALL_PREFIX=/usr .. && \ + cmake --build build --target install -j${NJOBS} && \ + rm -rf /tmp/adios2 + +# Install BLAS++ and LAPACK++ +FROM base AS blaspp_lapackpp + +RUN git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git /tmp/blaspp && \ + cd /tmp/blaspp && \ + cmake -S . -B build \ + -Duse_openmp=OFF \ + -Dgpu_backend=cuda \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DBLAS_LIBRARIES=/usr/lib/x86_64-linux-gnu/libblas.so \ + -DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/liblapack.so .. && \ + cmake --build build \ + --target install \ + -j${NJOBS} && \ + rm -rf /tmp/blaspp + +RUN git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git /tmp/lapackpp && \ + cd /tmp/lapackpp && \ + cmake -S . -B build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/liblapack.so .. && \ + cmake --build build \ + --target install -j${NJOBS} && \ + rm -rf /tmp/lapackpp + +# Final Image +FROM base AS final + +# Copy installed software from previous stages +COPY --from=c-blosc /usr /usr +COPY --from=adios2 /usr /usr +COPY --from=blaspp_lapackpp /usr /usr + +# Create and activate Python virtual environment +RUN python3 -m venv /opt/venv && \ + /opt/venv/bin/pip install --no-cache-dir \ + wheel \ + numpy \ + pandas \ + scipy \ + matplotlib \ + jupyter \ + scikit-learn \ + openpmd-api \ + yt \ + cupy-cuda12x \ + torch \ + optimas[all] \ + cython \ + packaging \ + build \ + setuptools + +# Set up the environment for the virtual environment +ENV PATH="/opt/venv/bin:${PATH}" + +# Set up entrypoint +ENTRYPOINT ["/bin/bash", "-c"] + +# Default command +CMD ["/bin/bash"] diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index 437300b8303..0ef14844493 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -31,7 +31,7 @@ fi # Remove old dependencies ##################################################### # -SW_DIR="${CFS}/${proj}/${USER}/sw/perlmutter/cpu" +SW_DIR="${PSCRATCH}/storage/sw/warpx/perlmutter/cpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} @@ -44,9 +44,29 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# build parallelism +PARALLEL=16 + # tmpfs build directory: avoids issues often seen with $HOME and is faster build_dir=$(mktemp -d) +# CCache +curl -Lo ccache.tar.xz https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz +tar -xf ccache.tar.xz +mv ccache-4.10.2-linux-x86_64 ${SW_DIR}/ccache-4.10.2 +rm -rf ccache.tar.xz + +# Boost (QED tables) +rm -rf $HOME/src/boost-temp +mkdir -p $HOME/src/boost-temp +curl -Lo $HOME/src/boost-temp/boost.tar.gz https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzf $HOME/src/boost-temp/boost.tar.gz -C $HOME/src/boost-temp +cd $HOME/src/boost-temp/boost_1_82_0 +./bootstrap.sh --with-libraries=math --prefix=${SW_DIR}/boost-1.82.0 +./b2 cxxflags="-std=c++17" install -j ${PARALLEL} +cd - +rm -rf $HOME/src/boost-temp + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -59,7 +79,7 @@ else fi rm -rf $HOME/src/c-blosc-pm-cpu-build cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel 16 +cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/c-blosc-pm-cpu-build # ADIOS2 @@ -74,7 +94,7 @@ else fi rm -rf $HOME/src/adios2-pm-cpu-build cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j 16 +cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j ${PARALLEL} rm -rf ${build_dir}/adios2-pm-cpu-build # BLAS++ (for PSATD+RZ) @@ -89,7 +109,7 @@ else fi rm -rf $HOME/src/blaspp-pm-cpu-build CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 -cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 +cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/blaspp-pm-cpu-build # LAPACK++ (for PSATD+RZ) @@ -104,48 +124,9 @@ else fi rm -rf $HOME/src/lapackpp-pm-cpu-build CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 +cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/lapackpp-pm-cpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-cpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-cpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ - -DHeffte_ENABLE_AVX=ON \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=ON \ - -DHeffte_ENABLE_CUDA=OFF \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-cpu-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index ac95ad9f3a0..ffa3d0f0714 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -31,7 +31,7 @@ fi # Remove old dependencies ##################################################### # -SW_DIR="${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu" +SW_DIR="${PSCRATCH}/storage/sw/warpx/perlmutter/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} @@ -44,9 +44,29 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# build parallelism +PARALLEL=16 + # tmpfs build directory: avoids issues often seen with $HOME and is faster build_dir=$(mktemp -d) +# CCache +curl -Lo ccache.tar.xz https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz +tar -xf ccache.tar.xz +mv ccache-4.10.2-linux-x86_64 ${SW_DIR}/ccache-4.10.2 +rm -rf ccache.tar.xz + +# Boost (QED tables) +rm -rf $HOME/src/boost-temp +mkdir -p $HOME/src/boost-temp +curl -Lo $HOME/src/boost-temp/boost.tar.gz https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzf $HOME/src/boost-temp/boost.tar.gz -C $HOME/src/boost-temp +cd $HOME/src/boost-temp/boost_1_82_0 +./bootstrap.sh --with-libraries=math --prefix=${SW_DIR}/boost-1.82.0 +./b2 cxxflags="-std=c++17" install -j ${PARALLEL} +cd - +rm -rf $HOME/src/boost-temp + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -59,7 +79,7 @@ else fi rm -rf $HOME/src/c-blosc-pm-gpu-build cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel 16 +cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/c-blosc-pm-gpu-build # ADIOS2 @@ -74,7 +94,7 @@ else fi rm -rf $HOME/src/adios2-pm-gpu-build cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j 16 +cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j ${PARALLEL} rm -rf ${build_dir}/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) @@ -89,7 +109,7 @@ else fi rm -rf $HOME/src/blaspp-pm-gpu-build CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 -cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 +cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) @@ -104,48 +124,9 @@ else fi rm -rf $HOME/src/lapackpp-pm-gpu-build CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 +cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel ${PARALLEL} rm -rf ${build_dir}/lapackpp-pm-gpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-gpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-gpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=ON \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-gpu-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip @@ -173,7 +154,7 @@ python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel # optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) python3 -m pip install --upgrade torch # CUDA 12 compatible wheel python3 -m pip install --upgrade optimas[all] - +python3 -m pip install --upgrade lasy # remove build temporary directory rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch index d13c7e3b4e5..84e93dbb8ea 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch @@ -13,6 +13,8 @@ #SBATCH -A #SBATCH -q regular #SBATCH -C cpu +# 8 cores per chiplet, 2x SMP +#SBATCH --cpus-per-task=16 #SBATCH --ntasks-per-node=16 #SBATCH --exclusive #SBATCH -o WarpX.o%j @@ -30,10 +32,9 @@ INPUTS=inputs_small # This will be our MPI rank assignment (2x8 is 16 ranks/node). # threads for OpenMP and threaded compressors per MPI rank -export SRUN_CPUS_PER_TASK=16 # 8 cores per chiplet, 2x SMP export OMP_PLACES=threads export OMP_PROC_BIND=spread -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} srun --cpu-bind=cores \ ${EXE} ${INPUTS} \ diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 94d598abf5b..fe665e87130 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -7,37 +7,39 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cpu -module load cmake/3.24.3 +module load cmake/3.30.2 module load cray-fftw/3.3.10.6 +# missing modules installed here +export SW_DIR=${PSCRATCH}/storage/sw/warpx/perlmutter/cpu + # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:${CMAKE_PREFIX_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:${LD_LIBRARY_PATH} # optional: for openPMD and PSATD+RZ support module load cray-hdf5-parallel/1.12.2.9 -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH} -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:${LD_LIBRARY_PATH} -export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH +export PATH=${SW_DIR}/ccache-4.10.2:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu" ] +if [ -d "${SW_DIR}/venvs/warpx-cpu" ] then - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate + source ${SW_DIR}/venvs/warpx-cpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch index f2ea5fa3e7f..bd47fa3bd2a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch @@ -17,6 +17,7 @@ # A100 80GB (256 nodes) #S BATCH -C gpu&hbm80g #SBATCH --exclusive +#SBATCH --cpus-per-task=32 # ideally single:1, but NERSC cgroups issue #SBATCH --gpu-bind=none #SBATCH --ntasks-per-node=4 @@ -33,8 +34,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) -export SRUN_CPUS_PER_TASK=16 -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=16 # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index da1d55964d1..dd78bc8ecf3 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -12,36 +12,38 @@ module load craype module load craype-x86-milan module load craype-accel-nvidia80 module load cudatoolkit -module load cmake/3.24.3 +module load cmake/3.30.2 + +# missing modules installed here +export SW_DIR=${PSCRATCH}/storage/sw/warpx/perlmutter/gpu # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:${CMAKE_PREFIX_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:${LD_LIBRARY_PATH} # optional: for openPMD and PSATD+RZ support module load cray-hdf5-parallel/1.12.2.9 -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH} -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:${LD_LIBRARY_PATH} -export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH +export PATH=${SW_DIR}/ccache-4.10.2:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu" ] +if [ -d "${SW_DIR}/venvs/warpx-gpu" ] then - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate + source ${SW_DIR}/venvs/warpx-gpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example index 333434c1b97..e1bd4e0fdd3 100644 --- a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -3,20 +3,24 @@ export proj="" # change me! # swap to GNU programming environment (with gcc 11.2) module swap PrgEnv-nvhpc PrgEnv-gnu -module swap gcc/12.2.0 gcc/11.2.0 -module load nvhpc-mixed/22.11 +module load gcc-native/12.3 +module load nvhpc-mixed/23.9 # swap to the Milan cray package -module swap craype-x86-rome craype-x86-milan +module load craype-x86-milan + +# extra modules +module use /soft/modulefiles +module load spack-pe-gnu # required dependencies -module load cmake/3.23.2 +module load cmake/3.27.7 # optional: for QED support with detailed tables -# module load boost/1.81.0 +module load boost # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.3 +module load hdf5/1.14.3 export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH @@ -30,7 +34,7 @@ export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31/lib64:$L export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load python/3.10.9 if [ -d "/home/${USER}/sw/polaris/gpu/venvs/warpx" ] then @@ -49,7 +53,7 @@ export CXXFLAGS="-march=znver3" export CFLAGS="-march=znver3" # compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) +export CC=$(which gcc-12) +export CXX=$(which g++-12) export CUDACXX=$(which nvcc) export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh deleted file mode 100755 index cfb01769384..00000000000 --- a/Tools/machines/quartz-llnl/install_dependencies.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was quartz_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your quartz_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="/usr/workspace/${USER}/quartz" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# tmpfs build directory: avoids issues often seen with ${HOME} and is faster -build_dir=$(mktemp -d) - -# c-blosc (I/O compression) -if [ -d ${HOME}/src/c-blosc ] -then - cd ${HOME}/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${HOME}/src/c-blosc -fi -cmake -S ${HOME}/src/c-blosc -B ${build_dir}/c-blosc-quartz-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-quartz-build --target install --parallel 6 - -# ADIOS2 -if [ -d ${HOME}/src/adios2 ] -then - cd ${HOME}/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${HOME}/src/adios2 -fi -cmake -S ${HOME}/src/adios2 -B ${build_dir}/adios2-quartz-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-quartz-build --target install -j 6 - -# BLAS++ (for PSATD+RZ) -if [ -d ${HOME}/src/blaspp ] -then - cd ${HOME}/src/blaspp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp -fi -cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 -cmake --build ${build_dir}/blaspp-quartz-build --target install --parallel 6 - -# LAPACK++ (for PSATD+RZ) -if [ -d ${HOME}/src/lapackpp ] -then - cd ${HOME}/src/lapackpp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp -fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -cmake --build ${build_dir}/lapackpp-quartz-build --target install --parallel 6 - - -# Python ###################################################################### -# -python3 -m pip install --upgrade --user virtualenv -rm -rf ${SW_DIR}/venvs/warpx-quartz -python3 -m venv ${SW_DIR}/venvs/warpx-quartz -source ${SW_DIR}/venvs/warpx-quartz/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip cache purge -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt - -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r ${HOME}/src/warpx/requirements.txt - -# ML dependencies -python3 -m pip install --upgrade torch - - -# remove build temporary directory ############################################ -# -rm -rf ${build_dir} diff --git a/Tools/machines/quartz-llnl/quartz.sbatch b/Tools/machines/quartz-llnl/quartz.sbatch deleted file mode 100644 index 4c1a82ff8e9..00000000000 --- a/Tools/machines/quartz-llnl/quartz.sbatch +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -l - -# Just increase this number of you need more nodes. -#SBATCH -N 2 -#SBATCH -t 24:00:00 -#SBATCH -A - -#SBATCH -J WarpX -#SBATCH -q pbatch -#SBATCH --qos=normal -#SBATCH --license=lustre1,lustre2 -#SBATCH --export=ALL -#SBATCH -e error.txt -#SBATCH -o output.txt -# one MPI rank per half-socket (see below) -#SBATCH --tasks-per-node=2 -# request all logical (virtual) cores per half-socket -#SBATCH --cpus-per-task=18 - - -# each Quartz node has 1 socket of Intel Xeon E5-2695 v4 -# each Xeon CPU is divided into 2 bus rings that each have direct L3 access -export WARPX_NMPI_PER_NODE=2 - -# each MPI rank per half-socket has 9 physical cores -# or 18 logical (virtual) cores -# over-subscribing each physical core with 2x -# hyperthreading led to a slight (3.5%) speedup on Cori's Intel Xeon E5-2698 v3, -# so we do the same here -# the settings below make sure threads are close to the -# controlling MPI rank (process) per half socket and -# distribute equally over close-by physical cores and, -# for N>9, also equally over close-by logical cores -export OMP_PROC_BIND=spread -export OMP_PLACES=threads -export OMP_NUM_THREADS=18 - -EXE="" # e.g. ./warpx - -srun --cpu_bind=cores -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) ${EXE} diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example deleted file mode 100644 index f296a0738ff..00000000000 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ /dev/null @@ -1,54 +0,0 @@ -# please set your project account -#export proj="" # edit this and comment in - -# required dependencies -module load cmake/3.23.1 -module load clang/14.0.6-magic -module load mvapich2/2.3.7 - -# optional: for PSATD support -module load fftw/3.3.10 - -# optional: for QED lookup table generation support -module load boost/1.80.0 - -# optional: for openPMD support -module load hdf5-parallel/1.14.0 - -SW_DIR="/usr/workspace/${USER}/quartz" -export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH -export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} - -# optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH - -# optional: for Python bindings -module load python/3.9.12 - -if [ -d "${SW_DIR}/venvs/warpx-quartz" ] -then - source ${SW_DIR}/venvs/warpx-quartz/bin/activate -fi - -# optional: an alias to request an interactive node for two hours -alias getNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=18 -p pdebug --pty bash" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -alias runNode="srun --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=18 -p pdebug" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CPU microarchitecture for Intel Xeon E5-2695 v4 -# note: the cc/CC/ftn wrappers below add those -export CXXFLAGS="-march=broadwell" -export CFLAGS="-march=broadwell" - -# compiler environment hints -export CC=$(which clang) -export CXX=$(which clang++) -export FC=$(which gfortran) diff --git a/Tools/machines/spock-olcf/spock_mi100.sbatch b/Tools/machines/spock-olcf/spock_mi100.sbatch deleted file mode 100644 index 4ee09201b3f..00000000000 --- a/Tools/machines/spock-olcf/spock_mi100.sbatch +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -#SBATCH -A -#SBATCH -J warpx -#SBATCH -o %x-%j.out -#SBATCH -t 00:10:00 -#SBATCH -p ecp -#SBATCH -N 1 - -export OMP_NUM_THREADS=1 -srun -n 4 -c 2 --ntasks-per-node=4 ./warpx inputs > output.txt diff --git a/Tools/machines/spock-olcf/spock_warpx.profile.example b/Tools/machines/spock-olcf/spock_warpx.profile.example deleted file mode 100644 index 6b10656c3bf..00000000000 --- a/Tools/machines/spock-olcf/spock_warpx.profile.example +++ /dev/null @@ -1,29 +0,0 @@ -# please set your project account -#export proj= - -# required dependencies -module load cmake/3.20.2 -module load craype-accel-amd-gfx908 -module load rocm/4.3.0 - -# optional: faster builds -module load ccache -module load ninja - -# optional: just an additional text editor -module load nano - -# optional: an alias to request an interactive node for one hour -alias getNode="salloc -A $proj -J warpx -t 01:00:00 -p ecp -N 1" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CUDA compilation for MI100 -export AMREX_AMD_ARCH=gfx908 - -# compiler environment hints -export CC=$ROCM_PATH/llvm/bin/clang -export CXX=$(which hipcc) -export LDFLAGS="-L${CRAYLIBS_X86_64} $(CC --cray-print-opts=libs) -lmpi" -# GPU aware MPI: ${PE_MPICH_GTL_DIR_gfx908} -lmpi_gtl_hsa diff --git a/Tools/machines/taurus-zih/taurus_warpx.profile.example b/Tools/machines/taurus-zih/taurus_warpx.profile.example index f564f696c4a..434d773067b 100644 --- a/Tools/machines/taurus-zih/taurus_warpx.profile.example +++ b/Tools/machines/taurus-zih/taurus_warpx.profile.example @@ -5,7 +5,7 @@ module load modenv/hiera module load foss/2021b module load CUDA/11.8.0 -module load CMake/3.22.1 +module load CMake/3.27.6 # optional: for QED support with detailed tables #module load Boost # TODO diff --git a/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh new file mode 100644 index 00000000000..95633549698 --- /dev/null +++ b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was tioga_mi300a_warpx.profile sourced and configured correctly? +# early access: not yet used! +#if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your tioga_mi300a_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SRC_DIR="/p/lustre1/${USER}/tioga/src" +SW_DIR="/p/lustre1/${USER}/tioga/warpx/mi300a" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) +build_procs=24 + +# C-Blosc2 (I/O compression) +if [ -d ${SRC_DIR}/c-blosc2 ] +then + cd ${SRC_DIR}/c-blosc2 + git fetch --prune + git checkout v2.15.1 + cd - +else + git clone -b v2.15.1 https://github.com/Blosc/c-blosc2.git ${SRC_DIR}/c-blosc2 +fi +cmake \ + --fresh \ + -S ${SRC_DIR}/c-blosc2 \ + -B ${build_dir}/c-blosc2-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_STATIC=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DDEACTIVATE_AVX512=OFF \ + -DWITH_SANITIZER=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-2.15.1 +cmake \ + --build ${build_dir}/c-blosc2-build \ + --target install \ + --parallel ${build_procs} +rm -rf ${build_dir}/c-blosc2-build + +# ADIOS2 +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +cmake \ + --fresh \ + -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-build \ + -DADIOS2_USE_Blosc2=ON \ + -DADIOS2_USE_Campaign=OFF \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake \ + --build ${build_dir}/adios2-build \ + --target install \ + --parallel ${build_procs} +rm -rf ${build_dir}/adios2-build + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ] +then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +cmake \ + --fresh \ + -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-tioga-mi300a-build \ + -Duse_openmp=OFF \ + -Dgpu_backend=hip \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake \ + --build ${build_dir}/blaspp-tioga-mi300a-build \ + --target install \ + --parallel ${build_procs} +rm -rf ${build_dir}/blaspp-tioga-mi300a-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ] +then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +cmake \ + --fresh \ + -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-tioga-mi300a-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dgpu_backend=hip \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake \ + --build ${build_dir}/lapackpp-tioga-mi300a-build \ + --target install \ + --parallel ${build_procs} +rm -rf ${build_dir}/lapackpp-tioga-mi300a-build + +# Python ###################################################################### +# +# sometimes, the Lassen PIP Index is down +export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" + +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +# python3 -m pip cache purge || true # Cache disabled on system +rm -rf ${SW_DIR}/venvs/warpx-trioga-mi300a +python3 -m venv ${SW_DIR}/venvs/warpx-trioga-mi300a +source ${SW_DIR}/venvs/warpx-trioga-mi300a/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade openpmd-viewer +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt +# cupy for ROCm +# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source +# https://docs.cupy.dev/en/stable/install.html#using-cupy-on-amd-gpu-experimental +# https://github.com/cupy/cupy/issues/7830 +# https://github.com/cupy/cupy/pull/8457 +# https://github.com/cupy/cupy/pull/8319 +#python3 -m pip install --upgrade "cython<3" +#HIPCC=${CXX} \ +#CXXFLAGS="-I${ROCM_PATH}/include/hipblas -I${ROCM_PATH}/include/hipsparse -I${ROCM_PATH}/include/hipfft -I${ROCM_PATH}/include/rocsolver -I${ROCM_PATH}/include/rccl" \ +#CUPY_INSTALL_USE_HIP=1 \ +#ROCM_HOME=${ROCM_PATH} \ +#HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ +# python3 -m pip install -v cupy +#python3 -m pip install --upgrade "cython>=3" + + +# for ML dependencies, see install_mi300a_ml.sh + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/tioga-llnl/install_mi300a_ml.sh b/Tools/machines/tioga-llnl/install_mi300a_ml.sh new file mode 100644 index 00000000000..178deed9975 --- /dev/null +++ b/Tools/machines/tioga-llnl/install_mi300a_ml.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was tioga_mi300a_warpx.profile sourced and configured correctly? +# early access: not yet used! +#if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your tioga_mi300a_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SRC_DIR="/p/lustre1/${USER}/tioga/src" +SW_DIR="/p/lustre1/${USER}/tioga/warpx/mi300a" + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qqq -y torch 2>/dev/null || true + + +# Python ML ################################################################### +# +# for basic python dependencies, see install_mi300a_dependencies.sh + +# sometimes, the Lassen PIP Index is down +export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" + +source ${SW_DIR}/venvs/warpx-trioga-mi300a/bin/activate + +python3 -m pip install --upgrade torch torchvision --index-url https://download.pytorch.org/whl/rocm6.1 +python3 -m pip install --upgrade scikit-learn +python3 -m pip install --upgrade "optimas[all]" diff --git a/Tools/machines/tioga-llnl/tioga_mi300a.sbatch b/Tools/machines/tioga-llnl/tioga_mi300a.sbatch new file mode 100644 index 00000000000..94ee97bc6a1 --- /dev/null +++ b/Tools/machines/tioga-llnl/tioga_mi300a.sbatch @@ -0,0 +1,44 @@ +#!/bin/bash -l + +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Joshua David Ludwig +# License: BSD-3-Clause-LBNL + +#SBATCH -t 00:30:00 +#SBATCH -N 1 +#SBATCH -J WarpX +#S BATCH -A # project name not needed yet +#SBATCH -p mi300a +#SBATCH --cpus-per-task=16 +#SBATCH --gpu-bind=none +#SBATCH --ntasks-per-node=4 +#SBATCH --gpus-per-node=4 +#SBATCH -o WarpX.o%j +#SBATCH -e WarpX.e%j + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=inputs + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +# threads for OpenMP and threaded compressors per MPI rank +# note: 16 avoids hyperthreading (32 virtual cores, 16 physical) +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# GPU-aware MPI optimizations +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# APUs share memory with the host: +# Do NOT pre-allocate a large heap in AMReX +APU_SHARED_MEMORY="amrex.the_arena_init_size=1" + +# MPI parallel processes +srun \ + ${EXE} ${INPUTS} \ + ${GPU_AWARE_MPI} ${APU_SHARED_MEMORY} \ + > output.txt diff --git a/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example new file mode 100644 index 00000000000..53fe21844c1 --- /dev/null +++ b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example @@ -0,0 +1,67 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +# early access: not yet used +# if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi + +# required dependencies +module purge +module load PrgEnv-cray-amd/8.5.0 +# module load rocmcc/6.1.2-cce-18.0.0-magic +module load craype-x86-genoa # CPU +module load craype-accel-amd-gfx942 # GPU +module load cmake/3.29.2 +module load cray-mpich +module load cray-libsci +module load rocm/6.1.2 + +# optional: faster builds +# ccache is system provided +module load ninja/1.10.2 + +# optional: for QED support with detailed tables +# TODO: no Boost module found + +# optional: for openPMD and PSATD+RZ support +SW_DIR="/p/lustre1/${USER}/tioga/warpx/mi300a" +module load cray-hdf5-parallel/1.12.2.11 +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-2.15.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-2.15.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# python +module load cray-python/3.11.7 + +if [ -d "${SW_DIR}/venvs/warpx-trioga-mi300a" ] +then + source ${SW_DIR}/venvs/warpx-trioga-mi300a/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 -p mi300a -t 1:00:00" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -p mi300a" + +# GPU-aware MPI +export MPICH_GPU_SUPPORT_ENABLED=1 + +# optimize ROCm/HIP compilation for MI300A +export AMREX_AMD_ARCH=gfx942 + +# compiler environment hints +export CC=$(which cc) +export CXX=$(which CC) +export FC=$(which ftn) +export HIPCXX=${CXX} diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 8536d08f67a..df00423e823 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -37,6 +37,7 @@ set(WarpX_OPENPMD_FOUND ${WarpX_OPENPMD}) # TODO #WarpX_ASCENT "Ascent in situ diagnostics" OFF) +#WarpX_CATALYST "Catalyst in situ diagnostics" OFF) #WarpX_FFT "FFT-based solvers" OFF) #WarpX_SENSEI "SENSEI in situ diagnostics" OFF) #WarpX_QED "QED support (requires PICSAR)" ON) diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index e05b4c3afc4..543d0cd0ce4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -301,12 +301,16 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".ASCENT") endif() + if(WarpX_CATALYST) + set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".CATALYST") + endif() + if(WarpX_OPENPMD) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".OPMD") endif() if(WarpX_FFT) - set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD") + set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".FFT") endif() if(WarpX_EB) @@ -441,6 +445,7 @@ function(warpx_print_summary) message(" Build options:") message(" APP: ${WarpX_APP}") message(" ASCENT: ${WarpX_ASCENT}") + message(" CATALYST: ${WarpX_CATALYST}") message(" COMPUTE: ${WarpX_COMPUTE}") message(" DIMS: ${WarpX_DIMS}") message(" Embedded Boundary: ${WarpX_EB}") @@ -453,7 +458,6 @@ function(warpx_print_summary) message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") message(" FFT Solvers: ${WarpX_FFT}") - message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f1e5b3f62e2..7a249cd6c5b 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -26,6 +26,11 @@ macro(find_amrex) set(AMReX_CONDUIT ON CACHE INTERNAL "") endif() + if(WarpX_CATALYST) + set(AMReX_CATALYST ON CACHE INTERNAL "") + set(AMReX_CONDUIT ON CACHE INTERNAL "") + endif() + if("${CMAKE_BUILD_TYPE}" MATCHES "Debug") set(AMReX_ASSERTIONS ON CACHE BOOL "") # note: floating-point exceptions can slow down debug runs a lot @@ -46,6 +51,12 @@ macro(find_amrex) set(AMReX_OMP OFF CACHE INTERNAL "") endif() + if(WarpX_FFT OR ABLASTR_FFT) + set(AMReX_FFT ON CACHE INTERNAL "") + else() + set(AMReX_FFT OFF CACHE INTERNAL "") + endif() + if(WarpX_EB) set(AMReX_EB ON CACHE INTERNAL "") else() @@ -87,6 +98,8 @@ macro(find_amrex) set(AMReX_PARTICLES ON CACHE INTERNAL "") set(AMReX_PROBINIT OFF CACHE INTERNAL "") set(AMReX_TINY_PROFILE ON CACHE BOOL "") + set(AMReX_LINEAR_SOLVERS_EM ON CACHE INTERNAL "") + set(AMReX_LINEAR_SOLVERS_INCFLO ON CACHE INTERNAL "") if(WarpX_ASCENT OR WarpX_SENSEI) set(AMReX_GPU_RDC ON CACHE BOOL "") @@ -140,22 +153,17 @@ macro(find_amrex) endif() add_subdirectory(${WarpX_amrex_src} _deps/localamrex-build/) else() + if(WarpX_COMPUTE STREQUAL CUDA) + enable_language(CUDA) + # AMReX 21.06+ supports CUDA_ARCHITECTURES + endif() FetchContent_Declare(fetchedamrex GIT_REPOSITORY ${WarpX_amrex_repo} GIT_TAG ${WarpX_amrex_branch} BUILD_IN_SOURCE 0 ) - FetchContent_GetProperties(fetchedamrex) - - if(NOT fetchedamrex_POPULATED) - FetchContent_Populate(fetchedamrex) - list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake") - if(WarpX_COMPUTE STREQUAL CUDA) - enable_language(CUDA) - # AMReX 21.06+ supports CUDA_ARCHITECTURES - endif() - add_subdirectory(${fetchedamrex_SOURCE_DIR} ${fetchedamrex_BINARY_DIR}) - endif() + FetchContent_MakeAvailable(fetchedamrex) + list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake") # advanced fetch options mark_as_advanced(FETCHCONTENT_BASE_DIR) @@ -200,6 +208,8 @@ macro(find_amrex) mark_as_advanced(AMReX_HYPRE) mark_as_advanced(AMReX_IPO) mark_as_advanced(AMReX_LINEAR_SOLVERS) + mark_as_advanced(AMReX_LINEAR_SOLVERS_INCFLO) + mark_as_advanced(AMReX_LINEAR_SOLVERS_EM) mark_as_advanced(AMReX_MEM_PROFILE) mark_as_advanced(AMReX_MPI) mark_as_advanced(AMReX_MPI_THREAD_MULTIPLE) @@ -226,6 +236,12 @@ macro(find_amrex) set(COMPONENT_ASCENT) endif() + if(WarpX_CATALYST) + set(COMPONENT_CATALYST CATALYST CONDUIT) + else() + set(COMPONENT_CATALYST) + endif() + set(WarpX_amrex_dim ${WarpX_DIMS}) # RZ is AMReX 2D list(TRANSFORM WarpX_amrex_dim REPLACE RZ 2) list(REMOVE_DUPLICATES WarpX_amrex_dim) @@ -233,6 +249,11 @@ macro(find_amrex) foreach(D IN LISTS WarpX_amrex_dim) set(COMPONENT_DIMS ${COMPONENT_DIMS} ${D}D) endforeach() + if(WarpX_FFT) + set(COMPONENT_FFT FFT) + else() + set(COMPONENT_FFT) + endif() if(WarpX_EB) set(COMPONENT_EB EB) else() @@ -250,7 +271,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 25.02 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} ${COMPONENT_FFT} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +294,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" +set(WarpX_amrex_branch "b364becad939a490bca4e7f8b23f7392c558a311" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/FFT.cmake b/cmake/dependencies/FFT.cmake index 571006e8530..df0ef11ae53 100644 --- a/cmake/dependencies/FFT.cmake +++ b/cmake/dependencies/FFT.cmake @@ -48,14 +48,20 @@ if(ABLASTR_FFT) # # cuFFT (CUDA) - # TODO: check if `find_package` search works + if(WarpX_COMPUTE STREQUAL CUDA) + # nothing to do (cuFFT is part of the CUDA SDK) + # TODO: check if `find_package` search works for cuFFT # rocFFT (HIP) - if(WarpX_COMPUTE STREQUAL HIP) + elseif(WarpX_COMPUTE STREQUAL HIP) find_package(rocfft REQUIRED) - # FFTW (NOACC, OMP, SYCL) - elseif(NOT WarpX_COMPUTE STREQUAL CUDA) + elseif(WarpX_COMPUTE STREQUAL SYCL) + # nothing to do (oneMKL is part of oneAPI) + # TODO: check if `find_package` search works for oneMKL + + # FFTW (NOACC, OMP) + else() # On Windows, try searching for FFTW3(f)Config.cmake files first # Installed .pc files wrongly and unconditionally add -lm # https://github.com/FFTW/fftw3/issues/236 @@ -106,6 +112,8 @@ if(ABLASTR_FFT) warpx_make_third_party_includes_system(cufft FFT) elseif(WarpX_COMPUTE STREQUAL HIP) warpx_make_third_party_includes_system(roc::rocfft FFT) + elseif(WarpX_COMPUTE STREQUAL SYCL) + warpx_make_third_party_includes_system(AMReX::SYCL FFT) else() if(WarpX_FFTW_SEARCH STREQUAL CMAKE) warpx_make_third_party_includes_system(FFTW3::fftw3${HFFTWp} FFT) diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake index 9086421356f..d5249b61641 100644 --- a/cmake/dependencies/PICSAR.cmake +++ b/cmake/dependencies/PICSAR.cmake @@ -53,19 +53,13 @@ function(find_picsar) get_source_version(PXRMP_QED ${WarpX_picsar_src}) else() FetchContent_Declare(fetchedpicsar - GIT_REPOSITORY ${WarpX_picsar_repo} - GIT_TAG ${WarpX_picsar_branch} + GIT_REPOSITORY ${WarpX_picsar_repo} + GIT_TAG ${WarpX_picsar_branch} BUILD_IN_SOURCE 0 + SOURCE_SUBDIR multi_physics/QED ) - FetchContent_GetProperties(fetchedpicsar) + FetchContent_MakeAvailable(fetchedpicsar) - if(NOT fetchedpicsar_POPULATED) - FetchContent_Populate(fetchedpicsar) - add_subdirectory( - ${fetchedpicsar_SOURCE_DIR}/multi_physics/QED - ${fetchedpicsar_BINARY_DIR} - ) - endif() get_source_version(PXRMP_QED ${fetchedpicsar_SOURCE_DIR}) if(NOT PXRMP_QED_GIT_VERSION) set(PXRMP_QED_GIT_VERSION "${WarpX_picsar_branch}" CACHE INTERNAL "") @@ -94,7 +88,7 @@ function(find_picsar) #message(STATUS "PICSAR: Using version '${PICSAR_VERSION}'") else() # not supported by PICSAR (yet) - #find_package(PICSAR 23.11 CONFIG REQUIRED QED) + #find_package(PICSAR 25.01 CONFIG REQUIRED QED) #message(STATUS "PICSAR: Found version '${PICSAR_VERSION}'") message(FATAL_ERROR "PICSAR: Cannot be used as externally installed " "library yet. " @@ -115,7 +109,7 @@ if(WarpX_QED) set(WarpX_picsar_repo "https://github.com/ECP-WarpX/picsar.git" CACHE STRING "Repository URI to pull and build PICSAR from if(WarpX_picsar_internal)") - set(WarpX_picsar_branch "aa54e985398c1d575abc7e6737cdbc660a13765f" + set(WarpX_picsar_branch "25.01" CACHE STRING "Repository branch for WarpX_picsar_repo if(WarpX_picsar_internal)") diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake index f58d37ee92e..a5a80f25790 100644 --- a/cmake/dependencies/openPMD.cmake +++ b/cmake/dependencies/openPMD.cmake @@ -13,7 +13,7 @@ function(find_openpmd) if(WarpX_openpmd_internal OR WarpX_openpmd_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - # see https://openpmd-api.readthedocs.io/en/0.15.2/dev/buildoptions.html + # see https://openpmd-api.readthedocs.io/en/0.16.1/dev/buildoptions.html set(openPMD_USE_ADIOS1 OFF CACHE INTERNAL "") set(openPMD_USE_MPI ${WarpX_MPI} CACHE INTERNAL "") set(openPMD_USE_PYTHON OFF CACHE INTERNAL "") @@ -32,12 +32,7 @@ function(find_openpmd) GIT_TAG ${WarpX_openpmd_branch} BUILD_IN_SOURCE 0 ) - FetchContent_GetProperties(fetchedopenpmd) - - if(NOT fetchedopenpmd_POPULATED) - FetchContent_Populate(fetchedopenpmd) - add_subdirectory(${fetchedopenpmd_SOURCE_DIR} ${fetchedopenpmd_BINARY_DIR}) - endif() + FetchContent_MakeAvailable(fetchedopenpmd) # advanced fetch options mark_as_advanced(FETCHCONTENT_BASE_DIR) @@ -76,7 +71,7 @@ function(find_openpmd) else() set(COMPONENT_WMPI NOMPI) endif() - find_package(openPMD 0.15.1 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI}) + find_package(openPMD 0.16.1 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI}) message(STATUS "openPMD-api: Found version '${openPMD_VERSION}'") endif() endfunction() @@ -92,7 +87,7 @@ if(WarpX_OPENPMD) set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" CACHE STRING "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") - set(WarpX_openpmd_branch "0.15.2" + set(WarpX_openpmd_branch "0.16.1" CACHE STRING "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 133747aaeae..be7c64acd69 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -47,12 +47,7 @@ function(find_pyamrex) GIT_TAG ${WarpX_pyamrex_branch} BUILD_IN_SOURCE 0 ) - FetchContent_GetProperties(fetchedpyamrex) - - if(NOT fetchedpyamrex_POPULATED) - FetchContent_Populate(fetchedpyamrex) - add_subdirectory(${fetchedpyamrex_SOURCE_DIR} ${fetchedpyamrex_BINARY_DIR}) - endif() + FetchContent_MakeAvailable(fetchedpyamrex) # advanced fetch options mark_as_advanced(FETCHCONTENT_BASE_DIR) @@ -64,7 +59,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.07 CONFIG REQUIRED) + find_package(pyAMReX 25.02 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +74,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" +set(WarpX_pyamrex_branch "3088ea12a1a6287246bf027c4235f10e92472450" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 0a7ec260493..e90b56b2d38 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -10,6 +10,11 @@ function(find_pybind11) message(STATUS "pybind11 repository: ${WarpX_pybind11_repo} (${WarpX_pybind11_branch})") include(FetchContent) endif() + + # rely on our find_package(Python ...) call + # https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake + set(PYBIND11_FINDPYTHON ON) + if(WarpX_pybind11_internal OR WarpX_pybind11_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) @@ -21,12 +26,7 @@ function(find_pybind11) GIT_TAG ${WarpX_pybind11_branch} BUILD_IN_SOURCE 0 ) - FetchContent_GetProperties(fetchedpybind11) - - if(NOT fetchedpybind11_POPULATED) - FetchContent_Populate(fetchedpybind11) - add_subdirectory(${fetchedpybind11_SOURCE_DIR} ${fetchedpybind11_BINARY_DIR}) - endif() + FetchContent_MakeAvailable(fetchedpybind11) # advanced fetch options mark_as_advanced(FETCHCONTENT_BASE_DIR) @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.12.0 CONFIG REQUIRED) + find_package(pybind11 2.13.0 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.12.0" +set(WarpX_pybind11_branch "v2.13.6" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") diff --git a/pyproject.toml b/pyproject.toml index f9e615f9b83..6210388f6e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,21 @@ requires = [ "setuptools>=42", "wheel", - "cmake>=3.20.0,<4.0.0", + "cmake>=3.24.0,<4.0.0", "packaging>=23", ] build-backend = "setuptools.build_meta" +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +select = ["E", "F", "I"] +ignore = ["E402", "E501", "F405"] + +[tool.ruff.lint.isort] +known-first-party = ["amrex", "picmistandard", "pywarpx", "warpx"] + [tool.isort] known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"] profile = "black" diff --git a/requirements.txt b/requirements.txt index 8e664ae3096..e44273328de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.29.0 +picmistandard==0.33.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/run_test.sh b/run_test.sh deleted file mode 100755 index 4efa86eb36c..00000000000 --- a/run_test.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -# Copyright 2018-2020 Axel Huebl, David Grote, Edoardo Zoni -# Luca Fedeli, Maxence Thevenet, Remi Lehe -# -# -# This file is part of WarpX. -# -# License: BSD-3-Clause-LBNL - -# This script runs some of WarpX's standard regression tests, but -# without comparing the output to previously run simulations. -# This checks that: -# - The code compiles and runs without error -# - For some of the tests, a Python script checks that the results are -# physically correct. - -# The tests can be influenced by environment variables: -# Use `export WARPX_CI_DIM=3` or `export WARPX_CI_DIM=2` in order to -# select only the tests that correspond to this dimension -# Use `export WARPX_TEST_ARCH=CPU` or `export WARPX_TEST_ARCH=GPU` in order -# to run the tests on CPU or GPU respectively. - -set -eu -o pipefail - -# Parse command line arguments: if test names are given as command line arguments, -# store them in variable tests_arg and define new command line argument to call -# regtest.py with option --tests (works also for single test) -tests_arg=$* -tests_run=${tests_arg:+--tests=${tests_arg}} - -# environment options -WARPX_CI_TMP=${WARPX_CI_TMP:-""} - -# Remove contents and link to a previous test directory (intentionally two arguments) -rm -rf test_dir/* test_dir -# Create a temporary test directory -if [ -z "${WARPX_CI_TMP}" ]; then - tmp_dir=$(mktemp --help >/dev/null 2>&1 && mktemp -d -t ci-XXXXXXXXXX || mktemp -d "${TMPDIR:-/tmp}"/ci-XXXXXXXXXX) - if [ $? -ne 0 ]; then - echo "Cannot create temporary directory" - exit 1 - fi -else - tmp_dir=${WARPX_CI_TMP} -fi - -# Copy WarpX into current test directory -rm -rf ${tmp_dir}/warpx -mkdir -p ${tmp_dir}/warpx -cp -r ./* ${tmp_dir}/warpx - -# Link the test directory -ln -s ${tmp_dir} test_dir - -# Switch to the test directory -cd test_dir -echo "cd $PWD" - -# Prepare a virtual environment -rm -rf py-venv -python3 -m venv py-venv -source py-venv/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build packaging setuptools wheel -python3 -m pip install --upgrade cmake -python3 -m pip install --upgrade -r warpx/Regression/requirements.txt -python3 -m pip cache purge - -# Clone AMReX and warpx-data -git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - -# warpx-data contains various required data sets -git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git -# openPMD-example-datasets contains various required data sets -mkdir -p openPMD-example-datasets -cd openPMD-example-datasets -curl -sOL https://github.com/openPMD/openPMD-example-datasets/raw/4ba1d257c5b4897c0a3cd57742bb0987343a902e/example-femm-thetaMode.h5 -curl -sOL https://github.com/openPMD/openPMD-example-datasets/raw/4ba1d257c5b4897c0a3cd57742bb0987343a902e/example-femm-3d.h5 -cd - - -# Clone the AMReX regression test utility -git clone https://github.com/AMReX-Codes/regression_testing.git - -# Prepare regression tests -mkdir -p rt-WarpX/WarpX-benchmarks -cd warpx/Regression -echo "cd $PWD" -python3 prepare_file_ci.py -cp ci-tests.ini ../../rt-WarpX -cp -r Checksum ../../regression_testing/ - -# Run tests -cd ../../regression_testing/ -echo "cd $PWD" -# run only tests specified in variable tests_arg (single test or multiple tests) -if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" -# run all tests (variables tests_arg and tests_run are empty) -else - python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all -fi - -# clean up python virtual environment -cd ../ -echo "cd $PWD" -deactivate diff --git a/setup.py b/setup.py index f05d5a71899..fae11aa0654 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def initialize_options(self): # clashes with directories many developers have in their source trees; # this can create confusing results with "pip install .", which clones # the whole source tree by default - self.build_base = '_tmppythonbuild' + self.build_base = os.path.join("_tmppythonbuild", "warpx") def run(self): # remove existing build directory @@ -38,8 +38,10 @@ def run(self): lib_path = os.path.join(PYWARPX_LIB_DIR, lib_name) libs_found.append(lib_path) if len(libs_found) == 0: - raise RuntimeError("Error: no pre-build WarpX modules found in " - "PYWARPX_LIB_DIR='{}'".format(PYWARPX_LIB_DIR)) + raise RuntimeError( + "Error: no pre-build WarpX modules found in " + "PYWARPX_LIB_DIR='{}'".format(PYWARPX_LIB_DIR) + ) # copy external libs into collection of files in a temporary build dir dst_path = os.path.join(self.build_lib, "pywarpx") @@ -48,7 +50,7 @@ def run(self): class CMakeExtension(Extension): - def __init__(self, name, sourcedir=''): + def __init__(self, name, sourcedir=""): Extension.__init__(self, name, sources=[]) self.sourcedir = os.path.abspath(sourcedir) @@ -58,140 +60,138 @@ def run(self): from packaging.version import parse try: - out = subprocess.check_output(['cmake', '--version']) + out = subprocess.check_output(["cmake", "--version"]) except OSError: raise RuntimeError( - "CMake 3.20.0+ must be installed to build the following " + - "extensions: " + - ", ".join(e.name for e in self.extensions)) + "CMake 3.24.0+ must be installed to build the following " + + "extensions: " + + ", ".join(e.name for e in self.extensions) + ) cmake_version = parse(re.search(r"version\s*([\d.]+)", out.decode()).group(1)) - if cmake_version < parse("3.20.0"): - raise RuntimeError("CMake >= 3.20.0 is required") + if cmake_version < parse("3.24.0"): + raise RuntimeError("CMake >= 3.24.0 is required") for ext in self.extensions: self.build_extension(ext) def build_extension(self, ext): - extdir = os.path.abspath(os.path.dirname( - self.get_ext_fullpath(ext.name) - )) + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) # required for auto-detection of auxiliary "native" libs if not extdir.endswith(os.path.sep): extdir += os.path.sep - r_dim = re.search(r'warpx_(1|2|rz|3)(?:d*)', ext.name) + r_dim = re.search(r"warpx_(1|2|rz|3)(?:d*)", ext.name) dims = r_dim.group(1).upper() + pyv = sys.version_info cmake_args = [ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + - os.path.join(extdir, "pywarpx"), - '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=' + extdir, - '-DWarpX_DIMS=' + dims, - '-DWarpX_APP:BOOL=OFF', + # Python: use the calling interpreter in CMake + # https://cmake.org/cmake/help/latest/module/FindPython.html#hints + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection + f"-DPython_ROOT_DIR={sys.prefix}", + f"-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}", + "-DPython_FIND_VERSION_EXACT=TRUE", + "-DPython_FIND_STRATEGY=LOCATION", + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + os.path.join(extdir, "pywarpx"), + "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=" + extdir, + "-DWarpX_DIMS=" + dims, + "-DWarpX_APP:BOOL=OFF", ## variants - '-DWarpX_COMPUTE=' + WARPX_COMPUTE, - '-DWarpX_MPI:BOOL=' + WARPX_MPI, - '-DWarpX_EB:BOOL=' + WARPX_EB, - '-DWarpX_OPENPMD:BOOL=' + WARPX_OPENPMD, - '-DWarpX_PRECISION=' + WARPX_PRECISION, - '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, - '-DWarpX_FFT:BOOL=' + WARPX_FFT, - '-DWarpX_HEFFTE:BOOL=' + WARPX_HEFFTE, - '-DWarpX_PYTHON:BOOL=ON', - '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, - '-DWarpX_QED:BOOL=' + WARPX_QED, - '-DWarpX_QED_TABLE_GEN:BOOL=' + WARPX_QED_TABLE_GEN, + "-DWarpX_COMPUTE=" + WARPX_COMPUTE, + "-DWarpX_MPI:BOOL=" + WARPX_MPI, + "-DWarpX_EB:BOOL=" + WARPX_EB, + "-DWarpX_OPENPMD:BOOL=" + WARPX_OPENPMD, + "-DWarpX_PRECISION=" + WARPX_PRECISION, + "-DWarpX_PARTICLE_PRECISION=" + WARPX_PARTICLE_PRECISION, + "-DWarpX_FFT:BOOL=" + WARPX_FFT, + "-DWarpX_PYTHON:BOOL=ON", + "-DWarpX_PYTHON_IPO:BOOL=" + WARPX_PYTHON_IPO, + "-DWarpX_QED:BOOL=" + WARPX_QED, + "-DWarpX_QED_TABLE_GEN:BOOL=" + WARPX_QED_TABLE_GEN, ## dependency control (developers & package managers) - '-DWarpX_amrex_internal=' + WARPX_AMREX_INTERNAL, + "-DWarpX_amrex_internal=" + WARPX_AMREX_INTERNAL, # PEP-440 conformant version from package "-DpyWarpX_VERSION_INFO=" + self.distribution.get_version(), # see PICSAR and openPMD below ## static/shared libs - '-DBUILD_SHARED_LIBS:BOOL=' + BUILD_SHARED_LIBS, + "-DBUILD_SHARED_LIBS:BOOL=" + BUILD_SHARED_LIBS, ## Unix: rpath to current dir when packaged ## needed for shared (here non-default) builds and ADIOS1 ## wrapper libraries - '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON', - '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF', + "-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON", + "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF", # Windows: has no RPath concept, all `.dll`s must be in %PATH% # or same dir as calling executable ] - if WARPX_QED.upper() in ['1', 'ON', 'TRUE', 'YES']: - cmake_args.append('-DWarpX_picsar_internal=' + WARPX_PICSAR_INTERNAL) - if WARPX_OPENPMD.upper() in ['1', 'ON', 'TRUE', 'YES']: + if WARPX_QED.upper() in ["1", "ON", "TRUE", "YES"]: + cmake_args.append("-DWarpX_picsar_internal=" + WARPX_PICSAR_INTERNAL) + if WARPX_OPENPMD.upper() in ["1", "ON", "TRUE", "YES"]: cmake_args += [ - '-DHDF5_USE_STATIC_LIBRARIES:BOOL=' + HDF5_USE_STATIC_LIBRARIES, - '-DADIOS_USE_STATIC_LIBS:BOOL=' + ADIOS_USE_STATIC_LIBS, - '-DWarpX_openpmd_internal=' + WARPX_OPENPMD_INTERNAL, + "-DHDF5_USE_STATIC_LIBRARIES:BOOL=" + HDF5_USE_STATIC_LIBRARIES, + "-DADIOS_USE_STATIC_LIBS:BOOL=" + ADIOS_USE_STATIC_LIBS, + "-DWarpX_openpmd_internal=" + WARPX_OPENPMD_INTERNAL, ] # further dependency control (developers & package managers) if WARPX_AMREX_SRC: - cmake_args.append('-DWarpX_amrex_src=' + WARPX_AMREX_SRC) + cmake_args.append("-DWarpX_amrex_src=" + WARPX_AMREX_SRC) if WARPX_AMREX_REPO: - cmake_args.append('-DWarpX_amrex_repo=' + WARPX_AMREX_REPO) + cmake_args.append("-DWarpX_amrex_repo=" + WARPX_AMREX_REPO) if WARPX_AMREX_BRANCH: - cmake_args.append('-DWarpX_amrex_branch=' + WARPX_AMREX_BRANCH) + cmake_args.append("-DWarpX_amrex_branch=" + WARPX_AMREX_BRANCH) if WARPX_OPENPMD_SRC: - cmake_args.append('-DWarpX_openpmd_src=' + WARPX_OPENPMD_SRC) + cmake_args.append("-DWarpX_openpmd_src=" + WARPX_OPENPMD_SRC) if WARPX_PICSAR_SRC: - cmake_args.append('-DWarpX_picsar_src=' + WARPX_PICSAR_SRC) + cmake_args.append("-DWarpX_picsar_src=" + WARPX_PICSAR_SRC) if WARPX_PYAMREX_SRC: - cmake_args.append('-DWarpX_pyamrex_src=' + WARPX_PYAMREX_SRC) + cmake_args.append("-DWarpX_pyamrex_src=" + WARPX_PYAMREX_SRC) if WARPX_PYAMREX_INTERNAL: - cmake_args.append('-DWarpX_pyamrex_internal=' + WARPX_PYAMREX_INTERNAL) + cmake_args.append("-DWarpX_pyamrex_internal=" + WARPX_PYAMREX_INTERNAL) if WARPX_PYBIND11_SRC: - cmake_args.append('-DWarpX_pybind11_src=' + WARPX_PYBIND11_SRC) + cmake_args.append("-DWarpX_pybind11_src=" + WARPX_PYBIND11_SRC) if WARPX_PYBIND11_INTERNAL: - cmake_args.append('-DWarpX_pybind11_internal=' + WARPX_PYBIND11_INTERNAL) + cmake_args.append("-DWarpX_pybind11_internal=" + WARPX_PYBIND11_INTERNAL) if WARPX_CCACHE_PROGRAM is not None: - cmake_args.append('-DCCACHE_PROGRAM=' + WARPX_CCACHE_PROGRAM) + cmake_args.append("-DCCACHE_PROGRAM=" + WARPX_CCACHE_PROGRAM) if sys.platform == "darwin": - cmake_args.append('-DCMAKE_INSTALL_RPATH=@loader_path') + cmake_args.append("-DCMAKE_INSTALL_RPATH=@loader_path") else: # values: linux*, aix, freebsd, ... # just as well win32 & cygwin (although Windows has no RPaths) - cmake_args.append('-DCMAKE_INSTALL_RPATH=$ORIGIN') + cmake_args.append("-DCMAKE_INSTALL_RPATH=$ORIGIN") - cfg = 'Debug' if self.debug else 'Release' - build_args = ['--config', cfg] + cfg = "Debug" if self.debug else "Release" + build_args = ["--config", cfg] if platform.system() == "Windows": cmake_args += [ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format( - cfg.upper(), - os.path.join(extdir, "pywarpx") + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format( + cfg.upper(), os.path.join(extdir, "pywarpx") ) ] if sys.maxsize > 2**32: - cmake_args += ['-A', 'x64'] + cmake_args += ["-A", "x64"] else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] - build_args += ['--parallel', BUILD_PARALLEL] + build_args += ["--parallel", BUILD_PARALLEL] build_dir = os.path.join(self.build_temp, dims) os.makedirs(build_dir, exist_ok=True) - subprocess.check_call( - ['cmake', ext.sourcedir] + cmake_args, - cwd=build_dir - ) - subprocess.check_call( - ['cmake', '--build', '.'] + build_args, - cwd=build_dir - ) + subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=build_dir) + subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=build_dir) # note that this does not call install; # we pick up artifacts directly from the build output dirs -with open('./README.md', encoding='utf-8') as f: +with open("./README.md", encoding="utf-8") as f: long_description = f.read() # Allow to control options via environment vars. # Work-around for https://github.com/pypa/setuptools/issues/1712 # Pick up existing WarpX libraries or... -PYWARPX_LIB_DIR = os.environ.get('PYWARPX_LIB_DIR') +PYWARPX_LIB_DIR = os.environ.get("PYWARPX_LIB_DIR") WARPX_PYTHON_IPO = os.environ.get("WARPX_PYTHON_IPO", "ON") @@ -200,143 +200,152 @@ def build_extension(self, ext): # note: changed default for SHARED, MPI, TESTING and EXAMPLES # note: we use all-uppercase variable names for environment control to be # consistent across platforms (especially Windows) -WARPX_COMPUTE = env.pop('WARPX_COMPUTE', 'OMP') -WARPX_MPI = env.pop('WARPX_MPI', 'OFF') -WARPX_EB = env.pop('WARPX_EB', 'OFF') -WARPX_OPENPMD = env.pop('WARPX_OPENPMD', 'ON') -WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') -WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) -WARPX_FFT = env.pop('WARPX_FFT', 'OFF') -WARPX_HEFFTE = env.pop('WARPX_HEFFTE', 'OFF') -WARPX_QED = env.pop('WARPX_QED', 'ON') -WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') -WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') -BUILD_PARALLEL = env.pop('BUILD_PARALLEL', '2') -BUILD_SHARED_LIBS = env.pop('WARPX_BUILD_SHARED_LIBS', - 'OFF') -#BUILD_TESTING = env.pop('WARPX_BUILD_TESTING', +WARPX_COMPUTE = env.pop("WARPX_COMPUTE", "OMP") +WARPX_MPI = env.pop("WARPX_MPI", "OFF") +WARPX_EB = env.pop("WARPX_EB", "ON") +WARPX_OPENPMD = env.pop("WARPX_OPENPMD", "ON") +WARPX_PRECISION = env.pop("WARPX_PRECISION", "DOUBLE") +WARPX_PARTICLE_PRECISION = env.pop("WARPX_PARTICLE_PRECISION", WARPX_PRECISION) +WARPX_FFT = env.pop("WARPX_FFT", "OFF") +WARPX_QED = env.pop("WARPX_QED", "ON") +WARPX_QED_TABLE_GEN = env.pop("WARPX_QED_TABLE_GEN", "OFF") +WARPX_DIMS = env.pop("WARPX_DIMS", "1;2;RZ;3") +BUILD_PARALLEL = env.pop("BUILD_PARALLEL", "2") +BUILD_SHARED_LIBS = env.pop("WARPX_BUILD_SHARED_LIBS", "OFF") +# BUILD_TESTING = env.pop('WARPX_BUILD_TESTING', # 'OFF') -#BUILD_EXAMPLES = env.pop('WARPX_BUILD_EXAMPLES', +# BUILD_EXAMPLES = env.pop('WARPX_BUILD_EXAMPLES', # 'OFF') # openPMD-api sub-control -HDF5_USE_STATIC_LIBRARIES = env.pop('HDF5_USE_STATIC_LIBRARIES', 'OFF') -ADIOS_USE_STATIC_LIBS = env.pop('ADIOS_USE_STATIC_LIBS', 'OFF') +HDF5_USE_STATIC_LIBRARIES = env.pop("HDF5_USE_STATIC_LIBRARIES", "OFF") +ADIOS_USE_STATIC_LIBS = env.pop("ADIOS_USE_STATIC_LIBS", "OFF") # CMake dependency control (developers & package managers) -WARPX_AMREX_SRC = env.pop('WARPX_AMREX_SRC', '') -WARPX_AMREX_REPO = env.pop('WARPX_AMREX_REPO', '') -WARPX_AMREX_BRANCH = env.pop('WARPX_AMREX_BRANCH', '') -WARPX_AMREX_INTERNAL = env.pop('WARPX_AMREX_INTERNAL', 'ON') -WARPX_OPENPMD_SRC = env.pop('WARPX_OPENPMD_SRC', '') -WARPX_OPENPMD_INTERNAL = env.pop('WARPX_OPENPMD_INTERNAL', 'ON') -WARPX_PICSAR_SRC = env.pop('WARPX_PICSAR_SRC', '') -WARPX_PICSAR_INTERNAL = env.pop('WARPX_PICSAR_INTERNAL', 'ON') -WARPX_PYAMREX_SRC = env.pop('WARPX_PYAMREX_SRC', '') -WARPX_PYAMREX_INTERNAL = env.pop('WARPX_PYAMREX_INTERNAL', 'ON') -WARPX_PYBIND11_SRC = env.pop('WARPX_PYBIND11_SRC', '') -WARPX_PYBIND11_INTERNAL = env.pop('WARPX_PYBIND11_INTERNAL', 'ON') -WARPX_CCACHE_PROGRAM = env.pop('WARPX_CCACHE_PROGRAM', None) +WARPX_AMREX_SRC = env.pop("WARPX_AMREX_SRC", "") +WARPX_AMREX_REPO = env.pop("WARPX_AMREX_REPO", "") +WARPX_AMREX_BRANCH = env.pop("WARPX_AMREX_BRANCH", "") +WARPX_AMREX_INTERNAL = env.pop("WARPX_AMREX_INTERNAL", "ON") +WARPX_OPENPMD_SRC = env.pop("WARPX_OPENPMD_SRC", "") +WARPX_OPENPMD_INTERNAL = env.pop("WARPX_OPENPMD_INTERNAL", "ON") +WARPX_PICSAR_SRC = env.pop("WARPX_PICSAR_SRC", "") +WARPX_PICSAR_INTERNAL = env.pop("WARPX_PICSAR_INTERNAL", "ON") +WARPX_PYAMREX_SRC = env.pop("WARPX_PYAMREX_SRC", "") +WARPX_PYAMREX_INTERNAL = env.pop("WARPX_PYAMREX_INTERNAL", "ON") +WARPX_PYBIND11_SRC = env.pop("WARPX_PYBIND11_SRC", "") +WARPX_PYBIND11_INTERNAL = env.pop("WARPX_PYBIND11_INTERNAL", "ON") +WARPX_CCACHE_PROGRAM = env.pop("WARPX_CCACHE_PROGRAM", None) for key in env.keys(): - if key.lower().startswith('warpx'): - print(f"\nWARNING: Found environment variable '{key}', which is not a recognized WarpX option\n") + if key.lower().startswith("warpx"): + print( + f"\nWARNING: Found environment variable '{key}', which is not a recognized WarpX option\n" + ) # https://cmake.org/cmake/help/v3.0/command/if.html -if WARPX_MPI.upper() in ['1', 'ON', 'TRUE', 'YES']: +if WARPX_MPI.upper() in ["1", "ON", "TRUE", "YES"]: WARPX_MPI = "ON" else: WARPX_MPI = "OFF" # Include embedded boundary functionality -if WARPX_EB.upper() in ['1', 'ON', 'TRUE', 'YES']: +if WARPX_EB.upper() in ["1", "ON", "TRUE", "YES"]: WARPX_EB = "ON" else: WARPX_EB = "OFF" # for CMake -cxx_modules = [] # values: warpx_1d, warpx_2d, warpx_rz, warpx_3d -cmdclass = {} # build extensions +cxx_modules = [] # values: warpx_1d, warpx_2d, warpx_rz, warpx_3d +cmdclass = {} # build extensions # externally pre-built: pick up pre-built WarpX libraries if PYWARPX_LIB_DIR: - cmdclass=dict(build=CopyPreBuild) + cmdclass = dict(build=CopyPreBuild) # CMake: build WarpX libraries ourselves else: cmdclass = dict(build_ext=CMakeBuild) - for dim in [x.lower() for x in WARPX_DIMS.split(';')]: + for dim in [x.lower() for x in WARPX_DIMS.split(";")]: name = dim if dim == "rz" else dim + "d" cxx_modules.append(CMakeExtension("warpx_" + name)) # Get the package requirements from the requirements.txt file install_requires = [] -with open('./requirements.txt') as f: - install_requires = [line.strip('\n') for line in f.readlines()] +with open("./requirements.txt") as f: + install_requires = [line.strip("\n") for line in f.readlines()] if WARPX_MPI == "ON": - install_requires.append('mpi4py>=2.1.0') + install_requires.append("mpi4py>=2.1.0") # keyword reference: # https://packaging.python.org/guides/distributing-packages-using-setuptools setup( - name='pywarpx', + name="pywarpx", # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.07', - packages = ['pywarpx'], - package_dir = {'pywarpx': 'Python/pywarpx'}, - author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', - author_email='jlvay@lbl.gov, grote1@llnl.gov, maxence.thevenet@desy.de, rlehe@lbl.gov, atmyers@lbl.gov, WeiqunZhang@lbl.gov, axelhuebl@lbl.gov', - maintainer='Axel Huebl, David P. Grote, Rémi Lehe', # wheel/pypi packages - maintainer_email='axelhuebl@lbl.gov, grote1@llnl.gov, rlehe@lbl.gov', - description='WarpX is an advanced electromagnetic Particle-In-Cell code.', + version="25.02", + packages=["pywarpx"], + package_dir={"pywarpx": "Python/pywarpx"}, + author="Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.", + author_email="jlvay@lbl.gov, grote1@llnl.gov, maxence.thevenet@desy.de, rlehe@lbl.gov, atmyers@lbl.gov, WeiqunZhang@lbl.gov, axelhuebl@lbl.gov", + maintainer="Axel Huebl, David P. Grote, Rémi Lehe", # wheel/pypi packages + maintainer_email="axelhuebl@lbl.gov, grote1@llnl.gov, rlehe@lbl.gov", + description="WarpX is an advanced electromagnetic Particle-In-Cell code.", long_description=long_description, - long_description_content_type='text/markdown', - keywords=('WarpX openscience mpi hpc research pic particle-in-cell ' - 'plasma laser-plasma accelerator modeling simulation'), - url='https://ecp-warpx.github.io', + long_description_content_type="text/markdown", + keywords=( + "WarpX openscience mpi hpc research pic particle-in-cell " + "plasma laser-plasma accelerator modeling simulation" + ), + url="https://ecp-warpx.github.io", project_urls={ - 'Documentation': 'https://warpx.readthedocs.io', - 'Doxygen': 'https://warpx.readthedocs.io/en/latest/_static/doxyhtml/index.html', + "Documentation": "https://warpx.readthedocs.io", + "Doxygen": "https://warpx.readthedocs.io/en/latest/_static/doxyhtml/index.html", #'Reference': 'https://doi.org/...', (Paper and/or Zenodo) - 'Source': 'https://github.com/ECP-WarpX/WarpX', - 'Tracker': 'https://github.com/ECP-WarpX/WarpX/issues', + "Source": "https://github.com/ECP-WarpX/WarpX", + "Tracker": "https://github.com/ECP-WarpX/WarpX/issues", }, # CMake: self-built as extension module ext_modules=cxx_modules, cmdclass=cmdclass, # scripts=['warpx_1d', 'warpx_2d', 'warpx_rz', 'warpx_3d'], zip_safe=False, - python_requires='>=3.8', + python_requires=">=3.8", # left for CI, truly ">=3.9" # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli - #entry_points={ + # entry_points={ # 'console_scripts': [ # 'warpx_3d = warpx.3d.__main__:main' # ] - #}, + # }, extras_require={ - 'all': ['openPMD-api~=0.15.1', 'openPMD-viewer~=1.1', 'yt>=4.1.0', 'matplotlib'], + "all": [ + "openPMD-api>=0.16.1", + "openPMD-viewer~=1.1", + "yt>=4.1.0", + "matplotlib", + ], }, # cmdclass={'test': PyTest}, # platforms='any', classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Natural Language :: English', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'Operating System :: OS Independent', - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Physics', - 'Programming Language :: C++', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - ('License :: OSI Approved :: ' - 'BSD License'), # TODO: use real SPDX: BSD-3-Clause-LBNL + "Development Status :: 5 - Production/Stable", + "Natural Language :: English", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Physics", + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + ( + "License :: OSI Approved :: BSD License" + ), # TODO: use real SPDX: BSD-3-Clause-LBNL ], # new PEP 639 format - license='BSD-3-Clause-LBNL', - license_files = ['LICENSE.txt', 'LEGAL.txt'], + license="BSD-3-Clause-LBNL", + license_files=["LICENSE.txt", "LEGAL.txt"], )