Skip to content

Commit 2e7b11b

Browse files
kodiakhq[bot]kodiakhq[bot]
authored
Refactor waLBerla bridge (#4864)
Fixes #4859, fixes #4855 Description of changes: - bugfix: - LB boundaries are now properly communicated in the ghost layer - see details in #4859 - performance: - the LB flag field is no longer communicated at every time step - the LB UBB field is no longer recalculated at every time step - LB boundary setters (node, slice, shape) now always trigger a full ghost communication - maintainability: - the waLBerla header files are no longer visible in the ESPResSo core and script interface - the Boost dependency is now checked at the CMake level to prevent building broken ESPResSo shared libraries (see details in #4856)
2 parents 5cc8361 + f4590fd commit 2e7b11b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+718
-3616
lines changed

‎.github/workflows/push_pull.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ permissions:
1010
jobs:
1111
macos:
1212
runs-on: macos-12
13-
if: ${{ github.repository == 'espressomd/espresso' }}
13+
if: false
1414
steps:
1515
- name: Checkout
1616
uses: actions/checkout@main

‎CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,9 @@ if(ESPRESSO_BUILD_TESTS)
435435
endif()
436436

437437
find_package(Boost 1.74.0 REQUIRED ${BOOST_COMPONENTS})
438+
if(${Boost_VERSION} VERSION_GREATER_EQUAL 1.84.0)
439+
message(FATAL_ERROR "Boost version ${Boost_VERSION} is unsupported.")
440+
endif()
438441

439442
#
440443
# Paths

‎maintainer/benchmarks/lb.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@
5353

5454
# process and check arguments
5555
n_iterations = 30
56-
assert args.volume_fraction > 0, "volume_fraction must be a positive number"
56+
assert args.volume_fraction > 0, "--volume_fraction must be a positive number"
5757
assert args.volume_fraction < np.pi / (3 * np.sqrt(2)), \
58-
"volume_fraction exceeds the physical limit of sphere packing (~0.74)"
58+
"--volume_fraction exceeds the physical limit of sphere packing (~0.74)"
5959
assert "box_l" not in args or args.particles_per_core == 0, \
60-
"Argument box_l requires particles_per_core=0"
60+
"Argument --box_l requires --particles_per_core=0"
6161

6262
required_features = ["LENNARD_JONES", "WALBERLA"]
6363
if args.gpu:
@@ -85,6 +85,7 @@
8585
if n_part == 0:
8686
box_l = args.box_l
8787
agrid = 1.
88+
lb_grid = args.box_l
8889
measurement_steps = 80
8990
else:
9091
# volume of N spheres with radius r: N * (4/3*pi*r^3)

‎maintainer/walberla_kernels/generate_ek_kernels.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -186,12 +186,6 @@ def replace_getData_with_uncheckedFastGetData(filename: str) -> None:
186186
index_shape=density_field.index_shape,
187187
target=target)
188188

189-
pystencils_walberla.generate_pack_info_from_kernel(
190-
ctx,
191-
f"DensityPackInfo_{precision_suffix}",
192-
ek_electrostatic.continuity(),
193-
target=target)
194-
195189
# ek reactions
196190
for i in range(1, max_num_reactants + 1):
197191
assignments = list(reaction_obj.generate_reaction(num_reactants=i))

‎maintainer/walberla_kernels/templates/Boundary.tmpl.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@
4040
#include <field/FlagField.h>
4141
#include <core/debug/Debug.h>
4242

43+
#include <cassert>
4344
#include <functional>
44-
#include <set>
45+
#include <memory>
4546
#include <vector>
4647

4748
{% for header in interface_spec.headers %}
@@ -122,7 +123,7 @@ class {{class_name}}
122123
{%- endif %}
123124
};
124125

125-
{{class_name}}( const shared_ptr<StructuredBlockForest> & blocks,
126+
{{class_name}}( const std::shared_ptr<StructuredBlockForest> & blocks,
126127
{{kernel|generate_constructor_parameters(['indexVector', 'indexVectorSize'])}}{{additional_data_handler.constructor_arguments}})
127128
:{{additional_data_handler.initialiser_list}} {{ kernel|generate_constructor_initializer_list(['indexVector', 'indexVectorSize']) }}
128129
{
@@ -177,7 +178,7 @@ class {{class_name}}
177178
}
178179

179180
template<typename FlagField_T>
180-
void fillFromFlagField( const shared_ptr<StructuredBlockForest> & blocks, ConstBlockDataID flagFieldID,
181+
void fillFromFlagField( const std::shared_ptr<StructuredBlockForest> & blocks, ConstBlockDataID flagFieldID,
181182
FlagUID boundaryFlagUID, FlagUID domainFlagUID)
182183
{
183184
for( auto blockIt = blocks->begin(); blockIt != blocks->end(); ++blockIt )
@@ -197,8 +198,8 @@ class {{class_name}}
197198
auto * flagField = block->getData< FlagField_T > ( flagFieldID );
198199
{{additional_data_handler.additional_field_data|indent(4)}}
199200

200-
if( !(flagField->flagExists(boundaryFlagUID) && flagField->flagExists(domainFlagUID) ))
201-
return;
201+
assert(flagField->flagExists(boundaryFlagUID and
202+
flagField->flagExists(domainFlagUID));
202203

203204
auto boundaryFlag = flagField->getFlag(boundaryFlagUID);
204205
auto domainFlag = flagField->getFlag(domainFlagUID);

‎samples/lb_circular_couette.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
cyl_center = agrid * (grid_size // 2 + 0.5) * [1, 1, 0]
5858
cylinder_in = espressomd.shapes.Cylinder(
5959
center=cyl_center, axis=[0, 0, 1], length=3 * system.box_l[2],
60-
radius=8.1 * agrid, direction=1)
60+
radius=8.6 * agrid, direction=1)
6161
cylinder_out = espressomd.shapes.Cylinder(
6262
center=cyl_center, axis=[0, 0, 1], length=3 * system.box_l[2],
6363
radius=14.5 * agrid, direction=-1)

‎src/core/electrostatics/p3m.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,9 +599,11 @@ double CoulombP3M::long_range_kernel(bool force_flag, bool energy_flag,
599599
}
600600
}
601601
energy *= prefactor;
602+
#ifdef NPT
602603
if (npt_flag) {
603604
npt_add_virial_contribution(energy);
604605
}
606+
#endif
605607
if (not energy_flag) {
606608
energy = 0.;
607609
}

‎src/core/lb/particle_coupling.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ Utils::Vector3d lb_drag_force(LB::Solver const &lb, double lb_gamma,
8686
/**
8787
* @brief Check if a position is within the local box + halo.
8888
*
89+
* @param local_box Local geometry
8990
* @param pos Position to check
9091
* @param halo Halo
9192
*

‎src/core/unit_tests/ek_interface_test.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ static auto make_ek_actor() {
8585
ek_lattice = std::make_shared<LatticeWalberla>(
8686
params.grid_dimensions, ::communicator.node_grid, n_ghost_layers);
8787
ek_container = std::make_shared<EK::EKWalberla::ek_container_type>(
88-
params.tau, new_ek_poisson_none(ek_lattice, single_precision));
88+
params.tau, walberla::new_ek_poisson_none(ek_lattice, single_precision));
8989
ek_reactions = std::make_shared<EK::EKWalberla::ek_reactions_type>();
9090
ek_instance = std::make_shared<EK::EKWalberla>(ek_container, ek_reactions);
9191
#endif
@@ -146,7 +146,7 @@ BOOST_AUTO_TEST_CASE(ek_interface_walberla) {
146146
auto constexpr single_precision = true;
147147
auto constexpr stoich = 1.;
148148
auto constexpr order = 2.;
149-
auto ek_species = new_ek_walberla(
149+
auto ek_species = walberla::new_ek_walberla(
150150
espresso::ek_lattice, params.diffusion, params.kT, params.valency,
151151
params.ext_efield, params.density, false, false, single_precision);
152152
auto ek_reactant = std::make_shared<EKReactant>(ek_species, stoich, order);

‎src/python/espressomd/lb.py

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -53,20 +53,6 @@ def __getitem__(self, key):
5353
def __str__(self):
5454
return f"{self.__class__.__name__}({self.get_params()})"
5555

56-
def _activate(self):
57-
self._activate_method()
58-
59-
def _deactivate(self):
60-
self._deactivate_method()
61-
62-
def _activate_method(self):
63-
self.call_method("activate")
64-
utils.handle_errors("HydrodynamicInteraction activation failed")
65-
66-
def _deactivate_method(self):
67-
self.call_method("deactivate")
68-
utils.handle_errors("HydrodynamicInteraction deactivation failed")
69-
7056
def validate_params(self, params):
7157
pass
7258

@@ -342,13 +328,6 @@ class LBFluidNodeWalberla(ScriptInterfaceHelper):
342328
def required_keys(self):
343329
return {"parent_sip", "index"}
344330

345-
def __init__(self, *args, **kwargs):
346-
if "sip" not in kwargs:
347-
super().__init__(*args, **kwargs)
348-
utils.handle_errors("LBFluidNode instantiation failed")
349-
else:
350-
super().__init__(**kwargs)
351-
352331
def __reduce__(self):
353332
raise NotImplementedError("Cannot serialize LB fluid node objects")
354333

@@ -494,7 +473,6 @@ def __init__(self, *args, **kwargs):
494473
slice_range, grid_size)
495474
node = LBFluidNodeWalberla(index=np.array([0, 0, 0]), **kwargs)
496475
super().__init__(*args, node_sip=node, **kwargs, **extra_kwargs)
497-
utils.handle_errors("LBFluidSliceWalberla instantiation failed")
498476

499477
def __iter__(self):
500478
lower, upper = self.call_method("get_slice_ranges")

‎src/script_interface/electrostatics/CoulombScafacos.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "script_interface/get_value.hpp"
3232
#include "script_interface/scafacos/scafacos.hpp"
3333

34+
#include <iomanip>
3435
#include <regex>
3536
#include <set>
3637
#include <sstream>

‎src/script_interface/walberla/EKFFT.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ class EKFFT : public EKPoissonSolver {
5555
auto const permittivity =
5656
get_value<double>(args, "permittivity") * m_conv_permittivity;
5757

58-
m_instance = new_ek_poisson_fft(m_lattice->lattice(), permittivity,
59-
m_single_precision);
58+
m_instance = ::walberla::new_ek_poisson_fft(
59+
m_lattice->lattice(), permittivity, m_single_precision);
6060

6161
add_parameters({
6262
{"permittivity",

‎src/script_interface/walberla/EKNone.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ class EKNone : public EKPoissonSolver {
4545
m_single_precision = get_value_or<bool>(args, "single_precision", false);
4646
m_lattice = get_value<std::shared_ptr<LatticeWalberla>>(args, "lattice");
4747

48-
m_instance = new_ek_poisson_none(m_lattice->lattice(), m_single_precision);
48+
m_instance = ::walberla::new_ek_poisson_none(m_lattice->lattice(),
49+
m_single_precision);
4950

5051
add_parameters({
5152
{"single_precision", AutoParameter::read_only,

‎src/script_interface/walberla/EKReaction.hpp

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@
2727
#include "LatticeIndices.hpp"
2828
#include "LatticeWalberla.hpp"
2929

30+
#include <walberla_bridge/electrokinetics/ek_walberla_init.hpp>
3031
#include <walberla_bridge/electrokinetics/reactions/EKReactionBase.hpp>
31-
#include <walberla_bridge/src/electrokinetics/reactions/EKReactionImplBulk.hpp>
32-
#include <walberla_bridge/src/electrokinetics/reactions/EKReactionImplIndexed.hpp>
32+
#include <walberla_bridge/electrokinetics/reactions/EKReactionBaseIndexed.hpp>
3333

3434
#include <script_interface/ScriptInterface.hpp>
3535
#include <script_interface/auto_parameters/AutoParameters.hpp>
@@ -80,22 +80,21 @@ class EKReaction : public AutoParameters<EKReaction, LatticeIndices> {
8080
return tau / std::pow(Utils::int_pow<3>(agrid), sum_alphas - 1.);
8181
}
8282

83-
template <typename T>
84-
std::shared_ptr<T> make_instance(VariantMap const &args) const {
83+
template <typename F>
84+
auto make_instance(VariantMap const &args, F &allocator) const {
8585
auto lattice = get_value<std::shared_ptr<LatticeWalberla>>(args, "lattice");
86-
auto reactant = get_value<std::vector<Variant>>(args, "reactants");
87-
auto output =
88-
std::vector<std::shared_ptr<::walberla::EKReactant>>(reactant.size());
86+
auto reactants = get_value<std::vector<Variant>>(args, "reactants");
87+
auto output = ::walberla::EKReactionBase::reactants_type(reactants.size());
8988
auto get_instance = [](Variant const &v) {
9089
return get_value<std::shared_ptr<EKReactant>>(v)->get_instance();
9190
};
92-
std::transform(reactant.begin(), reactant.end(), output.begin(),
91+
std::transform(reactants.begin(), reactants.end(), output.begin(),
9392
get_instance);
9493

9594
auto const coefficient =
9695
get_value<double>(args, "coefficient") * get_conversion_coefficient();
9796

98-
return std::make_shared<T>(lattice->lattice(), output, coefficient);
97+
return allocator(lattice->lattice(), output, coefficient);
9998
}
10099

101100
std::shared_ptr<::walberla::EKReactionBase> m_ekreaction;
@@ -118,7 +117,7 @@ class EKBulkReaction : public EKReaction {
118117

119118
void do_construct(VariantMap const &args) override {
120119
m_conv_coefficient = calculate_bulk_conversion_factor(args);
121-
m_ekreaction = make_instance<::walberla::EKReactionImplBulk>(args);
120+
m_ekreaction = make_instance(args, ::walberla::new_ek_reaction_bulk);
122121
}
123122
};
124123

@@ -143,10 +142,9 @@ class EKIndexedReaction : public EKReaction {
143142
void do_construct(VariantMap const &args) override {
144143
auto const agrid = get_agrid(args);
145144
m_conv_coefficient = calculate_bulk_conversion_factor(args) / agrid;
146-
m_ekreaction = make_instance<::walberla::EKReactionImplIndexed>(args);
147145
m_ekreaction_impl =
148-
std::dynamic_pointer_cast<::walberla::EKReactionImplIndexed>(
149-
get_instance());
146+
make_instance(args, ::walberla::new_ek_reaction_indexed);
147+
m_ekreaction = m_ekreaction_impl;
150148
}
151149

152150
[[nodiscard]] Variant do_call_method(std::string const &method,
@@ -170,7 +168,7 @@ class EKIndexedReaction : public EKReaction {
170168
}
171169

172170
private:
173-
std::shared_ptr<::walberla::EKReactionImplIndexed> m_ekreaction_impl;
171+
std::shared_ptr<::walberla::EKReactionBaseIndexed> m_ekreaction_impl;
174172
};
175173

176174
} // namespace ScriptInterface::walberla

‎src/script_interface/walberla/EKSpecies.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
#include "EKWalberlaNodeState.hpp"
2525
#include "WalberlaCheckpoint.hpp"
2626

27-
#include <walberla_bridge/LatticeWalberla.hpp>
2827
#include <walberla_bridge/electrokinetics/ek_walberla_init.hpp>
2928

3029
#include <boost/mpi.hpp>
@@ -119,7 +118,7 @@ void EKSpecies::do_construct(VariantMap const &args) {
119118
auto const ek_ext_efield = ext_efield * m_conv_ext_efield;
120119
auto const ek_density = m_density = density * m_conv_density;
121120
auto const ek_kT = kT * m_conv_energy;
122-
m_instance = new_ek_walberla(
121+
m_instance = ::walberla::new_ek_walberla(
123122
m_lattice->lattice(), ek_diffusion, ek_kT,
124123
get_value<double>(args, "valency"), ek_ext_efield, ek_density,
125124
get_value<bool>(args, "advection"),

‎src/script_interface/walberla/LBFluid.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@ Variant LBFluid::do_call_method(std::string const &name,
108108
}
109109
if (name == "clear_boundaries") {
110110
m_instance->clear_boundaries();
111-
m_instance->ghost_communication();
112111
::System::get_system().on_lb_boundary_conditions_change();
113112
return {};
114113
}
@@ -269,8 +268,8 @@ void LBFluid::load_checkpoint(std::string const &filename, int mode) {
269268
};
270269

271270
auto const on_success = [&lb_obj]() {
272-
lb_obj.reallocate_ubb_field();
273271
lb_obj.ghost_communication();
272+
lb_obj.reallocate_ubb_field();
274273
};
275274

276275
load_checkpoint_common(*context(), "LB", filename, mode, read_metadata,

‎src/script_interface/walberla/LBFluidNode.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@ Variant LBFluidNode::do_call_method(std::string const &name,
4747
if (name == "set_velocity_at_boundary") {
4848
if (is_none(params.at("value"))) {
4949
m_lb_fluid->remove_node_from_boundary(m_index);
50-
m_lb_fluid->ghost_communication();
5150
} else {
5251
auto const u =
5352
get_value<Utils::Vector3d>(params, "value") * m_conv_velocity;
5453
m_lb_fluid->set_node_velocity_at_boundary(m_index, u);
55-
m_lb_fluid->ghost_communication();
5654
}
55+
m_lb_fluid->ghost_communication();
56+
m_lb_fluid->reallocate_ubb_field();
5757
return {};
5858
}
5959
if (name == "get_velocity_at_boundary") {

‎src/script_interface/walberla/LBFluidSlice.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,10 @@ Variant LBFluidSlice::do_call_method(std::string const &name,
9999
1. / m_conv_velocity);
100100
}
101101
if (name == "set_velocity_at_boundary") {
102-
return call(&LatticeModel::set_slice_velocity_at_boundary, {1},
103-
m_conv_velocity);
102+
auto const retval = call(&LatticeModel::set_slice_velocity_at_boundary, {1},
103+
m_conv_velocity);
104+
m_lb_fluid->reallocate_ubb_field();
105+
return retval;
104106
}
105107
if (name == "get_pressure_tensor") {
106108
return call(&LatticeModel::get_slice_pressure_tensor, {3, 3},

‎src/walberla_bridge/CMakeLists.txt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ if(ESPRESSO_BUILD_WITH_CUDA AND WALBERLA_BUILD_WITH_CUDA)
4242
PRIVATE ${WALBERLA_LIBS})
4343
target_include_directories(espresso_walberla_cuda PUBLIC include)
4444
target_include_directories(
45-
espresso_walberla_cuda PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
46-
PRIVATE ${WALBERLA_INCLUDE_DIRS} ${walberla_BINARY_DIR}/src)
45+
espresso_walberla_cuda PRIVATE ${WALBERLA_INCLUDE_DIRS}
46+
${walberla_BINARY_DIR}/src)
4747
install(TARGETS espresso_walberla_cuda
4848
LIBRARY DESTINATION ${ESPRESSO_INSTALL_PYTHON}/espressomd)
4949
target_link_libraries(espresso_walberla PUBLIC espresso::walberla_cuda)
@@ -52,9 +52,8 @@ endif()
5252
target_link_libraries(
5353
espresso_walberla PUBLIC MPI::MPI_CXX espresso::utils
5454
PRIVATE espresso::cpp_flags espresso::walberla::cpp_flags ${WALBERLA_LIBS})
55-
target_include_directories(
56-
espresso_walberla PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
57-
PRIVATE ${WALBERLA_INCLUDE_DIRS} ${walberla_BINARY_DIR}/src)
55+
target_include_directories(espresso_walberla PRIVATE ${WALBERLA_INCLUDE_DIRS}
56+
${walberla_BINARY_DIR}/src)
5857

5958
add_subdirectory(src)
6059

‎src/walberla_bridge/include/walberla_bridge/electrokinetics/ek_poisson_fft_init.hpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
#pragma once
2121

2222
#include <walberla_bridge/LatticeWalberla.hpp>
23-
24-
#include "PoissonSolver/PoissonSolver.hpp"
23+
#include <walberla_bridge/electrokinetics/PoissonSolver/PoissonSolver.hpp>
2524

2625
#include <memory>
2726

27+
namespace walberla {
28+
2829
std::shared_ptr<walberla::PoissonSolver>
2930
new_ek_poisson_fft(std::shared_ptr<LatticeWalberla> const &lattice,
3031
double permittivity, bool single_precision);
32+
33+
} // namespace walberla

0 commit comments

Comments
 (0)