Skip to content

Commit 8b48fea

Browse files
franzpoeschelfranzpoeschel
committed
WIP: Filter partial patches
No actual filtering yet implemented
1 parent 3be5959 commit 8b48fea

File tree

1 file changed

+134
-52
lines changed

1 file changed

+134
-52
lines changed

‎include/picongpu/plugins/openPMD/restart/LoadSpecies.hpp‎

Lines changed: 134 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -121,74 +121,156 @@ namespace picongpu
121121
uint64_t* patchNumParticles = numParticlesShared.get();
122122
uint64_t* patchNumParticlesOffset = numParticlesOffsetShared.get();
123123

124-
uint64_t totalNumParticles = std::transform_reduce(
125-
fullMatches.begin(),
126-
fullMatches.end(),
127-
0,
128-
/* reduce = */ [](uint64_t left, uint64_t right) { return left + right; },
129-
/* transform = */ [patchNumParticles](size_t patchIdx) { return patchNumParticles[patchIdx]; });
124+
{
125+
uint64_t totalNumParticles = std::transform_reduce(
126+
fullMatches.begin(),
127+
fullMatches.end(),
128+
0,
129+
/* reduce = */ [](uint64_t left, uint64_t right) { return left + right; },
130+
/* transform = */ [patchNumParticles](size_t patchIdx)
131+
{ return patchNumParticles[patchIdx]; });
132+
133+
log<picLog::INPUT_OUTPUT>("openPMD: malloc mapped memory: %1%") % speciesName;
134+
135+
using FrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
136+
using BufferType = Frame<OperatorCreateAlpakaBuffer, NewParticleDescription>;
137+
138+
BufferType buffers;
139+
FrameType mappedFrame;
130140

131-
log<picLog::INPUT_OUTPUT>("openPMD: malloc mapped memory: %1%") % speciesName;
141+
uint64_t maxChunkSize = std::min(static_cast<uint64_t>(restartChunkSize), totalNumParticles);
132142

133-
using FrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
134-
using BufferType = Frame<OperatorCreateAlpakaBuffer, NewParticleDescription>;
143+
/*malloc mapped memory*/
144+
meta::ForEach<typename NewParticleDescription::ValueTypeSeq, MallocMappedMemory<boost::mpl::_1>>
145+
mallocMem;
146+
mallocMem(buffers, mappedFrame, maxChunkSize);
135147

136-
BufferType buffers;
137-
FrameType mappedFrame;
148+
for(size_t const patchIdx : fullMatches)
149+
{
150+
uint64_t particleOffset = patchNumParticlesOffset[patchIdx];
151+
uint64_t numParticles = patchNumParticles[patchIdx];
152+
153+
log<picLog::INPUT_OUTPUT>("openPMD: Loading %1% particles from offset %2%")
154+
% (long long unsigned) totalNumParticles % (long long unsigned) particleOffset;
138155

139-
uint64_t maxChunkSize = std::min(static_cast<uint64_t>(restartChunkSize), totalNumParticles);
140156

141-
/*malloc mapped memory*/
142-
meta::ForEach<typename NewParticleDescription::ValueTypeSeq, MallocMappedMemory<boost::mpl::_1>>
143-
mallocMem;
144-
mallocMem(buffers, mappedFrame, maxChunkSize);
157+
uint32_t const numLoadIterations
158+
= maxChunkSize == 0u ? 0u : alpaka::core::divCeil(numParticles, maxChunkSize);
159+
160+
for(uint64_t loadRound = 0u; loadRound < numLoadIterations; ++loadRound)
161+
{
162+
auto particleLoadOffset = particleOffset + loadRound * maxChunkSize;
163+
auto numLeftParticles = numParticles - loadRound * maxChunkSize;
164+
165+
auto numParticlesCurrentBatch = std::min(numLeftParticles, maxChunkSize);
166+
167+
log<picLog::INPUT_OUTPUT>("openPMD: (begin) load species %1% round: %2%/%3%") % speciesName
168+
% (loadRound + 1) % numLoadIterations;
169+
if(numParticlesCurrentBatch != 0)
170+
{
171+
meta::ForEach<
172+
typename NewParticleDescription::ValueTypeSeq,
173+
LoadParticleAttributesFromOpenPMD<boost::mpl::_1>>
174+
loadAttributes;
175+
loadAttributes(
176+
params,
177+
mappedFrame,
178+
particleSpecies,
179+
particleLoadOffset,
180+
numParticlesCurrentBatch);
181+
182+
pmacc::particles::operations::splitIntoListOfFrames(
183+
*speciesTmp,
184+
mappedFrame,
185+
numParticlesCurrentBatch,
186+
cellOffsetToTotalDomain,
187+
totalCellIdx_,
188+
*(params->cellDescription),
189+
picLog::INPUT_OUTPUT());
190+
}
191+
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species %1% round: %2%/%3%") % speciesName
192+
% (loadRound + 1) % numLoadIterations;
193+
}
194+
}
195+
}
145196

146-
for(size_t const patchIdx : fullMatches)
147197
{
148-
uint64_t particleOffset = patchNumParticlesOffset[patchIdx];
149-
uint64_t numParticles = patchNumParticles[patchIdx];
198+
SubGrid<simDim> const& subGrid = Environment<simDim>::get().SubGrid();
199+
pmacc::Selection<simDim> const localDomain = subGrid.getLocalDomain();
200+
pmacc::Selection<simDim> const globalDomain = subGrid.getGlobalDomain();
201+
202+
uint64_t totalNumParticles = std::transform_reduce(
203+
partialMatches.begin(),
204+
partialMatches.end(),
205+
0,
206+
/* reduce = */ [](uint64_t left, uint64_t right) { return left + right; },
207+
/* transform = */ [patchNumParticles](size_t patchIdx)
208+
{ return patchNumParticles[patchIdx]; });
150209

151-
log<picLog::INPUT_OUTPUT>("openPMD: Loading %1% particles from offset %2%")
152-
% (long long unsigned) totalNumParticles % (long long unsigned) particleOffset;
210+
log<picLog::INPUT_OUTPUT>("openPMD: malloc mapped memory for partial patches: %1%") % speciesName;
153211

212+
using FrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
213+
using BufferType = Frame<OperatorCreateAlpakaBuffer, NewParticleDescription>;
154214

155-
uint32_t const numLoadIterations
156-
= maxChunkSize == 0u ? 0u : alpaka::core::divCeil(numParticles, maxChunkSize);
215+
BufferType buffers;
216+
FrameType mappedFrame;
157217

158-
for(uint64_t loadRound = 0u; loadRound < numLoadIterations; ++loadRound)
218+
uint64_t maxChunkSize = std::min(static_cast<uint64_t>(restartChunkSize), totalNumParticles);
219+
220+
/*malloc mapped memory*/
221+
meta::ForEach<typename NewParticleDescription::ValueTypeSeq, MallocMappedMemory<boost::mpl::_1>>
222+
mallocMem;
223+
mallocMem(buffers, mappedFrame, maxChunkSize);
224+
for(size_t const patchIdx : partialMatches)
159225
{
160-
auto particleLoadOffset = particleOffset + loadRound * maxChunkSize;
161-
auto numLeftParticles = numParticles - loadRound * maxChunkSize;
226+
uint64_t particleOffset = patchNumParticlesOffset[patchIdx];
227+
uint64_t numParticles = patchNumParticles[patchIdx];
228+
229+
log<picLog::INPUT_OUTPUT>("openPMD: Loading up to %1% particles from offset %2%")
230+
% (long long unsigned) totalNumParticles % (long long unsigned) particleOffset;
231+
162232

163-
auto numParticlesCurrentBatch = std::min(numLeftParticles, maxChunkSize);
233+
uint32_t const numLoadIterations
234+
= maxChunkSize == 0u ? 0u : alpaka::core::divCeil(numParticles, maxChunkSize);
164235

165-
log<picLog::INPUT_OUTPUT>("openPMD: (begin) load species %1% round: %2%/%3%") % speciesName
166-
% (loadRound + 1) % numLoadIterations;
167-
if(numParticlesCurrentBatch != 0)
236+
for(uint64_t loadRound = 0u; loadRound < numLoadIterations; ++loadRound)
168237
{
169-
meta::ForEach<
170-
typename NewParticleDescription::ValueTypeSeq,
171-
LoadParticleAttributesFromOpenPMD<boost::mpl::_1>>
172-
loadAttributes;
173-
loadAttributes(
174-
params,
175-
mappedFrame,
176-
particleSpecies,
177-
particleLoadOffset,
178-
numParticlesCurrentBatch);
179-
180-
181-
pmacc::particles::operations::splitIntoListOfFrames(
182-
*speciesTmp,
183-
mappedFrame,
184-
numParticlesCurrentBatch,
185-
cellOffsetToTotalDomain,
186-
totalCellIdx_,
187-
*(params->cellDescription),
188-
picLog::INPUT_OUTPUT());
238+
auto particleLoadOffset = particleOffset + loadRound * maxChunkSize;
239+
auto numLeftParticles = numParticles - loadRound * maxChunkSize;
240+
241+
auto numParticlesCurrentBatch = std::min(numLeftParticles, maxChunkSize);
242+
243+
log<picLog::INPUT_OUTPUT>("openPMD: (begin) load species %1% round: %2%/%3%") % speciesName
244+
% (loadRound + 1) % numLoadIterations;
245+
if(numParticlesCurrentBatch != 0)
246+
{
247+
meta::ForEach<
248+
typename NewParticleDescription::ValueTypeSeq,
249+
LoadParticleAttributesFromOpenPMD<boost::mpl::_1>>
250+
loadAttributes;
251+
loadAttributes(
252+
params,
253+
mappedFrame,
254+
particleSpecies,
255+
particleLoadOffset,
256+
numParticlesCurrentBatch);
257+
258+
// now filter
259+
mappedFrame.getIdentifier(position());
260+
mappedFrame.getIdentifier(totalCellIdx()); // positionOffset
261+
262+
pmacc::particles::operations::splitIntoListOfFrames(
263+
*speciesTmp,
264+
mappedFrame,
265+
numParticlesCurrentBatch,
266+
cellOffsetToTotalDomain,
267+
totalCellIdx_,
268+
*(params->cellDescription),
269+
picLog::INPUT_OUTPUT());
270+
}
271+
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species %1% round: %2%/%3%") % speciesName
272+
% (loadRound + 1) % numLoadIterations;
189273
}
190-
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species %1% round: %2%/%3%") % speciesName
191-
% (loadRound + 1) % numLoadIterations;
192274
}
193275
}
194276
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species: %1%") % speciesName;

0 commit comments

Comments
 (0)