diff --git a/cmake/oomph_common.cmake b/cmake/oomph_common.cmake index 5dcdc152..733a0b68 100644 --- a/cmake/oomph_common.cmake +++ b/cmake/oomph_common.cmake @@ -8,9 +8,13 @@ mark_as_advanced(OOMPH_USE_FAST_PIMPL) # --------------------------------------------------------------------- # compiler and linker flags # --------------------------------------------------------------------- +#set(cxx_lang "$") function(oomph_target_compile_options target) set_target_properties(${target} PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE ON) target_compile_options(${target} PRIVATE -Wall -Wextra -Wpedantic) + #target_compile_options(${target} PRIVATE + # $<${cxx_lang}:$> + #) endfunction() function(oomph_target_link_options target) diff --git a/include/oomph/communicator.hpp b/include/oomph/communicator.hpp index 48f0c2f0..0e8fe1c7 100644 --- a/include/oomph/communicator.hpp +++ b/include/oomph/communicator.hpp @@ -23,12 +23,14 @@ namespace oomph { -class context; -class send_channel_base; -class recv_channel_base; - +class context_impl; class communicator_impl; +namespace detail +{ +communicator get_communicator(context_impl* c); +} // namespace detail + class communicator { public: @@ -42,9 +44,7 @@ class communicator static constexpr tag_type any_tag = -1; private: - friend class context; - friend class send_channel_base; - friend class recv_channel_base; + friend communicator detail::get_communicator(context_impl*); public: struct schedule @@ -198,7 +198,7 @@ class communicator // ==================== template - [[nodiscard]] recv_request recv(message_buffer& msg, rank_type src, tag_type tag) + recv_request recv(message_buffer& msg, rank_type src, tag_type tag) { assert(msg); auto& scheduled = m_schedule->scheduled_recvs; @@ -209,7 +209,7 @@ class communicator } template - [[nodiscard]] send_request send(message_buffer const& msg, rank_type dst, tag_type tag) + send_request send(message_buffer const& msg, rank_type dst, tag_type tag) { assert(msg); auto& scheduled = m_schedule->scheduled_sends; diff --git a/include/oomph/context.hpp b/include/oomph/context.hpp index b22e18b9..a982b853 100644 --- a/include/oomph/context.hpp +++ b/include/oomph/context.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -83,6 +84,10 @@ class context communicator get_communicator(); + template + tensor::map map_tensor( + tensor::vector const& extents, T* first, T* last); + private: detail::message_buffer make_buffer_core(std::size_t size); detail::message_buffer make_buffer_core(void* ptr, std::size_t size); diff --git a/include/oomph/tensor/buffer_cache.hpp b/include/oomph/tensor/buffer_cache.hpp new file mode 100644 index 00000000..16cd2e49 --- /dev/null +++ b/include/oomph/tensor/buffer_cache.hpp @@ -0,0 +1,31 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +template +struct buffer_cache +{ + std::shared_ptr> m_cache; + + buffer_cache() + : m_cache{std::make_shared>()} + { + } + buffer_cache(buffer_cache const&) = default; + buffer_cache& operator=(buffer_cache const&) = default; +}; +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/detail/buffer_cache.hpp b/include/oomph/tensor/detail/buffer_cache.hpp new file mode 100644 index 00000000..b6a58a24 --- /dev/null +++ b/include/oomph/tensor/detail/buffer_cache.hpp @@ -0,0 +1,104 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace oomph +{ +namespace tensor +{ +namespace detail +{ +template +struct buffer_cache +{ + std::vector>> m_messages; + std::map> m_available_idx; + + buffer_cache() noexcept = default; + buffer_cache(buffer_cache&&) noexcept = default; + buffer_cache& operator=(buffer_cache&&) noexcept = default; + + std::shared_ptr> operator()(communicator& comm, std::size_t size, Id id) + { + std::set* index_set = nullptr; + auto it = m_available_idx.find(id); + if (it == m_available_idx.end()) + { + index_set = &(m_available_idx[id]); + for (std::size_t i = 0; i < m_messages.size(); ++i) index_set->insert(i); + } + else + { + index_set = &(it->second); + } + + auto m_it = std::find_if(index_set->begin(), index_set->end(), + [this, size](std::size_t i) { return (m_messages[i]->size() == size); }); + + if (m_it == index_set->end()) + { + m_messages.push_back(std::make_shared>(comm.make_buffer(size))); + for (auto& s : m_available_idx) s.second.insert(m_messages.size() - 1); + index_set->erase(m_messages.size() - 1); + return m_messages.back(); + } + else + { + auto res = *m_it; + index_set->erase(m_it); + return m_messages[res]; + } + } + + template + std::shared_ptr> operator()(communicator& comm, std::size_t size, Id id, + buffer_cache& c2, Id2 id2) + { + std::set* index_set = nullptr; + auto it = m_available_idx.find(id); + if (it == m_available_idx.end()) + { + index_set = &(m_available_idx[id]); + for (std::size_t i = 0; i < m_messages.size(); ++i) index_set->insert(i); + } + else + { + index_set = &(it->second); + } + + auto m_it = std::find_if(index_set->begin(), index_set->end(), + [this, size](std::size_t i) { return (m_messages[i]->size() == size); }); + + if (m_it == index_set->end()) + { + auto res = c2(comm, size, id2); + m_messages.push_back(res); + for (auto& s : m_available_idx) s.second.insert(m_messages.size() - 1); + index_set->erase(m_messages.size() - 1); + return res; + } + else + { + auto res = *m_it; + index_set->erase(m_it); + return m_messages[res]; + } + } +}; + +} // namespace detail +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/detail/map.hpp b/include/oomph/tensor/detail/map.hpp new file mode 100644 index 00000000..946edca1 --- /dev/null +++ b/include/oomph/tensor/detail/map.hpp @@ -0,0 +1,74 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +namespace detail +{ +template +class map +{ + public: + static constexpr std::size_t dim() noexcept { return Layout::max_arg + 1; }; + + using vec = vector; + + protected: + static constexpr std::size_t s_stride_1_dim = Layout::find(dim() - 1); + + protected: + T* m_data; + std::size_t m_num_elements; + vec m_extents; + vec m_strides; + std::size_t m_line_size; + + public: + map(vec const& extents, T* first, T* last) + : m_data{first} + , m_num_elements{product(extents)} + , m_extents{extents} + { + auto const stride_1_extent = m_extents[s_stride_1_dim]; + auto const num_lines = m_num_elements / stride_1_extent; + auto const total_padding = (((last + 1) - first) - m_num_elements) * sizeof(T); + std::size_t const padding = (num_lines == 1) ? 0 : total_padding / (num_lines - 1); + m_strides[s_stride_1_dim] = 1; + std::size_t s = stride_1_extent * sizeof(T) + padding; + assert((s / sizeof(T)) * sizeof(T) == s); + s /= sizeof(T); + m_line_size = s; + for (std::size_t i = 1; i < dim(); ++i) + { + m_strides[Layout::find(dim() - 1 - i)] = s; + s *= m_extents[Layout::find(dim() - 1 - i)]; + } + } + + map(map const&) noexcept = default; + map(map&&) noexcept = default; + map& operator=(map const&) noexcept = default; + map& operator=(map&&) noexcept = default; + + public: + auto const& strides() const noexcept { return m_strides; } + vec const& extents() const noexcept { return m_extents; } + std::size_t line_size() const noexcept { return m_line_size; } + T* get_address(vec coord) const noexcept { return m_data + dot(coord, m_strides); } +}; + +} // namespace detail +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/detail/terminal.hpp b/include/oomph/tensor/detail/terminal.hpp new file mode 100644 index 00000000..c44da783 --- /dev/null +++ b/include/oomph/tensor/detail/terminal.hpp @@ -0,0 +1,275 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace oomph +{ +namespace tensor +{ +namespace detail +{ +template +class terminal; + +template +class terminal> +{ + public: + using map_type = map; + static constexpr std::size_t dim() noexcept { return map_type::dim(); }; + using vec = vector; + using range_type = range; + + static constexpr std::size_t last_dim = Layout::find(0); + + protected: + struct transport_range + { + range_type m_range; + std::shared_ptr> m_message; + int m_rank; + int m_tag; + bool m_direct; + }; + + struct serialization_range + { + range_type m_range; + T* m_ptr; + }; + + struct stage_t + { + std::vector m_transport_ranges; + std::vector m_serialization_ranges; + }; + + protected: + map_type m_map; + std::unique_ptr m_comm; + std::map m_stages; + std::shared_ptr> m_top_buffer_cache; + buffer_cache m_buffer_cache; + std::vector m_stage_lu; + bool m_connected = false; + + public: + template + terminal(Map& m) + : m_map{m} + , m_comm{std::make_unique(oomph::detail::get_communicator(m.m_context))} + , m_top_buffer_cache{std::make_shared>()} + { + } + + template + terminal(Map& m, std::shared_ptr> c) + : m_map{m} + , m_comm{std::make_unique(oomph::detail::get_communicator(m.m_context))} + , m_top_buffer_cache{c} + { + } + + terminal(terminal&&) noexcept = default; + terminal& operator=(terminal&&) noexcept = default; + + public: + void add_range(range_type const& view, int rank, int tag, std::size_t stage) + { + assert(!m_connected); + //x==*, y==1, z==1, w==1 -> direct + //x==Nx, y==*, z==1, w==1 -> direct + //x==Nx, y==Ny z==*, w==1 -> direct + //x==Nx, y==Ny z==Nz, w==* -> direct + + bool found_subset = (view.extents()[last_dim] != 1); + bool direct = true; + for (std::size_t d = 1; d < dim(); ++d) + { + auto const D = Layout::find(d); + if (found_subset && view.extents()[D] != m_map.extents()[D]) + { + direct = false; + break; + } + else if (view.extents()[D] != 1) + { + found_subset = true; + } + } + + auto& s = m_stages[stage]; + + if (direct) + { + auto ext = view.extents(); + ext[Layout::find(dim() - 1)] = m_map.line_size(); + auto const n_elements = product(ext); + s.m_transport_ranges.push_back(transport_range{view, + std::make_shared>( + m_comm->make_buffer(m_map.get_address(view.first()), n_elements)), + rank, tag, true}); + } + else + { + auto const n_elements = product(view.extents()); + auto const n_elements_slice = n_elements / view.extents()[last_dim]; + s.m_transport_ranges.push_back(transport_range{view, + //m_comm->make_buffer(n_elements), + //m_buffer_cache(*m_comm, n_elements, stage), + m_buffer_cache(*m_comm, n_elements, stage, *m_top_buffer_cache, m_comm.get()), rank, + tag, false}); + T* ptr = s.m_transport_ranges.back().m_message->data(); + + std::size_t const first_k = view.first()[last_dim]; + std::size_t const last_k = first_k + view.extents()[last_dim]; + for (std::size_t k = first_k; k < last_k; ++k) + { + auto first = view.first(); + first[last_dim] = k; + auto ext = view.extents(); + ext[last_dim] = 1; + + s.m_serialization_ranges.push_back( + serialization_range{range_type{first, ext}, ptr}); + ptr += n_elements_slice; + } + + std::sort(s.m_serialization_ranges.begin(), s.m_serialization_ranges.end(), + [](auto const& a, auto const& b) + { + auto const& first_a = a.m_range.first(); + auto const& first_b = b.m_range.first(); + for (std::size_t d = 0; d < dim(); ++d) + { + if (first_a[Layout::find(d)] < first_b[Layout::find(d)]) return true; + if (first_a[Layout::find(d)] > first_b[Layout::find(d)]) return false; + } + return true; + }); + } + } + + protected: + void connect() + { + assert(!m_connected); + assert(m_stages.size() > 0); + m_stage_lu = std::vector(m_stages.rbegin()->first + 1, nullptr); + for (auto& kvp : m_stages) m_stage_lu[kvp.first] = &(kvp.second); + m_connected = true; + } + + stage_t* get_stage(std::size_t stage) + { + assert(stage < m_stage_lu.size()); + return m_stage_lu[stage]; + } + + T* serialize(serialization_range const& r, T* dst, vec coord, + std::integral_constant) + { + static constexpr std::size_t D = Layout::find(dim() - 1); + T const* src = m_map.get_address(coord); + std::size_t const n = r.m_range.extents()[D]; + for (std::size_t i = 0; i < n; ++i) dst[i] = src[i]; + + return dst + n; + } + + T const* serialize(serialization_range const& r, T const* src, vec coord, + std::integral_constant) + { + static constexpr std::size_t D = Layout::find(dim() - 1); + T* dst = m_map.get_address(coord); + std::size_t const n = r.m_range.extents()[D]; + for (std::size_t i = 0; i < n; ++i) dst[i] = src[i]; + return src + n; + } + + template + Ptr serialize(serialization_range const& r, Ptr ptr, vec coord, + std::integral_constant) + { + static constexpr std::size_t D = Layout::find(N); + std::size_t const first = r.m_range.first()[D]; + std::size_t const last = first + r.m_range.extents()[D]; + for (; coord[D] < last; ++coord[D]) + ptr = serialize(r, ptr, coord, std::integral_constant{}); + return ptr; + } + + struct pack_handle + { + bool is_ready() const noexcept { return true; } + void wait() {} + }; + + pack_handle pack(std::size_t stage) + { + assert(m_connected); + stage_t* s = get_stage(stage); + if (!s) return {}; + for (auto& r : s->m_serialization_ranges) + serialize(r, r.m_ptr, r.m_range.first(), std::integral_constant()); + return {}; + } + + pack_handle unpack(std::size_t stage) + { + assert(m_connected); + stage_t* s = get_stage(stage); + if (!s) return {}; + for (auto& r : s->m_serialization_ranges) + serialize(r, (T const*)r.m_ptr, r.m_range.first(), + std::integral_constant()); + return {}; + } + + struct handle + { + communicator* m_comm; + bool is_ready() const noexcept { return m_comm->is_ready(); } + void progress() { m_comm->progress(); } + void wait() { m_comm->wait_all(); } + }; + + handle send(std::size_t stage) + { + assert(m_connected); + stage_t* s = get_stage(stage); + if (!s) return {m_comm.get()}; + for (auto& r : s->m_transport_ranges) m_comm->send(*r.m_message, r.m_rank, r.m_tag); + return {m_comm.get()}; + } + + handle recv(std::size_t stage) + { + assert(m_connected); + stage_t* s = get_stage(stage); + if (!s) return {m_comm.get()}; + for (auto& r : s->m_transport_ranges) m_comm->recv(*r.m_message, r.m_rank, r.m_tag); + return {m_comm.get()}; + } +}; + +} // namespace detail +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/layout.hpp b/include/oomph/tensor/layout.hpp new file mode 100644 index 00000000..f97ee68e --- /dev/null +++ b/include/oomph/tensor/layout.hpp @@ -0,0 +1,57 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +template +struct layout +{ + static constexpr std::size_t N = sizeof...(I); + static constexpr std::size_t max_arg = N - 1; + + using arg_list = boost::mp11::mp_list...>; + template + using sorter = boost::mp11::mp_bool<(A::value < B::value)>; + using sorted_arg_list = boost::mp11::mp_sort; + using lookup_list = boost::mp11::mp_iota_c; + static_assert(std::is_same>::value, + "arguments must be unique, contiguous and starting from 0"); + template + using F = boost::mp11::mp_find; + using reverse_lookup = boost::mp11::mp_transform; + + // Get the position of the element with value `i` in the layout + static constexpr std::size_t find(std::size_t i) + { + return find_impl(i, boost::mp11::make_index_sequence{}); + } + + // Get the value of the element at position `i` in the layout + static constexpr std::size_t at(std::size_t i) + { + std::size_t const ri[] = {I...}; + return ri[i]; + } + + template + static constexpr std::size_t find_impl(std::size_t i, boost::mp11::index_sequence) + { + std::size_t const ri[] = {boost::mp11::mp_at_c::value...}; + return ri[i]; + } +}; + +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/map.hpp b/include/oomph/tensor/map.hpp new file mode 100644 index 00000000..7dcb45e8 --- /dev/null +++ b/include/oomph/tensor/map.hpp @@ -0,0 +1,82 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include + +namespace oomph +{ +namespace tensor +{ +namespace detail +{ +template +class terminal; +} // namespace detail + +template +class map : public detail::map +{ + private: + friend class oomph::context; + friend class detail::terminal>; + using base = detail::map; + + public: + using base::dim; + using vec = typename base::vec; + + private: + context_impl* m_context; + + private: + map(context_impl* c, vec const& extents, T* first, T* last) + : base(extents, first, last) + , m_context{c} + { + } + + public: + map(map const&) = delete; + map& operator=(map const&) = delete; + + map(map&& other) noexcept + : base(std::move(other)) + , m_context{std::exchange(other.m_context, nullptr)} + { + } + + map& operator=(map&& other) noexcept + { + deregister(); + static_cast(*this) = std::move(other); + m_context = std::exchange(other.m_context, nullptr); + } + + ~map() { deregister(); } + + private: + void deregister() + { + if (m_context) {} + } +}; + +} // namespace tensor + +template +tensor::map +context::map_tensor(tensor::vector const& extents, T* first, + T* last) +{ + return {m.get(), extents, first, last}; +} +} // namespace oomph diff --git a/include/oomph/tensor/map_fwd.hpp b/include/oomph/tensor/map_fwd.hpp new file mode 100644 index 00000000..aab703b2 --- /dev/null +++ b/include/oomph/tensor/map_fwd.hpp @@ -0,0 +1,21 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +template +class map; +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/range.hpp b/include/oomph/tensor/range.hpp new file mode 100644 index 00000000..e875d5b1 --- /dev/null +++ b/include/oomph/tensor/range.hpp @@ -0,0 +1,48 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +template +class range +{ + public: + using vec = vector; + //template + //friend class oomph::detail::map; + + private: + vec m_first; + vec m_extents; + vec m_increments; + + public: + constexpr range(vec const& first, vec const& extents, + vec const& increments = make_uniform((std::size_t)1)) + : m_first{first} + , m_extents{extents} + , m_increments{increments} + { + } + + constexpr range(range const&) noexcept = default; + range& operator=(range const&) noexcept = default; + + vec const& first() const noexcept { return m_first; } + vec const& extents() const noexcept { return m_extents; } + vec const& increments() const noexcept { return m_increments; } +}; +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/receiver.hpp b/include/oomph/tensor/receiver.hpp new file mode 100644 index 00000000..52dcdd22 --- /dev/null +++ b/include/oomph/tensor/receiver.hpp @@ -0,0 +1,80 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include +#include + +namespace oomph +{ +namespace tensor +{ +template +class receiver; + +template +class receiver> : private detail::terminal> +{ + private: + using base = detail::terminal>; + + public: + using map_type = map; + using range_type = typename base::range_type; + using pack_handle = typename base::pack_handle; + using handle = typename base::handle; + + public: + receiver(map_type& m) + : base(m) + { + } + + receiver(map_type& m, buffer_cache const& c) + : base(m, c.m_cache) + { + } + + receiver(receiver&&) noexcept = default; + receiver& operator=(receiver&&) noexcept = default; + + receiver& add_src(range_type const& view, int rank, int tag, std::size_t stage = 0) + { + base::add_range(view, rank, tag, stage); + return *this; + } + + receiver& connect() + { + base::connect(); + return *this; + } + + pack_handle unpack(std::size_t stage = 0) { return base::unpack(stage); } + + handle recv(std::size_t stage = 0) { return base::recv(stage); } +}; + +template +receiver> +make_receiver(map& m) +{ + return {m}; +} + +template +receiver> +make_receiver(map& m, buffer_cache const& c) +{ + return {m, c}; +} +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/rt_layout.hpp b/include/oomph/tensor/rt_layout.hpp new file mode 100644 index 00000000..61d5a617 --- /dev/null +++ b/include/oomph/tensor/rt_layout.hpp @@ -0,0 +1,100 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +class rt_layout +{ + private: + std::vector const m; + + public: + rt_layout(std::initializer_list l) + : m{l} + { + } + + rt_layout(std::size_t* first, std::size_t* last) + : m(first, last) + { + } + + rt_layout(std::size_t* first, std::size_t count) + : rt_layout(first, first + count) + { + } + + rt_layout(rt_layout const&) = default; + + rt_layout(rt_layout&&) noexcept = default; + + public: + friend bool operator==(rt_layout const& a, rt_layout const& b) + { + std::size_t const s_a = a.size(); + std::size_t const s_b = b.size(); + if (s_a != s_b) return false; + for (std::size_t i = 0; i < s_a; ++i) + if (a.at(i) != b.at(i)) return false; + return true; + } + + template + bool is_equal() const noexcept + { + std::size_t const s = m.size(); + if (Layout::max_arg + 1 != s) return false; + for (std::size_t i = 0; i < s; ++i) + if (Layout::at(i) != m[i]) return false; + return true; + } + + public: + std::size_t const* data() const noexcept { return m.data(); } + std::size_t size() const noexcept { return m.size(); } + + public: + // Get the position of the element with value `i` in the layout + std::size_t find(std::size_t i) const noexcept + { + std::size_t const s = m.size(); + for (std::size_t j = 0; j < s; ++j) + if (m[j] == i) return j; + return s; + } + + // Get the value of the element at position `i` in the layout + std::size_t at(std::size_t i) const noexcept { return m[i]; } +}; + +namespace detail +{ +template +rt_layout make_rt_layout(std::index_sequence) noexcept +{ + return {{Layout::at(Is)...}}; +} + +} // namespace detail + +template +rt_layout +make_rt_layout() noexcept +{ + return detail::make_rt_layout(std::make_index_sequence{}); +} + +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/sender.hpp b/include/oomph/tensor/sender.hpp new file mode 100644 index 00000000..09892d56 --- /dev/null +++ b/include/oomph/tensor/sender.hpp @@ -0,0 +1,81 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include +#include + +namespace oomph +{ +namespace tensor +{ +template +class sender; + +template +class sender> : private detail::terminal> +{ + private: + using base = detail::terminal>; + + public: + using map_type = map; + using range_type = typename base::range_type; + using pack_handle = typename base::pack_handle; + using handle = typename base::handle; + + public: + sender(map_type& m) + : base(m) + { + } + + sender(map_type& m, buffer_cache const& c) + : base(m, c.m_cache) + { + } + + sender(sender&&) noexcept = default; + sender& operator=(sender&&) noexcept = default; + + sender& add_dst(range_type const& view, int rank, int tag, std::size_t stage = 0) + { + base::add_range(view, rank, tag, stage); + return *this; + } + + sender& connect() + { + base::connect(); + return *this; + } + + pack_handle pack(std::size_t stage = 0) { return base::pack(stage); } + + handle send(std::size_t stage = 0) { return base::send(stage); } +}; + +template +sender> +make_sender(map& m) +{ + return {m}; +} + +template +sender> +make_sender(map& m, buffer_cache const& c) +{ + return {m, c}; +} + +} // namespace tensor +} // namespace oomph diff --git a/include/oomph/tensor/vector.hpp b/include/oomph/tensor/vector.hpp new file mode 100644 index 00000000..0cd7a03b --- /dev/null +++ b/include/oomph/tensor/vector.hpp @@ -0,0 +1,228 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include + +namespace oomph +{ +namespace tensor +{ +template +struct vector +{ + T m_data[N]; + + static constexpr std::size_t size() noexcept { return N; } + + template + vector& operator=(vector const& v) + { + for (std::size_t i = 0; i < N; ++i) m_data[i] = v[i]; + } + + constexpr T const* cbegin() const { return m_data; } + constexpr T const* begin() const { return m_data; } + constexpr T* begin() { return m_data; } + + constexpr T const* cend() const { return cbegin() + N; } + constexpr T const* end() const { return begin() + N; } + constexpr T* end() { return begin() + N; } + + constexpr const T* data() const noexcept { return m_data; } + constexpr T* data() noexcept { return m_data; } + + constexpr T operator[](std::size_t i) const noexcept { return m_data[i]; } + T& operator[](std::size_t i) noexcept { return m_data[i]; } + + template + vector& operator+=(vector const& v) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] += v[i]; + return *this; + } + + template + vector& operator+=(U const& u) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] += u; + return *this; + } + + template + vector& operator-=(vector const& v) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] += v[i]; + return *this; + } + + template + vector& operator-=(U const& u) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] += u; + return *this; + } + + template + vector& operator*=(vector const& v) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] *= v[i]; + return *this; + } + + template + vector& operator*=(U const& u) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] *= u; + return *this; + } + + template + vector& operator/=(vector const& v) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] /= v[i]; + return *this; + } + + template + vector& operator/=(U const& u) noexcept + { + for (std::size_t i = 0; i < N; ++i) m_data[i] /= u; + return *this; + } +}; + +template +inline constexpr auto +add(vector const& a, vector const& b, std::index_sequence) noexcept +{ + using R = std::remove_reference_t() + std::declval())>; + return R{(a[I] + b[I])...}; +} + +template +inline constexpr auto +operator+(vector const& a, vector const& b) noexcept +{ + return add(a, b, std::make_index_sequence{}); +} + +template +inline constexpr auto +subtract(vector const& a, vector const& b, std::index_sequence) noexcept +{ + using R = std::remove_reference_t() - std::declval())>; + return R{(a[I] - b[I])...}; +} + +template +inline constexpr auto +operator-(vector const& a, vector const& b) noexcept +{ + return subtract(a, b, std::make_index_sequence{}); +} + +template +inline constexpr auto +multiply(vector const& a, vector const& b, std::index_sequence) noexcept +{ + using R = std::remove_reference_t() * std::declval())>; + return R{(a[I] * b[I])...}; +} + +template +inline constexpr auto +operator*(vector const& a, vector const& b) noexcept +{ + return multiply(a, b, std::make_index_sequence{}); +} + +template +inline constexpr auto +divide(vector const& a, vector const& b, std::index_sequence) noexcept +{ + using R = std::remove_reference_t() / std::declval())>; + return R{(a[I] / b[I])...}; +} + +template +inline constexpr auto +operator/(vector const& a, vector const& b) noexcept +{ + return divide(a, b, std::make_index_sequence{}); +} + +#if __cplusplus >= 201703L +template +inline constexpr auto +dot(vector const& a, vector const& b, std::index_sequence) noexcept +{ + return (... + (a[I] * b[I])); +} + +template +inline constexpr auto +dot(vector const& a, vector const& b) noexcept +{ + return dot(a, b, std::make_index_sequence{}); +} +#else +template +inline constexpr auto +dot(vector const& a, vector const& b) noexcept +{ + auto r = a[0] * b[0]; + for (std::size_t i = 1; i < N; ++i) r += a[i] * b[i]; + return r; +} +#endif + +#if __cplusplus >= 201703L +template +inline constexpr auto +product(vector const& v, std::index_sequence) noexcept +{ + return (... * v[I]); +} + +template +inline constexpr auto +product(vector const& v) noexcept +{ + return product(v, std::make_index_sequence{}); +} +#else +template +inline constexpr auto +product(vector const& v) noexcept +{ + T r = v[0]; + for (std::size_t i = 1; i < N; ++i) r *= v[i]; + return r; +} +#endif + +template +inline constexpr vector +make_uniform(T const& value, std::index_sequence) noexcept +{ + T const* const vref = &value; + return {vref[I - I]...}; +} + +template +inline constexpr vector +make_uniform(T const& value) noexcept +{ + return make_uniform(value, std::make_index_sequence{}); +} + +} // namespace tensor +} // namespace oomph diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57716f4c..237b54a5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,7 @@ target_sources(oomph_common PRIVATE utils.cpp) target_sources(oomph_common PRIVATE thread_id.cpp) target_sources(oomph_common PRIVATE rank_topology.cpp) +#target_sources(oomph_common PRIVATE tensor.cpp) if (OOMPH_WITH_MPI) add_subdirectory(mpi) diff --git a/src/src.cpp b/src/src.cpp index 3a515c4a..9771982c 100644 --- a/src/src.cpp +++ b/src/src.cpp @@ -63,7 +63,7 @@ context::~context() = default; communicator context::get_communicator() { - return {m->get_communicator()}; + return detail::get_communicator(m.get()); } /////////////////////////////// @@ -126,6 +126,15 @@ communicator::progress() m_impl->progress(); } +namespace detail +{ +communicator +get_communicator(context_impl* c) +{ + return {c->get_communicator()}; +} +} // namespace detail + #if OOMPH_ENABLE_BARRIER /////////////////////////////// // barrier // @@ -422,46 +431,4 @@ recv_request::cancel() return res; } -///////////////////////////////// -//// send_channel_base // -///////////////////////////////// -// -//send_channel_base::~send_channel_base() = default; -// -//void -//send_channel_base::connect() -//{ -// m_impl->connect(); -//} -// -///////////////////////////////// -//// recv_channel_base // -///////////////////////////////// -// -//recv_channel_base::~recv_channel_base() = default; -// -//void -//recv_channel_base::connect() -//{ -// m_impl->connect(); -//} -// -//std::size_t -//recv_channel_base::capacity() -//{ -// return m_impl->capacity(); -//} -// -//void* -//recv_channel_base::get(std::size_t& index) -//{ -// return m_impl->get(index); -//} -// -//recv_channel_impl* -//recv_channel_base::get_impl() noexcept -//{ -// return &(*m_impl); -//} - } // namespace oomph diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 274f9f7b..efd30dbe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,7 +7,8 @@ add_subdirectory(mpi_runner) # --------------------------------------------------------------------- # list of tests to be executed -set(parallel_tests test_context test_send_recv test_send_multi test_cancel test_locality) +set(parallel_tests test_context test_send_recv test_send_multi test_cancel test_locality + test_tensor) if (OOMPH_ENABLE_BARRIER) list(APPEND parallel_tests test_barrier) endif() diff --git a/test/test_tensor.cpp b/test/test_tensor.cpp new file mode 100644 index 00000000..e61c6c57 --- /dev/null +++ b/test/test_tensor.cpp @@ -0,0 +1,356 @@ +/* + * ghex-org + * + * Copyright (c) 2014-2021, ETH Zurich + * All rights reserved. + * + * Please, refer to the LICENSE file in the root directory. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +#include +#include "./mpi_runner/mpi_test_fixture.hpp" +#include +#include +#include +#include +#include + +template +void +test_layout() +{ + using namespace oomph; + using namespace oomph::tensor; + using layout_t = layout; + static_assert(layout_t::at(0) == I, ""); + static_assert(layout_t::at(1) == J, ""); + static_assert(layout_t::at(2) == K, ""); + + static constexpr std::size_t pos_0 = (I == 0) ? 0 : ((J == 0) ? 1 : 2); + static constexpr std::size_t pos_1 = (I == 1) ? 0 : ((J == 1) ? 1 : 2); + static constexpr std::size_t pos_2 = (I == 2) ? 0 : ((J == 2) ? 1 : 2); + + static_assert(layout_t::find(0) == pos_0, ""); + static_assert(layout_t::find(1) == pos_1, ""); + static_assert(layout_t::find(2) == pos_2, ""); + + auto l = make_rt_layout(); + + EXPECT_EQ(l.at(0), layout_t::at(0)); + EXPECT_EQ(l.at(1), layout_t::at(1)); + EXPECT_EQ(l.at(2), layout_t::at(2)); + + EXPECT_EQ(l.find(0), layout_t::find(0)); + EXPECT_EQ(l.find(1), layout_t::find(1)); + EXPECT_EQ(l.find(2), layout_t::find(2)); + + EXPECT_TRUE( l.template is_equal() ); + EXPECT_FALSE( (l.template is_equal>()) ); + EXPECT_FALSE( (l.template is_equal>()) ); + EXPECT_FALSE( (l.template is_equal>()) ); + EXPECT_FALSE( (l.template is_equal>()) ); + EXPECT_FALSE( (l.template is_equal>()) ); +} + +TEST(layout, ctor) +{ + using namespace oomph; + using namespace oomph::tensor; + + test_layout<0, 1, 2>(); + test_layout<0, 2, 1>(); + test_layout<1, 0, 2>(); + test_layout<1, 2, 0>(); + test_layout<2, 0, 1>(); + test_layout<2, 1, 0>(); + //test_layout<2,2,0>(); +} + +template +void +test_tensor_strides(std::size_t X, std::size_t Y, std::size_t Z, std::size_t padding = 0) +{ + using namespace oomph; + using T = double; + using layout_t = tensor::layout; + + std::size_t dims[] = {X, Y, Z}; + + std::vector data( + (layout_t::find(2) == 0 + ? (X + padding) * Y * Z + : (layout_t::find(2) == 1 ? (Y + padding) * X * Z : (Z + padding) * X * Y)) - + padding); + + tensor::detail::map m({X, Y, Z}, data.data(), (data.data() + data.size()) - 1); + + EXPECT_EQ(m.strides()[layout_t::find(2)], 1); + EXPECT_EQ(m.strides()[layout_t::find(1)], dims[layout_t::find(2)] + padding); + EXPECT_EQ(m.strides()[layout_t::find(0)], + dims[layout_t::find(1)] * (dims[layout_t::find(2)] + padding)); +} + +TEST(tensor, strides) +{ + test_tensor_strides<2, 1, 0>(2, 3, 5); + test_tensor_strides<2, 1, 0>(2, 3, 5, 1); + test_tensor_strides<2, 1, 0>(2, 3, 5, 3); + test_tensor_strides<2, 0, 1>(2, 3, 5); + test_tensor_strides<2, 0, 1>(2, 3, 5, 1); + test_tensor_strides<2, 0, 1>(2, 3, 5, 3); + test_tensor_strides<1, 2, 0>(2, 3, 5); + test_tensor_strides<1, 2, 0>(2, 3, 5, 1); + test_tensor_strides<1, 2, 0>(2, 3, 5, 3); + test_tensor_strides<1, 0, 2>(2, 3, 5); + test_tensor_strides<1, 0, 2>(2, 3, 5, 1); + test_tensor_strides<1, 0, 2>(2, 3, 5, 3); +} + +TEST_F(mpi_test_fixture, ctor) +{ + using namespace oomph; + auto ctxt = context(MPI_COMM_WORLD, false); + + std::size_t const halo = 2; + std::size_t const x = 2; + std::size_t const y = 3; + std::size_t const z = 5; + + std::vector data((x + 2 * halo) * (y + 2 * halo) * (z + 2 * halo), world_rank); + + data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 0))] = 9000; + data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 0))] = 9001; + data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 0))] = 9010; + data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 0))] = 9011; + data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 0))] = 9020; + data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 0))] = 9021; + + data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 1))] = 9100; + data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 1))] = 9101; + data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 1))] = 9110; + data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 1))] = 9111; + data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 1))] = 9120; + data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 1))] = 9121; + + data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 2))] = 9200; + data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 2))] = 9201; + data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 2))] = 9210; + data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 2))] = 9211; + data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 2))] = 9220; + data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 2))] = 9221; + + data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 3))] = 9300; + data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 3))] = 9301; + data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 3))] = 9310; + data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 3))] = 9311; + data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 3))] = 9320; + data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 3))] = 9321; + + data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 4))] = 9400; + data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 4))] = 9401; + data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 4))] = 9410; + data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 4))] = 9411; + data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 4))] = 9420; + data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 4))] = 9421; + + using layout_t = tensor::layout<2, 1, 0>; + using map_t = tensor::map; + + map_t t = ctxt.map_tensor({x + 2 * halo, y + 2 * halo, z + 2 * halo}, data.data(), + (data.data() + data.size()) - 1); + + tensor::buffer_cache s_cache; + tensor::buffer_cache r_cache; + tensor::sender> s = tensor::make_sender(t, s_cache); + tensor::receiver> r = tensor::make_receiver(t, r_cache); + + if (world_rank == 0) + { + s + // y-z plane, +x direction + .add_dst({{halo + x - halo, halo, halo}, {halo, y, z}}, 1, 0, 1) + // x-y plane, -z direction + .add_dst({{halo, halo, halo}, {x, y, halo}}, 2, 0, 0) + // whole x-y plane, -z direction + .add_dst({{0, 0, halo}, {x + 2 * halo, y + 2 * halo, halo}}, 3, 0, 2) + .connect(); + + s.pack(1).wait(); + s.send(1).wait(); + s.pack(0).wait(); + s.send(0).wait(); + s.pack(2).wait(); + s.send(2).wait(); + } + if (world_rank == 1) + { + r.add_src({{0, halo, halo}, {halo, y, z}}, 0, 0).connect(); + r.recv().wait(); + r.unpack().wait(); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 0))], 9000); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 0))], 9001); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 0))], 9010); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 0))], 9011); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 0))], 9020); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 0))], 9021); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 1))], 9100); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 1))], 9101); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 1))], 9110); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 1))], 9111); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 1))], 9120); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 1))], 9121); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 2))], 9200); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 2))], 9201); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 2))], 9210); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 2))], 9211); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 2))], 9220); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 2))], 9221); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 3))], 9300); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 3))], 9301); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 3))], 9310); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 3))], 9311); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 3))], 9320); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 3))], 9321); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 4))], 9400); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 4))], 9401); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 4))], 9410); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 4))], 9411); + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 4))], 9420); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 4))], 9421); + } + if (world_rank == 2) + { + r.add_src({{halo, halo, halo + z}, {x, y, halo}}, 0, 0).connect(); + r.recv().wait(); + r.unpack().wait(); + + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 9000); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 9001); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 9010); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 9011); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 9020); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 9021); + + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 9100); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 9101); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 9110); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 9111); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 9120); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 9121); + } + if (world_rank == 3) + { + r.add_src({{0, 0, halo + z}, {x + 2 * halo, y + 2 * halo, halo}}, 0, 0).connect(); + r.recv().wait(); + r.unpack().wait(); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 9000); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 9001); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 9010); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 9011); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 9020); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 9021); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 5))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (0 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (1 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 9100); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 9101); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 0 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 9110); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 9111); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 1 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 9120); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 9121); + EXPECT_EQ(data[4 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (halo + 2 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (5 + (y + 2 * halo) * (halo + 6))], 0); + + EXPECT_EQ(data[0 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[1 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 0 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[halo + 1 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[4 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + EXPECT_EQ(data[5 + (x + 2 * halo) * (6 + (y + 2 * halo) * (halo + 6))], 0); + } +}