From b8d7f0224d89504a6a9a59a4b968af3e7770bf69 Mon Sep 17 00:00:00 2001 From: Kulindu Cooray Date: Tue, 14 Nov 2023 05:49:42 -0500 Subject: [PATCH 1/2] Part of PythonAPI --- CMakeLists.txt | 6 ++ src/CMakeLists.txt | 8 ++ src/pcms/capi/client.cpp | 30 +----- src/pcms/capi/client_cpp.h | 38 +++++++ src/pcms/client.h | 2 +- src/pcms/pythonapi/CMakeLists.txt | 23 +++++ src/pcms/pythonapi/c_client_pybind.cpp | 101 +++++++++++++++++++ src/pcms/pythonapi/client_pybind.cpp | 133 +++++++++++++++++++++++++ 8 files changed, 312 insertions(+), 29 deletions(-) create mode 100644 src/pcms/capi/client_cpp.h create mode 100644 src/pcms/pythonapi/CMakeLists.txt create mode 100644 src/pcms/pythonapi/c_client_pybind.cpp create mode 100644 src/pcms/pythonapi/client_pybind.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e2851ebc..09a51e30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.19) project(pcms VERSION 0.0.5 LANGUAGES CXX) +#set(CMAKE_CXX_VISIBILITY_PRESET "default") include(GNUInstallDirs) include(CMakePackageConfigHelpers) @@ -16,11 +17,16 @@ option(PCMS_ENABLE_CLIENT "enable the coupling client implementation" ON) option(PCMS_ENABLE_XGC "enable xgc field adapter" ON) option(PCMS_ENABLE_OMEGA_H "enable Omega_h field adapter" OFF) option(PCMS_ENABLE_C "Enable pcms C api" ON) +option(PCMS_ENABLE_Python "Enable pcms Python api" OFF) # find package before fortran enabled, so we don't require the adios2 fortran interfaces # this is important because adios2 build with clang/gfortran is broken find_package(redev 4.3.0 REQUIRED) +if (PCMS_ENABLE_Python) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION FALSE) + set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +endif () if (PCMS_ENABLE_C) enable_language(C) option(PCMS_ENABLE_Fortran "Enable pcms fortran api" ON) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87dfd651..0129c418 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ # TODO split out the field transfer library + set(PCMS_HEADERS pcms.h pcms/arrays.h @@ -44,6 +45,7 @@ endif () find_package(Kokkos REQUIRED) find_package(perfstubs REQUIRED) +find_package(pybind11 REQUIRED) add_library(pcms_core ${PCMS_SOURCES}) set_target_properties(pcms_core PROPERTIES @@ -116,6 +118,12 @@ if(PCMS_ENABLE_Fortran) add_subdirectory(pcms/fortranapi) target_link_libraries(pcms_pcms INTERFACE pcms::fortranapi) endif() +if (PCMS_ENABLE_Python) + add_subdirectory(pcms/pythonapi) + #target_link_libraries(pcms_pcms INTERFACE pcms::pythonapi) +endif() + +#add_subdirectory(pcms/pythonapi) install( TARGETS pcms_pcms diff --git a/src/pcms/capi/client.cpp b/src/pcms/capi/client.cpp index 3a6ec813..6b98d38b 100644 --- a/src/pcms/capi/client.cpp +++ b/src/pcms/capi/client.cpp @@ -9,22 +9,7 @@ #include #include "pcms/xgc_reverse_classification.h" #include "pcms/dummy_field_adapter.h" -namespace pcms -{ -// Note that we have a closed set of types that can be used in the C interface -using FieldAdapterVariant = - std::variant, - pcms::XGCFieldAdapter, pcms::XGCFieldAdapter, - pcms::XGCFieldAdapter, - pcms::DummyFieldAdapter -//#ifdef PCMS_HAS_OMEGA_H -// , -// pcms::OmegaHFieldAdapter, -// pcms::OmegaHFieldAdapter -//#endif - >; - -} // namespace pcms +#include "client_cpp.h" [[nodiscard]] PcmsClientHandle pcms_create_client(const char* name, MPI_Comm comm) @@ -116,18 +101,7 @@ void pcms_receive_field(PcmsFieldHandle field_handle) PCMS_ALWAYS_ASSERT(field != nullptr); field->Receive(); } -template -void pcms_create_xgc_field_adapter_t( - const char* name, MPI_Comm comm, void* data, int size, - const pcms::ReverseClassificationVertex& reverse_classification, - in_overlap_function in_overlap, pcms::FieldAdapterVariant& field_adapter) -{ - PCMS_ALWAYS_ASSERT((size >0) ? (data!=nullptr) : true); - pcms::ScalarArrayView data_view( - reinterpret_cast(data), size); - field_adapter.emplace>( - name, comm, data_view, reverse_classification, in_overlap); -} + PcmsFieldAdapterHandle pcms_create_xgc_field_adapter( const char* name, MPI_Comm comm, void* data, int size, PcmsType data_type, const PcmsReverseClassificationHandle rc, in_overlap_function in_overlap) diff --git a/src/pcms/capi/client_cpp.h b/src/pcms/capi/client_cpp.h new file mode 100644 index 00000000..e032b3b5 --- /dev/null +++ b/src/pcms/capi/client_cpp.h @@ -0,0 +1,38 @@ +#ifndef PCMS_CLIENT_CPP_H +#define PCMS_CLIENT_CPP_H + +#include + +namespace pcms +{ + //namespace detail { +// Note that we have a closed set of types that can be used in the C interface + using FieldAdapterVariant = + std::variant, + pcms::XGCFieldAdapter, pcms::XGCFieldAdapter, + pcms::XGCFieldAdapter, + pcms::DummyFieldAdapter + //#ifdef PCMS_HAS_OMEGA_H + // , + // pcms::OmegaHFieldAdapter, + // pcms::OmegaHFieldAdapter + //#endif + >; + //} + +} // namespace pcms + +template +void pcms_create_xgc_field_adapter_t( + const char* name, MPI_Comm comm, void* data, int size, + const pcms::ReverseClassificationVertex& reverse_classification, + in_overlap_function in_overlap, pcms::FieldAdapterVariant& field_adapter) +{ + PCMS_ALWAYS_ASSERT((size >0) ? (data!=nullptr) : true); + pcms::ScalarArrayView data_view( + reinterpret_cast(data), size); + field_adapter.emplace>( + name, comm, data_view, reverse_classification, in_overlap); +} + +#endif \ No newline at end of file diff --git a/src/pcms/client.h b/src/pcms/client.h index 37021df1..00f3cddc 100644 --- a/src/pcms/client.h +++ b/src/pcms/client.h @@ -187,4 +187,4 @@ class CouplerClient }; } // namespace pcms -#endif // PCMS_COUPLING_CLIENT_H +#endif // PCMS_COUPLING_CLIENT_H \ No newline at end of file diff --git a/src/pcms/pythonapi/CMakeLists.txt b/src/pcms/pythonapi/CMakeLists.txt new file mode 100644 index 00000000..0373ee8c --- /dev/null +++ b/src/pcms/pythonapi/CMakeLists.txt @@ -0,0 +1,23 @@ +pybind11_add_module(c_pcms_python c_client_pybind.cpp ../capi/kokkos.cpp ../capi/client.cpp) +pybind11_add_module(pcms_python client_pybind.cpp ../capi/kokkos.cpp) + +set_target_properties(c_pcms_python pcms_python PROPERTIES CXX_STANDARD 17) +target_link_libraries(pcms_python PRIVATE Kokkos::kokkos pybind11::module MPI::MPI_C pcms::core) +target_link_libraries(c_pcms_python PRIVATE Kokkos::kokkos pybind11::module MPI::MPI_C pcms::core) + +install( + TARGETS c_pcms_python pcms_python + EXPORT pcms_python-targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/pythonapi/) +install( + EXPORT pcms_python-targets + NAMESPACE pcms:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcms) + + + + diff --git a/src/pcms/pythonapi/c_client_pybind.cpp b/src/pcms/pythonapi/c_client_pybind.cpp new file mode 100644 index 00000000..c5b641ad --- /dev/null +++ b/src/pcms/pythonapi/c_client_pybind.cpp @@ -0,0 +1,101 @@ +#include +#include +#include "pcms/xgc_field_adapter.h" +#include +#include +// #ifdef PCMS_HAS_OMEGA_H +// #include "pcms/omega_h_field.h" +// #endif +#include +#include "pcms/xgc_reverse_classification.h" +#include "pcms/dummy_field_adapter.h" +#include +#include +#include +#include + + +template +struct VALID_PCMS_TYPE : std::false_type {}; +template<> +struct VALID_PCMS_TYPE : std::true_type {}; +template<> +struct VALID_PCMS_TYPE : std::true_type {}; +template<> +struct VALID_PCMS_TYPE : std::true_type {}; +template<> +struct VALID_PCMS_TYPE : std::true_type {}; + + + +namespace py = pybind11; +namespace pcms +{ + +template +PcmsFieldAdapterHandle xgc_field_adapter_numpy(const char* name, MPI_Comm comm, +py::array_t data, const PcmsReverseClassificationHandle rc, in_overlap_function in_overlap) { + //Check T typing (SFINAE) + static_assert(VALID_PCMS_TYPE::value, "Passed type is invalid"); + + auto buffer_info = data.request(); // Get information about the NumPy array + // Get the size of the NumPy array + + auto field_adapter = new pcms::FieldAdapterVariant{}; + PCMS_ALWAYS_ASSERT(rc.pointer != nullptr); + + auto reverse_classification = reinterpret_cast(rc.pointer); + PCMS_ALWAYS_ASSERT(reverse_classification != nullptr); + + pcms_create_xgc_field_adapter_t(name, comm, buffer_info.ptr, buffer_info.size, *reverse_classification, in_overlap, *field_adapter); + + return PcmsFieldAdapterHandle{reinterpret_cast(field_adapter)}; +} +} + + + +PYBIND11_MODULE(c_pcms_python, m) +{ + m.doc() = "Python bindings for C PCMS client functions"; + + m.def("pcms_kokkos_initialize_without_args", &pcms_kokkos_initialize_without_args); + m.def("pcms_kokkos_finalize", &pcms_kokkos_finalize); + + py::enum_(m, "PcmsAdapterType") + .value("PCMS_ADAPTER_XGC", PCMS_ADAPTER_XGC) + .value("PCMS_ADAPTER_OMEGAH", PCMS_ADAPTER_OMEGAH) + .value("PCMS_ADAPTER_GENE", PCMS_ADAPTER_GENE) + .value("PCMS_ADAPTER_GEM", PCMS_ADAPTER_GEM); + + py::enum_(m, "PcmsType") + .value("PCMS_FLOAT", PCMS_FLOAT) + .value("PCMS_DOUBLE", PCMS_DOUBLE) + .value("PCMS_INT", PCMS_INT) + .value("PCMS_LONG_INT", PCMS_LONG_INT); + + + m.def("pcms_create_client", &pcms_create_client, "Create a PCMS client"); + m.def("pcms_destroy_client", &pcms_destroy_client, "Destroy a PCMS client"); + m.def("pcms_load_reverse_classification", &pcms_load_reverse_classification, "Load reverse classification data"); + m.def("pcms_destroy_reverse_classification", &pcms_destroy_reverse_classification, "Destroy reverse classification data"); + m.def("pcms_reverse_classification_count_verts", &pcms_reverse_classification_count_verts, "Count vertices in reverse classification"); + + m.def("pcms_create_xgc_field_adapter",&pcms::xgc_field_adapter_numpy, "Create an XGC field adapter for double"); + m.def("pcms_create_xgc_field_adapter",&pcms::xgc_field_adapter_numpy, "Create an XGC field adapter for float"); + m.def("pcms_create_xgc_field_adapter",&pcms::xgc_field_adapter_numpy, "Create an XGC field adapter for int"); + m.def("pcms_create_xgc_field_adapter",&pcms::xgc_field_adapter_numpy, "Create an XGC field adapter for long int"); + + m.def("pcms_create_xgc_field_adapter", &pcms_create_xgc_field_adapter, "Create an XGC field adapter"); + m.def("pcms_create_dummy_field_adapter", &pcms_create_dummy_field_adapter, "Create a dummy field adapter"); + m.def("pcms_destroy_field_adapter", &pcms_destroy_field_adapter, "Destroy a field adapter"); + m.def("pcms_add_field", &pcms_add_field, "Add a field"); + m.def("pcms_send_field_name", &pcms_send_field_name, "Send field name"); + m.def("pcms_receive_field_name", &pcms_receive_field_name, "Receive field name"); + m.def("pcms_send_field", &pcms_send_field, "Send field"); + m.def("pcms_receive_field", &pcms_receive_field, "Receive field"); + m.def("pcms_begin_send_phase", &pcms_begin_send_phase, "Begin send phase"); + m.def("pcms_end_send_phase", &pcms_end_send_phase, "End send phase"); + m.def("pcms_begin_receive_phase", &pcms_begin_receive_phase, "Begin receive phase"); + m.def("pcms_end_receive_phase", &pcms_end_receive_phase, "End receive phase"); +} diff --git a/src/pcms/pythonapi/client_pybind.cpp b/src/pcms/pythonapi/client_pybind.cpp new file mode 100644 index 00000000..b1cb32fa --- /dev/null +++ b/src/pcms/pythonapi/client_pybind.cpp @@ -0,0 +1,133 @@ +#include +#include +#include "../client.h" +#include "../xgc_field_adapter.h" + + +namespace py = pybind11; +using pcms::Mode; +using pcms::CoupledField; +using pcms::CouplerClient; + +namespace pcms{ +class PythonFieldAdapterConcept { + public: + + using memory_space = HostMemorySpace; + using value_type = double; + using coordinate_element_type = double; + + virtual int Serialize(ScalarArrayView buffer, + ScalarArrayView permutation) const = 0; + + virtual void Deserialize( + ScalarArrayView buffer, + ScalarArrayView permutation) const = 0; + + [[nodiscard]] virtual std::vector GetGids() const = 0; + + [[nodiscard]] virtual ReversePartitionMap GetReversePartitionMap( + const redev::Partition& partition) const = 0; + + [[nodiscard]] virtual bool RankParticipatesCouplingCommunication() const noexcept = 0; +}; + + +class PythonFieldAdapter { + public: + + PythonFieldAdapter(PythonFieldAdapterConcept& c) : adapter_(c) {}; + + PythonFieldAdapterConcept& adapter_; + using memory_space = HostMemorySpace; + using value_type = double; + using coordinate_element_type = double; + + int Serialize(ScalarArrayView buffer, + ScalarArrayView permutation) const { + adapter_.Serialize(buffer,permutation); + } + + void Deserialize( + ScalarArrayView buffer, + ScalarArrayView permutation) const { + adapter_.Deserialize(buffer,permutation); + } + + [[nodiscard]] std::vector GetGids() const { + adapter_.GetGids(); + } + + [[nodiscard]] ReversePartitionMap GetReversePartitionMap( + const redev::Partition& partition) const { + adapter_.GetReversePartitionMap(partition); + } + + [[nodiscard]] bool RankParticipatesCouplingCommunication() const noexcept { + adapter_.RankParticipatesCouplingCommunication(); + } +}; + +//Trampoline class +class PythonFieldAdapterTramp : public PythonFieldAdapterConcept { + public: + using PythonFieldAdapterConcept::PythonFieldAdapterConcept; + + int Serialize(ScalarArrayView buffer, + ScalarArrayView permutation) const override { PYBIND11_OVERRIDE_PURE(int, PythonFieldAdapterConcept, Serialize, buffer, permutation); } + + void Deserialize(ScalarArrayView buffer, + ScalarArrayView permutation) const override { PYBIND11_OVERRIDE_PURE(void, PythonFieldAdapterConcept, Deserialize, buffer, permutation); } + + std::vector GetGids() const override { PYBIND11_OVERRIDE_PURE(std::vector, PythonFieldAdapterConcept, GetGids); } + + ReversePartitionMap GetReversePartitionMap(const redev::Partition& partition) const override { PYBIND11_OVERRIDE_PURE(ReversePartitionMap, PythonFieldAdapterConcept, GetReversePartitionMap, partition); } + + bool RankParticipatesCouplingCommunication() const noexcept override { PYBIND11_OVERRIDE_PURE(bool, PythonFieldAdapterConcept, RankParticipatesCouplingCommunication ); } +}; +} // namespace pcms + + +PYBIND11_MODULE(pcms_python, m) { + m.doc() = "PCMS Coupling Client C++ Pybind11 Bindings"; + + m.def("pcms_kokkos_initialize_without_args", &pcms_kokkos_initialize_without_args); + m.def("pcms_kokkos_finalize", &pcms_kokkos_finalize); + + py::class_(m, "PythonFieldAdapterConcept") + .def(py::init<>()) + .def("Serialize", &pcms::PythonFieldAdapterConcept::Serialize) + .def("Deserialize", &pcms::PythonFieldAdapterConcept::Deserialize) + .def("GetGids", &pcms::PythonFieldAdapterConcept::GetGids) + .def("GetReversePartitionMap", &pcms::PythonFieldAdapterConcept::GetReversePartitionMap) + .def("RankParticipatesCouplingCommunication", &pcms::PythonFieldAdapterConcept::RankParticipatesCouplingCommunication); + + py::class_(m, "PythonFieldAdapter") + .def(py::init()); + + py::enum_(m, "Mode") + .value("Deferred", Mode::Deferred) + .value("Synchronous", Mode::Synchronous); + + py::class_(m, "CoupledField") + .def(py::init()) + .def("Send", &CoupledField::Send, py::arg("mode") = Mode::Synchronous) + .def("Receive", &CoupledField::Receive); + + py::class_(m, "CouplerClient") + .def(py::init()) + .def("GetPartition", &CouplerClient::GetPartition, py::return_value_policy::reference) //redev::Partition& + .def("AddField", &CouplerClient::AddField, + py::arg("name"), py::arg("field_adapter"), py::arg("participates") = true, + py::return_value_policy::reference) //CoupledField* + .def("SendField", &CouplerClient::SendField, py::arg("name"), py::arg("mode") = Mode::Synchronous) //void + .def("ReceiveField", &CouplerClient::ReceiveField, py::arg("name")) //void + .def("InSendPhase", &CouplerClient::InSendPhase) //bool + .def("InReceivePhase", &CouplerClient::InReceivePhase) //bool + .def("BeginSendPhase", &CouplerClient::BeginSendPhase) //void + .def("EndSendPhase", &CouplerClient::EndSendPhase) //void + .def("BeginReceivePhase", &CouplerClient::BeginReceivePhase) //void + .def("EndReceivePhase", &CouplerClient::EndReceivePhase); //void +} + + From 904f220d637e4203ef6a73140f73a803206cb11f Mon Sep 17 00:00:00 2001 From: Kulindu Cooray Date: Fri, 5 Jan 2024 13:01:15 -0500 Subject: [PATCH 2/2] Python API Draft --- src/pcms/pythonapi/CMakeLists.txt | 52 +++++ src/pcms/pythonapi/FindPythonModule.cmake | 92 ++++++++ src/pcms/pythonapi/PythonFieldAdapters.py | 88 ++++++++ .../c_test_proxy_coupling_xgc_client.py | 110 ++++++++++ src/pcms/pythonapi/client_pybind.cpp | 199 +++++++++++++++--- 5 files changed, 515 insertions(+), 26 deletions(-) create mode 100644 src/pcms/pythonapi/FindPythonModule.cmake create mode 100644 src/pcms/pythonapi/PythonFieldAdapters.py create mode 100644 src/pcms/pythonapi/c_test_proxy_coupling_xgc_client.py diff --git a/src/pcms/pythonapi/CMakeLists.txt b/src/pcms/pythonapi/CMakeLists.txt index 0373ee8c..848f9334 100644 --- a/src/pcms/pythonapi/CMakeLists.txt +++ b/src/pcms/pythonapi/CMakeLists.txt @@ -1,10 +1,62 @@ +find_package(Python REQUIRED) +include(FindPythonModule.cmake) +message(STATUS "Python_EXECUTABLE ${Python_EXECUTABLE}") +find_python_module(mpi4py REQUIRED) + +if(mpi4py_FOUND) + execute_process( + COMMAND + "${Python_EXECUTABLE}" "-c" + "import mpi4py as m; print(m.__version__); print(m.get_include());" + RESULT_VARIABLE + _mpi4py_SEARCH_SUCCESS + OUTPUT_VARIABLE + _mpi4py_VALUES + ERROR_VARIABLE + _mpi4py_ERROR_VALUE + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # Convert the process output into a list + string(REGEX REPLACE ";" "\\\\;" _mpi4py_VALUES ${_mpi4py_VALUES}) + string(REGEX REPLACE "\n" ";" _mpi4py_VALUES ${_mpi4py_VALUES}) + list(GET _mpi4py_VALUES 0 mpi4py_VERSION) + list(GET _mpi4py_VALUES 1 mpi4py_INCLUDE_DIRS) + + # Make sure all directory separators are '/' + string(REGEX REPLACE "\\\\" "/" mpi4py_INCLUDE_DIRS ${mpi4py_INCLUDE_DIRS}) + + # Get the major and minor version numbers + string(REGEX REPLACE "\\." ";" _mpi4py_VERSION_LIST ${mpi4py_VERSION}) + list(GET _mpi4py_VERSION_LIST 0 mpi4py_VERSION_MAJOR) + list(GET _mpi4py_VERSION_LIST 1 mpi4py_VERSION_MINOR) + list(GET _mpi4py_VERSION_LIST 2 mpi4py_VERSION_PATCH) + string(REGEX MATCH "[0-9]*" mpi4py_VERSION_PATCH ${mpi4py_VERSION_PATCH}) + math(EXPR mpi4py_VERSION_DECIMAL + "(${mpi4py_VERSION_MAJOR} * 10000) + (${mpi4py_VERSION_MINOR} * 100) + ${mpi4py_VERSION_PATCH}") +endif() + pybind11_add_module(c_pcms_python c_client_pybind.cpp ../capi/kokkos.cpp ../capi/client.cpp) pybind11_add_module(pcms_python client_pybind.cpp ../capi/kokkos.cpp) +target_include_directories(c_pcms_python + SYSTEM PRIVATE + ${mpi4py_INCLUDE_DIRS} +) + +target_include_directories(pcms_python + SYSTEM PRIVATE + ${mpi4py_INCLUDE_DIRS} +) + set_target_properties(c_pcms_python pcms_python PROPERTIES CXX_STANDARD 17) target_link_libraries(pcms_python PRIVATE Kokkos::kokkos pybind11::module MPI::MPI_C pcms::core) target_link_libraries(c_pcms_python PRIVATE Kokkos::kokkos pybind11::module MPI::MPI_C pcms::core) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/PythonFieldAdapters.py ${CMAKE_CURRENT_SOURCE_DIR}/c_test_proxy_coupling_xgc_client.py + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + + install( TARGETS c_pcms_python pcms_python EXPORT pcms_python-targets diff --git a/src/pcms/pythonapi/FindPythonModule.cmake b/src/pcms/pythonapi/FindPythonModule.cmake new file mode 100644 index 00000000..d0355fd7 --- /dev/null +++ b/src/pcms/pythonapi/FindPythonModule.cmake @@ -0,0 +1,92 @@ +# Downloaded from +# http://www.cmake.org/pipermail/cmake/2011-January/041666.html +# * Added FORCE to location var +# * Function to macro so module_FOUND shows up +# * Remove ``if(NOT PY_${module})`` so runs each time, also remove module caps +# * Module name, not path in fphsa +# * Added version handling via parse_version call + +#.rst: +# +# Find if a Python module is installed. +# Usage: find_python_module( [[ATLEAST | EXACT] version] [QUIET] [REQUIRED]) + +macro(find_python_module module) + cmake_parse_arguments(ARG + "QUIET;REQUIRED" # options + "ATLEAST;EXACT" # one-value arguments + "" # multi-value arguments + ${ARGN} # everything else + ) + + if(ARG_QUIET) + set(${module}_FIND_QUIETLY TRUE) + endif() + + if(ARG_REQUIRED) + set(${module}_FIND_REQUIRED TRUE) + endif() + + if(ARG_ATLEAST AND ARG_EXACT) + message(FATAL_ERROR "Can't be both ATLEAST and EXACT") + endif() + if(ARG_ATLEAST) + set(_op ">=") + set(${module}_tgtver ${ARG_ATLEAST}) + elseif(ARG_EXACT) + set(_op "==") + set(${module}_tgtver ${ARG_EXACT}) + else() + # deceive handle_standard_arguments into not caring about version + set(_${module}_requested_version_found "${Python_EXECUTABLE}") + endif() + + unset(PY_${module} CACHE) + unset(${module}_VERSION CACHE) + + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "import re; \ + import ${module}; \ + print(re.compile('/__init__.py.*').sub('', ${module}.__file__))" + RESULT_VARIABLE _${module}_status + OUTPUT_VARIABLE _${module}_location + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_status}) + set(PY_${module} ${_${module}_location} CACHE STRING "Location of Python module ${module}" FORCE) + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "import sys; \ + import ${module}; \ + print(${module}.__version__)" + RESULT_VARIABLE _${module}_ver_status + OUTPUT_VARIABLE _${module}_version + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_ver_status}) + set(${module}_VERSION ${_${module}_version} CACHE STRING + "Version of Python module ${module}" FORCE) + + if(${module}_tgtver) + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "from pkg_resources import parse_version; \ + print(parse_version('${${module}_VERSION}') ${_op} parse_version('${${module}_tgtver}'))" + RESULT_VARIABLE _${module}_verenuf_status + OUTPUT_VARIABLE _${module}_verenuf + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_verenuf_status}) + if(${_${module}_verenuf} STREQUAL "True") + set(_${module}_requested_version_found "${Python_EXECUTABLE}") + endif() + endif() + endif() + endif() + endif() + find_package_handle_standard_args(${module} DEFAULT_MSG PY_${module} _${module}_requested_version_found) +endmacro() diff --git a/src/pcms/pythonapi/PythonFieldAdapters.py b/src/pcms/pythonapi/PythonFieldAdapters.py new file mode 100644 index 00000000..350afda3 --- /dev/null +++ b/src/pcms/pythonapi/PythonFieldAdapters.py @@ -0,0 +1,88 @@ +import pcms_python as pcms +import numpy as np +from mpi4py import MPI + +memory_space = pcms.PythonFieldAdapterConcept.memory_space + +class PythonXGCFieldAdapter(pcms.PythonFieldAdapterConcept): + + def __init__(self, name, plane_communicator, data, reverse_classification, in_overlap): + self.name_ = name + self.plane_comm_ = plane_communicator + self.plane_rank_ = None + self.data_ = data + self.gids_ = [0] * data.size() + self.reverse_classification_ = None + self.mask_ = None + self.in_overlap_ = in_overlap + self.plane_root_ = 0 + self.memory_space = memory_space + + + self.plane_rank_ = self.plane_comm_.Get_rank() + if self.RankParticipatesCouplingCommunication(): + mask = pcms.PythonArrayMask(np.zeros_like(data, dtype=np.int8)) + assert callable(in_overlap) + py_rcv = pcms.PyReverseClassificationVertex(reverse_classification) + data = py_rcv.get_data() + for (dim, id), verts in data.items(): # geom would be an iterator + if in_overlap(dim,id): + for vert in verts: + assert vert < len(data) + mask[vert] = 1 + + self.reverse_classification_ = py_rcv + self.mask_ = mask + assert self.mask_.Size() != 0 + gids_ = np.arange(1, len(gids_) + 1, dtype=np.int64) + + def Serialize(self,buffer,permutation): + assert self.memory_space == memory_space, "gpu space unhandled" + if self.RankParticipatesCouplingCommunication(): + const_data = np.array(self.data_, dtype=self.data_.dtype, copy=False) + if buffer.size() > 0: + self.mask_.Apply(const_data, buffer, permutation) + return self.mask_.Size() + return 0 + + def Deserialize(self,buffer,permutation): + assert self.memory_space == memory_space, "gpu space unhandled" + if self.RankParticipatesCouplingCommunication(): + self.mask_.ToFullArray(buffer, self.data_, permutation) + + #Bcast line + pcms.Bcast(self.data_, self.plane_root_, self.plane_comm_) + + # Representing the ScalarArrayViews as numpy arrays + def GetGids(self): + if self.RankParticipatesCouplingCommunication(): + gids = np.zeros(self.mask_.Size(), dtype=np.int64) + self.mask_.Apply(self.gids_,gids) + return gids + return np.empty(0, dtype=np.int64) + + def GetReversePartitionMap(self, partition): + if self.RankParticipatesCouplingCommunication(): + reverse_partition = {} + assert callable(self.in_overlap_) + for (dim, id), verts in self.reverse_classification_.get_data().items(): + if self.in_overlap_(dim, id): + dr = pcms.RankParition(dim,id,partition) + it = reverse_partition.setdefault(dr, []) + inserted = not bool(it) #True if a new key-value pair was inserted, false otherwise + map_ = self.mask_.GetMap() + #Transformation + for v in verts: + idx = map_[v] + assert (idx > 0) + reverse_partition[dr].append(idx-1) + + for rank, idxs in reverse_partition.iterms(): + reverse_partition[rank] = sorted(idxs) + + return reverse_partition + return {} + + #optional + def RankParticipatesCouplingCommunication(self): + return self.plane_rank_ == self.plane_root_ \ No newline at end of file diff --git a/src/pcms/pythonapi/c_test_proxy_coupling_xgc_client.py b/src/pcms/pythonapi/c_test_proxy_coupling_xgc_client.py new file mode 100644 index 00000000..e75e8af4 --- /dev/null +++ b/src/pcms/pythonapi/c_test_proxy_coupling_xgc_client.py @@ -0,0 +1,110 @@ +import c_pcms_python as pcms +import numpy as np +from mpi4py import MPI +import sys + +def in_overlap(dimension, id): + if (id >= 22 and id <= 34): + if dimension == 2 or dimension == 1 or dimension == 0: + return 1 + return 0 + +# Give pcms the MPI comm through +if __name__ == '__main__': + MPI.Init() + pcms.pcms_kokkos_initialize_without_args() + world_rank, world_size, plane_rank, plane_size, client_rank, client_size = -1, -1, -1, -1, -1, -1 + + + world_rank = MPI.COMM_WORLD.Get_rank() + world_size = MPI.COMM_WORLD.Get_size() + + nplanes = 2 # You need to define the value of nplanes + if world_size % nplanes != 0: + print("Number of ranks must be divisible by the number of planes") + sys.exit("Aborting due to the invalid number of ranks") + + plane = world_rank % nplanes + print(f"PLANE {plane}") + + plane_comm = MPI.COMM_WORLD.Split(plane, world_rank) + plane_rank = plane_comm.rank + plane_size = plane_comm.size + client_comm = MPI.COMM_WORLD.Split(0 if plane_rank == 0 else MPI.UNDEFINED, world_rank) + + if client_comm != MPI.COMM_NULL: + client_rank = client_comm.rank + client_size = client_comm.size + + print(f"world: {world_rank} {world_size}; plane: {plane_rank} {plane_size}; client: {client_rank} {client_size}") + client = pcms.pcms_create_client("proxy_couple", MPI._handleof(client_comm)) # passes C object communicator client_comm + rc_file = sys.argv[1] + rc = pcms.pcms_load_reverse_classification(rc_file, MPI._handleof(MPI.COMM_WORLD)) # passes C object communicator + + nverts = pcms.pcms_reverse_classification_count_verts(rc) + data = np.zeros(nverts, dtype=np.int64) + + field = [] + field_adapters = [] + + for i in range(nplanes): + field_name = f"xgc_gids_plane_{i}" + print(field_name) + + communicating_rank = (i == plane) and (plane_rank == 0) + + if plane == i: + field_adapter = pcms.pcms_create_xgc_field_adapter( + "adapter1", plane_comm, data, nverts, pcms.PCMS_LONG_INT, rc, in_overlap + ) + else: + field_adapter = pcms.pcms_create_dummy_field_adapter() + field_adapters.append(field_adapter) + field.append(pcms.pcms_add_field(client, field_name, field_adapter, communicating_rank)) + + if plane_rank == 0: + for i in range(nverts): + data[i] = i + + pcms.pcms_begin_send_phase(client) + pcms.pcms_send_field(field[plane]) + pcms.pcms_end_send_phase(client) + pcms.pcms_begin_receive_phase(client) + pcms.pcms_receive_field(field[plane]) + pcms.pcms_end_receive_phase(client) + + for i in range(nverts): + if data[i] != i: + print(f"ERROR: data[{i}] = {data[i]}, should be {i}") + sys.exit() + + if plane_rank == 0: + for i in range(nverts): + data[i] *= 2 + + pcms.pcms_begin_send_phase(client); + pcms.pcms_send_field(field[plane]); + pcms.pcms_end_send_phase(client); + pcms.pcms_begin_receive_phase(client); + pcms.pcms_receive_field(field[plane]); + pcms.pcms_end_receive_phase(client); + + for i in range(nverts): + if data[i] != 2 * i: + print(f"ERROR: data[{i}] = {data[i]}, should be {2 * i}") + sys.exit() + + for field_adapter in field_adapters: + pcms.pcms_destroy_field_adapter(field_adapter) + + data = None + pcms.pcms_destroy_reverse_classification(rc) + pcms.pcms_destroy_client(client) + if(client_comm != MPI.COMM_NULL): + MPI.Comm.Free(client_comm) + + MPI.Comm.Free(plane_comm) + pcms.pcms_kokkos_finalize() + MPI.Finalize() + + diff --git a/src/pcms/pythonapi/client_pybind.cpp b/src/pcms/pythonapi/client_pybind.cpp index b1cb32fa..33709a2c 100644 --- a/src/pcms/pythonapi/client_pybind.cpp +++ b/src/pcms/pythonapi/client_pybind.cpp @@ -1,7 +1,12 @@ #include +#include +#include #include +#include #include "../client.h" #include "../xgc_field_adapter.h" +#include + namespace py = pybind11; @@ -9,7 +14,10 @@ using pcms::Mode; using pcms::CoupledField; using pcms::CouplerClient; + namespace pcms{ +using value_type = double; + class PythonFieldAdapterConcept { public: @@ -17,14 +25,13 @@ class PythonFieldAdapterConcept { using value_type = double; using coordinate_element_type = double; - virtual int Serialize(ScalarArrayView buffer, - ScalarArrayView permutation) const = 0; + virtual int Serialize(py::array_t& buffer, + py::array_t& permutation) const = 0; - virtual void Deserialize( - ScalarArrayView buffer, - ScalarArrayView permutation) const = 0; + virtual void Deserialize(py::array_t& buffer, + py::array_t& permutation) const = 0; - [[nodiscard]] virtual std::vector GetGids() const = 0; + [[nodiscard]] virtual py::array_t GetGids() const = 0; [[nodiscard]] virtual ReversePartitionMap GetReversePartitionMap( const redev::Partition& partition) const = 0; @@ -42,29 +49,78 @@ class PythonFieldAdapter { using memory_space = HostMemorySpace; using value_type = double; using coordinate_element_type = double; - + int Serialize(ScalarArrayView buffer, ScalarArrayView permutation) const { - adapter_.Serialize(buffer,permutation); + + py::buffer_info info_buff ( + buffer.data_handle(), + sizeof(value_type), + py::format_descriptor::format(), + 1, + { buffer.size() }, + { sizeof(value_type) } + ); + + py::buffer_info info_perm ( + const_cast(permutation.data_handle()), + sizeof(pcms::LO), + py::format_descriptor::format(), + 1, + { permutation.size() }, + { sizeof(pcms::LO) } + ); + + py::array_t buff(info_buff); + py::array_t perm(info_perm); + + return adapter_.Serialize(buff,perm); } void Deserialize( ScalarArrayView buffer, ScalarArrayView permutation) const { - adapter_.Deserialize(buffer,permutation); + + py::buffer_info info_buff ( + const_cast(buffer.data_handle()), + sizeof(value_type), + py::format_descriptor::format(), + 1, + { buffer.size() }, + { sizeof(value_type) } + ); + + py::buffer_info info_perm ( + const_cast(permutation.data_handle()), + sizeof(pcms::LO), + py::format_descriptor::format(), + 1, + { permutation.size() }, + { sizeof(pcms::LO) } + ); + + py::array_t buff(info_buff); + py::array_t perm(info_perm); + + return adapter_.Deserialize(buff,perm); } [[nodiscard]] std::vector GetGids() const { - adapter_.GetGids(); + py::array_t a = adapter_.GetGids(); + py::buffer_info info = a.request(); + + int size = info.size; + const GO* data = static_cast(info.ptr); + return std::vector(data, data+size); } [[nodiscard]] ReversePartitionMap GetReversePartitionMap( const redev::Partition& partition) const { - adapter_.GetReversePartitionMap(partition); + return adapter_.GetReversePartitionMap(partition); } [[nodiscard]] bool RankParticipatesCouplingCommunication() const noexcept { - adapter_.RankParticipatesCouplingCommunication(); + return adapter_.RankParticipatesCouplingCommunication(); } }; @@ -73,26 +129,100 @@ class PythonFieldAdapterTramp : public PythonFieldAdapterConcept { public: using PythonFieldAdapterConcept::PythonFieldAdapterConcept; - int Serialize(ScalarArrayView buffer, - ScalarArrayView permutation) const override { PYBIND11_OVERRIDE_PURE(int, PythonFieldAdapterConcept, Serialize, buffer, permutation); } + int Serialize(py::array_t& buffer, + py::array_t& permutation) const override { PYBIND11_OVERRIDE_PURE(int, PythonFieldAdapterConcept, Serialize, buffer, permutation); } + + void Deserialize(py::array_t& buffer, + py::array_t& permutation) const override { PYBIND11_OVERRIDE_PURE(void, PythonFieldAdapterConcept, Deserialize, buffer, permutation); } + + [[nodiscard]] py::array_t GetGids() const override { PYBIND11_OVERRIDE_PURE(py::array_t, PythonFieldAdapterConcept, GetGids); } + + [[nodiscard]] ReversePartitionMap GetReversePartitionMap(const redev::Partition& partition) const override { PYBIND11_OVERRIDE_PURE(ReversePartitionMap, PythonFieldAdapterConcept, GetReversePartitionMap, partition); } + + [[nodiscard]] bool RankParticipatesCouplingCommunication() const noexcept override { PYBIND11_OVERRIDE_PURE(bool, PythonFieldAdapterConcept, RankParticipatesCouplingCommunication ); } +}; + +template +class PythonArrayMask { + public: + + using memory_space = HostMemorySpace; + + PythonArrayMask(py::array_t& mask) : mask_(Convert(mask)) {} + + auto Apply(py::array_t& data, py::array_t& filtered_data, py::array_t& permutation) { + mask_.Apply(Convert(data),Convert(filtered_data),Convert(permutation)); + } - void Deserialize(ScalarArrayView buffer, - ScalarArrayView permutation) const override { PYBIND11_OVERRIDE_PURE(void, PythonFieldAdapterConcept, Deserialize, buffer, permutation); } + auto ToFullArray(py::array_t& filtered_data, py::array_t& output_array, py::array_t permutation = {}) { + mask_.ToFullArray(Convert(filtered_data),Convert(output_array),Convert(permutation)); + } - std::vector GetGids() const override { PYBIND11_OVERRIDE_PURE(std::vector, PythonFieldAdapterConcept, GetGids); } + int Size() { + return mask_.Size(); + } + + auto GetMap() const { + return mask_.GetMap(); + } - ReversePartitionMap GetReversePartitionMap(const redev::Partition& partition) const override { PYBIND11_OVERRIDE_PURE(ReversePartitionMap, PythonFieldAdapterConcept, GetReversePartitionMap, partition); } - bool RankParticipatesCouplingCommunication() const noexcept override { PYBIND11_OVERRIDE_PURE(bool, PythonFieldAdapterConcept, RankParticipatesCouplingCommunication ); } + private: + ArrayMask mask_; + + template + ScalarArrayView Convert(const py::array_t& arr) { + py::buffer_info arr_buf = arr.request(); + //PCMS_ASSERT + return ScalarArrayView{ + static_cast(arr_buf.ptr), arr_buf.size + }; + } +}; +//use ReadReverseClassificationVertex string version +class PyReverseClassificationVertex { +public: + PyReverseClassificationVertex(const ReverseClassificationVertex& rcv) : rcv_(rcv) {} + // Python-friendly method to get data + py::dict get_data() const { + py::dict result; + for (auto it = rcv_.begin(); it != rcv_.end(); ++it) { + const DimID& dimID = it->first; + const std::set& verts = it->second; + result[py::make_tuple(dimID.dim, dimID.id)] = py::cast(verts); + } + return result; + } + +private: + const ReverseClassificationVertex& rcv_; }; } // namespace pcms +static MPI_Comm* GetMPIComm(py::object py_comm) { + auto comm_ptr = PyMPIComm_Get(py_comm.ptr()); + + if(!comm_ptr) + throw py::error_already_set(); + + return comm_ptr; +} PYBIND11_MODULE(pcms_python, m) { m.doc() = "PCMS Coupling Client C++ Pybind11 Bindings"; m.def("pcms_kokkos_initialize_without_args", &pcms_kokkos_initialize_without_args); m.def("pcms_kokkos_finalize", &pcms_kokkos_finalize); + + m.def("RankPartition", [](const pcms::LO dim, const pcms::LO id, const redev::Partition& partition) { + return std::visit(pcms::detail::GetRank{pcms::DimID{dim,id}}, partition); + }); + + m.def("Bcast", [](py::array_t data, int plane_root, py::object plane_comm) { + auto buf_info = data.request(); + auto* comm_ptr = GetMPIComm(plane_comm); + MPI_Bcast(buf_info.ptr, buf_info.size, redev::getMpiType(pcms::value_type{}), plane_root, *comm_ptr); + }); py::class_(m, "PythonFieldAdapterConcept") .def(py::init<>()) @@ -103,23 +233,40 @@ PYBIND11_MODULE(pcms_python, m) { .def("RankParticipatesCouplingCommunication", &pcms::PythonFieldAdapterConcept::RankParticipatesCouplingCommunication); py::class_(m, "PythonFieldAdapter") - .def(py::init()); + .def(py::init()) + .def("Serialize", &pcms::PythonFieldAdapter::Serialize) + .def("Deserialize", &pcms::PythonFieldAdapter::Deserialize) + .def("GetGids", &pcms::PythonFieldAdapter::GetGids) + .def("GetReversePartitionMap", &pcms::PythonFieldAdapter::GetReversePartitionMap) + .def("RankParticipatesCouplingCommunication", &pcms::PythonFieldAdapter::RankParticipatesCouplingCommunication); + + + py::class_>(m, "PythonArrayMask") + .def(py::init&>()) + .def("Apply", &pcms::PythonArrayMask::Apply) + .def("ToFullArray", &pcms::PythonArrayMask::ToFullArray) + .def("Size", &pcms::PythonArrayMask::Size) + .def("GetMap", &pcms::PythonArrayMask::GetMap); + + py::class_(m, "PyReverseClassificationVertex") + .def(py::init()) + .def("get_data", &pcms::PyReverseClassificationVertex::get_data); py::enum_(m, "Mode") .value("Deferred", Mode::Deferred) .value("Synchronous", Mode::Synchronous); - - py::class_(m, "CoupledField") + + py::class_(m, "CoupledField") .def(py::init()) .def("Send", &CoupledField::Send, py::arg("mode") = Mode::Synchronous) .def("Receive", &CoupledField::Receive); - py::class_(m, "CouplerClient") + py::class_(m, "CouplerClient") .def(py::init()) .def("GetPartition", &CouplerClient::GetPartition, py::return_value_policy::reference) //redev::Partition& - .def("AddField", &CouplerClient::AddField, - py::arg("name"), py::arg("field_adapter"), py::arg("participates") = true, - py::return_value_policy::reference) //CoupledField* + .def("AddField",[](CouplerClient& self, const std::string& name, pcms::PythonFieldAdapter& fa, bool participates = true){ + self.AddField(name,fa,participates); + }) .def("SendField", &CouplerClient::SendField, py::arg("name"), py::arg("mode") = Mode::Synchronous) //void .def("ReceiveField", &CouplerClient::ReceiveField, py::arg("name")) //void .def("InSendPhase", &CouplerClient::InSendPhase) //bool