diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c411845..24cbfe29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,6 +338,7 @@ endif (USE_PYTHON) add_library(netgen_mpi INTERFACE) add_library(netgen_metis INTERFACE) if (USE_MPI) + set(MPI_DETERMINE_LIBRARY_VERSION TRUE) find_package(MPI REQUIRED) target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH}) target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} ) @@ -351,8 +352,8 @@ if (USE_MPI) if(USE_MPI4PY AND USE_PYTHON) execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE) find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED) - target_include_directories(netgen_metis INTERFACE ${MPI4PY_INCLUDE_DIR}) - target_compile_definitions(netgen_metis INTERFACE NG_MPI4PY ) + target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR}) + target_compile_definitions(netgen_python INTERFACE NG_MPI4PY ) message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}") endif(USE_MPI4PY AND USE_PYTHON) endif (USE_MPI) diff --git a/libsrc/core/CMakeLists.txt b/libsrc/core/CMakeLists.txt index 0ba4e353..eb03bbdb 100644 --- a/libsrc/core/CMakeLists.txt +++ b/libsrc/core/CMakeLists.txt @@ -104,7 +104,7 @@ add_dependencies(ngcore ng_generate_version_file) if(USE_PYTHON) pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp) - target_link_libraries(pyngcore PUBLIC ngcore netgen_python) + target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python) set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}") if(EMSCRIPTEN) target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS) @@ -112,22 +112,34 @@ if(USE_PYTHON) install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen) endif(USE_PYTHON) +function (build_mpi_variant) + set(target ng_${ARGV0}) + set(include_dir ${ARGV1}) + message("Building MPI variant: ${ARGV0} ${ARGV1}") + add_library(${target} SHARED ng_mpi.cpp) + target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python) + target_compile_definitions(${target} PUBLIC PARALLEL) + target_include_directories(${target} PRIVATE ${include_dir}) + install(TARGETS ${target} DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen) +endfunction() + if(USE_MPI) target_compile_definitions(ngcore PUBLIC PARALLEL) - # assume openmpi here - add_library(ng_openmpi SHARED ng_mpi.cpp) - target_link_libraries(ng_openmpi PUBLIC ngcore) - target_include_directories(ng_openmpi PRIVATE ${MPI_C_HEADER_DIR}) - install(TARGETS ng_openmpi DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen) + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*") + set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + endif() - # set(MPICH_INCLUDE_DIR /opt/mpich/include) + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*") + set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + endif() + + if(OPENMPI_INCLUDE_DIR) + build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR}) + endif() if(MPICH_INCLUDE_DIR) - add_library(ng_mpich SHARED ng_mpi.cpp) - target_include_directories(ng_mpich PRIVATE ${MPICH_INCLUDE_DIR}) - target_link_libraries(ng_mpich PUBLIC ngcore) - install(TARGETS ng_mpich DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen) - endif(MPICH_INCLUDE_DIR) + build_mpi_variant(mpich ${MPICH_INCLUDE_DIR}) + endif() endif(USE_MPI) diff --git a/libsrc/core/ng_mpi.cpp b/libsrc/core/ng_mpi.cpp index 11dc7562..92bac08f 100644 --- a/libsrc/core/ng_mpi.cpp +++ b/libsrc/core/ng_mpi.cpp @@ -5,6 +5,15 @@ #include #include "ngcore_api.hpp" +#include "pybind11/pytypes.h" + +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#include + +#include "python_ngcore.hpp" + +namespace py = pybind11; +#endif namespace ngcore { @@ -103,6 +112,31 @@ using namespace ngcore; NGCORE_API_EXPORT extern "C" void ng_init_mpi(); +static bool imported_mpi4py = false; + void ng_init_mpi() { +#if defined(NG_PYTHON) && defined(NG_MPI4PY) + NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + PyObject* py_src = src.ptr(); + auto type = Py_TYPE(py_src); + if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { + dst = mpi2ng(*PyMPIComm_Get(py_src)); + return !PyErr_Occurred(); + } + return false; + }; + NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + return py::handle(PyMPIComm_New(ng2mpi(src))); + }; +#endif + #include "ng_mpi_generated_init.hpp" } diff --git a/libsrc/core/ng_mpi.hpp b/libsrc/core/ng_mpi.hpp index cc988d24..83b80fe0 100644 --- a/libsrc/core/ng_mpi.hpp +++ b/libsrc/core/ng_mpi.hpp @@ -7,6 +7,12 @@ #include "ngcore_api.hpp" +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#include + +namespace py = pybind11; +#endif + namespace ngcore { NGCORE_API void InitMPI( @@ -69,6 +75,11 @@ struct NG_MPI_Aint { NG_MPI_Aint() = default; }; +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &); +NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm); +#endif + #include "ng_mpi_generated_declarations.hpp" } // namespace ngcore diff --git a/libsrc/core/ng_mpi_wrapper.cpp b/libsrc/core/ng_mpi_wrapper.cpp index b5752801..10b08ac2 100644 --- a/libsrc/core/ng_mpi_wrapper.cpp +++ b/libsrc/core/ng_mpi_wrapper.cpp @@ -1,11 +1,12 @@ +#include #include #include -#include #include "ng_mpi.hpp" #include "ngstream.hpp" #include "utils.hpp" +using std::cerr; using std::cout; using std::endl; @@ -13,7 +14,8 @@ namespace ngcore { static std::unique_ptr mpi_lib, ng_mpi_lib; -void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_dir) { +void InitMPI(std::filesystem::path mpi_lib_path, + std::filesystem::path ng_libs_dir) { if (ng_mpi_lib) return; cout << IM(3) << "InitMPI" << endl; @@ -25,58 +27,49 @@ void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_d init_handle mpi_init; mpi_initialized_handle mpi_initialized; get_version_handle get_version; + + mpi_lib = std::make_unique(mpi_lib_path, std::nullopt, true); + try { mpi_init = GetSymbol("MPI_Init"); - cout << IM(3) << "MPI already loaded " << mpi_init << endl; mpi_initialized = GetSymbol("MPI_Initialized"); get_version = GetSymbol("MPI_Get_library_version"); } catch (std::runtime_error &e) { - cout << IM(3) << "MPI not loaded" << endl; - mpi_lib = std::make_unique(mpi_lib_path, - std::nullopt, true); - mpi_init = mpi_lib->GetSymbol("MPI_Init"); - mpi_initialized = - mpi_lib->GetSymbol("MPI_Initialized"); - get_version = - mpi_lib->GetSymbol("MPI_Get_library_version"); + cerr << "Could not load MPI symbols: " << e.what() << endl; + throw e; } int flag = 0; mpi_initialized(&flag); if (!flag) { - cout << IM(3) << "Calling MPI_Init" << endl; - mpi_init(nullptr, nullptr); + typedef const char *pchar; + int argc = 1; + pchar args[] = {"netgen", nullptr}; + pchar *argv = &args[0]; + cout << IM(5) << "Calling MPI_Init" << endl; + mpi_init(&argc, (char ***)argv); } char version_string[65536]; int resultlen = 0; get_version(version_string, &resultlen); mpi_library_version = version_string; - cout << IM(3) << "MPI version: " << version_string << endl; + cout << IM(7) << "MPI version: " << version_string << endl; std::string libname = ""; if (mpi_library_version.substr(0, 8) == "Open MPI") { - cout << IM(3) << "Have Open MPI" << endl; + cout << IM(5) << "Have Open MPI" << endl; libname = std::string("libng_openmpi") + NETGEN_SHARED_LIBRARY_SUFFIX; } else if (mpi_library_version.substr(0, 5) == "MPICH") { - cout << IM(3) << "Have MPICH" << endl; - libname = std::string("libng_mpich.so") + NETGEN_SHARED_LIBRARY_SUFFIX; + cout << IM(5) << "Have MPICH" << endl; + libname = std::string("libng_mpich") + NETGEN_SHARED_LIBRARY_SUFFIX; } else - cout << IM(3) << "Unknown MPI" << endl; + cerr << "Unknown MPI version, skipping init: " << version_string<< endl; if (libname.size()) { - cout << IM(3) << "loading " << libname << endl; - cout << IM(3) << "NG_MPI_INT before " << NG_MPI_INT.value << endl; ng_mpi_lib = std::make_unique(libname); auto ng_init = ng_mpi_lib->GetSymbol("ng_init_mpi"); - cout << IM(3) << "have ng_init " << ng_init << endl; ng_init(); - cout << IM(3) << "NG_MPI_INT after " << NG_MPI_INT.value << endl; - - int size, rank; - NG_MPI_Comm_size(NG_MPI_COMM_WORLD, &size); - NG_MPI_Comm_rank(NG_MPI_COMM_WORLD, &rank); - cout << IM(3) << "Hello from " << rank << " of " << size << endl; } } @@ -86,6 +79,13 @@ static std::runtime_error no_mpi() { std::string mpi_library_version = ""; +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = + [](py::handle, NG_MPI_Comm &) -> bool { throw no_mpi(); }; +decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py = + [](NG_MPI_Comm) -> py::handle { throw no_mpi(); }; +#endif + #include "ng_mpi_generated_dummy_init.hpp" } // namespace ngcore diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index 748e5dd5..58fc5fb3 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -14,6 +14,8 @@ #include "flags.hpp" #include "ngcore_api.hpp" #include "profiler.hpp" +#include "ng_mpi.hpp" + namespace py = pybind11; namespace ngcore @@ -39,6 +41,27 @@ namespace ngcore namespace pybind11 { namespace detail { +#ifdef NG_MPI4PY +template <> struct type_caster { + public: + PYBIND11_TYPE_CASTER(ngcore::NG_MPI_Comm, _("mpi4py_comm")); + + // Python -> C++ + bool load(handle src, bool) { + return ngcore::NG_MPI_CommFromMPI4Py(src, value); + } + + // C++ -> Python + static handle cast(ngcore::NG_MPI_Comm src, + return_value_policy /* policy */, + handle /* parent */) + { + // Create an mpi4py handle + return ngcore::NG_MPI_CommToMPI4Py(src); + } +}; +#endif // NG_MPI4PY + template struct ngcore_list_caster { using value_conv = make_caster; diff --git a/libsrc/core/python_ngcore_export.cpp b/libsrc/core/python_ngcore_export.cpp index 3952d1a1..f973f000 100644 --- a/libsrc/core/python_ngcore_export.cpp +++ b/libsrc/core/python_ngcore_export.cpp @@ -7,6 +7,11 @@ using namespace ngcore; using namespace std; using namespace pybind11::literals; +namespace pybind11 { namespace detail { +}} // namespace pybind11::detail + + + PYBIND11_MODULE(pyngcore, m) // NOLINT { @@ -31,7 +36,10 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT ExportArray(m); ExportTable(m); - + + py::class_ (m, "_NG_MPI_Comm") + ; + py::class_> (m, "BitArray") .def(py::init([] (size_t n) { return make_shared(n); }),py::arg("n")) .def(py::init([] (const BitArray& a) { return make_shared(a); } ), py::arg("ba")) diff --git a/libsrc/core/utils.cpp b/libsrc/core/utils.cpp index b0a9b683..98e010a9 100644 --- a/libsrc/core/utils.cpp +++ b/libsrc/core/utils.cpp @@ -117,7 +117,7 @@ namespace ngcore const std::chrono::time_point wall_time_start = TClock::now(); - int printmessage_importance = 0; + int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0; bool NGSOStream :: glob_active = true; NGCORE_API int GetCompiledSIMDSize() diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index 7917518a..c8e5545f 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -1,3 +1,4 @@ +#include "pybind11/pytypes.h" #ifdef NG_PYTHON #include @@ -23,52 +24,6 @@ public: ~ClearSolutionClass() { Ng_ClearSolutionData(); } }; -#undef NG_MPI4PY -#ifdef NG_MPI4PY -#include - -struct mpi4py_comm { - mpi4py_comm() = default; - mpi4py_comm(MPI_Comm value) : value(value) {} - operator MPI_Comm () { return value; } - - MPI_Comm value; -}; - -namespace pybind11 { namespace detail { - template <> struct type_caster { - public: - PYBIND11_TYPE_CASTER(mpi4py_comm, _("mpi4py_comm")); - - // Python -> C++ - bool load(handle src, bool) { - PyObject *py_src = src.ptr(); - // Check that we have been passed an mpi4py communicator - if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { - // Convert to regular MPI communicator - value.value = *PyMPIComm_Get(py_src); - } else { - return false; - } - - return !PyErr_Occurred(); - } - - // C++ -> Python - static handle cast(mpi4py_comm src, - return_value_policy /* policy */, - handle /* parent */) - { - // Create an mpi4py handle - return PyMPIComm_New(src.value); - } - }; -}} // namespace pybind11::detail - -#endif // NG_MPI4PY - - - using namespace netgen; @@ -101,9 +56,6 @@ static Transformation<3> global_trafo(Vec<3> (0,0,0)); DLL_HEADER void ExportNetgenMeshing(py::module &m) { -#ifdef NG_MPI4PY - import_mpi4py(); -#endif // NG_MPI4PY py::register_exception(m, "NgException"); m.attr("_netgen_executable_started") = py::cast(netgen::netgen_executable_started); string script; @@ -133,13 +85,10 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) py::class_ (m, "MPI_Comm") .def(py::init([] () { return NgMPI_Comm(NG_MPI_COMM_WORLD); })) -#ifdef NG_MPI4PY - .def(py::init([] (mpi4py_comm comm) - { - return NgMPI_Comm(comm); - })) - .def_property_readonly ("mpi4py", [] (NgMPI_Comm comm) { return mpi4py_comm(comm); }) -#endif // NG_MPI4PY + .def(py::init([] (NG_MPI_Comm comm) { return NgMPI_Comm(comm); })) + .def_property_readonly ("mpi4py", [](NgMPI_Comm & self) { + return static_cast(self); + }) .def_property_readonly ("rank", &NgMPI_Comm::Rank) .def_property_readonly ("size", &NgMPI_Comm::Size) .def("Barrier", &NgMPI_Comm::Barrier) @@ -169,9 +118,7 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) ; -#ifdef NG_MPI4PY - py::implicitly_convertible(); -#endif // NG_MPI4PY + py::implicitly_convertible(); py::class_(m, "NGDummyArgument")