Fix mpi4py converters, some cleanup

CMakeLists.txt

@@ -338,6 +338,7 @@ endif (USE_PYTHON)
 add_library(netgen_mpi INTERFACE)
 add_library(netgen_metis INTERFACE)
 if (USE_MPI)
+    set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
     find_package(MPI REQUIRED)
     target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH})
     target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} )
@@ -351,8 +352,8 @@ if (USE_MPI)
     if(USE_MPI4PY AND USE_PYTHON)
       execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE)
       find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED)
-      target_include_directories(netgen_metis INTERFACE ${MPI4PY_INCLUDE_DIR})
+      target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR})
-      target_compile_definitions(netgen_metis INTERFACE NG_MPI4PY )
+      target_compile_definitions(netgen_python INTERFACE NG_MPI4PY )
       message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}")
     endif(USE_MPI4PY AND USE_PYTHON)
 endif (USE_MPI)

libsrc/core/CMakeLists.txt

@@ -104,7 +104,7 @@ add_dependencies(ngcore ng_generate_version_file)
 
 if(USE_PYTHON)
   pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp)
-  target_link_libraries(pyngcore PUBLIC ngcore netgen_python)
+  target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python)
   set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}")
   if(EMSCRIPTEN)
     target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS)
@@ -112,22 +112,34 @@ if(USE_PYTHON)
   install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen)
 endif(USE_PYTHON)
 
+function (build_mpi_variant)
+  set(target ng_${ARGV0})
+  set(include_dir ${ARGV1})
+  message("Building MPI variant: ${ARGV0} ${ARGV1}")
+  add_library(${target} SHARED ng_mpi.cpp)
+  target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python)
+  target_compile_definitions(${target} PUBLIC PARALLEL)
+  target_include_directories(${target} PRIVATE ${include_dir})
+  install(TARGETS ${target} DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
+endfunction()
+
 if(USE_MPI)
   target_compile_definitions(ngcore PUBLIC PARALLEL)
 
-  # assume openmpi here
+  if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*")
-  add_library(ng_openmpi SHARED ng_mpi.cpp)
+    set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
-  target_link_libraries(ng_openmpi PUBLIC ngcore)
+  endif()
-  target_include_directories(ng_openmpi PRIVATE ${MPI_C_HEADER_DIR})
-  install(TARGETS ng_openmpi DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
 
-  # set(MPICH_INCLUDE_DIR /opt/mpich/include)
+  if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*")
+    set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
+  endif()
+
+  if(OPENMPI_INCLUDE_DIR)
+    build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR})
+  endif()
   if(MPICH_INCLUDE_DIR)
-    add_library(ng_mpich SHARED ng_mpi.cpp)
+    build_mpi_variant(mpich ${MPICH_INCLUDE_DIR})
-    target_include_directories(ng_mpich PRIVATE ${MPICH_INCLUDE_DIR})
+  endif()
-    target_link_libraries(ng_mpich PUBLIC ngcore)
-    install(TARGETS ng_mpich DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
-  endif(MPICH_INCLUDE_DIR)
 
 endif(USE_MPI)
 

libsrc/core/ng_mpi.cpp

@@ -5,6 +5,15 @@
 #include <type_traits>
 
 #include "ngcore_api.hpp"
+#include "pybind11/pytypes.h"
+
+#if defined(NG_PYTHON) && defined(NG_MPI4PY)
+#include <mpi4py.h>
+
+#include "python_ngcore.hpp"
+
+namespace py = pybind11;
+#endif
 
 namespace ngcore {
 
@@ -103,6 +112,31 @@ using namespace ngcore;
 
 NGCORE_API_EXPORT extern "C" void ng_init_mpi();
 
+static bool imported_mpi4py = false;
+
 void ng_init_mpi() {
+#if defined(NG_PYTHON) && defined(NG_MPI4PY)
+  NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool {
+    if (!imported_mpi4py) {
+      import_mpi4py();
+      imported_mpi4py = true;
+    }
+    PyObject* py_src = src.ptr();
+    auto type = Py_TYPE(py_src);
+    if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
+      dst = mpi2ng(*PyMPIComm_Get(py_src));
+      return !PyErr_Occurred();
+    }
+    return false;
+  };
+  NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle {
+    if (!imported_mpi4py) {
+      import_mpi4py();
+      imported_mpi4py = true;
+    }
+    return py::handle(PyMPIComm_New(ng2mpi(src)));
+  };
+#endif
+
 #include "ng_mpi_generated_init.hpp"
 }

libsrc/core/ng_mpi.hpp

@@ -7,6 +7,12 @@
 
 #include "ngcore_api.hpp"
 
+#if defined(NG_PYTHON) && defined(NG_MPI4PY)
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+#endif
+
 namespace ngcore {
 
 NGCORE_API void InitMPI(
@@ -69,6 +75,11 @@ struct NG_MPI_Aint {
   NG_MPI_Aint() = default;
 };
 
+#if defined(NG_PYTHON) && defined(NG_MPI4PY)
+NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
+NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
+#endif
+
 #include "ng_mpi_generated_declarations.hpp"
 
 }  // namespace ngcore

libsrc/core/ng_mpi_wrapper.cpp

@@ -1,11 +1,12 @@
+#include <filesystem>
 #include <iostream>
 #include <stdexcept>
-#include <filesystem>
 
 #include "ng_mpi.hpp"
 #include "ngstream.hpp"
 #include "utils.hpp"
 
+using std::cerr;
 using std::cout;
 using std::endl;
 
@@ -13,7 +14,8 @@ namespace ngcore {
 
 static std::unique_ptr<SharedLibrary> mpi_lib, ng_mpi_lib;
 
-void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_dir) {
+void InitMPI(std::filesystem::path mpi_lib_path,
+             std::filesystem::path ng_libs_dir) {
   if (ng_mpi_lib) return;
   cout << IM(3) << "InitMPI" << endl;
 
@@ -25,58 +27,49 @@ void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_d
   init_handle mpi_init;
   mpi_initialized_handle mpi_initialized;
   get_version_handle get_version;
+
+  mpi_lib = std::make_unique<SharedLibrary>(mpi_lib_path, std::nullopt, true);
+
   try {
     mpi_init = GetSymbol<init_handle>("MPI_Init");
-    cout << IM(3) << "MPI already loaded " << mpi_init << endl;
     mpi_initialized = GetSymbol<mpi_initialized_handle>("MPI_Initialized");
     get_version = GetSymbol<get_version_handle>("MPI_Get_library_version");
   } catch (std::runtime_error &e) {
-    cout << IM(3) << "MPI not loaded" << endl;
+    cerr << "Could not load MPI symbols: " << e.what() << endl;
-    mpi_lib = std::make_unique<SharedLibrary>(mpi_lib_path,
+    throw e;
-                                              std::nullopt, true);
-    mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
-    mpi_initialized =
-        mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
-    get_version =
-        mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
   }
 
   int flag = 0;
   mpi_initialized(&flag);
   if (!flag) {
-    cout << IM(3) << "Calling MPI_Init" << endl;
+    typedef const char *pchar;
-    mpi_init(nullptr, nullptr);
+    int argc = 1;
+    pchar args[] = {"netgen", nullptr};
+    pchar *argv = &args[0];
+    cout << IM(5) << "Calling MPI_Init" << endl;
+    mpi_init(&argc, (char ***)argv);
   }
 
   char version_string[65536];
   int resultlen = 0;
   get_version(version_string, &resultlen);
   mpi_library_version = version_string;
-  cout << IM(3) << "MPI version: " << version_string << endl;
+  cout << IM(7) << "MPI version: " << version_string << endl;
 
   std::string libname = "";
   if (mpi_library_version.substr(0, 8) == "Open MPI") {
-    cout << IM(3) << "Have Open MPI" << endl;
+    cout << IM(5) << "Have Open MPI" << endl;
     libname = std::string("libng_openmpi") + NETGEN_SHARED_LIBRARY_SUFFIX;
   } else if (mpi_library_version.substr(0, 5) == "MPICH") {
-    cout << IM(3) << "Have MPICH" << endl;
+    cout << IM(5) << "Have MPICH" << endl;
-    libname = std::string("libng_mpich.so") + NETGEN_SHARED_LIBRARY_SUFFIX;
+    libname = std::string("libng_mpich") + NETGEN_SHARED_LIBRARY_SUFFIX;
   } else
-    cout << IM(3) << "Unknown MPI" << endl;
+    cerr << "Unknown MPI version, skipping init: " << version_string<< endl;
 
   if (libname.size()) {
-    cout << IM(3) << "loading " << libname << endl;
-    cout << IM(3) << "NG_MPI_INT before " << NG_MPI_INT.value << endl;
     ng_mpi_lib = std::make_unique<SharedLibrary>(libname);
     auto ng_init = ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi");
-    cout << IM(3) << "have ng_init " << ng_init << endl;
     ng_init();
-    cout << IM(3) << "NG_MPI_INT after  " << NG_MPI_INT.value << endl;
-
-    int size, rank;
-    NG_MPI_Comm_size(NG_MPI_COMM_WORLD, &size);
-    NG_MPI_Comm_rank(NG_MPI_COMM_WORLD, &rank);
-    cout << IM(3) << "Hello from " << rank << " of " << size << endl;
   }
 }
 
@@ -86,6 +79,13 @@ static std::runtime_error no_mpi() {
 
 std::string mpi_library_version = "";
 
+#if defined(NG_PYTHON) && defined(NG_MPI4PY)
+decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
+    [](py::handle, NG_MPI_Comm &) -> bool { throw no_mpi(); };
+decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
+    [](NG_MPI_Comm) -> py::handle { throw no_mpi(); };
+#endif
+
 #include "ng_mpi_generated_dummy_init.hpp"
 
 }  // namespace ngcore

libsrc/core/python_ngcore.hpp

@@ -14,6 +14,8 @@
 #include "flags.hpp"
 #include "ngcore_api.hpp"
 #include "profiler.hpp"
+#include "ng_mpi.hpp"
+
 namespace py = pybind11;
 
 namespace ngcore
@@ -39,6 +41,27 @@ namespace ngcore
 namespace pybind11 {
 namespace detail {
 
+#ifdef NG_MPI4PY
+template <> struct type_caster<ngcore::NG_MPI_Comm> {
+  public:
+  PYBIND11_TYPE_CASTER(ngcore::NG_MPI_Comm, _("mpi4py_comm"));
+
+    // Python -> C++
+    bool load(handle src, bool) {
+      return ngcore::NG_MPI_CommFromMPI4Py(src, value);
+    }
+
+    // C++ -> Python
+    static handle cast(ngcore::NG_MPI_Comm src,
+                       return_value_policy /* policy */,
+                       handle /* parent */)
+    {
+      // Create an mpi4py handle
+      return ngcore::NG_MPI_CommToMPI4Py(src);
+    }
+};
+#endif // NG_MPI4PY
+
 template <typename Type, typename Value> struct ngcore_list_caster {
     using value_conv = make_caster<Value>;
 

libsrc/core/python_ngcore_export.cpp

@@ -7,6 +7,11 @@ using namespace ngcore;
 using namespace std;
 using namespace pybind11::literals;
 
+namespace pybind11 { namespace detail {
+}} // namespace pybind11::detail
+
+
+
 
 PYBIND11_MODULE(pyngcore, m) // NOLINT
 {
@@ -32,6 +37,9 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
 
   ExportTable<int>(m);
 
+  py::class_<NG_MPI_Comm> (m, "_NG_MPI_Comm")
+          ;
+
   py::class_<BitArray, shared_ptr<BitArray>> (m, "BitArray")
     .def(py::init([] (size_t n) { return make_shared<BitArray>(n); }),py::arg("n"))
     .def(py::init([] (const BitArray& a) { return make_shared<BitArray>(a); } ), py::arg("ba"))

libsrc/core/utils.cpp

@@ -117,7 +117,7 @@ namespace ngcore
 
   const std::chrono::time_point<TClock> wall_time_start = TClock::now();
 
-  int printmessage_importance = 0;
+  int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0;
   bool NGSOStream :: glob_active = true;
 
   NGCORE_API int GetCompiledSIMDSize()

libsrc/meshing/python_mesh.cpp

@@ -1,3 +1,4 @@
+#include "pybind11/pytypes.h"
 #ifdef NG_PYTHON
 
 #include <regex>
@@ -23,52 +24,6 @@ public:
   ~ClearSolutionClass() { Ng_ClearSolutionData(); }
 };
 
-#undef NG_MPI4PY
-#ifdef NG_MPI4PY
-#include <mpi4py.h>
-
-struct mpi4py_comm {
-  mpi4py_comm() = default;
-  mpi4py_comm(MPI_Comm value) : value(value) {}
-  operator MPI_Comm () { return value; }
-
-  MPI_Comm value;
-};
-
-namespace pybind11 { namespace detail {
-  template <> struct type_caster<mpi4py_comm> {
-    public:
-    PYBIND11_TYPE_CASTER(mpi4py_comm, _("mpi4py_comm"));
-
-      // Python -> C++
-      bool load(handle src, bool) {
-        PyObject *py_src = src.ptr();
-        // Check that we have been passed an mpi4py communicator
-        if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
-          // Convert to regular MPI communicator
-          value.value = *PyMPIComm_Get(py_src);
-        } else {
-          return false;
-        }
-
-        return !PyErr_Occurred();
-      }
-
-      // C++ -> Python
-      static handle cast(mpi4py_comm src,
-                         return_value_policy /* policy */,
-                         handle /* parent */)
-      {
-        // Create an mpi4py handle
-        return PyMPIComm_New(src.value);
-      }
-  };
-}} // namespace pybind11::detail
-
-#endif // NG_MPI4PY
-
-
-
 
 
 using namespace netgen;
@@ -101,9 +56,6 @@ static Transformation<3> global_trafo(Vec<3> (0,0,0));
 
 DLL_HEADER void ExportNetgenMeshing(py::module &m) 
 {
-#ifdef NG_MPI4PY
-  import_mpi4py();
-#endif // NG_MPI4PY
   py::register_exception<NgException>(m, "NgException");
   m.attr("_netgen_executable_started") = py::cast(netgen::netgen_executable_started);
   string script;
@@ -133,13 +85,10 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m)
 
   py::class_<NgMPI_Comm> (m, "MPI_Comm")
     .def(py::init([] () { return NgMPI_Comm(NG_MPI_COMM_WORLD); }))
-#ifdef NG_MPI4PY
+    .def(py::init([] (NG_MPI_Comm comm) { return NgMPI_Comm(comm); }))
-    .def(py::init([] (mpi4py_comm comm)
+    .def_property_readonly ("mpi4py", [](NgMPI_Comm & self) {
-                  {
+        return static_cast<NG_MPI_Comm>(self);
-                    return NgMPI_Comm(comm);
+          })
-                  }))
-    .def_property_readonly ("mpi4py", [] (NgMPI_Comm comm) { return mpi4py_comm(comm); })
-#endif // NG_MPI4PY
     .def_property_readonly ("rank", &NgMPI_Comm::Rank)
     .def_property_readonly ("size", &NgMPI_Comm::Size)
     .def("Barrier", &NgMPI_Comm::Barrier)
@@ -169,9 +118,7 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m)
   ;
 
 
-#ifdef NG_MPI4PY
+  py::implicitly_convertible<NG_MPI_Comm, NgMPI_Comm>();
-  py::implicitly_convertible<mpi4py_comm, NgMPI_Comm>();
-#endif // NG_MPI4PY
 
   
   py::class_<NGDummyArgument>(m, "NGDummyArgument")