Some windows fixes, update Metis version

cmake/SuperBuild.cmake

@@ -215,7 +215,6 @@ endif(USE_CGNS)
 
 #######################################################################
 if(USE_MPI)
-  if(UNIX)
     if (METIS_DIR)
       message(STATUS "Using external METIS at: ${METIS_DIR}")
     else (METIS_DIR)
@@ -226,9 +225,6 @@ if(USE_MPI)
 	include(cmake/external_projects/metis.cmake)
       endif(NOT METIS_FOUND)
     endif(METIS_DIR)
-  else(UNIX)
-    find_package(METIS REQUIRED)
-  endif(UNIX)
 endif(USE_MPI)
 
 

cmake/external_projects/metis.cmake

@@ -3,8 +3,8 @@ set(METIS_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/metis)
 
 ExternalProject_Add(project_metis
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies
-  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p6.tar.gz
-  URL_MD5 55fc654bb838846b856ba898795143f1
+  URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p12.tar.gz
+  URL_MD5 6cd66f75f88dfa2cf043de011f85d8bc
   DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies
   CMAKE_ARGS
          -DGKLIB_PATH=${METIS_SRC_DIR}/GKlib

libsrc/core/CMakeLists.txt

@@ -120,12 +120,19 @@ function (build_mpi_variant)
   target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python)
   target_compile_definitions(${target} PUBLIC PARALLEL)
   target_include_directories(${target} PRIVATE ${include_dir})
-  install(TARGETS ${target} DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
+  set_target_properties(${target} PROPERTIES PREFIX "")
+  install(TARGETS ${target} RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
 endfunction()
 
 if(USE_MPI)
   target_compile_definitions(ngcore PUBLIC PARALLEL)
 
+  message(STATUS "Found MPI version\n${MPI_C_LIBRARY_VERSION_STRING}")
+
+  if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Microsoft MPI.*")
+    set(MICROSOFT_MPI_INCLUDE_DIR ${MPI_C_HEADER_DIR})
+  endif()
+
   if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*")
     set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
   endif()
@@ -140,6 +147,10 @@ if(USE_MPI)
   if(MPICH_INCLUDE_DIR)
     build_mpi_variant(mpich ${MPICH_INCLUDE_DIR})
   endif()
+  if(MICROSOFT_MPI_INCLUDE_DIR)
+    build_mpi_variant(microsoft_mpi ${MICROSOFT_MPI_INCLUDE_DIR})
+    target_link_libraries(ng_microsoft_mpi PUBLIC ${MPI_msmpi_LIBRARY})
+  endif()
 
 endif(USE_MPI)
 

libsrc/core/ng_mpi.cpp

@@ -15,6 +15,15 @@
 namespace py = pybind11;
 #endif
 
+#ifdef MSMPI_VER
+int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2,
+                          MPI_Comm* arg3) {
+  throw std::runtime_error(
+      "MPI_Comm_create_group not supported on Microsoft MPI");
+}
+static MPI_Datatype MPI_CXX_DOUBLE_COMPLEX;
+#endif  // MSMPI_VER
+
 namespace ngcore {
 
 static_assert(sizeof(MPI_Status) <= sizeof(NG_MPI_Status), "Size mismatch");
@@ -29,7 +38,7 @@ NG_MPI_Status* mpi2ng(MPI_Status* status) {
   return reinterpret_cast<NG_MPI_Status*>(status);
 }
 
-#ifndef MPICH
+#if !defined(MPICH) && !defined(MSMPI_VER)
 NG_MPI_Comm mpi2ng(MPI_Comm comm) { return reinterpret_cast<uintptr_t>(comm); }
 #endif
 
@@ -140,7 +149,9 @@ int ng2mpi(int value) { return value; }
 
 using namespace ngcore;
 
-NGCORE_API_EXPORT extern "C" void ng_init_mpi();
+extern "C" {
+NGCORE_API_EXPORT void ng_init_mpi();
+}
 
 static bool imported_mpi4py = false;
 

libsrc/core/ng_mpi.hpp

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <optional>
 #include <string>
 
 #include "ngcore_api.hpp"
@@ -18,9 +19,7 @@ namespace py = pybind11;
 namespace ngcore {
 
 NGCORE_API void InitMPI(
-    std::filesystem::path mpi_lib_path = std::string("libmpi") +
-                                         NETGEN_SHARED_LIBRARY_SUFFIX);
-NGCORE_API extern std::string mpi_library_version;
+    std::optional<std::filesystem::path> mpi_lib_path = std::nullopt);
 
 inline void not_implemented() { throw std::runtime_error("Not implemented"); }
 

libsrc/core/ng_mpi_wrapper.cpp

@@ -6,6 +6,7 @@
 
 #include "ng_mpi.hpp"
 #include "ngstream.hpp"
+#include "python_ngcore.hpp"
 #include "utils.hpp"
 
 using std::cerr;
@@ -26,71 +27,96 @@ struct MPIFinalizer {
   }
 } mpi_finalizer;
 
-void InitMPI(std::filesystem::path mpi_lib_path) {
+void InitMPI(std::optional<std::filesystem::path> mpi_lib_path) {
   if (ng_mpi_lib) return;
+
   cout << IM(3) << "InitMPI" << endl;
 
-  typedef void (*get_version_handle)(char *, int *);
-  typedef int (*init_handle)(int *, char ***);
-  typedef int (*mpi_initialized_handle)(int *);
+  std::string vendor = "";
+  std::string ng_lib_name = "";
+  std::string mpi4py_lib_file = "";
+
+  if (mpi_lib_path) {
+    // Dynamic load of given shared MPI library
+    // Then call MPI_Init, read the library version and set the vender name
+    try {
+      typedef int (*init_handle)(int *, char ***);
+      typedef int (*mpi_initialized_handle)(int *);
+      mpi_lib =
+          std::make_unique<SharedLibrary>(*mpi_lib_path, std::nullopt, true);
+      auto mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
+      auto mpi_initialized =
+          mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
+
+      int flag = 0;
+      mpi_initialized(&flag);
+      if (!flag) {
+        typedef const char *pchar;
+        int argc = 1;
+        pchar args[] = {"netgen", nullptr};
+        pchar *argv = &args[0];
+        cout << IM(5) << "Calling MPI_Init" << endl;
+        mpi_init(&argc, (char ***)argv);
+        need_mpi_finalize = true;
+      }
+
+      char c_version_string[65536];
+      c_version_string[0] = '\0';
+      int result_len = 0;
+      typedef void (*get_version_handle)(char *, int *);
+      auto get_version =
+          mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
+      get_version(c_version_string, &result_len);
+      std::string version = c_version_string;
+
+      if (version.substr(0, 8) == "Open MPI")
+        vendor = "Open MPI";
+      else if (version.substr(0, 5) == "MPICH")
+        vendor = "MPICH";
+      else if (version.substr(0, 5) == "Microsoft MPI")
+        vendor = "Microsoft MPI";
+      else
+        throw std::runtime_error(
+            std::string("Unknown MPI version: " + version));
+    } catch (std::runtime_error &e) {
+      cerr << "Could not load MPI: " << e.what() << endl;
+      throw e;
+    }
+  } else {
+    // Use mpi4py to init MPI library and get the vendor name
+    auto mpi4py = py::module::import("mpi4py.MPI");
+    vendor = mpi4py.attr("get_vendor")()[py::int_(0)].cast<std::string>();
+
+#ifndef WIN32
+    // Load mpi4py library (it exports all MPI symbols) to have all MPI symbols
+    // available before the ng_mpi wrapper is loaded This is not necessary on
+    // windows as the matching mpi dll is linked to the ng_mpi wrapper directly
+    mpi4py_lib_file = mpi4py.attr("__file__").cast<std::string>();
+    mpi_lib =
+        std::make_unique<SharedLibrary>(mpi4py_lib_file, std::nullopt, true);
+#endif  // WIN32
+  }
+
+  if (vendor == "Open MPI")
+    ng_lib_name = "ng_openmpi";
+  else if (vendor == "MPICH")
+    ng_lib_name = "ng_mpich";
+  else if (vendor == "Microsoft MPI")
+    ng_lib_name = "ng_msmpi";
+  else
+    throw std::runtime_error("Unknown MPI vendor: " + vendor);
+
+  // Load the ng_mpi wrapper and call ng_init_mpi to set all function pointers
   typedef void (*ng_init_handle)();
-
-  init_handle mpi_init;
-  mpi_initialized_handle mpi_initialized;
-  get_version_handle get_version;
-
-  mpi_lib = std::make_unique<SharedLibrary>(mpi_lib_path, std::nullopt, true);
-
-  try {
-    mpi_init = GetSymbol<init_handle>("MPI_Init");
-    mpi_initialized = GetSymbol<mpi_initialized_handle>("MPI_Initialized");
-    get_version = GetSymbol<get_version_handle>("MPI_Get_library_version");
-  } catch (std::runtime_error &e) {
-    cerr << "Could not load MPI symbols: " << e.what() << endl;
-    throw e;
-  }
-
-  int flag = 0;
-  mpi_initialized(&flag);
-  if (!flag) {
-    typedef const char *pchar;
-    int argc = 1;
-    pchar args[] = {"netgen", nullptr};
-    pchar *argv = &args[0];
-    cout << IM(5) << "Calling MPI_Init" << endl;
-    mpi_init(&argc, (char ***)argv);
-    need_mpi_finalize = true;
-  }
-
-  char version_string[65536];
-  int resultlen = 0;
-  get_version(version_string, &resultlen);
-  mpi_library_version = version_string;
-  cout << IM(7) << "MPI version: " << version_string << endl;
-
-  std::string libname = "";
-  if (mpi_library_version.substr(0, 8) == "Open MPI") {
-    cout << IM(5) << "Have Open MPI" << endl;
-    libname = std::string("libng_openmpi") + NETGEN_SHARED_LIBRARY_SUFFIX;
-  } else if (mpi_library_version.substr(0, 5) == "MPICH") {
-    cout << IM(5) << "Have MPICH" << endl;
-    libname = std::string("libng_mpich") + NETGEN_SHARED_LIBRARY_SUFFIX;
-  } else
-    cerr << "Unknown MPI version, skipping init: " << version_string << endl;
-
-  if (libname.size()) {
-    ng_mpi_lib = std::make_unique<SharedLibrary>(libname);
-    auto ng_init = ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi");
-    ng_init();
-  }
+  ng_mpi_lib = std::make_unique<SharedLibrary>(ng_lib_name);
+  ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi")();
+  std::cout << IM(3) << "MPI wrapper loaded, vendor: " << vendor << endl;
 }
 
 static std::runtime_error no_mpi() {
   return std::runtime_error("MPI not enabled");
 }
 
-std::string mpi_library_version = "";
-
 #if defined(NG_PYTHON) && defined(NG_MPI4PY)
 decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
     [](py::handle, NG_MPI_Comm &) -> bool { throw no_mpi(); };

libsrc/core/python_ngcore_export.cpp

@@ -40,7 +40,7 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
   #ifdef PARALLEL
   py::class_<NG_MPI_Comm> (m, "_NG_MPI_Comm")
           ;
-  m.def("InitMPI", &InitMPI);
+  m.def("InitMPI", &InitMPI, py::arg("mpi_library_path")=nullopt);
   #endif // PARALLEL
 
   py::class_<BitArray, shared_ptr<BitArray>> (m, "BitArray")

libsrc/core/utils.cpp

@@ -178,7 +178,7 @@ namespace ngcore
     Unload();
     lib_name = lib_name_;
 #ifdef WIN32
-    lib = LoadLibrary(lib_name.wstring().c_str());
+    lib = LoadLibraryW(lib_name.wstring().c_str());
     if (!lib) throw std::runtime_error(std::string("Could not load library ") + lib_name.string());
 #else // WIN32
     auto flags = RTLD_NOW;