diff --git a/cmake/SuperBuild.cmake b/cmake/SuperBuild.cmake index 27315f04..13bbfad0 100644 --- a/cmake/SuperBuild.cmake +++ b/cmake/SuperBuild.cmake @@ -215,7 +215,6 @@ endif(USE_CGNS) ####################################################################### if(USE_MPI) - if(UNIX) if (METIS_DIR) message(STATUS "Using external METIS at: ${METIS_DIR}") else (METIS_DIR) @@ -226,9 +225,6 @@ if(USE_MPI) include(cmake/external_projects/metis.cmake) endif(NOT METIS_FOUND) endif(METIS_DIR) - else(UNIX) - find_package(METIS REQUIRED) - endif(UNIX) endif(USE_MPI) diff --git a/cmake/external_projects/metis.cmake b/cmake/external_projects/metis.cmake index 1711b5df..5501a0f5 100644 --- a/cmake/external_projects/metis.cmake +++ b/cmake/external_projects/metis.cmake @@ -3,8 +3,8 @@ set(METIS_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/metis) ExternalProject_Add(project_metis PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies - URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p6.tar.gz - URL_MD5 55fc654bb838846b856ba898795143f1 + URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p12.tar.gz + URL_MD5 6cd66f75f88dfa2cf043de011f85d8bc DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies CMAKE_ARGS -DGKLIB_PATH=${METIS_SRC_DIR}/GKlib diff --git a/libsrc/core/CMakeLists.txt b/libsrc/core/CMakeLists.txt index eb03bbdb..b11c0817 100644 --- a/libsrc/core/CMakeLists.txt +++ b/libsrc/core/CMakeLists.txt @@ -120,12 +120,19 @@ function (build_mpi_variant) target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python) target_compile_definitions(${target} PUBLIC PARALLEL) target_include_directories(${target} PRIVATE ${include_dir}) - install(TARGETS ${target} DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen) + set_target_properties(${target} PROPERTIES PREFIX "") + install(TARGETS ${target} RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen) endfunction() if(USE_MPI) target_compile_definitions(ngcore PUBLIC PARALLEL) + message(STATUS "Found MPI version\n${MPI_C_LIBRARY_VERSION_STRING}") + + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Microsoft MPI.*") + set(MICROSOFT_MPI_INCLUDE_DIR ${MPI_C_HEADER_DIR}) + endif() + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*") set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) endif() @@ -140,6 +147,10 @@ if(USE_MPI) if(MPICH_INCLUDE_DIR) build_mpi_variant(mpich ${MPICH_INCLUDE_DIR}) endif() + if(MICROSOFT_MPI_INCLUDE_DIR) + build_mpi_variant(microsoft_mpi ${MICROSOFT_MPI_INCLUDE_DIR}) + target_link_libraries(ng_microsoft_mpi PUBLIC ${MPI_msmpi_LIBRARY}) + endif() endif(USE_MPI) diff --git a/libsrc/core/ng_mpi.cpp b/libsrc/core/ng_mpi.cpp index daf1410a..6ede6c7a 100644 --- a/libsrc/core/ng_mpi.cpp +++ b/libsrc/core/ng_mpi.cpp @@ -15,6 +15,15 @@ namespace py = pybind11; #endif +#ifdef MSMPI_VER +int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2, + MPI_Comm* arg3) { + throw std::runtime_error( + "MPI_Comm_create_group not supported on Microsoft MPI"); +} +static MPI_Datatype MPI_CXX_DOUBLE_COMPLEX; +#endif // MSMPI_VER + namespace ngcore { static_assert(sizeof(MPI_Status) <= sizeof(NG_MPI_Status), "Size mismatch"); @@ -29,7 +38,7 @@ NG_MPI_Status* mpi2ng(MPI_Status* status) { return reinterpret_cast(status); } -#ifndef MPICH +#if !defined(MPICH) && !defined(MSMPI_VER) NG_MPI_Comm mpi2ng(MPI_Comm comm) { return reinterpret_cast(comm); } #endif @@ -140,7 +149,9 @@ int ng2mpi(int value) { return value; } using namespace ngcore; -NGCORE_API_EXPORT extern "C" void ng_init_mpi(); +extern "C" { +NGCORE_API_EXPORT void ng_init_mpi(); +} static bool imported_mpi4py = false; diff --git a/libsrc/core/ng_mpi.hpp b/libsrc/core/ng_mpi.hpp index a6e63d60..29c97956 100644 --- a/libsrc/core/ng_mpi.hpp +++ b/libsrc/core/ng_mpi.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include "ngcore_api.hpp" @@ -18,9 +19,7 @@ namespace py = pybind11; namespace ngcore { NGCORE_API void InitMPI( - std::filesystem::path mpi_lib_path = std::string("libmpi") + - NETGEN_SHARED_LIBRARY_SUFFIX); -NGCORE_API extern std::string mpi_library_version; + std::optional mpi_lib_path = std::nullopt); inline void not_implemented() { throw std::runtime_error("Not implemented"); } diff --git a/libsrc/core/ng_mpi_wrapper.cpp b/libsrc/core/ng_mpi_wrapper.cpp index d24f9c68..328e5ac7 100644 --- a/libsrc/core/ng_mpi_wrapper.cpp +++ b/libsrc/core/ng_mpi_wrapper.cpp @@ -6,6 +6,7 @@ #include "ng_mpi.hpp" #include "ngstream.hpp" +#include "python_ngcore.hpp" #include "utils.hpp" using std::cerr; @@ -26,71 +27,96 @@ struct MPIFinalizer { } } mpi_finalizer; -void InitMPI(std::filesystem::path mpi_lib_path) { +void InitMPI(std::optional mpi_lib_path) { if (ng_mpi_lib) return; + cout << IM(3) << "InitMPI" << endl; - typedef void (*get_version_handle)(char *, int *); - typedef int (*init_handle)(int *, char ***); - typedef int (*mpi_initialized_handle)(int *); + std::string vendor = ""; + std::string ng_lib_name = ""; + std::string mpi4py_lib_file = ""; + + if (mpi_lib_path) { + // Dynamic load of given shared MPI library + // Then call MPI_Init, read the library version and set the vender name + try { + typedef int (*init_handle)(int *, char ***); + typedef int (*mpi_initialized_handle)(int *); + mpi_lib = + std::make_unique(*mpi_lib_path, std::nullopt, true); + auto mpi_init = mpi_lib->GetSymbol("MPI_Init"); + auto mpi_initialized = + mpi_lib->GetSymbol("MPI_Initialized"); + + int flag = 0; + mpi_initialized(&flag); + if (!flag) { + typedef const char *pchar; + int argc = 1; + pchar args[] = {"netgen", nullptr}; + pchar *argv = &args[0]; + cout << IM(5) << "Calling MPI_Init" << endl; + mpi_init(&argc, (char ***)argv); + need_mpi_finalize = true; + } + + char c_version_string[65536]; + c_version_string[0] = '\0'; + int result_len = 0; + typedef void (*get_version_handle)(char *, int *); + auto get_version = + mpi_lib->GetSymbol("MPI_Get_library_version"); + get_version(c_version_string, &result_len); + std::string version = c_version_string; + + if (version.substr(0, 8) == "Open MPI") + vendor = "Open MPI"; + else if (version.substr(0, 5) == "MPICH") + vendor = "MPICH"; + else if (version.substr(0, 5) == "Microsoft MPI") + vendor = "Microsoft MPI"; + else + throw std::runtime_error( + std::string("Unknown MPI version: " + version)); + } catch (std::runtime_error &e) { + cerr << "Could not load MPI: " << e.what() << endl; + throw e; + } + } else { + // Use mpi4py to init MPI library and get the vendor name + auto mpi4py = py::module::import("mpi4py.MPI"); + vendor = mpi4py.attr("get_vendor")()[py::int_(0)].cast(); + +#ifndef WIN32 + // Load mpi4py library (it exports all MPI symbols) to have all MPI symbols + // available before the ng_mpi wrapper is loaded This is not necessary on + // windows as the matching mpi dll is linked to the ng_mpi wrapper directly + mpi4py_lib_file = mpi4py.attr("__file__").cast(); + mpi_lib = + std::make_unique(mpi4py_lib_file, std::nullopt, true); +#endif // WIN32 + } + + if (vendor == "Open MPI") + ng_lib_name = "ng_openmpi"; + else if (vendor == "MPICH") + ng_lib_name = "ng_mpich"; + else if (vendor == "Microsoft MPI") + ng_lib_name = "ng_msmpi"; + else + throw std::runtime_error("Unknown MPI vendor: " + vendor); + + // Load the ng_mpi wrapper and call ng_init_mpi to set all function pointers typedef void (*ng_init_handle)(); - - init_handle mpi_init; - mpi_initialized_handle mpi_initialized; - get_version_handle get_version; - - mpi_lib = std::make_unique(mpi_lib_path, std::nullopt, true); - - try { - mpi_init = GetSymbol("MPI_Init"); - mpi_initialized = GetSymbol("MPI_Initialized"); - get_version = GetSymbol("MPI_Get_library_version"); - } catch (std::runtime_error &e) { - cerr << "Could not load MPI symbols: " << e.what() << endl; - throw e; - } - - int flag = 0; - mpi_initialized(&flag); - if (!flag) { - typedef const char *pchar; - int argc = 1; - pchar args[] = {"netgen", nullptr}; - pchar *argv = &args[0]; - cout << IM(5) << "Calling MPI_Init" << endl; - mpi_init(&argc, (char ***)argv); - need_mpi_finalize = true; - } - - char version_string[65536]; - int resultlen = 0; - get_version(version_string, &resultlen); - mpi_library_version = version_string; - cout << IM(7) << "MPI version: " << version_string << endl; - - std::string libname = ""; - if (mpi_library_version.substr(0, 8) == "Open MPI") { - cout << IM(5) << "Have Open MPI" << endl; - libname = std::string("libng_openmpi") + NETGEN_SHARED_LIBRARY_SUFFIX; - } else if (mpi_library_version.substr(0, 5) == "MPICH") { - cout << IM(5) << "Have MPICH" << endl; - libname = std::string("libng_mpich") + NETGEN_SHARED_LIBRARY_SUFFIX; - } else - cerr << "Unknown MPI version, skipping init: " << version_string << endl; - - if (libname.size()) { - ng_mpi_lib = std::make_unique(libname); - auto ng_init = ng_mpi_lib->GetSymbol("ng_init_mpi"); - ng_init(); - } + ng_mpi_lib = std::make_unique(ng_lib_name); + ng_mpi_lib->GetSymbol("ng_init_mpi")(); + std::cout << IM(3) << "MPI wrapper loaded, vendor: " << vendor << endl; } static std::runtime_error no_mpi() { return std::runtime_error("MPI not enabled"); } -std::string mpi_library_version = ""; - #if defined(NG_PYTHON) && defined(NG_MPI4PY) decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = [](py::handle, NG_MPI_Comm &) -> bool { throw no_mpi(); }; diff --git a/libsrc/core/python_ngcore_export.cpp b/libsrc/core/python_ngcore_export.cpp index ff0b4857..0a6b82f1 100644 --- a/libsrc/core/python_ngcore_export.cpp +++ b/libsrc/core/python_ngcore_export.cpp @@ -40,7 +40,7 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT #ifdef PARALLEL py::class_ (m, "_NG_MPI_Comm") ; - m.def("InitMPI", &InitMPI); + m.def("InitMPI", &InitMPI, py::arg("mpi_library_path")=nullopt); #endif // PARALLEL py::class_> (m, "BitArray") diff --git a/libsrc/core/utils.cpp b/libsrc/core/utils.cpp index c65c57f9..4ab08f36 100644 --- a/libsrc/core/utils.cpp +++ b/libsrc/core/utils.cpp @@ -178,7 +178,7 @@ namespace ngcore Unload(); lib_name = lib_name_; #ifdef WIN32 - lib = LoadLibrary(lib_name.wstring().c_str()); + lib = LoadLibraryW(lib_name.wstring().c_str()); if (!lib) throw std::runtime_error(std::string("Could not load library ") + lib_name.string()); #else // WIN32 auto flags = RTLD_NOW;