diff --git a/libsrc/core/generate_mpi_sources.py b/libsrc/core/generate_mpi_sources.py index dd5ee8c0..da494a05 100644 --- a/libsrc/core/generate_mpi_sources.py +++ b/libsrc/core/generate_mpi_sources.py @@ -16,6 +16,7 @@ functions = [ ("int", "MPI_Get_processor_name", "char*", "int*"), ("int", "MPI_Group_incl", "MPI_Group", "int", "int*", "MPI_Group*"), ("int", "MPI_Init", "int*", "char***"), + ("int", "MPI_Init_thread", "int*", "char***", "int", "int*"), ("int", "MPI_Initialized", "int*"), ("int", "MPI_Iprobe", "int", "int", "MPI_Comm", "int*", "MPI_Status*"), ("int", "MPI_Irecv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"), @@ -42,26 +43,30 @@ functions = [ constants = [ ("MPI_Comm", "MPI_COMM_WORLD"), - ("MPI_Status*", "MPI_STATUS_IGNORE"), - ("MPI_Status*", "MPI_STATUSES_IGNORE"), - ("MPI_Datatype", "MPI_INT"), - ("MPI_Datatype", "MPI_SHORT"), ("MPI_Datatype", "MPI_CHAR"), - ("MPI_Datatype", "MPI_UINT64_T"), - ("MPI_Datatype", "MPI_DOUBLE"), + ("MPI_Datatype", "MPI_CXX_DOUBLE_COMPLEX"), ("MPI_Datatype", "MPI_C_BOOL"), ("MPI_Datatype", "MPI_DATATYPE_NULL"), - ("MPI_Datatype", "MPI_CXX_DOUBLE_COMPLEX"), - ("void*", "MPI_IN_PLACE"), - ("int", "MPI_MAX_PROCESSOR_NAME"), - ("int", "MPI_ANY_SOURCE"), - ("int", "MPI_ROOT"), - ("int", "MPI_PROC_NULL"), - ("int", "MPI_ANY_TAG"), + ("MPI_Datatype", "MPI_DOUBLE"), + ("MPI_Datatype", "MPI_INT"), + ("MPI_Datatype", "MPI_SHORT"), + ("MPI_Datatype", "MPI_UINT64_T"), + ("MPI_Op", "MPI_LOR"), ("MPI_Op", "MPI_MAX"), ("MPI_Op", "MPI_MIN"), ("MPI_Op", "MPI_SUM"), - ("MPI_Op", "MPI_LOR"), + ("MPI_Status*", "MPI_STATUSES_IGNORE"), + ("MPI_Status*", "MPI_STATUS_IGNORE"), + ("int", "MPI_ANY_SOURCE"), + ("int", "MPI_ANY_TAG"), + ("int", "MPI_MAX_PROCESSOR_NAME"), + ("int", "MPI_PROC_NULL"), + ("int", "MPI_ROOT"), + ("int", "MPI_SUBVERSION"), + ("int", "MPI_THREAD_MULTIPLE"), + ("int", "MPI_THREAD_SINGLE"), + ("int", "MPI_VERSION"), + ("void*", "MPI_IN_PLACE"), ] def get_args(f): diff --git a/libsrc/core/ng_mpi.hpp b/libsrc/core/ng_mpi.hpp index 4af75bc8..a6e63d60 100644 --- a/libsrc/core/ng_mpi.hpp +++ b/libsrc/core/ng_mpi.hpp @@ -19,8 +19,7 @@ namespace ngcore { NGCORE_API void InitMPI( std::filesystem::path mpi_lib_path = std::string("libmpi") + - NETGEN_SHARED_LIBRARY_SUFFIX, - std::filesystem::path ng_libs_dir = ""); + NETGEN_SHARED_LIBRARY_SUFFIX); NGCORE_API extern std::string mpi_library_version; inline void not_implemented() { throw std::runtime_error("Not implemented"); } @@ -86,5 +85,5 @@ NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm); } // namespace ngcore -#endif // PARALLEL +#endif // PARALLEL #endif // NG_MPI_HPP_INCLUDED diff --git a/libsrc/core/ng_mpi_generated_declarations.hpp b/libsrc/core/ng_mpi_generated_declarations.hpp index bc54a63b..24c48cc4 100644 --- a/libsrc/core/ng_mpi_generated_declarations.hpp +++ b/libsrc/core/ng_mpi_generated_declarations.hpp @@ -15,6 +15,7 @@ NGCORE_API extern int (*NG_MPI_Get_count)(NG_MPI_Status*, NG_MPI_Datatype, int*) NGCORE_API extern int (*NG_MPI_Get_processor_name)(char*, int*); NGCORE_API extern int (*NG_MPI_Group_incl)(NG_MPI_Group, int, int*, NG_MPI_Group*); NGCORE_API extern int (*NG_MPI_Init)(int*, char***); +NGCORE_API extern int (*NG_MPI_Init_thread)(int*, char***, int, int*); NGCORE_API extern int (*NG_MPI_Initialized)(int*); NGCORE_API extern int (*NG_MPI_Iprobe)(int, int, NG_MPI_Comm, int*, NG_MPI_Status*); NGCORE_API extern int (*NG_MPI_Irecv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*); @@ -38,23 +39,27 @@ NGCORE_API extern int (*NG_MPI_Wait)(NG_MPI_Request*, NG_MPI_Status*); NGCORE_API extern int (*NG_MPI_Waitall)(int, NG_MPI_Request*, NG_MPI_Status*); NGCORE_API extern int (*NG_MPI_Waitany)(int, NG_MPI_Request*, int*, NG_MPI_Status*); NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_WORLD; -NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE; -NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE; -NGCORE_API extern NG_MPI_Datatype NG_MPI_INT; -NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT; NGCORE_API extern NG_MPI_Datatype NG_MPI_CHAR; -NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T; -NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE; +NGCORE_API extern NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX; NGCORE_API extern NG_MPI_Datatype NG_MPI_C_BOOL; NGCORE_API extern NG_MPI_Datatype NG_MPI_DATATYPE_NULL; -NGCORE_API extern NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX; -NGCORE_API extern void* NG_MPI_IN_PLACE; -NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME; -NGCORE_API extern int NG_MPI_ANY_SOURCE; -NGCORE_API extern int NG_MPI_ROOT; -NGCORE_API extern int NG_MPI_PROC_NULL; -NGCORE_API extern int NG_MPI_ANY_TAG; +NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE; +NGCORE_API extern NG_MPI_Datatype NG_MPI_INT; +NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT; +NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T; +NGCORE_API extern NG_MPI_Op NG_MPI_LOR; NGCORE_API extern NG_MPI_Op NG_MPI_MAX; NGCORE_API extern NG_MPI_Op NG_MPI_MIN; NGCORE_API extern NG_MPI_Op NG_MPI_SUM; -NGCORE_API extern NG_MPI_Op NG_MPI_LOR; +NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE; +NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE; +NGCORE_API extern int NG_MPI_ANY_SOURCE; +NGCORE_API extern int NG_MPI_ANY_TAG; +NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME; +NGCORE_API extern int NG_MPI_PROC_NULL; +NGCORE_API extern int NG_MPI_ROOT; +NGCORE_API extern int NG_MPI_SUBVERSION; +NGCORE_API extern int NG_MPI_THREAD_MULTIPLE; +NGCORE_API extern int NG_MPI_THREAD_SINGLE; +NGCORE_API extern int NG_MPI_VERSION; +NGCORE_API extern void* NG_MPI_IN_PLACE; diff --git a/libsrc/core/ng_mpi_generated_dummy_init.hpp b/libsrc/core/ng_mpi_generated_dummy_init.hpp index e8047b50..396a1f42 100644 --- a/libsrc/core/ng_mpi_generated_dummy_init.hpp +++ b/libsrc/core/ng_mpi_generated_dummy_init.hpp @@ -15,6 +15,7 @@ decltype(NG_MPI_Get_count) NG_MPI_Get_count = [](NG_MPI_Status*, NG_MPI_Datatype decltype(NG_MPI_Get_processor_name) NG_MPI_Get_processor_name = [](char*, int*)->int { throw no_mpi(); }; decltype(NG_MPI_Group_incl) NG_MPI_Group_incl = [](NG_MPI_Group, int, int*, NG_MPI_Group*)->int { throw no_mpi(); }; decltype(NG_MPI_Init) NG_MPI_Init = [](int*, char***)->int { throw no_mpi(); }; +decltype(NG_MPI_Init_thread) NG_MPI_Init_thread = [](int*, char***, int, int*)->int { throw no_mpi(); }; decltype(NG_MPI_Initialized) NG_MPI_Initialized = [](int*)->int { throw no_mpi(); }; decltype(NG_MPI_Iprobe) NG_MPI_Iprobe = [](int, int, NG_MPI_Comm, int*, NG_MPI_Status*)->int { throw no_mpi(); }; decltype(NG_MPI_Irecv) NG_MPI_Irecv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); }; @@ -38,23 +39,27 @@ decltype(NG_MPI_Wait) NG_MPI_Wait = [](NG_MPI_Request*, NG_MPI_Status*)->int { t decltype(NG_MPI_Waitall) NG_MPI_Waitall = [](int, NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); }; decltype(NG_MPI_Waitany) NG_MPI_Waitany = [](int, NG_MPI_Request*, int*, NG_MPI_Status*)->int { throw no_mpi(); }; NG_MPI_Comm NG_MPI_COMM_WORLD = 0; -NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0; -NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0; -NG_MPI_Datatype NG_MPI_INT = 0; -NG_MPI_Datatype NG_MPI_SHORT = 0; NG_MPI_Datatype NG_MPI_CHAR = 0; -NG_MPI_Datatype NG_MPI_UINT64_T = 0; -NG_MPI_Datatype NG_MPI_DOUBLE = 0; +NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX = 0; NG_MPI_Datatype NG_MPI_C_BOOL = 0; NG_MPI_Datatype NG_MPI_DATATYPE_NULL = 0; -NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX = 0; -void* NG_MPI_IN_PLACE = 0; -int NG_MPI_MAX_PROCESSOR_NAME = 0; -int NG_MPI_ANY_SOURCE = 0; -int NG_MPI_ROOT = 0; -int NG_MPI_PROC_NULL = 0; -int NG_MPI_ANY_TAG = 0; +NG_MPI_Datatype NG_MPI_DOUBLE = 0; +NG_MPI_Datatype NG_MPI_INT = 0; +NG_MPI_Datatype NG_MPI_SHORT = 0; +NG_MPI_Datatype NG_MPI_UINT64_T = 0; +NG_MPI_Op NG_MPI_LOR = 0; NG_MPI_Op NG_MPI_MAX = 0; NG_MPI_Op NG_MPI_MIN = 0; NG_MPI_Op NG_MPI_SUM = 0; -NG_MPI_Op NG_MPI_LOR = 0; +NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0; +NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0; +int NG_MPI_ANY_SOURCE = 0; +int NG_MPI_ANY_TAG = 0; +int NG_MPI_MAX_PROCESSOR_NAME = 0; +int NG_MPI_PROC_NULL = 0; +int NG_MPI_ROOT = 0; +int NG_MPI_SUBVERSION = 0; +int NG_MPI_THREAD_MULTIPLE = 0; +int NG_MPI_THREAD_SINGLE = 0; +int NG_MPI_VERSION = 0; +void* NG_MPI_IN_PLACE = 0; diff --git a/libsrc/core/ng_mpi_generated_init.hpp b/libsrc/core/ng_mpi_generated_init.hpp index a8e6cb0f..3b3b93ef 100644 --- a/libsrc/core/ng_mpi_generated_init.hpp +++ b/libsrc/core/ng_mpi_generated_init.hpp @@ -15,6 +15,7 @@ NG_MPI_Get_count = [](NG_MPI_Status* arg0, NG_MPI_Datatype arg1, int* arg2)->int NG_MPI_Get_processor_name = [](char* arg0, int* arg1)->int { return MPI_Get_processor_name( ng2mpi(arg0), ng2mpi(arg1)); }; NG_MPI_Group_incl = [](NG_MPI_Group arg0, int arg1, int* arg2, NG_MPI_Group* arg3)->int { return MPI_Group_incl( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); }; NG_MPI_Init = [](int* arg0, char*** arg1)->int { return MPI_Init( ng2mpi(arg0), ng2mpi(arg1)); }; +NG_MPI_Init_thread = [](int* arg0, char*** arg1, int arg2, int* arg3)->int { return MPI_Init_thread( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); }; NG_MPI_Initialized = [](int* arg0)->int { return MPI_Initialized( ng2mpi(arg0)); }; NG_MPI_Iprobe = [](int arg0, int arg1, NG_MPI_Comm arg2, int* arg3, NG_MPI_Status* arg4)->int { return MPI_Iprobe( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3), ng2mpi(arg4)); }; NG_MPI_Irecv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Irecv( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3), ng2mpi(arg4), ng2mpi(arg5), ng2mpi(arg6)); }; @@ -38,23 +39,27 @@ NG_MPI_Wait = [](NG_MPI_Request* arg0, NG_MPI_Status* arg1)->int { return MPI_Wa NG_MPI_Waitall = [](int arg0, NG_MPI_Request* arg1, NG_MPI_Status* arg2)->int { return MPI_Waitall( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2)); }; NG_MPI_Waitany = [](int arg0, NG_MPI_Request* arg1, int* arg2, NG_MPI_Status* arg3)->int { return MPI_Waitany( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); }; NG_MPI_COMM_WORLD = mpi2ng(MPI_COMM_WORLD); -NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE); -NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE); -NG_MPI_INT = mpi2ng(MPI_INT); -NG_MPI_SHORT = mpi2ng(MPI_SHORT); NG_MPI_CHAR = mpi2ng(MPI_CHAR); -NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T); -NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE); +NG_MPI_CXX_DOUBLE_COMPLEX = mpi2ng(MPI_CXX_DOUBLE_COMPLEX); NG_MPI_C_BOOL = mpi2ng(MPI_C_BOOL); NG_MPI_DATATYPE_NULL = mpi2ng(MPI_DATATYPE_NULL); -NG_MPI_CXX_DOUBLE_COMPLEX = mpi2ng(MPI_CXX_DOUBLE_COMPLEX); -NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE); -NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME); -NG_MPI_ANY_SOURCE = mpi2ng(MPI_ANY_SOURCE); -NG_MPI_ROOT = mpi2ng(MPI_ROOT); -NG_MPI_PROC_NULL = mpi2ng(MPI_PROC_NULL); -NG_MPI_ANY_TAG = mpi2ng(MPI_ANY_TAG); +NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE); +NG_MPI_INT = mpi2ng(MPI_INT); +NG_MPI_SHORT = mpi2ng(MPI_SHORT); +NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T); +NG_MPI_LOR = mpi2ng(MPI_LOR); NG_MPI_MAX = mpi2ng(MPI_MAX); NG_MPI_MIN = mpi2ng(MPI_MIN); NG_MPI_SUM = mpi2ng(MPI_SUM); -NG_MPI_LOR = mpi2ng(MPI_LOR); +NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE); +NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE); +NG_MPI_ANY_SOURCE = mpi2ng(MPI_ANY_SOURCE); +NG_MPI_ANY_TAG = mpi2ng(MPI_ANY_TAG); +NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME); +NG_MPI_PROC_NULL = mpi2ng(MPI_PROC_NULL); +NG_MPI_ROOT = mpi2ng(MPI_ROOT); +NG_MPI_SUBVERSION = mpi2ng(MPI_SUBVERSION); +NG_MPI_THREAD_MULTIPLE = mpi2ng(MPI_THREAD_MULTIPLE); +NG_MPI_THREAD_SINGLE = mpi2ng(MPI_THREAD_SINGLE); +NG_MPI_VERSION = mpi2ng(MPI_VERSION); +NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE); diff --git a/libsrc/core/ng_mpi_wrapper.cpp b/libsrc/core/ng_mpi_wrapper.cpp index d7191ef1..d24f9c68 100644 --- a/libsrc/core/ng_mpi_wrapper.cpp +++ b/libsrc/core/ng_mpi_wrapper.cpp @@ -15,9 +15,18 @@ using std::endl; namespace ngcore { static std::unique_ptr mpi_lib, ng_mpi_lib; +static bool need_mpi_finalize = false; -void InitMPI(std::filesystem::path mpi_lib_path, - std::filesystem::path ng_libs_dir) { +struct MPIFinalizer { + ~MPIFinalizer() { + if (need_mpi_finalize) { + cout << IM(5) << "Calling MPI_Finalize" << endl; + NG_MPI_Finalize(); + } + } +} mpi_finalizer; + +void InitMPI(std::filesystem::path mpi_lib_path) { if (ng_mpi_lib) return; cout << IM(3) << "InitMPI" << endl; @@ -50,6 +59,7 @@ void InitMPI(std::filesystem::path mpi_lib_path, pchar *argv = &args[0]; cout << IM(5) << "Calling MPI_Init" << endl; mpi_init(&argc, (char ***)argv); + need_mpi_finalize = true; } char version_string[65536]; diff --git a/libsrc/core/utils.cpp b/libsrc/core/utils.cpp index a93b1372..b82f972c 100644 --- a/libsrc/core/utils.cpp +++ b/libsrc/core/utils.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "ngstream.hpp" @@ -178,7 +179,7 @@ namespace ngcore lib_name = lib_name_; #ifdef WIN32 lib = LoadLibrary(lib_name.wstring().c_str()); - if (!lib) throw std::runtime_error(string("Could not load library ") + lib_name.string()); + if (!lib) throw std::runtime_error(std::wstring("Could not load library ") + lib_name.wstring()); #else // WIN32 auto flags = RTLD_NOW; if (global) flags |= RTLD_GLOBAL; diff --git a/ng/ngappinit.cpp b/ng/ngappinit.cpp index 92b05a6f..cfcf5998 100644 --- a/ng/ngappinit.cpp +++ b/ng/ngappinit.cpp @@ -9,12 +9,7 @@ #include #include #include - -#ifdef PARALLEL -#include - -// extern void ParallelRun(); -#endif +#include #include "../libsrc/interface/writeuser.hpp" @@ -66,22 +61,20 @@ int main(int argc, char ** argv) netgen::netgen_executable_started = true; #ifdef PARALLEL - int mpi_required = MPI_THREAD_MULTIPLE; + int mpi_required = netgen::NG_MPI_THREAD_MULTIPLE; #ifdef VTRACE - mpi_required = MPI_THREAD_SINGLE; + mpi_required = NG_MPI_THREAD_SINGLE; #endif int mpi_provided; - MPI_Init_thread(&argc, &argv, mpi_required, &mpi_provided); + netgen::InitMPI(); + netgen::NG_MPI_Init_thread(&argc, &argv, mpi_required, &mpi_provided); - MPI_Comm_size(MPI_COMM_WORLD, &netgen::ntasks); - MPI_Comm_rank(MPI_COMM_WORLD, &netgen::id); + netgen::NG_MPI_Comm_size(netgen::NG_MPI_COMM_WORLD, &netgen::ntasks); + netgen::NG_MPI_Comm_rank(netgen::NG_MPI_COMM_WORLD, &netgen::id); if(netgen::ntasks!=1) throw ngcore::Exception("Netgen GUI cannot run MPI-parallel"); - // MPI_COMM_WORLD is just a local communicator - // netgen::ng_comm = ngcore::NgMPI_Comm{MPI_COMM_WORLD, false}; - #endif if ( netgen::id == 0 ) @@ -113,7 +106,7 @@ int main(int argc, char ** argv) #ifdef PARALLEL - cout << "Including MPI version " << MPI_VERSION << '.' << MPI_SUBVERSION << endl; + cout << "Including MPI version " << netgen::NG_MPI_VERSION << '.' << netgen::NG_MPI_SUBVERSION << endl; #endif } @@ -287,7 +280,7 @@ int main(int argc, char ** argv) else { // ParallelRun(); - MPI_Finalize(); + netgen::NG_MPI_Finalize(); } #endif