diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c411845..71ac2d86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ option( USE_PYTHON "build with python interface" ON ) cmake_dependent_option( PREFER_SYSTEM_PYBIND11 "Use system wide PyBind11" OFF "USE_PYTHON" OFF) option( USE_MPI "enable mpi parallelization" OFF ) option( USE_MPI4PY "enable mpi4py interface" ON ) +option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" ON ) option( USE_OCC "build with OpenCascade geometry kernel interface" ON) option( USE_STLGEOM "build with STL geometry support" ON) option( USE_CSG "build with CSG kernel" ON) @@ -335,13 +336,10 @@ if (USE_PYTHON) endif (USE_PYTHON) ####################################################################### -add_library(netgen_mpi INTERFACE) add_library(netgen_metis INTERFACE) if (USE_MPI) - find_package(MPI REQUIRED) - target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH}) - target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} ) - target_compile_definitions(netgen_mpi INTERFACE PARALLEL ) + set(MPI_DETERMINE_LIBRARY_VERSION TRUE) + find_package(MPI) find_package(METIS REQUIRED) target_include_directories(netgen_metis INTERFACE ${METIS_INCLUDE_DIR}) @@ -351,12 +349,11 @@ if (USE_MPI) if(USE_MPI4PY AND USE_PYTHON) execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE) find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED) - target_include_directories(netgen_metis INTERFACE ${MPI4PY_INCLUDE_DIR}) - target_compile_definitions(netgen_metis INTERFACE NG_MPI4PY ) + target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR}) + target_compile_definitions(netgen_python INTERFACE NG_MPI4PY ) message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}") endif(USE_MPI4PY AND USE_PYTHON) endif (USE_MPI) -install(TARGETS netgen_mpi netgen_metis ${NG_INSTALL_DIR}) ####################################################################### add_library(occ_libs INTERFACE IMPORTED) diff --git a/cmake/SuperBuild.cmake b/cmake/SuperBuild.cmake index 27315f04..8b895cc7 100644 --- a/cmake/SuperBuild.cmake +++ b/cmake/SuperBuild.cmake @@ -215,7 +215,6 @@ endif(USE_CGNS) ####################################################################### if(USE_MPI) - if(UNIX) if (METIS_DIR) message(STATUS "Using external METIS at: ${METIS_DIR}") else (METIS_DIR) @@ -226,9 +225,6 @@ if(USE_MPI) include(cmake/external_projects/metis.cmake) endif(NOT METIS_FOUND) endif(METIS_DIR) - else(UNIX) - find_package(METIS REQUIRED) - endif(UNIX) endif(USE_MPI) @@ -246,6 +242,7 @@ set_vars( NETGEN_CMAKE_ARGS USE_GUI USE_PYTHON USE_MPI + USE_MPI_WRAPPER USE_VT USE_VTUNE USE_NUMA diff --git a/cmake/external_projects/metis.cmake b/cmake/external_projects/metis.cmake index 1711b5df..5501a0f5 100644 --- a/cmake/external_projects/metis.cmake +++ b/cmake/external_projects/metis.cmake @@ -3,8 +3,8 @@ set(METIS_DIR ${CMAKE_CURRENT_BINARY_DIR}/dependencies/metis) ExternalProject_Add(project_metis PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dependencies - URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p6.tar.gz - URL_MD5 55fc654bb838846b856ba898795143f1 + URL https://bitbucket.org/petsc/pkg-metis/get/v5.1.0-p12.tar.gz + URL_MD5 6cd66f75f88dfa2cf043de011f85d8bc DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external_dependencies CMAKE_ARGS -DGKLIB_PATH=${METIS_SRC_DIR}/GKlib diff --git a/cmake/generate_version_file.cmake b/cmake/generate_version_file.cmake index 82ab0446..780c287b 100644 --- a/cmake/generate_version_file.cmake +++ b/cmake/generate_version_file.cmake @@ -106,6 +106,7 @@ file(GENERATE OUTPUT netgen_config.hpp CONTENT #define NETGEN_USE_CHECK_RANGE $ #define NETGEN_BUILD_STUB_FILES $ #define NETGEN_BUILD_FOR_CONDA $ +#define NETGEN_SHARED_LIBRARY_SUFFIX \"${CMAKE_SHARED_LIBRARY_SUFFIX}\" #endif // NETGEN_CONFIG_HPP_INCLUDED___ ") diff --git a/libsrc/core/.clang-tidy b/libsrc/core/.clang-tidy index 9ceddce8..086e97b8 100644 --- a/libsrc/core/.clang-tidy +++ b/libsrc/core/.clang-tidy @@ -1,4 +1,4 @@ -Checks: '*,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers' +Checks: '*,-cppcoreguidelines-avoid-non-const-global-variables,-llvmlibc-restrict-system-libc-headers,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers' CheckOptions: - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor value: 1 diff --git a/libsrc/core/CMakeLists.txt b/libsrc/core/CMakeLists.txt index d0e37213..71cbf0bc 100644 --- a/libsrc/core/CMakeLists.txt +++ b/libsrc/core/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(ngcore ${NGCORE_LIBRARY_TYPE} taskmanager.cpp utils.cpp version.cpp + ng_mpi_wrapper.cpp ) string(REPLACE "|" ";" ng_compile_flags_replace_sep "${NG_COMPILE_FLAGS}") @@ -57,6 +58,8 @@ if(WIN32) get_WIN32_WINNT(ver) target_compile_definitions(ngcore PUBLIC _WIN32_WINNT=${ver} WNT WNT_WINDOW NOMINMAX MSVC_EXPRESS _CRT_SECURE_NO_WARNINGS HAVE_STRUCT_TIMESPEC WIN32) target_link_options(ngcore PUBLIC /ignore:4273 /ignore:4217 /ignore:4049) +else(WIN32) + target_link_libraries(ngcore PUBLIC dl) endif(WIN32) target_compile_definitions(ngcore PRIVATE NGCORE_EXPORTS) @@ -82,7 +85,7 @@ endif(USE_NUMA) install(TARGETS ngcore DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen) -target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE "$" ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(ngcore PRIVATE "$" ${CMAKE_THREAD_LIBS_INIT}) install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp memtracer.hpp exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp @@ -90,6 +93,7 @@ install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp register_archive.hpp autodiff.hpp autodiffdiff.hpp + ng_mpi.hpp ng_mpi_generated_declarations.hpp DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel) if(ENABLE_CPP_CORE_GUIDELINES_CHECK) @@ -100,7 +104,7 @@ add_dependencies(ngcore ng_generate_version_file) if(USE_PYTHON) pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp) - target_link_libraries(pyngcore PUBLIC ngcore netgen_python) + target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python) set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}") if(EMSCRIPTEN) target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS) @@ -108,3 +112,61 @@ if(USE_PYTHON) install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen) endif(USE_PYTHON) +function (build_mpi_variant) + set(target ng_${ARGV0}) + set(include_dir ${ARGV1}) + message("1Building MPI variant: ${ARGV0} ${ARGV1}") + add_library(${target} SHARED ng_mpi.cpp) + target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python) + target_compile_definitions(${target} PUBLIC PARALLEL NG_MPI_WRAPPER) + target_include_directories(${target} PRIVATE ${include_dir}) + set_target_properties(${target} PROPERTIES PREFIX "") + install(TARGETS ${target} RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen) +endfunction() + +if(USE_MPI) + target_compile_definitions(ngcore PUBLIC PARALLEL) + + message(STATUS "Found MPI version\n${MPI_C_LIBRARY_VERSION_STRING}") + + if(USE_MPI_WRAPPER) + target_compile_definitions(ngcore PUBLIC NG_MPI_WRAPPER) + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Microsoft MPI.*") + set(MICROSOFT_MPI_INCLUDE_DIR ${MPI_C_HEADER_DIR}) + set(MICROSOFT_MPI_LIBRARY ${MPI_msmpi_LIBRARY}) + endif() + + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*") + set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + endif() + + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*") + set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + endif() + + if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Intel.*") + set(INTEL_MPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + endif() + + if(OPENMPI_INCLUDE_DIR) + build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR}) + endif() + if(MPICH_INCLUDE_DIR) + build_mpi_variant(mpich ${MPICH_INCLUDE_DIR}) + endif() + if(INTEL_MPI_INCLUDE_DIR) + build_mpi_variant(intel_mpi ${INTEL_MPI_INCLUDE_DIR}) + if(WIN32) + target_link_libraries(ng_intel_mpi PUBLIC ${INTEL_MPI_LIBRARY}) + endif() + endif() + if(MICROSOFT_MPI_INCLUDE_DIR) + build_mpi_variant(microsoft_mpi ${MICROSOFT_MPI_INCLUDE_DIR}) + target_link_libraries(ng_microsoft_mpi PUBLIC ${MICROSOFT_MPI_LIBRARY}) + endif() + else() + target_link_libraries(ngcore PUBLIC ${MPI_C_LIBRARIES}) + endif(USE_MPI_WRAPPER) + +endif(USE_MPI) + diff --git a/libsrc/core/generate_mpi_sources.py b/libsrc/core/generate_mpi_sources.py new file mode 100644 index 00000000..1a491065 --- /dev/null +++ b/libsrc/core/generate_mpi_sources.py @@ -0,0 +1,164 @@ +functions = [ + ("double", "MPI_Wtime"), + ("int", "MPI_Allgather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"), + ("int", "MPI_Allreduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "MPI_Comm"), + ("int", "MPI_Alltoall", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "MPI_Comm"), + ("int", "MPI_Barrier", "MPI_Comm"), + ("int", "MPI_Bcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"), + ("int", "MPI_Comm_create_group", "MPI_Comm", "MPI_Group", "int", "MPI_Comm*"), + ("int", "MPI_Comm_free", "MPI_Comm*"), + ("int", "MPI_Comm_group", "MPI_Comm", "MPI_Group*"), + ("int", "MPI_Comm_rank", "MPI_Comm", "int*"), + ("int", "MPI_Comm_size", "MPI_Comm", "int*"), + ("int", "MPI_Finalize"), + ("int", "MPI_Gather", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"), + ("int", "MPI_Get_count", "MPI_Status*", "MPI_Datatype", "int*"), + ("int", "MPI_Get_processor_name", "char*", "int*"), + ("int", "MPI_Group_incl", "MPI_Group", "int", "int*", "MPI_Group*"), + ("int", "MPI_Init", "int*", "char***"), + ("int", "MPI_Init_thread", "int*", "char***", "int", "int*"), + ("int", "MPI_Initialized", "int*"), + ("int", "MPI_Iprobe", "int", "int", "MPI_Comm", "int*", "MPI_Status*"), + ("int", "MPI_Irecv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"), + ("int", "MPI_Isend", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Request*"), + ("int", "MPI_Probe", "int", "int", "MPI_Comm", "MPI_Status*"), + ("int", "MPI_Query_thread", "int*"), + ("int", "MPI_Recv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Status*"), + ("int", "MPI_Reduce", "void*", "void*", "int", "MPI_Datatype", "MPI_Op", "int", "MPI_Comm"), + ("int", "MPI_Reduce_local", "void*", "void*", "int", "MPI_Datatype", "MPI_Op"), + ("int", "MPI_Request_free", "MPI_Request*"), + ("int", "MPI_Scatter", "void*", "int", "MPI_Datatype", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"), + ("int", "MPI_Send", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm"), + ("int", "MPI_Type_commit", "MPI_Datatype*"), + ("int", "MPI_Type_contiguous", "int", "MPI_Datatype", "MPI_Datatype*"), + ("int", "MPI_Type_create_resized", "MPI_Datatype", "MPI_Aint", "MPI_Aint", "MPI_Datatype*"), + ("int", "MPI_Type_create_struct", "int", "int*:0", "MPI_Aint*:0", "MPI_Datatype*:0", "MPI_Datatype*"), + ("int", "MPI_Type_free", "MPI_Datatype*"), + ("int", "MPI_Type_get_extent", "MPI_Datatype", "MPI_Aint*", "MPI_Aint*"), + ("int", "MPI_Type_indexed", "int", "int*:0", "int*:0", "MPI_Datatype", "MPI_Datatype*"), + ("int", "MPI_Type_size", "MPI_Datatype", "int*"), + ("int", "MPI_Wait", "MPI_Request*", "MPI_Status*"), + ("int", "MPI_Waitall", "int", "MPI_Request*:0", "MPI_Status*"), + ("int", "MPI_Waitany", "int", "MPI_Request*:0", "int*", "MPI_Status*"), + ] + +constants = [ + ("MPI_Comm", "MPI_COMM_WORLD"), + ("MPI_Datatype", "MPI_CHAR"), + ("MPI_Datatype", "MPI_CXX_DOUBLE_COMPLEX"), + ("MPI_Datatype", "MPI_C_BOOL"), + ("MPI_Datatype", "MPI_DATATYPE_NULL"), + ("MPI_Datatype", "MPI_DOUBLE"), + ("MPI_Datatype", "MPI_INT"), + ("MPI_Datatype", "MPI_SHORT"), + ("MPI_Datatype", "MPI_UINT64_T"), + ("MPI_Op", "MPI_LOR"), + ("MPI_Op", "MPI_MAX"), + ("MPI_Op", "MPI_MIN"), + ("MPI_Op", "MPI_SUM"), + ("MPI_Status*", "MPI_STATUSES_IGNORE"), + ("MPI_Status*", "MPI_STATUS_IGNORE"), + ("int", "MPI_ANY_SOURCE"), + ("int", "MPI_ANY_TAG"), + ("int", "MPI_MAX_PROCESSOR_NAME"), + ("int", "MPI_PROC_NULL"), + ("int", "MPI_ROOT"), + ("int", "MPI_SUBVERSION"), + ("int", "MPI_THREAD_MULTIPLE"), + ("int", "MPI_THREAD_SINGLE"), + ("int", "MPI_VERSION"), + ("void*", "MPI_IN_PLACE"), +] + +def get_args(f, counts=False): + args = [] + for arg in f[2:]: + has_count = ':' in arg + if has_count: + s, count = arg.split(':') + count = int(count) + else: + s = arg + count = None + if s.startswith("MPI_"): + s = "NG_" + s + if counts: + args.append((s, count)) + else: + args.append(s) + return args + +def generate_declarations(): + code = "" + nowrapper_code = "" + for f in functions: + ret = f[0] + name = f[1] + args = ", ".join(get_args(f)) + code += f"NGCORE_API extern {ret} (*NG_{name})({args});\n" + nowrapper_code += f"static const auto NG_{name} = {name};\n" + + for typ, name in constants: + if typ.startswith("MPI_"): + typ = "NG_" + typ + code += f"NGCORE_API extern {typ} NG_{name};\n" + nowrapper_code += f"static const decltype({name}) NG_{name} = {name};\n" + + with open("ng_mpi_generated_declarations.hpp", "w") as f: + f.write("#ifdef NG_MPI_WRAPPER\n") + f.write(code) + f.write("#else // NG_MPI_WRAPPER\n") + f.write(nowrapper_code) + f.write("#endif // NG_MPI_WRAPPER\n") + +def generate_dummy_init(): + code = "" + for f in functions: + ret = f[0] + name = f[1] + args = ", ".join(get_args(f)) + code += f"decltype(NG_{name}) NG_{name} = []({args})->{ret} {{ throw no_mpi(); }};\n" + + for typ, name in constants: + if typ.startswith("MPI_"): + typ = "NG_" + typ + code += f"{typ} NG_{name} = 0;\n" + + with open("ng_mpi_generated_dummy_init.hpp", "w") as f: + f.write(code) + +def generate_init(): + code = "" + for f in functions: + ret = f[0] + name = f[1] + args = get_args(f, counts=True) + in_args ='' + call_args = '' + for i in range(len(args)): + arg, count = args[i] + if i > 0: + in_args += ', ' + call_args += ', ' + in_args += arg + f" arg{i}" + if not arg.startswith("NG_"): + # plain type (like int, int *, etc.), just pass the argument along + call_args += f" arg{i}" + elif count is None: + # MPI type (by value or pointer), but just one object, no arrays + call_args += f" ng2mpi(arg{i})" + else: + # arrays of MPI types, we need to copy them due to incompatible size + call_args += f" ng2mpi(arg{i}, arg{count})" + code += f"NG_{name} = []({in_args})->{ret} {{ return {name}({call_args}); }};\n" + + for _, name in constants: + code += f"NG_{name} = mpi2ng({name});\n" + + with open("ng_mpi_generated_init.hpp", "w") as f: + f.write(code) + +if __name__ == "__main__": + generate_declarations() + generate_dummy_init() + generate_init() diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index 8cfe31d2..53f3ae43 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -3,16 +3,14 @@ #include -#ifdef PARALLEL -#define OMPI_SKIP_MPICXX -#include -#endif +#include #include "array.hpp" #include "table.hpp" #include "exception.hpp" #include "profiler.hpp" #include "ngstream.hpp" +#include "ng_mpi.hpp" namespace ngcore { @@ -22,67 +20,70 @@ namespace ngcore template struct MPI_typetrait { }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_INT; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_INT; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_SHORT; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_SHORT; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_CHAR; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_CHAR; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_CHAR; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_UINT64_T; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_UINT64_T; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_DOUBLE; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_DOUBLE; } }; + + template <> struct MPI_typetrait> { + static NG_MPI_Datatype MPIType () { return NG_MPI_CXX_DOUBLE_COMPLEX; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_C_BOOL; } }; + static NG_MPI_Datatype MPIType () { return NG_MPI_C_BOOL; } }; template struct MPI_typetrait> { - static MPI_Datatype MPIType () + static NG_MPI_Datatype MPIType () { - static MPI_Datatype MPI_T = 0; - if (!MPI_T) + static NG_MPI_Datatype NG_MPI_T = 0; + if (!NG_MPI_T) { - MPI_Type_contiguous ( S, MPI_typetrait::MPIType(), &MPI_T); - MPI_Type_commit ( &MPI_T ); + NG_MPI_Type_contiguous ( S, MPI_typetrait::MPIType(), &NG_MPI_T); + NG_MPI_Type_commit ( &NG_MPI_T ); } - return MPI_T; + return NG_MPI_T; } }; template ::MPIType())> - inline MPI_Datatype GetMPIType () { + inline NG_MPI_Datatype GetMPIType () { return MPI_typetrait::MPIType(); } template - inline MPI_Datatype GetMPIType (T &) { + inline NG_MPI_Datatype GetMPIType (T &) { return GetMPIType(); } - inline void MyMPI_WaitAll (FlatArray requests) + inline void MyMPI_WaitAll (FlatArray requests) { static Timer t("MPI - WaitAll"); RegionTimer reg(t); if (!requests.Size()) return; - MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE); + NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE); } - inline int MyMPI_WaitAny (FlatArray requests) + inline int MyMPI_WaitAny (FlatArray requests) { int nr; - MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE); + NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE); return nr; } @@ -91,7 +92,7 @@ namespace ngcore class NgMPI_Comm { protected: - MPI_Comm comm; + NG_MPI_Comm comm; bool valid_comm; int * refcount; int rank, size; @@ -100,11 +101,11 @@ namespace ngcore : valid_comm(false), refcount(nullptr), rank(0), size(1) { ; } - NgMPI_Comm (MPI_Comm _comm, bool owns = false) + NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false) : comm(_comm), valid_comm(true) { int flag; - MPI_Initialized (&flag); + NG_MPI_Initialized (&flag); if (!flag) { valid_comm = false; @@ -119,8 +120,8 @@ namespace ngcore else refcount = new int{1}; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); + NG_MPI_Comm_rank(comm, &rank); + NG_MPI_Comm_size(comm, &size); } NgMPI_Comm (const NgMPI_Comm & c) @@ -141,7 +142,7 @@ namespace ngcore { if (refcount) if (--(*refcount) == 0) - MPI_Comm_free(&comm); + NG_MPI_Comm_free(&comm); } bool ValidCommunicator() const @@ -153,7 +154,7 @@ namespace ngcore { if (refcount) if (--(*refcount) == 0) - MPI_Comm_free(&comm); + NG_MPI_Comm_free(&comm); refcount = c.refcount; if (refcount) (*refcount)++; @@ -169,7 +170,7 @@ namespace ngcore InvalidCommException() : Exception("Do not have a valid communicator") { ; } }; - operator MPI_Comm() const { + operator NG_MPI_Comm() const { if (!valid_comm) throw InvalidCommException(); return comm; } @@ -178,7 +179,7 @@ namespace ngcore int Size() const { return size; } void Barrier() const { static Timer t("MPI - Barrier"); RegionTimer reg(t); - if (size > 1) MPI_Barrier (comm); + if (size > 1) NG_MPI_Barrier (comm); } @@ -186,82 +187,82 @@ namespace ngcore template())> void Send (T & val, int dest, int tag) const { - MPI_Send (&val, 1, GetMPIType(), dest, tag, comm); + NG_MPI_Send (&val, 1, GetMPIType(), dest, tag, comm); } void Send (const std::string & s, int dest, int tag) const { - MPI_Send( const_cast (&s[0]), s.length(), MPI_CHAR, dest, tag, comm); + NG_MPI_Send( const_cast (&s[0]), s.length(), NG_MPI_CHAR, dest, tag, comm); } template())> void Send(FlatArray s, int dest, int tag) const { - MPI_Send (s.Data(), s.Size(), GetMPIType(), dest, tag, comm); + NG_MPI_Send (s.Data(), s.Size(), GetMPIType(), dest, tag, comm); } template())> void Recv (T & val, int src, int tag) const { - MPI_Recv (&val, 1, GetMPIType(), src, tag, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv (&val, 1, GetMPIType(), src, tag, comm, NG_MPI_STATUS_IGNORE); } void Recv (std::string & s, int src, int tag) const { - MPI_Status status; + NG_MPI_Status status; int len; - MPI_Probe (src, tag, comm, &status); - MPI_Get_count (&status, MPI_CHAR, &len); + NG_MPI_Probe (src, tag, comm, &status); + NG_MPI_Get_count (&status, NG_MPI_CHAR, &len); // s.assign (len, ' '); s.resize (len); - MPI_Recv( &s[0], len, MPI_CHAR, src, tag, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv( &s[0], len, NG_MPI_CHAR, src, tag, comm, NG_MPI_STATUS_IGNORE); } template ())> void Recv (FlatArray s, int src, int tag) const { - MPI_Recv (s.Data(), s.Size(), GetMPIType (), src, tag, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv (s.Data(), s.Size(), GetMPIType (), src, tag, comm, NG_MPI_STATUS_IGNORE); } template ())> void Recv (Array & s, int src, int tag) const { - MPI_Status status; + NG_MPI_Status status; int len; - const MPI_Datatype MPI_T = GetMPIType (); - MPI_Probe (src, tag, comm, &status); - MPI_Get_count (&status, MPI_T, &len); + const NG_MPI_Datatype NG_MPI_T = GetMPIType (); + NG_MPI_Probe (src, tag, comm, &status); + NG_MPI_Get_count (&status, NG_MPI_T, &len); s.SetSize (len); - MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv (s.Data(), len, NG_MPI_T, src, tag, comm, NG_MPI_STATUS_IGNORE); } /** --- non-blocking P2P --- **/ template())> - MPI_Request ISend (T & val, int dest, int tag) const + NG_MPI_Request ISend (T & val, int dest, int tag) const { - MPI_Request request; - MPI_Isend (&val, 1, GetMPIType(), dest, tag, comm, &request); + NG_MPI_Request request; + NG_MPI_Isend (&val, 1, GetMPIType(), dest, tag, comm, &request); return request; } template())> - MPI_Request ISend (FlatArray s, int dest, int tag) const + NG_MPI_Request ISend (FlatArray s, int dest, int tag) const { - MPI_Request request; - MPI_Isend (s.Data(), s.Size(), GetMPIType(), dest, tag, comm, &request); + NG_MPI_Request request; + NG_MPI_Isend (s.Data(), s.Size(), GetMPIType(), dest, tag, comm, &request); return request; } template())> - MPI_Request IRecv (T & val, int dest, int tag) const + NG_MPI_Request IRecv (T & val, int dest, int tag) const { - MPI_Request request; - MPI_Irecv (&val, 1, GetMPIType(), dest, tag, comm, &request); + NG_MPI_Request request; + NG_MPI_Irecv (&val, 1, GetMPIType(), dest, tag, comm, &request); return request; } template())> - MPI_Request IRecv (FlatArray s, int src, int tag) const + NG_MPI_Request IRecv (FlatArray s, int src, int tag) const { - MPI_Request request; - MPI_Irecv (s.Data(), s.Size(), GetMPIType(), src, tag, comm, &request); + NG_MPI_Request request; + NG_MPI_Irecv (s.Data(), s.Size(), GetMPIType(), src, tag, comm, &request); return request; } @@ -269,41 +270,41 @@ namespace ngcore /** --- collectives --- **/ template ())> - T Reduce (T d, const MPI_Op & op, int root = 0) const + T Reduce (T d, const NG_MPI_Op & op, int root = 0) const { static Timer t("MPI - Reduce"); RegionTimer reg(t); if (size == 1) return d; T global_d; - MPI_Reduce (&d, &global_d, 1, GetMPIType(), op, root, comm); + NG_MPI_Reduce (&d, &global_d, 1, GetMPIType(), op, root, comm); return global_d; } template ())> - T AllReduce (T d, const MPI_Op & op) const + T AllReduce (T d, const NG_MPI_Op & op) const { static Timer t("MPI - AllReduce"); RegionTimer reg(t); if (size == 1) return d; T global_d; - MPI_Allreduce ( &d, &global_d, 1, GetMPIType(), op, comm); + NG_MPI_Allreduce ( &d, &global_d, 1, GetMPIType(), op, comm); return global_d; } template ())> - void AllReduce (FlatArray d, const MPI_Op & op) const + void AllReduce (FlatArray d, const NG_MPI_Op & op) const { static Timer t("MPI - AllReduce Array"); RegionTimer reg(t); if (size == 1) return; - MPI_Allreduce (MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType(), op, comm); + NG_MPI_Allreduce (NG_MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType(), op, comm); } template ())> void Bcast (T & s, int root = 0) const { if (size == 1) return; static Timer t("MPI - Bcast"); RegionTimer reg(t); - MPI_Bcast (&s, 1, GetMPIType(), root, comm); + NG_MPI_Bcast (&s, 1, GetMPIType(), root, comm); } @@ -316,7 +317,7 @@ namespace ngcore Bcast (ds, root); if (Rank() != root) d.SetSize (ds); if (ds != 0) - MPI_Bcast (d.Data(), ds, GetMPIType(), root, comm); + NG_MPI_Bcast (d.Data(), ds, GetMPIType(), root, comm); } @@ -326,13 +327,13 @@ namespace ngcore int len = s.length(); Bcast (len, root); if (rank != 0) s.resize (len); - MPI_Bcast (&s[0], len, MPI_CHAR, root, comm); + NG_MPI_Bcast (&s[0], len, NG_MPI_CHAR, root, comm); } template void AllToAll (FlatArray send, FlatArray recv) const { - MPI_Alltoall (send.Data(), 1, GetMPIType(), + NG_MPI_Alltoall (send.Data(), 1, GetMPIType(), recv.Data(), 1, GetMPIType(), comm); } @@ -341,15 +342,15 @@ namespace ngcore void ScatterRoot (FlatArray send) const { if (size == 1) return; - MPI_Scatter (send.Data(), 1, GetMPIType(), - MPI_IN_PLACE, -1, GetMPIType(), 0, comm); + NG_MPI_Scatter (send.Data(), 1, GetMPIType(), + NG_MPI_IN_PLACE, -1, GetMPIType(), 0, comm); } template void Scatter (T & recv) const { if (size == 1) return; - MPI_Scatter (NULL, 0, GetMPIType(), + NG_MPI_Scatter (NULL, 0, GetMPIType(), &recv, 1, GetMPIType(), 0, comm); } @@ -358,7 +359,7 @@ namespace ngcore { recv[0] = T(0); if (size == 1) return; - MPI_Gather (MPI_IN_PLACE, 1, GetMPIType(), + NG_MPI_Gather (NG_MPI_IN_PLACE, 1, GetMPIType(), recv.Data(), 1, GetMPIType(), 0, comm); } @@ -366,7 +367,7 @@ namespace ngcore void Gather (T send) const { if (size == 1) return; - MPI_Gather (&send, 1, GetMPIType(), + NG_MPI_Gather (&send, 1, GetMPIType(), NULL, 1, GetMPIType(), 0, comm); } @@ -379,7 +380,7 @@ namespace ngcore recv[0] = val; return; } - MPI_Allgather (&val, 1, GetMPIType(), + NG_MPI_Allgather (&val, 1, GetMPIType(), recv.Data(), 1, GetMPIType(), comm); } @@ -400,7 +401,7 @@ namespace ngcore recv_data = DynamicTable (recv_sizes, true); - Array requests; + Array requests; for (int dest = 0; dest < size; dest++) if (dest != rank && send_data[dest].Size()) requests.Append (ISend (FlatArray(send_data[dest]), dest, tag)); @@ -418,11 +419,11 @@ namespace ngcore NgMPI_Comm SubCommunicator (FlatArray procs) const { - MPI_Comm subcomm; - MPI_Group gcomm, gsubcomm; - MPI_Comm_group(comm, &gcomm); - MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm); - MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm); + NG_MPI_Comm subcomm; + NG_MPI_Group gcomm, gsubcomm; + NG_MPI_Comm_group(comm, &gcomm); + NG_MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm); + NG_MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm); return NgMPI_Comm(subcomm, true); } @@ -440,16 +441,16 @@ namespace ngcore MyMPI(int argc, char ** argv) { int is_init = -1; - MPI_Initialized(&is_init); + NG_MPI_Initialized(&is_init); if (!is_init) { - MPI_Init (&argc, &argv); + NG_MPI_Init (&argc, &argv); initialized_by_me = true; } else initialized_by_me = false; - NgMPI_Comm comm(MPI_COMM_WORLD); + NgMPI_Comm comm(NG_MPI_COMM_WORLD); NGSOStream::SetGlobalActive (comm.Rank() == 0); if (comm.Size() > 1) @@ -459,7 +460,7 @@ namespace ngcore ~MyMPI() { if (initialized_by_me) - MPI_Finalize (); + NG_MPI_Finalize (); } }; @@ -468,42 +469,42 @@ namespace ngcore #else // PARALLEL - class MPI_Comm { + class NG_MPI_Comm { int nr; public: - MPI_Comm (int _nr = 0) : nr(_nr) { ; } + NG_MPI_Comm (int _nr = 0) : nr(_nr) { ; } operator int() const { return nr; } - bool operator== (MPI_Comm c2) const { return nr == c2.nr; } + bool operator== (NG_MPI_Comm c2) const { return nr == c2.nr; } }; - static MPI_Comm MPI_COMM_WORLD = 12345, MPI_COMM_NULL = 10000; + static NG_MPI_Comm NG_MPI_COMM_WORLD = 12345, NG_MPI_COMM_NULL = 10000; - typedef int MPI_Op; - typedef int MPI_Datatype; - typedef int MPI_Request; + typedef int NG_MPI_Op; + typedef int NG_MPI_Datatype; + typedef int NG_MPI_Request; - enum { MPI_SUM = 0, MPI_MIN = 1, MPI_MAX = 2, MPI_LOR = 4711 }; + enum { NG_MPI_SUM = 0, NG_MPI_MIN = 1, NG_MPI_MAX = 2, NG_MPI_LOR = 4711 }; - inline void MPI_Type_contiguous ( int, MPI_Datatype, MPI_Datatype*) { ; } - inline void MPI_Type_commit ( MPI_Datatype * ) { ; } + inline void NG_MPI_Type_contiguous ( int, NG_MPI_Datatype, NG_MPI_Datatype*) { ; } + inline void NG_MPI_Type_commit ( NG_MPI_Datatype * ) { ; } template struct MPI_typetrait { - static MPI_Datatype MPIType () { return -1; } + static NG_MPI_Datatype MPIType () { return -1; } }; template - inline MPI_Datatype GetMPIType () { return -1; } + inline NG_MPI_Datatype GetMPIType () { return -1; } class NgMPI_Comm { public: NgMPI_Comm () { ; } - NgMPI_Comm (MPI_Comm _comm, bool owns = false) { ; } + NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false) { ; } size_t Rank() const { return 0; } size_t Size() const { return 1; } bool ValidCommunicator() const { return false; } void Barrier() const { ; } - operator MPI_Comm() const { return MPI_Comm(); } + operator NG_MPI_Comm() const { return NG_MPI_Comm(); } template void Send( T & val, int dest, int tag) const { ; } @@ -521,25 +522,25 @@ namespace ngcore void Recv (Array & s, int src, int tag) const { ; } template - MPI_Request ISend (T & val, int dest, int tag) const { return 0; } + NG_MPI_Request ISend (T & val, int dest, int tag) const { return 0; } template - MPI_Request ISend (FlatArray s, int dest, int tag) const { return 0; } + NG_MPI_Request ISend (FlatArray s, int dest, int tag) const { return 0; } template - MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } + NG_MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } template - MPI_Request IRecv (FlatArray s, int src, int tag) const { return 0; } + NG_MPI_Request IRecv (FlatArray s, int src, int tag) const { return 0; } template - T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; } + T Reduce (T d, const NG_MPI_Op & op, int root = 0) const { return d; } template - T AllReduce (T d, const MPI_Op & op) const { return d; } + T AllReduce (T d, const NG_MPI_Op & op) const { return d; } template - void AllReduce (FlatArray d, const MPI_Op & op) const { ; } + void AllReduce (FlatArray d, const NG_MPI_Op & op) const { ; } template void Bcast (T & s, int root = 0) const { ; } @@ -562,8 +563,8 @@ namespace ngcore { return *this; } }; - inline void MyMPI_WaitAll (FlatArray requests) { ; } - inline int MyMPI_WaitAny (FlatArray requests) { return 0; } + inline void MyMPI_WaitAll (FlatArray requests) { ; } + inline int MyMPI_WaitAny (FlatArray requests) { return 0; } class MyMPI { diff --git a/libsrc/core/ng_mpi.cpp b/libsrc/core/ng_mpi.cpp new file mode 100644 index 00000000..9dce98c8 --- /dev/null +++ b/libsrc/core/ng_mpi.cpp @@ -0,0 +1,183 @@ +#define OMPI_SKIP_MPICXX + +#include "ng_mpi.hpp" + +#include + +#include + +#include "ngcore_api.hpp" +#include "pybind11/pytypes.h" + +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#include + +#include "python_ngcore.hpp" + +namespace py = pybind11; +#endif + +#ifdef MSMPI_VER +int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2, + MPI_Comm* arg3) { + throw std::runtime_error( + "MPI_Comm_create_group not supported on Microsoft MPI"); +} +static MPI_Datatype MPI_CXX_DOUBLE_COMPLEX; +#endif // MSMPI_VER + +namespace ngcore { + +static_assert(sizeof(MPI_Status) <= sizeof(NG_MPI_Status), "Size mismatch"); +static_assert(alignof(MPI_Status) <= alignof(NG_MPI_Status), "Size mismatch"); + +int mpi2ng(int value) { return value; } +void* mpi2ng(void* ptr) { return ptr; } + +NG_MPI_Status* mpi2ng(MPI_Status* status) { + return reinterpret_cast(status); +} + +#if !defined(MPICH) && !defined(MSMPI_VER) +NG_MPI_Comm mpi2ng(MPI_Comm comm) { return reinterpret_cast(comm); } +#endif + +template +void gather_strided_array(size_t count, char* data) { + static_assert(size <= stride, "Size must be less than or equal to stride"); + if constexpr (size < stride) { + char* dst = data; + char* src = data; + for (auto i : Range(count)) { + memcpy(dst, src, size); + dst += size; + src += stride; + } + } +} + +template +T cast_ng2mpi(uintptr_t obj) { + if constexpr (std::is_pointer_v) + return reinterpret_cast(obj); + else + return static_cast(obj); +} + +template +T cast_ng2mpi(uintptr_t* ptr) { + if constexpr (std::is_pointer_v) + return reinterpret_cast(ptr); + else + return static_cast(ptr); +} + +template +T* cast_ng2mpi(TSrc* ptr, int count) { + gather_strided_array(count, + reinterpret_cast(ptr)); + return reinterpret_cast(ptr); +} + +MPI_Comm ng2mpi(NG_MPI_Comm comm) { + static_assert(sizeof(MPI_Comm) <= sizeof(comm.value), "Size mismatch"); + static_assert(alignof(MPI_Comm) <= alignof(NG_MPI_Comm), "Size mismatch"); + return cast_ng2mpi(comm.value); +} + +MPI_Group ng2mpi(NG_MPI_Group group) { + static_assert(sizeof(MPI_Group) <= sizeof(group.value), "Size mismatch"); + static_assert(alignof(MPI_Group) <= alignof(NG_MPI_Group), "Size mismatch"); + return cast_ng2mpi(group.value); +} + +MPI_Comm* ng2mpi(NG_MPI_Comm* comm) { + return cast_ng2mpi(&comm->value); +} +MPI_Group* ng2mpi(NG_MPI_Group* group) { + return cast_ng2mpi(&group->value); +} +MPI_Datatype* ng2mpi(NG_MPI_Datatype* type) { + return cast_ng2mpi(&type->value); +} +MPI_Datatype* ng2mpi(NG_MPI_Datatype* type, int count) { + return cast_ng2mpi(&type->value, count); +} +MPI_Request* ng2mpi(NG_MPI_Request* request) { + return cast_ng2mpi(&request->value); +} +MPI_Request* ng2mpi(NG_MPI_Request* request, int count) { + return cast_ng2mpi(&request->value, count); +} +MPI_Status* ng2mpi(NG_MPI_Status* status) { + return reinterpret_cast(status); +} +MPI_Aint* ng2mpi(NG_MPI_Aint* aint) { + return reinterpret_cast(aint); +} +MPI_Aint* ng2mpi(NG_MPI_Aint* aint, int count) { + return cast_ng2mpi(aint, count); +} + +MPI_Datatype ng2mpi(NG_MPI_Datatype type) { + static_assert(sizeof(MPI_Datatype) <= sizeof(type.value), "Size mismatch"); + return cast_ng2mpi(type.value); +} + +MPI_Request ng2mpi(NG_MPI_Request request) { + static_assert(sizeof(MPI_Request) <= sizeof(request.value), "Size mismatch"); + return cast_ng2mpi(request.value); +} + +MPI_Op ng2mpi(NG_MPI_Op op) { + static_assert(sizeof(MPI_Op) <= sizeof(op.value), "Size mismatch"); + return cast_ng2mpi(op.value); +} + +MPI_Aint ng2mpi(NG_MPI_Aint aint) { + static_assert(sizeof(MPI_Aint) <= sizeof(aint.value), "Size mismatch"); + return cast_ng2mpi(aint.value); +} + +void* ng2mpi(void* ptr) { return ptr; } +char* ng2mpi(char* ptr) { return ptr; } +char*** ng2mpi(char*** ptr) { return ptr; } +int* ng2mpi(int* ptr) { return ptr; } +int ng2mpi(int value) { return value; } + +} // namespace ngcore + +using namespace ngcore; + +extern "C" { +NGCORE_API_EXPORT void ng_init_mpi(); +} + +static bool imported_mpi4py = false; + +void ng_init_mpi() { +#if defined(NG_PYTHON) && defined(NG_MPI4PY) + NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + PyObject* py_src = src.ptr(); + auto type = Py_TYPE(py_src); + if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { + dst = mpi2ng(*PyMPIComm_Get(py_src)); + return !PyErr_Occurred(); + } + return false; + }; + NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + return py::handle(PyMPIComm_New(ng2mpi(src))); + }; +#endif + +#include "ng_mpi_generated_init.hpp" +} diff --git a/libsrc/core/ng_mpi.hpp b/libsrc/core/ng_mpi.hpp new file mode 100644 index 00000000..17f29a34 --- /dev/null +++ b/libsrc/core/ng_mpi.hpp @@ -0,0 +1,105 @@ +#ifndef NG_MPI_HPP_INCLUDED +#define NG_MPI_HPP_INCLUDED + +#ifdef PARALLEL + +#include +#include +#include + +#include "ngcore_api.hpp" + +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#include + +namespace py = pybind11; +#endif + +#ifndef NG_MPI_WRAPPER +#include +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#include +#endif +#endif // NG_MPI_WRAPPER + +namespace ngcore { + +NGCORE_API void InitMPI( + std::optional mpi_lib_path = std::nullopt); + +#ifdef NG_MPI_WRAPPER +inline void not_implemented() { throw std::runtime_error("Not implemented"); } + +struct NG_MPI_Status { + uintptr_t data[4]; +}; + +struct NG_MPI_Comm { + uintptr_t value; + NG_MPI_Comm() { value = 0; } + NG_MPI_Comm(uintptr_t value_) : value(value_) {} + NG_MPI_Comm(const NG_MPI_Comm &comm) : value(comm.value) {} + + void operator=(int value_) { value = value_; } + void operator=(uintptr_t value_) { value = value_; } + bool operator==(const NG_MPI_Comm &comm) const { return value == comm.value; } + bool operator!=(const NG_MPI_Comm &comm) const { return value != comm.value; } +}; + +struct NG_MPI_Datatype { + uintptr_t value = 0; + NG_MPI_Datatype() = default; + NG_MPI_Datatype(uintptr_t value_) : value(value_) {} + operator bool() const { return value != 0; } + void operator=(NG_MPI_Datatype type) { value = type.value; } + void operator=(uintptr_t value_) { value = value_; } + void operator=(void *value_) { value = reinterpret_cast(value_); } +}; + +struct NG_MPI_Request { + uintptr_t value = 0; + NG_MPI_Request() = default; + NG_MPI_Request(uintptr_t value_) : value(value_) {} + void operator=(uintptr_t value_) { value = value_; } +}; + +struct NG_MPI_Op { + uintptr_t value; + NG_MPI_Op(uintptr_t value_) : value(value_) {} + void operator=(uintptr_t value_) { value = value_; } + void operator=(void *value_) { value = reinterpret_cast(value_); } +}; + +struct NG_MPI_Group { + uintptr_t value = 0; + NG_MPI_Group(uintptr_t value_) : value(value_) {} + NG_MPI_Group() = default; +}; + +struct NG_MPI_Aint { + intptr_t value = 0; + NG_MPI_Aint(intptr_t value_) : value(value_) {} + NG_MPI_Aint() = default; +}; + +#else +using NG_MPI_Status = MPI_Status; +using NG_MPI_Comm = MPI_Comm; +using NG_MPI_Datatype = MPI_Datatype; +using NG_MPI_Request = MPI_Request; +using NG_MPI_Op = MPI_Op; +using NG_MPI_Group = MPI_Group; +using NG_MPI_Aint = MPI_Aint; +#endif + +#include "ng_mpi_generated_declarations.hpp" + +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &); +NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm); +#endif + +} // namespace ngcore + +#endif // PARALLEL +#endif // NG_MPI_HPP_INCLUDED diff --git a/libsrc/core/ng_mpi_generated_declarations.hpp b/libsrc/core/ng_mpi_generated_declarations.hpp new file mode 100644 index 00000000..d6a9e33f --- /dev/null +++ b/libsrc/core/ng_mpi_generated_declarations.hpp @@ -0,0 +1,135 @@ +#ifdef NG_MPI_WRAPPER +NGCORE_API extern double (*NG_MPI_Wtime)(); +NGCORE_API extern int (*NG_MPI_Allgather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Allreduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Alltoall)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Barrier)(NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Bcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Comm_create_group)(NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*); +NGCORE_API extern int (*NG_MPI_Comm_free)(NG_MPI_Comm*); +NGCORE_API extern int (*NG_MPI_Comm_group)(NG_MPI_Comm, NG_MPI_Group*); +NGCORE_API extern int (*NG_MPI_Comm_rank)(NG_MPI_Comm, int*); +NGCORE_API extern int (*NG_MPI_Comm_size)(NG_MPI_Comm, int*); +NGCORE_API extern int (*NG_MPI_Finalize)(); +NGCORE_API extern int (*NG_MPI_Gather)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Get_count)(NG_MPI_Status*, NG_MPI_Datatype, int*); +NGCORE_API extern int (*NG_MPI_Get_processor_name)(char*, int*); +NGCORE_API extern int (*NG_MPI_Group_incl)(NG_MPI_Group, int, int*, NG_MPI_Group*); +NGCORE_API extern int (*NG_MPI_Init)(int*, char***); +NGCORE_API extern int (*NG_MPI_Init_thread)(int*, char***, int, int*); +NGCORE_API extern int (*NG_MPI_Initialized)(int*); +NGCORE_API extern int (*NG_MPI_Iprobe)(int, int, NG_MPI_Comm, int*, NG_MPI_Status*); +NGCORE_API extern int (*NG_MPI_Irecv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*); +NGCORE_API extern int (*NG_MPI_Isend)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*); +NGCORE_API extern int (*NG_MPI_Probe)(int, int, NG_MPI_Comm, NG_MPI_Status*); +NGCORE_API extern int (*NG_MPI_Query_thread)(int*); +NGCORE_API extern int (*NG_MPI_Recv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*); +NGCORE_API extern int (*NG_MPI_Reduce)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Reduce_local)(void*, void*, int, NG_MPI_Datatype, NG_MPI_Op); +NGCORE_API extern int (*NG_MPI_Request_free)(NG_MPI_Request*); +NGCORE_API extern int (*NG_MPI_Scatter)(void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Send)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm); +NGCORE_API extern int (*NG_MPI_Type_commit)(NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_contiguous)(int, NG_MPI_Datatype, NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_create_resized)(NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_create_struct)(int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_free)(NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_get_extent)(NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*); +NGCORE_API extern int (*NG_MPI_Type_indexed)(int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*); +NGCORE_API extern int (*NG_MPI_Type_size)(NG_MPI_Datatype, int*); +NGCORE_API extern int (*NG_MPI_Wait)(NG_MPI_Request*, NG_MPI_Status*); +NGCORE_API extern int (*NG_MPI_Waitall)(int, NG_MPI_Request*, NG_MPI_Status*); +NGCORE_API extern int (*NG_MPI_Waitany)(int, NG_MPI_Request*, int*, NG_MPI_Status*); +NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_WORLD; +NGCORE_API extern NG_MPI_Datatype NG_MPI_CHAR; +NGCORE_API extern NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX; +NGCORE_API extern NG_MPI_Datatype NG_MPI_C_BOOL; +NGCORE_API extern NG_MPI_Datatype NG_MPI_DATATYPE_NULL; +NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE; +NGCORE_API extern NG_MPI_Datatype NG_MPI_INT; +NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT; +NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T; +NGCORE_API extern NG_MPI_Op NG_MPI_LOR; +NGCORE_API extern NG_MPI_Op NG_MPI_MAX; +NGCORE_API extern NG_MPI_Op NG_MPI_MIN; +NGCORE_API extern NG_MPI_Op NG_MPI_SUM; +NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE; +NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE; +NGCORE_API extern int NG_MPI_ANY_SOURCE; +NGCORE_API extern int NG_MPI_ANY_TAG; +NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME; +NGCORE_API extern int NG_MPI_PROC_NULL; +NGCORE_API extern int NG_MPI_ROOT; +NGCORE_API extern int NG_MPI_SUBVERSION; +NGCORE_API extern int NG_MPI_THREAD_MULTIPLE; +NGCORE_API extern int NG_MPI_THREAD_SINGLE; +NGCORE_API extern int NG_MPI_VERSION; +NGCORE_API extern void* NG_MPI_IN_PLACE; +#else // NG_MPI_WRAPPER +static const auto NG_MPI_Wtime = MPI_Wtime; +static const auto NG_MPI_Allgather = MPI_Allgather; +static const auto NG_MPI_Allreduce = MPI_Allreduce; +static const auto NG_MPI_Alltoall = MPI_Alltoall; +static const auto NG_MPI_Barrier = MPI_Barrier; +static const auto NG_MPI_Bcast = MPI_Bcast; +static const auto NG_MPI_Comm_create_group = MPI_Comm_create_group; +static const auto NG_MPI_Comm_free = MPI_Comm_free; +static const auto NG_MPI_Comm_group = MPI_Comm_group; +static const auto NG_MPI_Comm_rank = MPI_Comm_rank; +static const auto NG_MPI_Comm_size = MPI_Comm_size; +static const auto NG_MPI_Finalize = MPI_Finalize; +static const auto NG_MPI_Gather = MPI_Gather; +static const auto NG_MPI_Get_count = MPI_Get_count; +static const auto NG_MPI_Get_processor_name = MPI_Get_processor_name; +static const auto NG_MPI_Group_incl = MPI_Group_incl; +static const auto NG_MPI_Init = MPI_Init; +static const auto NG_MPI_Init_thread = MPI_Init_thread; +static const auto NG_MPI_Initialized = MPI_Initialized; +static const auto NG_MPI_Iprobe = MPI_Iprobe; +static const auto NG_MPI_Irecv = MPI_Irecv; +static const auto NG_MPI_Isend = MPI_Isend; +static const auto NG_MPI_Probe = MPI_Probe; +static const auto NG_MPI_Query_thread = MPI_Query_thread; +static const auto NG_MPI_Recv = MPI_Recv; +static const auto NG_MPI_Reduce = MPI_Reduce; +static const auto NG_MPI_Reduce_local = MPI_Reduce_local; +static const auto NG_MPI_Request_free = MPI_Request_free; +static const auto NG_MPI_Scatter = MPI_Scatter; +static const auto NG_MPI_Send = MPI_Send; +static const auto NG_MPI_Type_commit = MPI_Type_commit; +static const auto NG_MPI_Type_contiguous = MPI_Type_contiguous; +static const auto NG_MPI_Type_create_resized = MPI_Type_create_resized; +static const auto NG_MPI_Type_create_struct = MPI_Type_create_struct; +static const auto NG_MPI_Type_free = MPI_Type_free; +static const auto NG_MPI_Type_get_extent = MPI_Type_get_extent; +static const auto NG_MPI_Type_indexed = MPI_Type_indexed; +static const auto NG_MPI_Type_size = MPI_Type_size; +static const auto NG_MPI_Wait = MPI_Wait; +static const auto NG_MPI_Waitall = MPI_Waitall; +static const auto NG_MPI_Waitany = MPI_Waitany; +static const decltype(MPI_COMM_WORLD) NG_MPI_COMM_WORLD = MPI_COMM_WORLD; +static const decltype(MPI_CHAR) NG_MPI_CHAR = MPI_CHAR; +static const decltype(MPI_CXX_DOUBLE_COMPLEX) NG_MPI_CXX_DOUBLE_COMPLEX = MPI_CXX_DOUBLE_COMPLEX; +static const decltype(MPI_C_BOOL) NG_MPI_C_BOOL = MPI_C_BOOL; +static const decltype(MPI_DATATYPE_NULL) NG_MPI_DATATYPE_NULL = MPI_DATATYPE_NULL; +static const decltype(MPI_DOUBLE) NG_MPI_DOUBLE = MPI_DOUBLE; +static const decltype(MPI_INT) NG_MPI_INT = MPI_INT; +static const decltype(MPI_SHORT) NG_MPI_SHORT = MPI_SHORT; +static const decltype(MPI_UINT64_T) NG_MPI_UINT64_T = MPI_UINT64_T; +static const decltype(MPI_LOR) NG_MPI_LOR = MPI_LOR; +static const decltype(MPI_MAX) NG_MPI_MAX = MPI_MAX; +static const decltype(MPI_MIN) NG_MPI_MIN = MPI_MIN; +static const decltype(MPI_SUM) NG_MPI_SUM = MPI_SUM; +static const decltype(MPI_STATUSES_IGNORE) NG_MPI_STATUSES_IGNORE = MPI_STATUSES_IGNORE; +static const decltype(MPI_STATUS_IGNORE) NG_MPI_STATUS_IGNORE = MPI_STATUS_IGNORE; +static const decltype(MPI_ANY_SOURCE) NG_MPI_ANY_SOURCE = MPI_ANY_SOURCE; +static const decltype(MPI_ANY_TAG) NG_MPI_ANY_TAG = MPI_ANY_TAG; +static const decltype(MPI_MAX_PROCESSOR_NAME) NG_MPI_MAX_PROCESSOR_NAME = MPI_MAX_PROCESSOR_NAME; +static const decltype(MPI_PROC_NULL) NG_MPI_PROC_NULL = MPI_PROC_NULL; +static const decltype(MPI_ROOT) NG_MPI_ROOT = MPI_ROOT; +static const decltype(MPI_SUBVERSION) NG_MPI_SUBVERSION = MPI_SUBVERSION; +static const decltype(MPI_THREAD_MULTIPLE) NG_MPI_THREAD_MULTIPLE = MPI_THREAD_MULTIPLE; +static const decltype(MPI_THREAD_SINGLE) NG_MPI_THREAD_SINGLE = MPI_THREAD_SINGLE; +static const decltype(MPI_VERSION) NG_MPI_VERSION = MPI_VERSION; +static const decltype(MPI_IN_PLACE) NG_MPI_IN_PLACE = MPI_IN_PLACE; +#endif // NG_MPI_WRAPPER diff --git a/libsrc/core/ng_mpi_generated_dummy_init.hpp b/libsrc/core/ng_mpi_generated_dummy_init.hpp new file mode 100644 index 00000000..6050bdee --- /dev/null +++ b/libsrc/core/ng_mpi_generated_dummy_init.hpp @@ -0,0 +1,66 @@ +decltype(NG_MPI_Wtime) NG_MPI_Wtime = []()->double { throw no_mpi(); }; +decltype(NG_MPI_Allgather) NG_MPI_Allgather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Allreduce) NG_MPI_Allreduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Alltoall) NG_MPI_Alltoall = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Barrier) NG_MPI_Barrier = [](NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Bcast) NG_MPI_Bcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Comm_create_group) NG_MPI_Comm_create_group = [](NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*)->int { throw no_mpi(); }; +decltype(NG_MPI_Comm_free) NG_MPI_Comm_free = [](NG_MPI_Comm*)->int { throw no_mpi(); }; +decltype(NG_MPI_Comm_group) NG_MPI_Comm_group = [](NG_MPI_Comm, NG_MPI_Group*)->int { throw no_mpi(); }; +decltype(NG_MPI_Comm_rank) NG_MPI_Comm_rank = [](NG_MPI_Comm, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Comm_size) NG_MPI_Comm_size = [](NG_MPI_Comm, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Finalize) NG_MPI_Finalize = []()->int { throw no_mpi(); }; +decltype(NG_MPI_Gather) NG_MPI_Gather = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Get_count) NG_MPI_Get_count = [](NG_MPI_Status*, NG_MPI_Datatype, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Get_processor_name) NG_MPI_Get_processor_name = [](char*, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Group_incl) NG_MPI_Group_incl = [](NG_MPI_Group, int, int*, NG_MPI_Group*)->int { throw no_mpi(); }; +decltype(NG_MPI_Init) NG_MPI_Init = [](int*, char***)->int { throw no_mpi(); }; +decltype(NG_MPI_Init_thread) NG_MPI_Init_thread = [](int*, char***, int, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Initialized) NG_MPI_Initialized = [](int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Iprobe) NG_MPI_Iprobe = [](int, int, NG_MPI_Comm, int*, NG_MPI_Status*)->int { throw no_mpi(); }; +decltype(NG_MPI_Irecv) NG_MPI_Irecv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); }; +decltype(NG_MPI_Isend) NG_MPI_Isend = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Request*)->int { throw no_mpi(); }; +decltype(NG_MPI_Probe) NG_MPI_Probe = [](int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); }; +decltype(NG_MPI_Query_thread) NG_MPI_Query_thread = [](int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Recv) NG_MPI_Recv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); }; +decltype(NG_MPI_Reduce) NG_MPI_Reduce = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op, int, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Reduce_local) NG_MPI_Reduce_local = [](void*, void*, int, NG_MPI_Datatype, NG_MPI_Op)->int { throw no_mpi(); }; +decltype(NG_MPI_Request_free) NG_MPI_Request_free = [](NG_MPI_Request*)->int { throw no_mpi(); }; +decltype(NG_MPI_Scatter) NG_MPI_Scatter = [](void*, int, NG_MPI_Datatype, void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Send) NG_MPI_Send = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_commit) NG_MPI_Type_commit = [](NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_contiguous) NG_MPI_Type_contiguous = [](int, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_create_resized) NG_MPI_Type_create_resized = [](NG_MPI_Datatype, NG_MPI_Aint, NG_MPI_Aint, NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_create_struct) NG_MPI_Type_create_struct = [](int, int*, NG_MPI_Aint*, NG_MPI_Datatype*, NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_free) NG_MPI_Type_free = [](NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_get_extent) NG_MPI_Type_get_extent = [](NG_MPI_Datatype, NG_MPI_Aint*, NG_MPI_Aint*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_indexed) NG_MPI_Type_indexed = [](int, int*, int*, NG_MPI_Datatype, NG_MPI_Datatype*)->int { throw no_mpi(); }; +decltype(NG_MPI_Type_size) NG_MPI_Type_size = [](NG_MPI_Datatype, int*)->int { throw no_mpi(); }; +decltype(NG_MPI_Wait) NG_MPI_Wait = [](NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); }; +decltype(NG_MPI_Waitall) NG_MPI_Waitall = [](int, NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); }; +decltype(NG_MPI_Waitany) NG_MPI_Waitany = [](int, NG_MPI_Request*, int*, NG_MPI_Status*)->int { throw no_mpi(); }; +NG_MPI_Comm NG_MPI_COMM_WORLD = 0; +NG_MPI_Datatype NG_MPI_CHAR = 0; +NG_MPI_Datatype NG_MPI_CXX_DOUBLE_COMPLEX = 0; +NG_MPI_Datatype NG_MPI_C_BOOL = 0; +NG_MPI_Datatype NG_MPI_DATATYPE_NULL = 0; +NG_MPI_Datatype NG_MPI_DOUBLE = 0; +NG_MPI_Datatype NG_MPI_INT = 0; +NG_MPI_Datatype NG_MPI_SHORT = 0; +NG_MPI_Datatype NG_MPI_UINT64_T = 0; +NG_MPI_Op NG_MPI_LOR = 0; +NG_MPI_Op NG_MPI_MAX = 0; +NG_MPI_Op NG_MPI_MIN = 0; +NG_MPI_Op NG_MPI_SUM = 0; +NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0; +NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0; +int NG_MPI_ANY_SOURCE = 0; +int NG_MPI_ANY_TAG = 0; +int NG_MPI_MAX_PROCESSOR_NAME = 0; +int NG_MPI_PROC_NULL = 0; +int NG_MPI_ROOT = 0; +int NG_MPI_SUBVERSION = 0; +int NG_MPI_THREAD_MULTIPLE = 0; +int NG_MPI_THREAD_SINGLE = 0; +int NG_MPI_VERSION = 0; +void* NG_MPI_IN_PLACE = 0; diff --git a/libsrc/core/ng_mpi_generated_init.hpp b/libsrc/core/ng_mpi_generated_init.hpp new file mode 100644 index 00000000..b44bc2a7 --- /dev/null +++ b/libsrc/core/ng_mpi_generated_init.hpp @@ -0,0 +1,66 @@ +NG_MPI_Wtime = []()->double { return MPI_Wtime(); }; +NG_MPI_Allgather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Allgather( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), ng2mpi(arg6)); }; +NG_MPI_Allreduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, NG_MPI_Comm arg5)->int { return MPI_Allreduce( arg0, arg1, arg2, ng2mpi(arg3), ng2mpi(arg4), ng2mpi(arg5)); }; +NG_MPI_Alltoall = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, NG_MPI_Comm arg6)->int { return MPI_Alltoall( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), ng2mpi(arg6)); }; +NG_MPI_Barrier = [](NG_MPI_Comm arg0)->int { return MPI_Barrier( ng2mpi(arg0)); }; +NG_MPI_Bcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4)->int { return MPI_Bcast( arg0, arg1, ng2mpi(arg2), arg3, ng2mpi(arg4)); }; +NG_MPI_Comm_create_group = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, int arg2, NG_MPI_Comm* arg3)->int { return MPI_Comm_create_group( ng2mpi(arg0), ng2mpi(arg1), arg2, ng2mpi(arg3)); }; +NG_MPI_Comm_free = [](NG_MPI_Comm* arg0)->int { return MPI_Comm_free( ng2mpi(arg0)); }; +NG_MPI_Comm_group = [](NG_MPI_Comm arg0, NG_MPI_Group* arg1)->int { return MPI_Comm_group( ng2mpi(arg0), ng2mpi(arg1)); }; +NG_MPI_Comm_rank = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_rank( ng2mpi(arg0), arg1); }; +NG_MPI_Comm_size = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_size( ng2mpi(arg0), arg1); }; +NG_MPI_Finalize = []()->int { return MPI_Finalize(); }; +NG_MPI_Gather = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Gather( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), arg6, ng2mpi(arg7)); }; +NG_MPI_Get_count = [](NG_MPI_Status* arg0, NG_MPI_Datatype arg1, int* arg2)->int { return MPI_Get_count( ng2mpi(arg0), ng2mpi(arg1), arg2); }; +NG_MPI_Get_processor_name = [](char* arg0, int* arg1)->int { return MPI_Get_processor_name( arg0, arg1); }; +NG_MPI_Group_incl = [](NG_MPI_Group arg0, int arg1, int* arg2, NG_MPI_Group* arg3)->int { return MPI_Group_incl( ng2mpi(arg0), arg1, arg2, ng2mpi(arg3)); }; +NG_MPI_Init = [](int* arg0, char*** arg1)->int { return MPI_Init( arg0, arg1); }; +NG_MPI_Init_thread = [](int* arg0, char*** arg1, int arg2, int* arg3)->int { return MPI_Init_thread( arg0, arg1, arg2, arg3); }; +NG_MPI_Initialized = [](int* arg0)->int { return MPI_Initialized( arg0); }; +NG_MPI_Iprobe = [](int arg0, int arg1, NG_MPI_Comm arg2, int* arg3, NG_MPI_Status* arg4)->int { return MPI_Iprobe( arg0, arg1, ng2mpi(arg2), arg3, ng2mpi(arg4)); }; +NG_MPI_Irecv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Irecv( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), ng2mpi(arg6)); }; +NG_MPI_Isend = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Request* arg6)->int { return MPI_Isend( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), ng2mpi(arg6)); }; +NG_MPI_Probe = [](int arg0, int arg1, NG_MPI_Comm arg2, NG_MPI_Status* arg3)->int { return MPI_Probe( arg0, arg1, ng2mpi(arg2), ng2mpi(arg3)); }; +NG_MPI_Query_thread = [](int* arg0)->int { return MPI_Query_thread( arg0); }; +NG_MPI_Recv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Status* arg6)->int { return MPI_Recv( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), ng2mpi(arg6)); }; +NG_MPI_Reduce = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4, int arg5, NG_MPI_Comm arg6)->int { return MPI_Reduce( arg0, arg1, arg2, ng2mpi(arg3), ng2mpi(arg4), arg5, ng2mpi(arg6)); }; +NG_MPI_Reduce_local = [](void* arg0, void* arg1, int arg2, NG_MPI_Datatype arg3, NG_MPI_Op arg4)->int { return MPI_Reduce_local( arg0, arg1, arg2, ng2mpi(arg3), ng2mpi(arg4)); }; +NG_MPI_Request_free = [](NG_MPI_Request* arg0)->int { return MPI_Request_free( ng2mpi(arg0)); }; +NG_MPI_Scatter = [](void* arg0, int arg1, NG_MPI_Datatype arg2, void* arg3, int arg4, NG_MPI_Datatype arg5, int arg6, NG_MPI_Comm arg7)->int { return MPI_Scatter( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5), arg6, ng2mpi(arg7)); }; +NG_MPI_Send = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5)->int { return MPI_Send( arg0, arg1, ng2mpi(arg2), arg3, arg4, ng2mpi(arg5)); }; +NG_MPI_Type_commit = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_commit( ng2mpi(arg0)); }; +NG_MPI_Type_contiguous = [](int arg0, NG_MPI_Datatype arg1, NG_MPI_Datatype* arg2)->int { return MPI_Type_contiguous( arg0, ng2mpi(arg1), ng2mpi(arg2)); }; +NG_MPI_Type_create_resized = [](NG_MPI_Datatype arg0, NG_MPI_Aint arg1, NG_MPI_Aint arg2, NG_MPI_Datatype* arg3)->int { return MPI_Type_create_resized( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); }; +NG_MPI_Type_create_struct = [](int arg0, int* arg1, NG_MPI_Aint* arg2, NG_MPI_Datatype* arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_create_struct( arg0, arg1, ng2mpi(arg2, arg0), ng2mpi(arg3, arg0), ng2mpi(arg4)); }; +NG_MPI_Type_free = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_free( ng2mpi(arg0)); }; +NG_MPI_Type_get_extent = [](NG_MPI_Datatype arg0, NG_MPI_Aint* arg1, NG_MPI_Aint* arg2)->int { return MPI_Type_get_extent( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2)); }; +NG_MPI_Type_indexed = [](int arg0, int* arg1, int* arg2, NG_MPI_Datatype arg3, NG_MPI_Datatype* arg4)->int { return MPI_Type_indexed( arg0, arg1, arg2, ng2mpi(arg3), ng2mpi(arg4)); }; +NG_MPI_Type_size = [](NG_MPI_Datatype arg0, int* arg1)->int { return MPI_Type_size( ng2mpi(arg0), arg1); }; +NG_MPI_Wait = [](NG_MPI_Request* arg0, NG_MPI_Status* arg1)->int { return MPI_Wait( ng2mpi(arg0), ng2mpi(arg1)); }; +NG_MPI_Waitall = [](int arg0, NG_MPI_Request* arg1, NG_MPI_Status* arg2)->int { return MPI_Waitall( arg0, ng2mpi(arg1, arg0), ng2mpi(arg2)); }; +NG_MPI_Waitany = [](int arg0, NG_MPI_Request* arg1, int* arg2, NG_MPI_Status* arg3)->int { return MPI_Waitany( arg0, ng2mpi(arg1, arg0), arg2, ng2mpi(arg3)); }; +NG_MPI_COMM_WORLD = mpi2ng(MPI_COMM_WORLD); +NG_MPI_CHAR = mpi2ng(MPI_CHAR); +NG_MPI_CXX_DOUBLE_COMPLEX = mpi2ng(MPI_CXX_DOUBLE_COMPLEX); +NG_MPI_C_BOOL = mpi2ng(MPI_C_BOOL); +NG_MPI_DATATYPE_NULL = mpi2ng(MPI_DATATYPE_NULL); +NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE); +NG_MPI_INT = mpi2ng(MPI_INT); +NG_MPI_SHORT = mpi2ng(MPI_SHORT); +NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T); +NG_MPI_LOR = mpi2ng(MPI_LOR); +NG_MPI_MAX = mpi2ng(MPI_MAX); +NG_MPI_MIN = mpi2ng(MPI_MIN); +NG_MPI_SUM = mpi2ng(MPI_SUM); +NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE); +NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE); +NG_MPI_ANY_SOURCE = mpi2ng(MPI_ANY_SOURCE); +NG_MPI_ANY_TAG = mpi2ng(MPI_ANY_TAG); +NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME); +NG_MPI_PROC_NULL = mpi2ng(MPI_PROC_NULL); +NG_MPI_ROOT = mpi2ng(MPI_ROOT); +NG_MPI_SUBVERSION = mpi2ng(MPI_SUBVERSION); +NG_MPI_THREAD_MULTIPLE = mpi2ng(MPI_THREAD_MULTIPLE); +NG_MPI_THREAD_SINGLE = mpi2ng(MPI_THREAD_SINGLE); +NG_MPI_VERSION = mpi2ng(MPI_VERSION); +NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE); diff --git a/libsrc/core/ng_mpi_wrapper.cpp b/libsrc/core/ng_mpi_wrapper.cpp new file mode 100644 index 00000000..8b193562 --- /dev/null +++ b/libsrc/core/ng_mpi_wrapper.cpp @@ -0,0 +1,191 @@ +#ifdef PARALLEL + +#include +#include +#include + +#include "ng_mpi.hpp" +#include "ngstream.hpp" +#include "python_ngcore.hpp" +#include "utils.hpp" + +using std::cerr; +using std::cout; +using std::endl; + +namespace ngcore { + +#ifdef NG_MPI_WRAPPER +static std::unique_ptr mpi_lib, ng_mpi_lib; +static bool need_mpi_finalize = false; + +struct MPIFinalizer { + ~MPIFinalizer() { + if (need_mpi_finalize) { + cout << IM(5) << "Calling MPI_Finalize" << endl; + NG_MPI_Finalize(); + } + } +} mpi_finalizer; + +void InitMPI(std::optional mpi_lib_path) { + if (ng_mpi_lib) return; + + cout << IM(3) << "InitMPI" << endl; + + std::string vendor = ""; + std::string mpi4py_lib_file = ""; + + if (mpi_lib_path) { + // Dynamic load of given shared MPI library + // Then call MPI_Init, read the library version and set the vender name + try { + typedef int (*init_handle)(int *, char ***); + typedef int (*mpi_initialized_handle)(int *); + mpi_lib = + std::make_unique(*mpi_lib_path, std::nullopt, true); + auto mpi_init = mpi_lib->GetSymbol("MPI_Init"); + auto mpi_initialized = + mpi_lib->GetSymbol("MPI_Initialized"); + + int flag = 0; + mpi_initialized(&flag); + if (!flag) { + typedef const char *pchar; + int argc = 1; + pchar args[] = {"netgen", nullptr}; + pchar *argv = &args[0]; + cout << IM(5) << "Calling MPI_Init" << endl; + mpi_init(&argc, (char ***)argv); + need_mpi_finalize = true; + } + + char c_version_string[65536]; + c_version_string[0] = '\0'; + int result_len = 0; + typedef void (*get_version_handle)(char *, int *); + auto get_version = + mpi_lib->GetSymbol("MPI_Get_library_version"); + get_version(c_version_string, &result_len); + std::string version = c_version_string; + + if (version.substr(0, 8) == "Open MPI") + vendor = "Open MPI"; + else if (version.substr(0, 5) == "MPICH") + vendor = "MPICH"; + else if (version.substr(0, 13) == "Microsoft MPI") + vendor = "Microsoft MPI"; + else if (version.substr(0, 12) == "Intel(R) MPI") + vendor = "Intel MPI"; + else + throw std::runtime_error( + std::string("Unknown MPI version: " + version)); + } catch (std::runtime_error &e) { + cerr << "Could not load MPI: " << e.what() << endl; + throw e; + } + } else { + // Use mpi4py to init MPI library and get the vendor name + auto mpi4py = py::module::import("mpi4py.MPI"); + vendor = mpi4py.attr("get_vendor")()[py::int_(0)].cast(); + +#ifndef WIN32 + // Load mpi4py library (it exports all MPI symbols) to have all MPI symbols + // available before the ng_mpi wrapper is loaded This is not necessary on + // windows as the matching mpi dll is linked to the ng_mpi wrapper directly + mpi4py_lib_file = mpi4py.attr("__file__").cast(); + mpi_lib = + std::make_unique(mpi4py_lib_file, std::nullopt, true); +#endif // WIN32 + } + + std::string ng_lib_name = ""; + if (vendor == "Open MPI") + ng_lib_name = "ng_openmpi"; + else if (vendor == "MPICH") + ng_lib_name = "ng_mpich"; + else if (vendor == "Microsoft MPI") + ng_lib_name = "ng_microsoft_mpi"; + else if (vendor == "Intel MPI") + ng_lib_name = "ng_intel_mpi"; + else + throw std::runtime_error("Unknown MPI vendor: " + vendor); + + ng_lib_name += NETGEN_SHARED_LIBRARY_SUFFIX; + + // Load the ng_mpi wrapper and call ng_init_mpi to set all function pointers + typedef void (*ng_init_handle)(); + ng_mpi_lib = std::make_unique(ng_lib_name); + ng_mpi_lib->GetSymbol("ng_init_mpi")(); + std::cout << IM(3) << "MPI wrapper loaded, vendor: " << vendor << endl; +} + +static std::runtime_error no_mpi() { + return std::runtime_error("MPI not enabled"); +} + +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = + [](py::handle py_obj, NG_MPI_Comm &ng_comm) -> bool { + // If this gets called, it means that we want to convert an mpi4py + // communicator to a Netgen MPI communicator, but the Netgen MPI wrapper + // runtime was not yet initialized. + + // store the current address of this function + auto old_converter_address = NG_MPI_CommFromMPI4Py; + + // initialize the MPI wrapper runtime, this sets all the function pointers + InitMPI(); + + // if the initialization was successful, the function pointer should have + // changed + // -> call the actual conversion function + if (NG_MPI_CommFromMPI4Py != old_converter_address) + return NG_MPI_CommFromMPI4Py(py_obj, ng_comm); + + // otherwise, something strange happened + throw no_mpi(); +}; +decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py = + [](NG_MPI_Comm) -> py::handle { throw no_mpi(); }; +#endif + +#include "ng_mpi_generated_dummy_init.hpp" +#else // NG_MPI_WRAPPER + +static bool imported_mpi4py = false; +#if defined(NG_PYTHON) && defined(NG_MPI4PY) +decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = + [](py::handle src, NG_MPI_Comm &dst) -> bool { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + PyObject *py_src = src.ptr(); + auto type = Py_TYPE(py_src); + if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { + dst = *PyMPIComm_Get(py_src); + return !PyErr_Occurred(); + } + return false; +}; + +decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py = + [](NG_MPI_Comm src) -> py::handle { + if (!imported_mpi4py) { + import_mpi4py(); + imported_mpi4py = true; + } + return py::handle(PyMPIComm_New(src)); +}; + +#endif + +void InitMPI(std::optional) {} + +#endif // NG_MPI_WRAPPER + + +} // namespace ngcore + +#endif // PARALLEL diff --git a/libsrc/core/ngcore.hpp b/libsrc/core/ngcore.hpp index 285a34e2..44f59f9e 100644 --- a/libsrc/core/ngcore.hpp +++ b/libsrc/core/ngcore.hpp @@ -10,7 +10,7 @@ #include "hashtable.hpp" #include "localheap.hpp" #include "logging.hpp" -#include "mpi_wrapper.hpp" +// #include "mpi_wrapper.hpp" #include "profiler.hpp" #include "signal.hpp" #include "simd.hpp" diff --git a/libsrc/core/ngcore_api.hpp b/libsrc/core/ngcore_api.hpp index e66e9b87..35f222d4 100644 --- a/libsrc/core/ngcore_api.hpp +++ b/libsrc/core/ngcore_api.hpp @@ -1,6 +1,8 @@ #ifndef NETGEN_CORE_NGCORE_API_HPP #define NETGEN_CORE_NGCORE_API_HPP +#include "netgen_config.hpp" + #ifdef WIN32 // This function or variable may be unsafe. Consider using _ftime64_s instead. To disable deprecation, use _CRT_SECURE_NO_WARNINGS. See online help for details. diff --git a/libsrc/core/paje_trace.cpp b/libsrc/core/paje_trace.cpp index 1b81395e..f3f9e944 100644 --- a/libsrc/core/paje_trace.cpp +++ b/libsrc/core/paje_trace.cpp @@ -13,6 +13,7 @@ extern const char *header; constexpr int MPI_PAJE_WRITER = 1; +constexpr int ASSUMED_MPI_MAX_PROCESSOR_NAME = 256; namespace ngcore { @@ -24,7 +25,7 @@ namespace ngcore if(id1) @@ -496,9 +497,9 @@ namespace ngcore nthreads = nranks; thread_aliases.reserve(nthreads); - std::array ahostname; + std::array ahostname; int len; - MPI_Get_processor_name(ahostname.data(), &len); + NG_MPI_Get_processor_name(ahostname.data(), &len); std::string hostname = ahostname.data(); std::map host_map; @@ -854,15 +855,15 @@ namespace ngcore { #ifdef PARALLEL // Hostname - NgMPI_Comm comm(MPI_COMM_WORLD); + NgMPI_Comm comm(NG_MPI_COMM_WORLD); // auto rank = comm.Rank(); // auto nranks = comm.Size(); std::string hostname; { - std::array ahostname; + std::array ahostname; int len; - MPI_Get_processor_name(ahostname.data(), &len); + NG_MPI_Get_processor_name(ahostname.data(), &len); hostname = ahostname.data(); } diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index 748e5dd5..cd3d14fb 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -14,6 +14,8 @@ #include "flags.hpp" #include "ngcore_api.hpp" #include "profiler.hpp" +#include "ng_mpi.hpp" + namespace py = pybind11; namespace ngcore @@ -31,6 +33,16 @@ namespace ngcore static constexpr bool value = decltype(check((T*) nullptr))::value; }; } // namespace detail + + struct mpi4py_comm { + mpi4py_comm() = default; +#ifdef PARALLEL + mpi4py_comm(NG_MPI_Comm value) : value(value) {} + operator NG_MPI_Comm () { return value; } + + NG_MPI_Comm value; +#endif // PARALLEL + }; } // namespace ngcore @@ -39,6 +51,27 @@ namespace ngcore namespace pybind11 { namespace detail { +#ifdef NG_MPI4PY +template <> struct type_caster { + public: + PYBIND11_TYPE_CASTER(ngcore::mpi4py_comm, _("mpi4py_comm")); + + // Python -> C++ + bool load(handle src, bool) { + return ngcore::NG_MPI_CommFromMPI4Py(src, value.value); + } + + // C++ -> Python + static handle cast(ngcore::mpi4py_comm src, + return_value_policy /* policy */, + handle /* parent */) + { + // Create an mpi4py handle + return ngcore::NG_MPI_CommToMPI4Py(src.value); + } +}; +#endif // NG_MPI4PY + template struct ngcore_list_caster { using value_conv = make_caster; diff --git a/libsrc/core/python_ngcore_export.cpp b/libsrc/core/python_ngcore_export.cpp index 21d25038..8c9e1a0e 100644 --- a/libsrc/core/python_ngcore_export.cpp +++ b/libsrc/core/python_ngcore_export.cpp @@ -1,11 +1,18 @@ #include "python_ngcore.hpp" #include "bitarray.hpp" #include "taskmanager.hpp" +#include "mpi_wrapper.hpp" using namespace ngcore; using namespace std; using namespace pybind11::literals; +namespace pybind11 { namespace detail { +}} // namespace pybind11::detail + + + + PYBIND11_MODULE(pyngcore, m) // NOLINT { try @@ -29,7 +36,13 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT ExportArray(m); ExportTable(m); - + + #ifdef PARALLEL + py::class_ (m, "_NG_MPI_Comm") + ; + m.def("InitMPI", &InitMPI, py::arg("mpi_library_path")=nullopt); + #endif // PARALLEL + py::class_> (m, "BitArray") .def(py::init([] (size_t n) { return make_shared(n); }),py::arg("n")) .def(py::init([] (const BitArray& a) { return make_shared(a); } ), py::arg("ba")) @@ -328,4 +341,35 @@ threads : int }, "Returns list of timers" ); m.def("ResetTimers", &NgProfiler::Reset); + + py::class_ (m, "MPI_Comm") +#ifdef PARALLEL + .def(py::init([] (mpi4py_comm comm) { return NgMPI_Comm(comm); })) + .def("WTime", [](NgMPI_Comm & c) { return NG_MPI_Wtime(); }) + .def_property_readonly ("mpi4py", [](NgMPI_Comm & self) { return NG_MPI_CommToMPI4Py(self); }) +#endif // PARALLEL + .def_property_readonly ("rank", &NgMPI_Comm::Rank) + .def_property_readonly ("size", &NgMPI_Comm::Size) + .def("Barrier", &NgMPI_Comm::Barrier) + .def("Sum", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, NG_MPI_SUM); }) + .def("Min", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, NG_MPI_MIN); }) + .def("Max", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, NG_MPI_MAX); }) + .def("Sum", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, NG_MPI_SUM); }) + .def("Min", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, NG_MPI_MIN); }) + .def("Max", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, NG_MPI_MAX); }) + .def("Sum", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, NG_MPI_SUM); }) + .def("Min", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, NG_MPI_MIN); }) + .def("Max", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, NG_MPI_MAX); }) + .def("SubComm", [](NgMPI_Comm & c, std::vector proc_list) { + Array procs(proc_list.size()); + for (int i = 0; i < procs.Size(); i++) + { procs[i] = proc_list[i]; } + if (!procs.Contains(c.Rank())) + { throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); } + return c.SubCommunicator(procs); + }, py::arg("procs")); + ; + + + py::implicitly_convertible(); } diff --git a/libsrc/core/utils.cpp b/libsrc/core/utils.cpp index eb03eb98..4ab08f36 100644 --- a/libsrc/core/utils.cpp +++ b/libsrc/core/utils.cpp @@ -3,16 +3,25 @@ #include "logging.hpp" #include "simd_generic.hpp" -#ifndef WIN32 +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#undef WIN32_LEAN_AND_MEAN +#else // WIN32 #include -#endif +#include +#endif //WIN32 +// #include #include #include #include +#include +#include #include "ngstream.hpp" + namespace ngcore { namespace detail @@ -109,7 +118,7 @@ namespace ngcore const std::chrono::time_point wall_time_start = TClock::now(); - int printmessage_importance = 0; + int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0; bool NGSOStream :: glob_active = true; NGCORE_API int GetCompiledSIMDSize() @@ -134,5 +143,91 @@ namespace ngcore return path; } + + SharedLibrary :: SharedLibrary(const std::filesystem::path & lib_name_, std::optional directory_to_delete_, bool global ) + : lib_name(lib_name_),directory_to_delete(directory_to_delete_) + { + Load(lib_name, global); + } + + SharedLibrary :: ~SharedLibrary() + { + Unload(); + if(directory_to_delete) + for([[maybe_unused]] auto i : Range(5)) + { + // on Windows, a (detached?) child process of the compiler/linker might still block the directory + // wait for it to finish (up to a second) + try + { + std::filesystem::remove_all(*directory_to_delete); + directory_to_delete = std::nullopt; + break; + } + catch(const std::exception &e) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + } + if(directory_to_delete) + std::cerr << "Could not delete " << directory_to_delete->string() << std::endl; + } + + void SharedLibrary :: Load( const std::filesystem::path & lib_name_, bool global ) + { + Unload(); + lib_name = lib_name_; +#ifdef WIN32 + lib = LoadLibraryW(lib_name.wstring().c_str()); + if (!lib) throw std::runtime_error(std::string("Could not load library ") + lib_name.string()); +#else // WIN32 + auto flags = RTLD_NOW; + if (global) flags |= RTLD_GLOBAL; + lib = dlopen(lib_name.c_str(), flags); + if(lib == nullptr) throw std::runtime_error(dlerror()); +#endif // WIN32 + } + + void SharedLibrary :: Unload() { + if(lib) + { +#ifdef WIN32 + FreeLibrary((HMODULE)lib); +#else // WIN32 + int rc = dlclose(lib); + if(rc != 0) std::cerr << "Failed to close library " << lib_name << std::endl; +#endif // WIN32 + } + } + + void* SharedLibrary :: GetRawSymbol( std::string func_name ) + { +#ifdef WIN32 + void* func = GetProcAddress((HMODULE)lib, func_name.c_str()); + if(func == nullptr) + throw std::runtime_error(std::string("Could not find function ") + func_name + " in library " + lib_name.string()); +#else // WIN32 + void* func = dlsym(lib, func_name.c_str()); + if(func == nullptr) + throw std::runtime_error(dlerror()); +#endif // WIN32 + + return func; + } + + void* GetRawSymbol( std::string func_name ) + { + void * func = nullptr; +#ifdef WIN32 + throw std::runtime_error("GetRawSymbol not implemented on WIN32"); +#else // WIN32 + func = dlsym(RTLD_DEFAULT, func_name.c_str()); + if(func == nullptr) + throw std::runtime_error(dlerror()); +#endif // WIN32 + return func; + } + + } // namespace ngcore diff --git a/libsrc/core/utils.hpp b/libsrc/core/utils.hpp index 79d919c0..323fe67a 100644 --- a/libsrc/core/utils.hpp +++ b/libsrc/core/utils.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -347,6 +348,41 @@ namespace ngcore NGCORE_API std::filesystem::path GetTempFilename(); + NGCORE_API void* GetRawSymbol( std::string func_name ); + + template + TFunc GetSymbol( std::string func_name ) + { + return reinterpret_cast(GetRawSymbol(func_name)); + } + + // Class to handle/load shared libraries + class NGCORE_API SharedLibrary + { + std::filesystem::path lib_name; + std::optional directory_to_delete = std::nullopt; + void *lib = nullptr; + + public: + SharedLibrary() = default; + SharedLibrary(const std::filesystem::path & lib_name_, std::optional directory_to_delete_ = std::nullopt, bool global = false ); + + SharedLibrary(const SharedLibrary &) = delete; + SharedLibrary & operator =(const SharedLibrary &) = delete; + + ~SharedLibrary(); + + template + TFunc GetSymbol( std::string func_name ) + { + return reinterpret_cast(GetRawSymbol(func_name)); + } + + void Load( const std::filesystem::path & lib_name_, bool global = true); + void Unload(); + void* GetRawSymbol( std::string func_name ); + }; + } // namespace ngcore #endif // NETGEN_CORE_UTILS_HPP diff --git a/libsrc/general/mpi_interface.cpp b/libsrc/general/mpi_interface.cpp deleted file mode 100644 index 8fd6fb96..00000000 --- a/libsrc/general/mpi_interface.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/**************************************************************************/ -/* File: mpi_interface.cpp */ -/* Author: Joachim Schoeberl */ -/* Date: 04. Apr. 97 */ -/**************************************************************************/ - -#ifdef OLD -#include -#include - - -namespace netgen -{ - - -#ifdef PARALLEL - - void MyMPI_SendCmd (const char * cmd) - { - int ntasks; - MPI_Comm_size(MPI_COMM_WORLD, &ntasks); - - if(ntasks==1) - return; - - for (int dest = 1; dest < ntasks; dest++) - MPI_Send( (void*)cmd, (strlen(cmd)+1), MPI_CHAR, dest, MPI_TAG_CMD, MPI_COMM_WORLD); - } - - string MyMPI_RecvCmd () - { - MPI_Status status; - int flag; - int size_of_msg = -1; - - MPI_Probe(0, MPI_TAG_CMD, MPI_COMM_WORLD, &status); - MPI_Get_count(&status, MPI_CHAR, &size_of_msg); - - //char* buf = (char*)malloc(size_of_msg*sizeof(char)); - char buf[100000]; //1MB should be enough... - - MPI_Recv( &buf, size_of_msg, MPI_CHAR, 0, MPI_TAG_CMD, MPI_COMM_WORLD, &status); - - return string(buf); - } - - // #else - // MPI_Comm MPI_COMM_WORLD, MPI_COMM_NULL; -#endif -} - - - -#endif diff --git a/libsrc/general/mpi_interface.hpp b/libsrc/general/mpi_interface.hpp deleted file mode 100644 index edc9c5f1..00000000 --- a/libsrc/general/mpi_interface.hpp +++ /dev/null @@ -1,336 +0,0 @@ -braucht keiner mehr - - -#ifdef XXXXXX - -#ifndef FILE_PARALLEL -#define FILE_PARALLEL - - -#ifdef VTRACE -#include "vt_user.h" -#else - #define VT_USER_START(n) - #define VT_USER_END(n) - #define VT_TRACER(n) -#endif - - -namespace netgen -{ - -#ifdef OLD -#ifdef PARALLEL - template - inline MPI_Datatype MyGetMPIType ( ) - { cerr << "ERROR in GetMPIType() -- no type found" << endl;return 0; } - template <> - inline MPI_Datatype MyGetMPIType ( ) - { return MPI_INT; } - template <> - inline MPI_Datatype MyGetMPIType ( ) - { return MPI_DOUBLE; } - template <> - inline MPI_Datatype MyGetMPIType ( ) - { return MPI_CHAR; } - template<> - inline MPI_Datatype MyGetMPIType ( ) - { return MPI_UINT64_T; } -#else - typedef int MPI_Datatype; - template inline MPI_Datatype MyGetMPIType ( ) { return 0; } -#endif -#endif - - - enum { MPI_TAG_CMD = 110 }; - enum { MPI_TAG_MESH = 210 }; - enum { MPI_TAG_VIS = 310 }; - -#ifdef PARALLEL - - [[deprecated("mympi_send int, use comm.Send instead")]] - inline void MyMPI_Send (int i, int dest, int tag, MPI_Comm comm) - { - int hi = i; - MPI_Send( &hi, 1, MPI_INT, dest, tag, comm); - } - - [[deprecated("mympi_revc int, use comm.Recv instead")]] - inline void MyMPI_Recv (int & i, int src, int tag, MPI_Comm comm) - { - MPI_Status status; - MPI_Recv( &i, 1, MPI_INT, src, tag, comm, &status); - } - - [[deprecated("mympi_send string, use comm.Send instead")]] - inline void MyMPI_Send (const string & s, int dest, int tag, MPI_Comm comm) - { - MPI_Send( const_cast (s.c_str()), s.length(), MPI_CHAR, dest, tag, comm); - } - - [[deprecated("mympi_revc string, use comm.Recv instead")]] - inline void MyMPI_Recv (string & s, int src, int tag, MPI_Comm comm) - { - MPI_Status status; - int len; - MPI_Probe (src, tag, MPI_COMM_WORLD, &status); - MPI_Get_count (&status, MPI_CHAR, &len); - s.assign (len, ' '); - MPI_Recv( &s[0], len, MPI_CHAR, src, tag, comm, &status); - } - - - - template - [[deprecated("mympi_send ngflatarray, use comm.send instead")]] - inline void MyMPI_Send (NgFlatArray s, int dest, int tag, MPI_Comm comm) - { - MPI_Send( &s.First(), s.Size(), GetMPIType(), dest, tag, comm); - } - - template - [[deprecated("mympi_recv ngflatarray, use comm.Recv instead")]] - inline void MyMPI_Recv ( NgFlatArray s, int src, int tag, MPI_Comm comm) - { - MPI_Status status; - MPI_Recv( &s.First(), s.Size(), GetMPIType(), src, tag, comm, &status); - } - - template - [[deprecated("use ngcore - Array instead")]] - inline void MyMPI_Recv ( NgArray & s, int src, int tag, MPI_Comm comm) - { - MPI_Status status; - int len; - MPI_Probe (src, tag, comm, &status); - MPI_Get_count (&status, GetMPIType(), &len); - - s.SetSize (len); - MPI_Recv( &s.First(), len, GetMPIType(), src, tag, comm, &status); - } - - template - [[deprecated("use ngcore - Array instead")]] - inline int MyMPI_Recv ( NgArray & s, int tag, MPI_Comm comm) - { - MPI_Status status; - int len; - MPI_Probe (MPI_ANY_SOURCE, tag, comm, &status); - - int src = status.MPI_SOURCE; - - MPI_Get_count (&status, GetMPIType(), &len); - - s.SetSize (len); - MPI_Recv( &s.First(), len, GetMPIType(), src, tag, comm, &status); - - return src; - } - - - /* - template - inline void MyMPI_ISend (NgFlatArray s, int dest, int tag, MPI_Request & request) - { - MPI_Isend( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, & request); - } - - - template - inline void MyMPI_IRecv (NgFlatArray s, int dest, int tag, MPI_Request & request) - { - MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, & request); - } - */ - - template - [[deprecated("mympi_isend ngflatarray, use comm.send instead")]] - [[deprecated("use ngcore - Array instead")]] - inline MPI_Request MyMPI_ISend (NgFlatArray s, int dest, int tag, MPI_Comm comm) - { - MPI_Request request; - MPI_Isend( &s.First(), s.Size(), GetMPIType(), dest, tag, comm, &request); - return request; - } - - template - [[deprecated("mympi_irecv ngflatarray, use comm.recv instead")]] - inline MPI_Request MyMPI_IRecv (NgFlatArray s, int dest, int tag, MPI_Comm comm) - { - MPI_Request request; - MPI_Irecv( &s.First(), s.Size(), GetMPIType(), dest, tag, comm, &request); - return request; - } - - /* - template - inline void MyMPI_ISend (NgFlatArray s, int dest, int tag) - { - MPI_Request request; - MPI_Isend( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, &request); - MPI_Request_free (&request); - } - - - template - inline void MyMPI_IRecv (NgFlatArray s, int dest, int tag) - { - MPI_Request request; - MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, &request); - MPI_Request_free (&request); - } - */ - - - - /* - send a table entry to each of the processes in the group ... - receive-table entries will be set - */ - - template - [[deprecated("do we need that ? ")]] - inline void MyMPI_ExchangeTable (TABLE & send_data, - TABLE & recv_data, int tag, - const NgMPI_Comm & comm) - { - int rank = comm.Rank(); - int ntasks = comm.Size(); - - Array send_sizes(ntasks); - Array recv_sizes(ntasks); - for (int i = 0; i < ntasks; i++) - send_sizes[i] = send_data[i].Size(); - - comm.AllToAll (send_sizes, recv_sizes); - - for (int i = 0; i < ntasks; i++) - recv_data.SetEntrySize (i, recv_sizes[i], sizeof(T)); - - Array requests; - for (int dest = 0; dest < ntasks; dest++) - if (dest != rank && send_data[dest].Size()) - requests.Append (comm.ISend (FlatArray(send_data[dest]), dest, tag)); - - for (int dest = 0; dest < ntasks; dest++) - if (dest != rank && recv_data[dest].Size()) - requests.Append (comm.IRecv (FlatArray(recv_data[dest]), dest, tag)); - - MyMPI_WaitAll (requests); - } - - - template - [[deprecated("do we need that ? ")]] - inline void MyMPI_ExchangeTable (DynamicTable & send_data, - DynamicTable & recv_data, int tag, - const NgMPI_Comm & comm) - { - int rank = comm.Rank(); - int ntasks = comm.Size(); - - Array send_sizes(ntasks); - Array recv_sizes(ntasks); - for (int i = 0; i < ntasks; i++) - send_sizes[i] = send_data[i].Size(); - - comm.AllToAll (send_sizes, recv_sizes); - - // for (int i = 0; i < ntasks; i++) - // recv_data.SetEntrySize (i, recv_sizes[i], sizeof(T)); - recv_data = DynamicTable (recv_sizes, true); - - Array requests; - for (int dest = 0; dest < ntasks; dest++) - if (dest != rank && send_data[dest].Size()) - requests.Append (comm.ISend (FlatArray(send_data[dest]), dest, tag)); - - for (int dest = 0; dest < ntasks; dest++) - if (dest != rank && recv_data[dest].Size()) - requests.Append (comm.IRecv (FlatArray(recv_data[dest]), dest, tag)); - - MyMPI_WaitAll (requests); - } - - - - - [[deprecated("do we still send commands?")]] - DLL_HEADER void MyMPI_SendCmd (const char * cmd); - [[deprecated("do we still send commands?")]] - extern string MyMPI_RecvCmd (); - - - template - [[deprecated("use comm.BCast instead")]] - inline void MyMPI_Bcast (T & s, MPI_Comm comm) - { - MPI_Bcast (&s, 1, GetMPIType(), 0, comm); - } - - template - [[deprecated("use comm.BCast instead")]] - inline void MyMPI_Bcast (NgArray & s, NgMPI_Comm comm) - { - int size = s.Size(); - // MyMPI_Bcast (size, comm); - comm.Bcast(size); - // if (MyMPI_GetId(comm) != 0) s.SetSize (size); - if (comm.Rank() != 0) s.SetSize (size); - MPI_Bcast (&s[0], size, GetMPIType(), 0, comm); - } - - template - [[deprecated("use comm.BCast instead")]] - inline void MyMPI_Bcast (NgArray & s, int root, MPI_Comm comm) - { - int id; - MPI_Comm_rank(comm, &id); - - int size = s.Size(); - MPI_Bcast (&size, 1, MPI_INT, root, comm); - if (id != root) s.SetSize (size); - if ( !size ) return; - MPI_Bcast (&s[0], size, GetMPIType(), root, comm); - } - - template - [[deprecated("mympi_allgather deprecated, use comm.allgather")]] - inline void MyMPI_Allgather (const T & send, NgFlatArray recv, MPI_Comm comm) - { - MPI_Allgather( const_cast (&send), 1, GetMPIType(), &recv[0], 1, GetMPIType(), comm); - } - - template - [[deprecated("mympi_alltoall deprecated, use comm.alltoall")]] - inline void MyMPI_Alltoall (NgFlatArray send, NgFlatArray recv, MPI_Comm comm) - { - MPI_Alltoall( &send[0], 1, GetMPIType(), &recv[0], 1, GetMPIType(), comm); - } - - -#else - template - [[deprecated("do we need that ? ")]] - inline void MyMPI_ExchangeTable (TABLE & send_data, - TABLE & recv_data, int tag, - const NgMPI_Comm & comm) - { - ; - } - - template - [[deprecated("do we need that ? ")]] - inline void MyMPI_ExchangeTable (DynamicTable & send_data, - DynamicTable & recv_data, int tag, - const NgMPI_Comm & comm) - { ; } -#endif // PARALLEL - -} - -#endif - - -#endif diff --git a/libsrc/include/mystdlib.h b/libsrc/include/mystdlib.h index 17b498eb..b23a06ab 100644 --- a/libsrc/include/mystdlib.h +++ b/libsrc/include/mystdlib.h @@ -26,25 +26,6 @@ #include #include -#ifdef PARALLEL -// #undef SEEK_SET -// #undef SEEK_CUR -// #undef SEEK_END -#include -#include // for usleep (only for parallel) -#endif - - - -/* -#ifdef METIS -namespace metis { extern "C" { -#include -} } -#endif -*/ - - #ifndef M_PI #define M_PI 3.14159265358979323846 diff --git a/libsrc/include/nginterface_v2.hpp b/libsrc/include/nginterface_v2.hpp index 788f7395..5c48fe68 100644 --- a/libsrc/include/nginterface_v2.hpp +++ b/libsrc/include/nginterface_v2.hpp @@ -9,6 +9,7 @@ /**************************************************************************/ #include "mydefs.hpp" +#include /* C++ interface to Netgen diff --git a/libsrc/interface/nginterface.cpp b/libsrc/interface/nginterface.cpp index cf779603..4fb229d9 100644 --- a/libsrc/interface/nginterface.cpp +++ b/libsrc/interface/nginterface.cpp @@ -31,7 +31,7 @@ void RunParallel ( void * (*fun)(void *), void * in) #ifdef PARALLEL int provided; - MPI_Query_thread(&provided); + netgen::NG_MPI_Query_thread(&provided); if (provided < 3) if (netgen::ntasks > 1) parthread = false; // cout << "runparallel = " << parthread << endl; diff --git a/libsrc/meshing/curvedelems.cpp b/libsrc/meshing/curvedelems.cpp index e711ddb9..829aca7e 100644 --- a/libsrc/meshing/curvedelems.cpp +++ b/libsrc/meshing/curvedelems.cpp @@ -561,7 +561,7 @@ namespace netgen auto comm = mesh.GetCommunicator(); #ifdef PARALLEL - enum { MPI_TAG_CURVE = MPI_TAG_MESH+20 }; + enum { NG_MPI_TAG_CURVE = NG_MPI_TAG_MESH+20 }; const ParallelMeshTopology & partop = mesh.GetParallelTopology (); #endif int ntasks = comm.Size(); @@ -654,8 +654,8 @@ namespace netgen } if (ntasks > 1) - // MyMPI_ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE, comm); - comm.ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE); + // MyMPI_ExchangeTable (send_orders, recv_orders, NG_MPI_TAG_CURVE, comm); + comm.ExchangeTable (send_orders, recv_orders, NG_MPI_TAG_CURVE); if (ntasks > 1 && working) { @@ -770,8 +770,8 @@ namespace netgen } } - // MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm); - comm.ExchangeTable (senddata, recvdata, MPI_TAG_CURVE); + // MyMPI_ExchangeTable (senddata, recvdata, NG_MPI_TAG_CURVE, comm); + comm.ExchangeTable (senddata, recvdata, NG_MPI_TAG_CURVE); NgArray cnt(ntasks); cnt = 0; @@ -976,8 +976,8 @@ namespace netgen } } - // MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm); - comm.ExchangeTable (senddata, recvdata, MPI_TAG_CURVE); + // MyMPI_ExchangeTable (senddata, recvdata, NG_MPI_TAG_CURVE, comm); + comm.ExchangeTable (senddata, recvdata, NG_MPI_TAG_CURVE); NgArray cnt(ntasks); cnt = 0; @@ -1166,8 +1166,8 @@ namespace netgen } if (ntasks > 1) - // MyMPI_ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE, comm); - comm.ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE); + // MyMPI_ExchangeTable (send_surfnr, recv_surfnr, NG_MPI_TAG_CURVE, comm); + comm.ExchangeTable (send_surfnr, recv_surfnr, NG_MPI_TAG_CURVE); if (ntasks > 1 && working) { diff --git a/libsrc/meshing/meshclass.cpp b/libsrc/meshing/meshclass.cpp index af292c52..256e48a1 100644 --- a/libsrc/meshing/meshclass.cpp +++ b/libsrc/meshing/meshclass.cpp @@ -1710,7 +1710,7 @@ namespace netgen maxglob = max(globnum[pi], maxglob); } - maxglob = comm.AllReduce (maxglob, MPI_MAX); + maxglob = comm.AllReduce (maxglob, NG_MPI_MAX); int numglob = maxglob+1-PointIndex::BASE; if (comm.Rank() > 0) { diff --git a/libsrc/meshing/meshclass.hpp b/libsrc/meshing/meshclass.hpp index 134ca8bc..0c7fd57c 100644 --- a/libsrc/meshing/meshclass.hpp +++ b/libsrc/meshing/meshclass.hpp @@ -25,7 +25,7 @@ namespace netgen class NetgenGeometry; using namespace std; - static constexpr int MPI_TAG_MESH = 210; + static constexpr int NG_MPI_TAG_MESH = 210; enum resthtype { RESTRICTH_FACE, RESTRICTH_EDGE, diff --git a/libsrc/meshing/meshtype.cpp b/libsrc/meshing/meshtype.cpp index dfc8663a..5cd45321 100644 --- a/libsrc/meshing/meshtype.cpp +++ b/libsrc/meshing/meshtype.cpp @@ -18,125 +18,125 @@ namespace netgen #ifdef PARALLEL - MPI_Datatype MeshPoint :: MyGetMPIType ( ) + NG_MPI_Datatype MeshPoint :: MyGetMPIType ( ) { - static MPI_Datatype type = MPI_DATATYPE_NULL; - static MPI_Datatype htype = MPI_DATATYPE_NULL; - if (type == MPI_DATATYPE_NULL) + static NG_MPI_Datatype type = NG_MPI_DATATYPE_NULL; + static NG_MPI_Datatype htype = NG_MPI_DATATYPE_NULL; + if (type == NG_MPI_DATATYPE_NULL) { MeshPoint hp; int blocklen[] = { 3, 1, 1 }; - MPI_Aint displ[] = { (char*)&hp.x[0] - (char*)&hp, + NG_MPI_Aint displ[] = { (char*)&hp.x[0] - (char*)&hp, (char*)&hp.layer - (char*)&hp, (char*)&hp.singular - (char*)&hp }; - MPI_Datatype types[] = { MPI_DOUBLE, MPI_INT, MPI_DOUBLE }; + NG_MPI_Datatype types[] = { NG_MPI_DOUBLE, NG_MPI_INT, NG_MPI_DOUBLE }; // *testout << "displ = " << displ[0] << ", " << displ[1] << ", " << displ[2] << endl; // *testout << "sizeof = " << sizeof (MeshPoint) << endl; - MPI_Type_create_struct (3, blocklen, displ, types, &htype); - MPI_Type_commit ( &htype ); - MPI_Aint lb, ext; - MPI_Type_get_extent (htype, &lb, &ext); + NG_MPI_Type_create_struct (3, blocklen, displ, types, &htype); + NG_MPI_Type_commit ( &htype ); + NG_MPI_Aint lb, ext; + NG_MPI_Type_get_extent (htype, &lb, &ext); // *testout << "lb = " << lb << endl; // *testout << "ext = " << ext << endl; ext = sizeof (MeshPoint); - MPI_Type_create_resized (htype, lb, ext, &type); - MPI_Type_commit ( &type ); + NG_MPI_Type_create_resized (htype, lb, ext, &type); + NG_MPI_Type_commit ( &type ); } return type; } - MPI_Datatype Element2d :: MyGetMPIType ( ) + NG_MPI_Datatype Element2d :: MyGetMPIType ( ) { - static MPI_Datatype type = MPI_DATATYPE_NULL; - static MPI_Datatype htype = MPI_DATATYPE_NULL; - if (type == MPI_DATATYPE_NULL) + static NG_MPI_Datatype type = NG_MPI_DATATYPE_NULL; + static NG_MPI_Datatype htype = NG_MPI_DATATYPE_NULL; + if (type == NG_MPI_DATATYPE_NULL) { Element2d hel; int blocklen[] = { ELEMENT2D_MAXPOINTS, 1, 1, 1 }; - MPI_Aint displ[] = + NG_MPI_Aint displ[] = { (char*)&hel.pnum[0] - (char*)&hel, (char*)&hel.index - (char*)&hel, (char*)&hel.typ - (char*)&hel, (char*)&hel.np - (char*)&hel }; - MPI_Datatype types[] = { GetMPIType(), GetMPIType(hel.index), + NG_MPI_Datatype types[] = { GetMPIType(), GetMPIType(hel.index), GetMPIType(hel.typ), GetMPIType(hel.np) }; // *testout << "displ = " << displ[0] << ", " << displ[1] << ", " << displ[2] << endl; // *testout << "sizeof = " << sizeof (MeshPoint) << endl; - MPI_Type_create_struct (4, blocklen, displ, types, &htype); - MPI_Type_commit ( &htype ); - MPI_Aint lb, ext; - MPI_Type_get_extent (htype, &lb, &ext); + NG_MPI_Type_create_struct (4, blocklen, displ, types, &htype); + NG_MPI_Type_commit ( &htype ); + NG_MPI_Aint lb, ext; + NG_MPI_Type_get_extent (htype, &lb, &ext); // *testout << "lb = " << lb << endl; // *testout << "ext = " << ext << endl; ext = sizeof (Element2d); - MPI_Type_create_resized (htype, lb, ext, &type); - MPI_Type_commit ( &type ); + NG_MPI_Type_create_resized (htype, lb, ext, &type); + NG_MPI_Type_commit ( &type ); } return type; } - MPI_Datatype Element :: MyGetMPIType ( ) + NG_MPI_Datatype Element :: MyGetMPIType ( ) { - static MPI_Datatype type = MPI_DATATYPE_NULL; - static MPI_Datatype htype = MPI_DATATYPE_NULL; - if (type == MPI_DATATYPE_NULL) + static NG_MPI_Datatype type = NG_MPI_DATATYPE_NULL; + static NG_MPI_Datatype htype = NG_MPI_DATATYPE_NULL; + if (type == NG_MPI_DATATYPE_NULL) { Element hel; int blocklen[] = { ELEMENT_MAXPOINTS, 1, 1, 1 }; - MPI_Aint displ[] = + NG_MPI_Aint displ[] = { (char*)&hel.pnum[0] - (char*)&hel, (char*)&hel.index - (char*)&hel, (char*)&hel.typ - (char*)&hel, (char*)&hel.np - (char*)&hel }; - MPI_Datatype types[] = { GetMPIType(), GetMPIType(hel.index), + NG_MPI_Datatype types[] = { GetMPIType(), GetMPIType(hel.index), GetMPIType(hel.typ), GetMPIType(hel.np) }; // *testout << "displ = " << displ[0] << ", " << displ[1] << ", " << displ[2] << endl; // *testout << "sizeof = " << sizeof (MeshPoint) << endl; - MPI_Type_create_struct (4, blocklen, displ, types, &htype); - MPI_Type_commit ( &htype ); - MPI_Aint lb, ext; - MPI_Type_get_extent (htype, &lb, &ext); + NG_MPI_Type_create_struct (4, blocklen, displ, types, &htype); + NG_MPI_Type_commit ( &htype ); + NG_MPI_Aint lb, ext; + NG_MPI_Type_get_extent (htype, &lb, &ext); // *testout << "lb = " << lb << endl; // *testout << "ext = " << ext << endl; ext = sizeof (Element); - MPI_Type_create_resized (htype, lb, ext, &type); - MPI_Type_commit ( &type ); + NG_MPI_Type_create_resized (htype, lb, ext, &type); + NG_MPI_Type_commit ( &type ); } return type; } - MPI_Datatype Segment :: MyGetMPIType ( ) + NG_MPI_Datatype Segment :: MyGetMPIType ( ) { - static MPI_Datatype type = MPI_DATATYPE_NULL; - static MPI_Datatype htype = MPI_DATATYPE_NULL; - if (type == MPI_DATATYPE_NULL) + static NG_MPI_Datatype type = NG_MPI_DATATYPE_NULL; + static NG_MPI_Datatype htype = NG_MPI_DATATYPE_NULL; + if (type == NG_MPI_DATATYPE_NULL) { Segment hel; int blocklen[] = { 3, 1, 1, 1 }; - MPI_Aint displ[] = + NG_MPI_Aint displ[] = { (char*)&hel.pnums[0] - (char*)&hel, (char*)&hel.edgenr - (char*)&hel, (char*)&hel.cd2i - (char*)&hel, (char*)&hel.si - (char*)&hel }; - MPI_Datatype types[] = { + NG_MPI_Datatype types[] = { GetMPIType(), GetMPIType(hel.edgenr), GetMPIType(hel.cd2i), GetMPIType(hel.si) }; // *testout << "displ = " << displ[0] << ", " << displ[1] << ", " << displ[2] << endl; // *testout << "sizeof = " << sizeof (MeshPoint) << endl; - MPI_Type_create_struct (4, blocklen, displ, types, &htype); - MPI_Type_commit ( &htype ); - MPI_Aint lb, ext; - MPI_Type_get_extent (htype, &lb, &ext); + NG_MPI_Type_create_struct (4, blocklen, displ, types, &htype); + NG_MPI_Type_commit ( &htype ); + NG_MPI_Aint lb, ext; + NG_MPI_Type_get_extent (htype, &lb, &ext); // *testout << "lb = " << lb << endl; // *testout << "ext = " << ext << endl; ext = sizeof (Segment); - MPI_Type_create_resized (htype, lb, ext, &type); - MPI_Type_commit ( &type ); + NG_MPI_Type_create_resized (htype, lb, ext, &type); + NG_MPI_Type_commit ( &type ); } return type; } diff --git a/libsrc/meshing/meshtype.hpp b/libsrc/meshing/meshtype.hpp index c9528c61..d765980f 100644 --- a/libsrc/meshing/meshtype.hpp +++ b/libsrc/meshing/meshtype.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -372,7 +373,7 @@ namespace netgen bool IsSingular() const { return (singular != 0.0); } #ifdef PARALLEL - static MPI_Datatype MyGetMPIType ( ); + static NG_MPI_Datatype MyGetMPIType ( ); #endif void DoArchive (Archive & ar) @@ -583,7 +584,7 @@ namespace netgen } #ifdef PARALLEL - static MPI_Datatype MyGetMPIType(); + static NG_MPI_Datatype MyGetMPIType(); #endif @@ -886,7 +887,7 @@ namespace netgen } #ifdef PARALLEL - static MPI_Datatype MyGetMPIType(); + static NG_MPI_Datatype MyGetMPIType(); #endif /// @@ -1138,7 +1139,7 @@ namespace netgen void DoArchive (Archive & ar); #ifdef PARALLEL - static MPI_Datatype MyGetMPIType(); + static NG_MPI_Datatype MyGetMPIType(); #endif }; @@ -1642,25 +1643,25 @@ namespace netgen namespace ngcore { template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_INT; } + static NG_MPI_Datatype MPIType () { return NG_MPI_INT; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return MPI_CHAR; } + static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return netgen::MeshPoint::MyGetMPIType(); } + static NG_MPI_Datatype MPIType () { return netgen::MeshPoint::MyGetMPIType(); } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return netgen::Element::MyGetMPIType(); } + static NG_MPI_Datatype MPIType () { return netgen::Element::MyGetMPIType(); } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return netgen::Element2d::MyGetMPIType(); } + static NG_MPI_Datatype MPIType () { return netgen::Element2d::MyGetMPIType(); } }; template <> struct MPI_typetrait { - static MPI_Datatype MPIType () { return netgen::Segment::MyGetMPIType(); } + static NG_MPI_Datatype MPIType () { return netgen::Segment::MyGetMPIType(); } }; } diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 37d3e941..997eaad5 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -53,15 +53,15 @@ namespace ngcore }; // class SurfPointPackage template<> struct MPI_typetrait { - static MPI_Datatype MPIType () { - static MPI_Datatype MPI_T = 0; + static NG_MPI_Datatype MPIType () { + static NG_MPI_Datatype MPI_T = 0; if (!MPI_T) { int block_len[2] = { 2, 2 }; - MPI_Aint displs[3] = { 0, 2*sizeof(int) }; - MPI_Datatype types[2] = { MPI_INT, MPI_DOUBLE }; - MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); - MPI_Type_commit(&MPI_T); + NG_MPI_Aint displs[3] = { 0, 2*sizeof(int) }; + NG_MPI_Datatype types[2] = { NG_MPI_INT, NG_MPI_DOUBLE }; + NG_MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); + NG_MPI_Type_commit(&MPI_T); } return MPI_T; } @@ -119,15 +119,15 @@ namespace ngcore }; // class SelPackage template<> struct MPI_typetrait { - static MPI_Datatype MPIType () { - static MPI_Datatype MPI_T = 0; + static NG_MPI_Datatype MPIType () { + static NG_MPI_Datatype MPI_T = 0; if (!MPI_T) { int block_len[2] = { 3, ELEMENT2D_MAXPOINTS }; - MPI_Aint displs[3] = { 0, 3*sizeof(int) }; - MPI_Datatype types[2] = { MPI_INT, GetMPIType() }; - MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); - MPI_Type_commit(&MPI_T); + NG_MPI_Aint displs[3] = { 0, 3*sizeof(int) }; + NG_MPI_Datatype types[2] = { NG_MPI_INT, GetMPIType() }; + NG_MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); + NG_MPI_Type_commit(&MPI_T); } return MPI_T; } @@ -145,15 +145,15 @@ namespace ngcore }; // class PointElPackage template<> struct MPI_typetrait { - static MPI_Datatype MPIType () { - static MPI_Datatype MPI_T = 0; + static NG_MPI_Datatype MPIType () { + static NG_MPI_Datatype MPI_T = 0; if (!MPI_T) { int block_len[2] = { 1, 1 }; - MPI_Aint displs[3] = { 0, sizeof(netgen::PointIndex) }; - MPI_Datatype types[2] = { GetMPIType(), MPI_INT }; - MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); - MPI_Type_commit(&MPI_T); + NG_MPI_Aint displs[3] = { 0, sizeof(netgen::PointIndex) }; + NG_MPI_Datatype types[2] = { GetMPIType(), NG_MPI_INT }; + NG_MPI_Type_create_struct(2, block_len, displs, types, &MPI_T); + NG_MPI_Type_commit(&MPI_T); } return MPI_T; } @@ -222,7 +222,7 @@ namespace netgen int dim = GetDimension(); comm.Bcast(dim); - Array sendrequests(8*(ntasks-1)); + Array sendrequests(8*(ntasks-1)); sendrequests.SetSize0(); // If the topology is not already updated, we do not need to @@ -452,27 +452,27 @@ namespace netgen tbuildvertex.Stop(); PrintMessage ( 3, "Sending Vertices - vertices"); - Array point_types(ntasks-1); + Array point_types(ntasks-1); for (int dest = 1; dest < ntasks; dest++) { NgFlatArray verts = verts_of_proc[dest]; - // sendrequests.Append (MyMPI_ISend (verts, dest, MPI_TAG_MESH+1, comm)); - sendrequests.Append (comm.ISend (FlatArray(verts), dest, MPI_TAG_MESH+1)); + // sendrequests.Append (MyMPI_ISend (verts, dest, NG_MPI_TAG_MESH+1, comm)); + sendrequests.Append (comm.ISend (FlatArray(verts), dest, NG_MPI_TAG_MESH+1)); - MPI_Datatype mptype = MeshPoint::MyGetMPIType(); + NG_MPI_Datatype mptype = MeshPoint::MyGetMPIType(); int numv = verts.Size(); NgArray blocklen (numv); blocklen = 1; - MPI_Type_indexed (numv, (numv == 0) ? nullptr : &blocklen[0], + NG_MPI_Type_indexed (numv, (numv == 0) ? nullptr : &blocklen[0], (numv == 0) ? nullptr : reinterpret_cast (&verts[0]), mptype, &point_types[dest-1]); - MPI_Type_commit (&point_types[dest-1]); + NG_MPI_Type_commit (&point_types[dest-1]); - MPI_Request request; - MPI_Isend( points.Data(), 1, point_types[dest-1], dest, MPI_TAG_MESH+1, comm, &request); + NG_MPI_Request request; + NG_MPI_Isend( points.Data(), 1, point_types[dest-1], dest, NG_MPI_TAG_MESH+1, comm, &request); sendrequests.Append (request); } @@ -533,10 +533,10 @@ namespace netgen } } } - Array req_per; + Array req_per; for(int dest = 1; dest < ntasks; dest++) - // req_per.Append(MyMPI_ISend(pp_data[dest], dest, MPI_TAG_MESH+1, comm)); - req_per.Append(comm.ISend(FlatArray(pp_data[dest]), dest, MPI_TAG_MESH+1)); + // req_per.Append(MyMPI_ISend(pp_data[dest], dest, NG_MPI_TAG_MESH+1, comm)); + req_per.Append(comm.ISend(FlatArray(pp_data[dest]), dest, NG_MPI_TAG_MESH+1)); MyMPI_WaitAll(req_per); PrintMessage ( 3, "Sending Vertices - distprocs"); @@ -570,7 +570,7 @@ namespace netgen tbuilddistpnums.Stop(); for ( int dest = 1; dest < ntasks; dest ++ ) - sendrequests.Append (comm.ISend (distpnums[dest], dest, MPI_TAG_MESH+1)); + sendrequests.Append (comm.ISend (distpnums[dest], dest, NG_MPI_TAG_MESH+1)); @@ -604,7 +604,7 @@ namespace netgen tbuildelementtable.Stop(); for (int dest = 1; dest < ntasks; dest ++ ) - sendrequests.Append (comm.ISend (elementarrays[dest], dest, MPI_TAG_MESH+2)); + sendrequests.Append (comm.ISend (elementarrays[dest], dest, NG_MPI_TAG_MESH+2)); PrintMessage ( 3, "Sending Face Descriptors" ); @@ -621,7 +621,7 @@ namespace netgen } for (int dest = 1; dest < ntasks; dest++) - sendrequests.Append (comm.ISend (fddata, dest, MPI_TAG_MESH+3)); + sendrequests.Append (comm.ISend (fddata, dest, NG_MPI_TAG_MESH+3)); /** Surface Elements **/ @@ -697,7 +697,7 @@ namespace netgen }); // distribute sel data for (int dest = 1; dest < ntasks; dest++) - sendrequests.Append (comm.ISend(selbuf[dest], dest, MPI_TAG_MESH+4)); + sendrequests.Append (comm.ISend(selbuf[dest], dest, NG_MPI_TAG_MESH+4)); /** Segments **/ @@ -849,7 +849,7 @@ namespace netgen }); // distribute segment data for (int dest = 1; dest < ntasks; dest++) - sendrequests.Append (comm.ISend(segm_buf[dest], dest, MPI_TAG_MESH+5)); + sendrequests.Append (comm.ISend(segm_buf[dest], dest, NG_MPI_TAG_MESH+5)); /** Point-Elements **/ PrintMessage ( 3, "Point-Elements ..."); @@ -870,7 +870,7 @@ namespace netgen iterate_zdes([&](const auto & pack, auto dest) { zde_buf.Add(dest, pack); }); for (int dest = 1; dest < ntasks; dest++) - { sendrequests.Append (comm.ISend(zde_buf[dest], dest, MPI_TAG_MESH+6)); } + { sendrequests.Append (comm.ISend(zde_buf[dest], dest, NG_MPI_TAG_MESH+6)); } PrintMessage ( 3, "now wait ..."); @@ -878,7 +878,7 @@ namespace netgen // clean up MPI-datatypes we allocated earlier for (auto t : point_types) - { MPI_Type_free(&t); } + { NG_MPI_Type_free(&t); } paralleltop -> SetNV_Loc2Glob (0); paralleltop -> SetNV (0); @@ -895,8 +895,8 @@ namespace netgen nnames[3] = GetNCD3Names(); int tot_nn = nnames[0] + nnames[1] + nnames[2] + nnames[3]; for( int k = 1; k < ntasks; k++) - sendrequests[k] = comm.ISend(nnames, k, MPI_TAG_MESH+7); - // (void) MPI_Isend(nnames, 4, MPI_INT, k, MPI_TAG_MESH+6, comm, &sendrequests[k]); + sendrequests[k] = comm.ISend(nnames, k, NG_MPI_TAG_MESH+7); + // (void) NG_MPI_Isend(nnames, 4, NG_MPI_INT, k, NG_MPI_TAG_MESH+6, comm, &sendrequests[k]); auto iterate_names = [&](auto func) { for (int k = 0; k < nnames[0]; k++) func(materials[k]); for (int k = 0; k < nnames[1]; k++) func(bcnames[k]); @@ -908,7 +908,7 @@ namespace netgen tot_nn = 0; iterate_names([&](auto ptr) { name_sizes[tot_nn++] = (ptr==NULL) ? 0 : ptr->size(); }); for( int k = 1; k < ntasks; k++) - (void) MPI_Isend(&name_sizes[0], tot_nn, MPI_INT, k, MPI_TAG_MESH+7, comm, &sendrequests[ntasks+k]); + (void) NG_MPI_Isend(&name_sizes[0], tot_nn, NG_MPI_INT, k, NG_MPI_TAG_MESH+7, comm, &sendrequests[ntasks+k]); // names int strs = 0; iterate_names([&](auto ptr) { strs += (ptr==NULL) ? 0 : ptr->size(); }); @@ -920,7 +920,7 @@ namespace netgen for (int j=0; j < name.size(); j++) compiled_names[strs++] = name[j]; }); for( int k = 1; k < ntasks; k++) - (void) MPI_Isend(&(compiled_names[0]), strs, MPI_CHAR, k, MPI_TAG_MESH+7, comm, &sendrequests[2*ntasks+k]); + (void) NG_MPI_Isend(&(compiled_names[0]), strs, NG_MPI_CHAR, k, NG_MPI_TAG_MESH+7, comm, &sendrequests[2*ntasks+k]); PrintMessage ( 3, "wait for names"); @@ -1006,7 +1006,7 @@ namespace netgen timer_pts.Start(); Array verts; - comm.Recv (verts, 0, MPI_TAG_MESH+1); + comm.Recv (verts, 0, NG_MPI_TAG_MESH+1); int numvert = verts.Size(); paralleltop -> SetNV (numvert); @@ -1026,12 +1026,12 @@ namespace netgen for (int i = 0; i < numvert; i++) AddPoint (netgen::Point<3> (0,0,0)); - MPI_Datatype mptype = MeshPoint::MyGetMPIType(); - MPI_Status status; - MPI_Recv( points.Data(), numvert, mptype, 0, MPI_TAG_MESH+1, comm, &status); + NG_MPI_Datatype mptype = MeshPoint::MyGetMPIType(); + NG_MPI_Status status; + NG_MPI_Recv( points.Data(), numvert, mptype, 0, NG_MPI_TAG_MESH+1, comm, &status); Array pp_data; - comm.Recv(pp_data, 0, MPI_TAG_MESH+1); + comm.Recv(pp_data, 0, NG_MPI_TAG_MESH+1); int maxidentnr = pp_data[0]; auto & idents = GetIdentifications(); @@ -1052,7 +1052,7 @@ namespace netgen } Array dist_pnums; - comm.Recv (dist_pnums, 0, MPI_TAG_MESH+1); + comm.Recv (dist_pnums, 0, NG_MPI_TAG_MESH+1); for (int hi = 0; hi < dist_pnums.Size(); hi += 3) paralleltop -> @@ -1067,7 +1067,7 @@ namespace netgen RegionTimer reg(timer_els); Array elarray; - comm.Recv (elarray, 0, MPI_TAG_MESH+2); + comm.Recv (elarray, 0, NG_MPI_TAG_MESH+2); for (int ind = 0, elnum = 1; ind < elarray.Size(); elnum++) { @@ -1086,7 +1086,7 @@ namespace netgen { Array fddata; - comm.Recv (fddata, 0, MPI_TAG_MESH+3); + comm.Recv (fddata, 0, NG_MPI_TAG_MESH+3); for (int i = 0; i < fddata.Size(); i += 6) { int faceind = AddFaceDescriptor @@ -1101,7 +1101,7 @@ namespace netgen RegionTimer reg(timer_sels); Array selbuf; - comm.Recv ( selbuf, 0, MPI_TAG_MESH+4); + comm.Recv ( selbuf, 0, NG_MPI_TAG_MESH+4); int nlocsel = selbuf.Size(); paralleltop -> SetNSE ( nlocsel ); @@ -1124,9 +1124,9 @@ namespace netgen { // NgArray segmbuf; - // MyMPI_Recv ( segmbuf, 0, MPI_TAG_MESH+5, comm); + // MyMPI_Recv ( segmbuf, 0, NG_MPI_TAG_MESH+5, comm); Array segmbuf; - comm.Recv (segmbuf, 0, MPI_TAG_MESH+5); + comm.Recv (segmbuf, 0, NG_MPI_TAG_MESH+5); Segment seg; int globsegi; @@ -1170,7 +1170,7 @@ namespace netgen { /** 0d-Elements **/ Array zdes; - comm.Recv ( zdes, 0, MPI_TAG_MESH+6); + comm.Recv ( zdes, 0, NG_MPI_TAG_MESH+6); pointelements.SetSize(zdes.Size()); for (auto k : Range(pointelements)) { auto & el = pointelements[k]; @@ -1183,8 +1183,8 @@ namespace netgen paralleltop -> EnumeratePointsGlobally(); /** Recv bc-names **/ ArrayMem nnames{0,0,0,0}; - // MPI_Recv(nnames, 4, MPI_INT, 0, MPI_TAG_MESH+6, comm, MPI_STATUS_IGNORE); - comm.Recv(nnames, 0, MPI_TAG_MESH+7); + // NG_MPI_Recv(nnames, 4, NG_MPI_INT, 0, NG_MPI_TAG_MESH+6, comm, NG_MPI_STATUS_IGNORE); + comm.Recv(nnames, 0, NG_MPI_TAG_MESH+7); // cout << "nnames = " << FlatArray(nnames) << endl; materials.SetSize(nnames[0]); bcnames.SetSize(nnames[1]); @@ -1193,12 +1193,12 @@ namespace netgen int tot_nn = nnames[0] + nnames[1] + nnames[2] + nnames[3]; NgArray name_sizes(tot_nn); - MPI_Recv(&name_sizes[0], tot_nn, MPI_INT, 0, MPI_TAG_MESH+7, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv(&name_sizes[0], tot_nn, NG_MPI_INT, 0, NG_MPI_TAG_MESH+7, comm, NG_MPI_STATUS_IGNORE); int tot_size = 0; for (int k = 0; k < tot_nn; k++) tot_size += name_sizes[k]; NgArray compiled_names(tot_size); - MPI_Recv(&(compiled_names[0]), tot_size, MPI_CHAR, 0, MPI_TAG_MESH+7, comm, MPI_STATUS_IGNORE); + NG_MPI_Recv(&(compiled_names[0]), tot_size, NG_MPI_CHAR, 0, NG_MPI_TAG_MESH+7, comm, NG_MPI_STATUS_IGNORE); tot_nn = tot_size = 0; auto write_names = [&] (auto & array) { diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index 88c0ab39..155aba06 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -94,8 +94,8 @@ namespace netgen Array first_master_point(comm.Size()); comm.AllGather (num_master_points, first_master_point); - auto max_oldv = comm.AllReduce (Max (glob_vert.Range(0, oldnv)), MPI_MAX); - if (comm.AllReduce (oldnv, MPI_SUM) == 0) + auto max_oldv = comm.AllReduce (Max (glob_vert.Range(0, oldnv)), NG_MPI_MAX); + if (comm.AllReduce (oldnv, NG_MPI_SUM) == 0) max_oldv = PointIndex::BASE-1; size_t num_glob_points = max_oldv+1; @@ -138,7 +138,7 @@ namespace netgen for (auto p : dps) send_data[p][nsend[p]++] = L2G(pi); - Array requests; + Array requests; for (int i = 0; i < comm.Size(); i++) { if (nsend[i]) @@ -342,10 +342,10 @@ namespace netgen sendarray.Append (topology.GetSurfaceElementFace (el)); } - Array sendrequests; + Array sendrequests; for (int dest = 1; dest < ntasks; dest++) - // sendrequests.Append (MyMPI_ISend (*sendarrays[dest], dest, MPI_TAG_MESH+10, comm)); - sendrequests.Append (comm.ISend (FlatArray(*sendarrays[dest]), dest, MPI_TAG_MESH+10)); + // sendrequests.Append (MyMPI_ISend (*sendarrays[dest], dest, NG_MPI_TAG_MESH+10, comm)); + sendrequests.Append (comm.ISend (FlatArray(*sendarrays[dest]), dest, NG_MPI_TAG_MESH+10)); MyMPI_WaitAll (sendrequests); for (int dest = 1; dest < ntasks; dest++) @@ -356,9 +356,9 @@ namespace netgen { // NgArray recvarray; - // MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10, comm); + // MyMPI_Recv (recvarray, 0, NG_MPI_TAG_MESH+10, comm); Array recvarray; - comm.Recv (recvarray, 0, MPI_TAG_MESH+10); // MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10, comm); + comm.Recv (recvarray, 0, NG_MPI_TAG_MESH+10); // MyMPI_Recv (recvarray, 0, NG_MPI_TAG_MESH+10, comm); int ii = 0; @@ -413,7 +413,7 @@ namespace netgen Array cnt_send(ntasks); - int maxsize = comm.AllReduce (mesh.mlbetweennodes.Size(), MPI_MAX); + int maxsize = comm.AllReduce (mesh.mlbetweennodes.Size(), NG_MPI_MAX); // update new vertices after mesh-refinement if (maxsize > 0) { @@ -500,8 +500,8 @@ namespace netgen } DynamicTable recv_verts(ntasks); - // MyMPI_ExchangeTable (send_verts, recv_verts, MPI_TAG_MESH+9, comm); - comm.ExchangeTable (send_verts, recv_verts, MPI_TAG_MESH+9); + // MyMPI_ExchangeTable (send_verts, recv_verts, NG_MPI_TAG_MESH+9, comm); + comm.ExchangeTable (send_verts, recv_verts, NG_MPI_TAG_MESH+9); for (int dest = 0; dest < ntasks; dest++) if (dest != id) @@ -533,7 +533,7 @@ namespace netgen } } - changed = comm.AllReduce (changed, MPI_LOR); + changed = comm.AllReduce (changed, NG_MPI_LOR); } } @@ -558,8 +558,8 @@ namespace netgen for (int dist : GetDistantProcs(pi)) dest2vert.Add (dist, pi); - // MPI_Group_free(&MPI_LocalGroup); - // MPI_Comm_free(&MPI_LocalComm); + // NG_MPI_Group_free(&NG_MPI_LocalGroup); + // NG_MPI_Comm_free(&NG_MPI_LocalComm); } @@ -696,8 +696,8 @@ namespace netgen // cout << "UpdateCoarseGrid - edges mpi-exchange" << endl; // TABLE recv_edges(ntasks); DynamicTable recv_edges(ntasks); - // MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, comm); - comm.ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9); + // MyMPI_ExchangeTable (send_edges, recv_edges, NG_MPI_TAG_MESH+9, comm); + comm.ExchangeTable (send_edges, recv_edges, NG_MPI_TAG_MESH+9); // cout << "UpdateCoarseGrid - edges mpi-exchange done" << endl; for (int dest = 0; dest < ntasks; dest++) @@ -806,8 +806,8 @@ namespace netgen // cout << "UpdateCoarseGrid - faces mpi-exchange" << endl; // TABLE recv_faces(ntasks); DynamicTable recv_faces(ntasks); - // MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+9, comm); - comm.ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+9); + // MyMPI_ExchangeTable (send_faces, recv_faces, NG_MPI_TAG_MESH+9, comm); + comm.ExchangeTable (send_faces, recv_faces, NG_MPI_TAG_MESH+9); // cout << "UpdateCoarseGrid - faces mpi-exchange done" << endl; for (int dest = 0; dest < ntasks; dest++) @@ -839,8 +839,8 @@ namespace netgen // EnumeratePointsGlobally(); is_updated = true; - // MPI_Group_free(&MPI_LocalGroup); - // MPI_Comm_free(&MPI_LocalComm); + // NG_MPI_Group_free(&NG_MPI_LocalGroup); + // NG_MPI_Comm_free(&NG_MPI_LocalComm); } } diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index 15a29c47..f93ed62f 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -1,3 +1,4 @@ +#include "pybind11/pytypes.h" #ifdef NG_PYTHON #include @@ -24,52 +25,6 @@ public: }; -#ifdef NG_MPI4PY -#include - -struct mpi4py_comm { - mpi4py_comm() = default; - mpi4py_comm(MPI_Comm value) : value(value) {} - operator MPI_Comm () { return value; } - - MPI_Comm value; -}; - -namespace pybind11 { namespace detail { - template <> struct type_caster { - public: - PYBIND11_TYPE_CASTER(mpi4py_comm, _("mpi4py_comm")); - - // Python -> C++ - bool load(handle src, bool) { - PyObject *py_src = src.ptr(); - // Check that we have been passed an mpi4py communicator - if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { - // Convert to regular MPI communicator - value.value = *PyMPIComm_Get(py_src); - } else { - return false; - } - - return !PyErr_Occurred(); - } - - // C++ -> Python - static handle cast(mpi4py_comm src, - return_value_policy /* policy */, - handle /* parent */) - { - // Create an mpi4py handle - return PyMPIComm_New(src.value); - } - }; -}} // namespace pybind11::detail - -#endif // NG_MPI4PY - - - - using namespace netgen; @@ -101,9 +56,6 @@ static Transformation<3> global_trafo(Vec<3> (0,0,0)); DLL_HEADER void ExportNetgenMeshing(py::module &m) { -#ifdef NG_MPI4PY - import_mpi4py(); -#endif // NG_MPI4PY py::register_exception(m, "NgException"); m.attr("_netgen_executable_started") = py::cast(netgen::netgen_executable_started); string script; @@ -131,47 +83,6 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) py::implicitly_convertible(); - py::class_ (m, "MPI_Comm") -#ifdef NG_MPI4PY - .def(py::init([] (mpi4py_comm comm) - { - return NgMPI_Comm(comm); - })) - .def_property_readonly ("mpi4py", [] (NgMPI_Comm comm) { return mpi4py_comm(comm); }) -#endif // NG_MPI4PY - .def_property_readonly ("rank", &NgMPI_Comm::Rank) - .def_property_readonly ("size", &NgMPI_Comm::Size) - .def("Barrier", &NgMPI_Comm::Barrier) - -#ifdef PARALLEL - .def("WTime", [](NgMPI_Comm & c) { return MPI_Wtime(); }) -#else - .def("WTime", [](NgMPI_Comm & c) { return -1.0; }) -#endif - .def("Sum", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, MPI_SUM); }) - .def("Min", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, MPI_MIN); }) - .def("Max", [](NgMPI_Comm & c, double x) { return c.AllReduce(x, MPI_MAX); }) - .def("Sum", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, MPI_SUM); }) - .def("Min", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, MPI_MIN); }) - .def("Max", [](NgMPI_Comm & c, int x) { return c.AllReduce(x, MPI_MAX); }) - .def("Sum", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, MPI_SUM); }) - .def("Min", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, MPI_MIN); }) - .def("Max", [](NgMPI_Comm & c, size_t x) { return c.AllReduce(x, MPI_MAX); }) - .def("SubComm", [](NgMPI_Comm & c, std::vector proc_list) { - Array procs(proc_list.size()); - for (int i = 0; i < procs.Size(); i++) - { procs[i] = proc_list[i]; } - if (!procs.Contains(c.Rank())) - { throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); } - return c.SubCommunicator(procs); - }, py::arg("procs")); - ; - - -#ifdef NG_MPI4PY - py::implicitly_convertible(); -#endif // NG_MPI4PY - py::class_(m, "NGDummyArgument") .def("__bool__", []( NGDummyArgument &self ) { return false; } ) diff --git a/libsrc/visualization/CMakeLists.txt b/libsrc/visualization/CMakeLists.txt index d1b8b184..1e7a2938 100644 --- a/libsrc/visualization/CMakeLists.txt +++ b/libsrc/visualization/CMakeLists.txt @@ -7,7 +7,7 @@ target_sources(nggui PRIVATE vssolution.cpp visualpkg.cpp ) -target_link_libraries( nggui PUBLIC "$" ${MPI_CXX_LIBRARIES} ${OPENGL_LIBRARIES} nglib) +target_link_libraries( nggui PUBLIC "$" ${OPENGL_LIBRARIES} nglib) install(FILES meshdoc.hpp mvdraw.hpp visual_api.hpp diff --git a/libsrc/visualization/vssolution.cpp b/libsrc/visualization/vssolution.cpp index 84cd31ee..16dd17d0 100644 --- a/libsrc/visualization/vssolution.cpp +++ b/libsrc/visualization/vssolution.cpp @@ -1229,7 +1229,7 @@ namespace netgen MyMPI_SendCmd ("solsurfellist"); for ( int dest = 1; dest < ntasks; dest++ ) - MyMPI_Recv (par_surfellists[dest], dest, MPI_TAG_VIS); + MyMPI_Recv (par_surfellists[dest], dest, NG_MPI_TAG_VIS); if (surfellist) glDeleteLists (surfellist, 1); @@ -1760,7 +1760,7 @@ namespace netgen #ifdef PARALLELGL glFinish(); if (id > 0) - MyMPI_Send (surfellist, 0, MPI_TAG_VIS); + MyMPI_Send (surfellist, 0, NG_MPI_TAG_VIS); #endif } @@ -1780,7 +1780,7 @@ namespace netgen MyMPI_SendCmd ("solsurfellinelist"); for ( int dest = 1; dest < ntasks; dest++ ) - MyMPI_Recv (par_surfellists[dest], dest, MPI_TAG_VIS); + MyMPI_Recv (par_surfellists[dest], dest, NG_MPI_TAG_VIS); if (linelist) glDeleteLists (linelist, 1); @@ -1864,7 +1864,7 @@ namespace netgen #ifdef PARALLELGL glFinish(); if (id > 0) - MyMPI_Send (linelist, 0, MPI_TAG_VIS); + MyMPI_Send (linelist, 0, NG_MPI_TAG_VIS); #endif } @@ -2725,8 +2725,8 @@ namespace netgen if (ntasks > 1) { double hmin, hmax; - MPI_Reduce (&minv, &hmin, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); - MPI_Reduce (&maxv, &hmax, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + NG_MPI_Reduce (&minv, &hmin, 1, NG_MPI_DOUBLE, NG_MPI_MIN, 0, NG_MPI_COMM_WORLD); + NG_MPI_Reduce (&maxv, &hmax, 1, NG_MPI_DOUBLE, NG_MPI_MAX, 0, NG_MPI_COMM_WORLD); minv = hmin; maxv = hmax; } @@ -4370,7 +4370,7 @@ namespace netgen MyMPI_SendCmd ("clipplanetrigs"); for ( int dest = 1; dest < ntasks; dest++ ) - MyMPI_Recv (parlists[dest], dest, MPI_TAG_VIS); + MyMPI_Recv (parlists[dest], dest, NG_MPI_TAG_VIS); if (clipplanelist_scal) glDeleteLists (clipplanelist_scal, 1); @@ -4515,7 +4515,7 @@ namespace netgen #ifdef PARALLELGLGL glFinish(); if (id > 0) - MyMPI_Send (clipplanelist_scal, 0, MPI_TAG_VIS); + MyMPI_Send (clipplanelist_scal, 0, NG_MPI_TAG_VIS); #endif } @@ -4928,7 +4928,7 @@ namespace netgen void VisualSceneSolution :: Broadcast () { - MPI_Datatype type; + NG_MPI_Datatype type; int blocklen[] = { 1, 1, 1, 1, @@ -4937,7 +4937,7 @@ namespace netgen 1, 4, 1, 1, 1 }; - MPI_Aint displ[] = { (char*)&usetexture - (char*)this, + NG_MPI_Aint displ[] = { (char*)&usetexture - (char*)this, (char*)&clipsolution - (char*)this, (char*)&scalfunction - (char*)this, (char*)&scalcomp - (char*)this, @@ -4961,19 +4961,19 @@ namespace netgen }; - MPI_Datatype types[] = { - MPI_INT, MPI_INT, MPI_INT, MPI_INT, - MPI_INT, MPI_INT, MPI_INT, MPI_INT, - MPI_DOUBLE, MPI_DOUBLE, MPI_INT, MPI_INT, - MPI_INT, MPI_DOUBLE, MPI_INT, MPI_INT, - MPI_DOUBLE + NG_MPI_Datatype types[] = { + NG_MPI_INT, NG_MPI_INT, NG_MPI_INT, NG_MPI_INT, + NG_MPI_INT, NG_MPI_INT, NG_MPI_INT, NG_MPI_INT, + NG_MPI_DOUBLE, NG_MPI_DOUBLE, NG_MPI_INT, NG_MPI_INT, + NG_MPI_INT, NG_MPI_DOUBLE, NG_MPI_INT, NG_MPI_INT, + NG_MPI_DOUBLE }; - MPI_Type_create_struct (17, blocklen, displ, types, &type); - MPI_Type_commit ( &type ); + NG_MPI_Type_create_struct (17, blocklen, displ, types, &type); + NG_MPI_Type_commit ( &type ); - MPI_Bcast (this, 1, type, 0, MPI_COMM_WORLD); - MPI_Type_free (&type); + NG_MPI_Bcast (this, 1, type, 0, NG_MPI_COMM_WORLD); + NG_MPI_Type_free (&type); } #endif diff --git a/ng/CMakeLists.txt b/ng/CMakeLists.txt index 2acecf34..b5a1985b 100644 --- a/ng/CMakeLists.txt +++ b/ng/CMakeLists.txt @@ -37,7 +37,7 @@ endif(USE_GUI) if(USE_PYTHON) add_library(ngpy SHARED netgenpy.cpp) target_link_libraries( ngpy PUBLIC nglib PRIVATE "$" ) - target_link_libraries( ngpy PRIVATE ${MPI_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${JPEG_LIBRARIES} ${MKL_LIBRARIES} ${ZLIB_LIBRARIES} occ_libs netgen_cgns ) + target_link_libraries( ngpy PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${JPEG_LIBRARIES} ${MKL_LIBRARIES} ${ZLIB_LIBRARIES} occ_libs netgen_cgns ) if(APPLE) set_target_properties( ngpy PROPERTIES SUFFIX ".so") elseif(WIN32) diff --git a/ng/ngappinit.cpp b/ng/ngappinit.cpp index 92b05a6f..cfcf5998 100644 --- a/ng/ngappinit.cpp +++ b/ng/ngappinit.cpp @@ -9,12 +9,7 @@ #include #include #include - -#ifdef PARALLEL -#include - -// extern void ParallelRun(); -#endif +#include #include "../libsrc/interface/writeuser.hpp" @@ -66,22 +61,20 @@ int main(int argc, char ** argv) netgen::netgen_executable_started = true; #ifdef PARALLEL - int mpi_required = MPI_THREAD_MULTIPLE; + int mpi_required = netgen::NG_MPI_THREAD_MULTIPLE; #ifdef VTRACE - mpi_required = MPI_THREAD_SINGLE; + mpi_required = NG_MPI_THREAD_SINGLE; #endif int mpi_provided; - MPI_Init_thread(&argc, &argv, mpi_required, &mpi_provided); + netgen::InitMPI(); + netgen::NG_MPI_Init_thread(&argc, &argv, mpi_required, &mpi_provided); - MPI_Comm_size(MPI_COMM_WORLD, &netgen::ntasks); - MPI_Comm_rank(MPI_COMM_WORLD, &netgen::id); + netgen::NG_MPI_Comm_size(netgen::NG_MPI_COMM_WORLD, &netgen::ntasks); + netgen::NG_MPI_Comm_rank(netgen::NG_MPI_COMM_WORLD, &netgen::id); if(netgen::ntasks!=1) throw ngcore::Exception("Netgen GUI cannot run MPI-parallel"); - // MPI_COMM_WORLD is just a local communicator - // netgen::ng_comm = ngcore::NgMPI_Comm{MPI_COMM_WORLD, false}; - #endif if ( netgen::id == 0 ) @@ -113,7 +106,7 @@ int main(int argc, char ** argv) #ifdef PARALLEL - cout << "Including MPI version " << MPI_VERSION << '.' << MPI_SUBVERSION << endl; + cout << "Including MPI version " << netgen::NG_MPI_VERSION << '.' << netgen::NG_MPI_SUBVERSION << endl; #endif } @@ -287,7 +280,7 @@ int main(int argc, char ** argv) else { // ParallelRun(); - MPI_Finalize(); + netgen::NG_MPI_Finalize(); } #endif diff --git a/nglib/CMakeLists.txt b/nglib/CMakeLists.txt index b1036ea1..aa9c101b 100644 --- a/nglib/CMakeLists.txt +++ b/nglib/CMakeLists.txt @@ -11,7 +11,7 @@ if(EMSCRIPTEN) target_include_directories(nglib PUBLIC $) else(EMSCRIPTEN) target_link_libraries(nglib PUBLIC ngcore) - target_link_libraries( nglib PRIVATE ${MPI_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${JPEG_LIBRARIES} ${MKL_LIBRARIES} ${ZLIB_LIBRARIES} occ_libs netgen_cgns ) + target_link_libraries( nglib PRIVATE ${CMAKE_THREAD_LIBS_INIT} ${JPEG_LIBRARIES} ${MKL_LIBRARIES} ${ZLIB_LIBRARIES} occ_libs netgen_cgns ) endif(EMSCRIPTEN) install(TARGETS nglib netgen_cgns ${NG_INSTALL_DIR}) diff --git a/nglib/nglib.cpp b/nglib/nglib.cpp index c72e067b..11a43ddb 100644 --- a/nglib/nglib.cpp +++ b/nglib/nglib.cpp @@ -32,12 +32,6 @@ namespace netgen { -#ifdef PARALLEL -#include - -#endif - - /* namespace netgen { @@ -989,7 +983,7 @@ namespace netgen { #ifdef PARALLEL int id = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &id); + NG_MPI_Comm_rank(NG_MPI_COMM_WORLD, &id); if (id != 0) return; #endif (*mycout) << s << flush; diff --git a/python/__init__.py b/python/__init__.py index 4f71b212..df3e979a 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -35,8 +35,7 @@ if sys.platform.startswith('win'): v = sys.version_info if v.major == 3 and v.minor >= 8: os.add_dll_directory(_netgen_bin_dir) - else: - os.environ['PATH'] += ';'+_netgen_bin_dir + os.environ['PATH'] += ';'+_netgen_bin_dir del sys del os diff --git a/python/meshing.py b/python/meshing.py index 9b912b66..72be7947 100644 --- a/python/meshing.py +++ b/python/meshing.py @@ -1,4 +1,5 @@ from .libngpy._meshing import * +from pyngcore import MPI_Comm class _MeshsizeObject: @property diff --git a/setup.py b/setup.py index 4314ec3b..90b06d0f 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,8 @@ from subprocess import check_output setup_requires = ['pybind11-stubgen==2.5'] +pyprefix = pathlib.Path(sys.prefix).as_posix() + def install_filter(cmake_manifest): print(cmake_manifest) return cmake_manifest @@ -59,6 +61,7 @@ if 'NETGEN_CCACHE' in os.environ: packages = ['netgen', 'pyngcore'] +have_mpi = False if 'darwin' in sys.platform: cmake_args += [ '-DNG_INSTALL_DIR_LIB=netgen', @@ -68,6 +71,11 @@ if 'darwin' in sys.platform: '-DNG_INSTALL_DIR_INCLUDE=netgen/include', '-DNG_INSTALL_DIR_RES=share', ] + if os.path.exists('/usr/local/include/mpi.h'): + have_mpi = True + cmake_args += [ + '-DOPENMPI_INCLUDE_DIR=/usr/local/include', + ] elif 'win' in sys.platform: cmake_args += [ '-A Win64', @@ -77,6 +85,15 @@ elif 'win' in sys.platform: '-DNG_INSTALL_DIR_CMAKE=netgen/cmake', '-DNG_INSTALL_DIR_INCLUDE=netgen/include', ] + py_libdir = pathlib.Path(sys.prefix) / 'Library' + lib_file = py_libdir / 'lib' / 'impi.lib' + include_dir = py_libdir / 'include' + if lib_file.exists(): + have_mpi = True + cmake_args += [ + f'-DINTEL_MPI_INCLUDE_DIR={include_dir.as_posix()}', + f'-DINTEL_MPI_LIBRARY={lib_file.as_posix()}', + ] elif 'linux' in sys.platform: name_dir = name.replace('-','_') cmake_args += [ @@ -86,8 +103,25 @@ elif 'linux' in sys.platform: '-DTCL_INCLUDE_PATH=/usr/include', '-DTK_INCLUDE_PATH=/usr/include', ] + mpich_include = '/opt/mpich/include' + openmpi_include = '/opt/openmpi/include' + if os.path.exists(mpich_include+'/mpi.h'): + have_mpi = True + cmake_args += [ + f'-DMPICH_INCLUDE_DIR={mpich_include}', + ] + if os.path.exists(openmpi_include+'/mpi.h'): + have_mpi = True + cmake_args += [ + f'-DOPENMPI_INCLUDE_DIR={openmpi_include}', + ] packages = [] +if have_mpi: + cmake_args += [ + '-DUSE_MPI=ON', + ] + cmake_args += [ '-DUSE_SUPERBUILD:BOOL=ON', '-DUSE_CCACHE:BOOL=ON', @@ -101,7 +135,6 @@ cmake_args += [ '-DBUILD_STUB_FILES=ON', ] -pyprefix = pathlib.Path(sys.prefix).as_posix() cmake_args += [f'-DCMAKE_PREFIX_PATH={pyprefix}', f'-DPython3_ROOT_DIR={pyprefix}'] setup( diff --git a/tests/build_pip.sh b/tests/build_pip.sh index 0c620f56..614a56d3 100755 --- a/tests/build_pip.sh +++ b/tests/build_pip.sh @@ -1,7 +1,18 @@ set -e ulimit -n 1024000 # lower open file limit, seems to affect performance yum -y update -yum -y install ninja-build fontconfig-devel tk-devel tcl-devel libXmu-devel mesa-libGLU-devel ccache +yum -y install ninja-build fontconfig-devel tk-devel tcl-devel libXmu-devel mesa-libGLU-devel ccache dpkg + + +curl http://ftp.de.debian.org/debian/pool/main/o/openmpi/libopenmpi-dev_4.1.6-13.3_amd64.deb -o openmpi-dev.deb +dpkg-deb -R openmpi-dev.deb /opt/openmpi +mv /opt/openmpi/usr/lib/x86_64-linux-gnu/openmpi/include /opt/openmpi/include + + +curl http://ftp.de.debian.org/debian/pool/main/m/mpich/libmpich-dev_4.2.0-5.1_amd64.deb -o mpich.deb +dpkg-deb -R mpich.deb /opt/mpich +mv /opt/mpich/usr/lib/x86_64-linux-gnu/mpich/include /opt/mpich/include + rm -rf wheelhouse export NETGEN_CCACHE=1 @@ -13,6 +24,7 @@ do export PYDIR="/opt/python/cp${pyversion}-cp${pyversion}/bin" echo $PYDIR $PYDIR/pip install -U pytest-check numpy wheel scikit-build pybind11-stubgen + $PYDIR/pip install -i https://pypi.anaconda.org/mpi4py/simple/ --pre mpi4py rm -rf _skbuild NETGEN_ARCH=avx2 $PYDIR/pip wheel . diff --git a/tests/build_pip_mac.sh b/tests/build_pip_mac.sh index b7a41fdb..a21ec64a 100755 --- a/tests/build_pip_mac.sh +++ b/tests/build_pip_mac.sh @@ -1,4 +1,5 @@ set -e + rm -rf _skbuild dist export PYDIR=/Library/Frameworks/Python.framework/Versions/$1/bin