ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2025-01-27 21:30:35 +05:00
Code Issues Packages Projects Releases Wiki Activity

MPI wrapper to decide at run-time about MPI version

Browse Source
This commit is contained in:
Matthias Hochsteger 2024-05-02 20:23:15 +02:00
parent 08eec4460c
commit f7467c4b34
14 changed files with 684 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libsrc/core/.clang-tidy
View File

@ -1,4 +1,4 @@
Checks: '*,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
Checks: '*,-cppcoreguidelines-avoid-non-const-global-variables,-llvmlibc-restrict-system-libc-headers,-clang-analyzer-alpha.*,-*braces-around-statements,-fuchsia-*,-google-runtime-references,-readability-implicit-bool-conversion,-google-explicit-constructor,-hicpp-explicit-conversions,-google-runtime-int,-llvm-header-guard,-modernize-pass-by-value,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-readability-magic-numbers,-cppcoreguidelines-avoid-magic-numbers'
CheckOptions:
- key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
value: 1

24
libsrc/core/CMakeLists.txt
View File

@ -12,6 +12,7 @@ add_library(ngcore ${NGCORE_LIBRARY_TYPE}
taskmanager.cpp
utils.cpp
version.cpp
ng_mpi_wrapper.cpp
)
string(REPLACE "|" ";" ng_compile_flags_replace_sep "${NG_COMPILE_FLAGS}")
@ -57,6 +58,8 @@ if(WIN32)
get_WIN32_WINNT(ver)
target_compile_definitions(ngcore PUBLIC _WIN32_WINNT=${ver} WNT WNT_WINDOW NOMINMAX MSVC_EXPRESS _CRT_SECURE_NO_WARNINGS HAVE_STRUCT_TIMESPEC WIN32)
target_link_options(ngcore PUBLIC /ignore:4273 /ignore:4217 /ignore:4049)
else(WIN32)
target_link_libraries(ngcore PUBLIC dl)
endif(WIN32)
target_compile_definitions(ngcore PRIVATE NGCORE_EXPORTS)
@ -82,7 +85,7 @@ endif(USE_NUMA)
install(TARGETS ngcore DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE "$<BUILD_INTERFACE:netgen_python>" ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(ngcore PRIVATE "$<BUILD_INTERFACE:netgen_python>" ${CMAKE_THREAD_LIBS_INIT})
install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp memtracer.hpp
exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp
@ -90,6 +93,7 @@ install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp
xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp
simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp
register_archive.hpp autodiff.hpp autodiffdiff.hpp
ng_mpi.hpp ng_mpi_generated_declarations.hpp
DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
if(ENABLE_CPP_CORE_GUIDELINES_CHECK)
@ -108,3 +112,21 @@ if(USE_PYTHON)
install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen)
endif(USE_PYTHON)
if(USE_MPI)
target_compile_definitions(ngcore PUBLIC PARALLEL)
# assume openmpi here
add_library(ng_openmpi SHARED ng_mpi.cpp)
target_link_libraries(ng_openmpi PUBLIC ngcore)
install(TARGETS ng_openmpi DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
# set(MPICH_INCLUDE_DIR /opt/mpich/include)
if(MPICH_INCLUDE_DIR)
add_library(ng_mpich SHARED ng_mpi.cpp)
target_include_directories(ng_mpich PRIVATE ${MPICH_INCLUDE_DIR})
target_link_libraries(ng_mpich PUBLIC ngcore)
install(TARGETS ng_mpich DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
endif(MPICH_INCLUDE_DIR)
endif(USE_MPI)

98
libsrc/core/generate_mpi_sources.py Normal file
View File

@ -0,0 +1,98 @@
functions = [
("int", "MPI_Init", "int*", "char***"),
("int", "MPI_Initialized", "int*"),
("int", "MPI_Finalize"),
("int", "MPI_Barrier", "MPI_Comm"),
("int", "MPI_Comm_free", "MPI_Comm*"),
("int", "MPI_Comm_rank", "MPI_Comm", "int*"),
("int", "MPI_Comm_size", "MPI_Comm", "int*"),
("int", "MPI_Type_commit", "MPI_Datatype*"),
("int", "MPI_Waitall", "int", "MPI_Request*", "MPI_Status*"),
("int", "MPI_Waitany", "int", "MPI_Request*", "int*", "MPI_Status*"),
("int", "MPI_Wait", "MPI_Request*", "MPI_Status*"),
("int", "MPI_Send", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm"),
("int", "MPI_Probe", "int", "int", "MPI_Comm", "MPI_Status*"),
("int", "MPI_Recv", "void*", "int", "MPI_Datatype", "int", "int", "MPI_Comm", "MPI_Status*"),
("int", "MPI_Get_count", "MPI_Status*", "MPI_Datatype", "int*"),
("int", "MPI_Bcast", "void*", "int", "MPI_Datatype", "int", "MPI_Comm"),
("int", "MPI_Comm_group", "MPI_Comm", "MPI_Group*"),
("int", "MPI_Group_incl", "MPI_Group", "int", "int*", "MPI_Group*"),
("int", "MPI_Comm_create_group", "MPI_Comm", "MPI_Group", "int", "MPI_Comm*"),
("int", "MPI_Get_processor_name", "char*", "int*"),
]
constants = [
("MPI_Comm", "MPI_COMM_WORLD"),
("MPI_Status*", "MPI_STATUS_IGNORE"),
("MPI_Status*", "MPI_STATUSES_IGNORE"),
("MPI_Datatype", "MPI_INT"),
("MPI_Datatype", "MPI_SHORT"),
("MPI_Datatype", "MPI_CHAR"),
("MPI_Datatype", "MPI_UINT64_T"),
("MPI_Datatype", "MPI_DOUBLE"),
("MPI_Datatype", "MPI_C_BOOL"),
("void*", "MPI_IN_PLACE"),
("int", "MPI_MAX_PROCESSOR_NAME"),
]
def get_args(f):
return ["NG_"+a if a.startswith("MPI_") else a for a in f[2:]]
def generate_declarations():
code = ""
for f in functions:
ret = f[0]
name = f[1]
args = ", ".join(get_args(f))
code += f"NGCORE_API extern {ret} (*NG_{name})({args});\n"
for typ, name in constants:
if typ.startswith("MPI_"):
typ = "NG_" + typ
code += f"NGCORE_API extern {typ} NG_{name};\n"
with open("ng_mpi_generated_declarations.hpp", "w") as f:
f.write(code)
def generate_dummy_init():
code = ""
for f in functions:
ret = f[0]
name = f[1]
args = ", ".join(get_args(f))
code += f"decltype(NG_{name}) NG_{name} = []({args})->{ret} {{ throw no_mpi(); }};\n"
for typ, name in constants:
if typ.startswith("MPI_"):
typ = "NG_" + typ
code += f"{typ} NG_{name} = 0;\n"
with open("ng_mpi_generated_dummy_init.hpp", "w") as f:
f.write(code)
def generate_init():
code = ""
for f in functions:
ret = f[0]
name = f[1]
args = get_args(f)
in_args =''
call_args = ''
for i, a in enumerate(args):
if i > 0:
in_args += ', '
call_args += ', '
in_args += a + f" arg{i}"
call_args += f" ng2mpi(arg{i})"
code += f"NG_{name} = []({in_args})->{ret} {{ return {name}({call_args}); }};\n"
for _, name in constants:
code += f"NG_{name} = mpi2ng({name});\n"
with open("ng_mpi_generated_init.hpp", "w") as f:
f.write(code)
if __name__ == "__main__":
generate_declarations()
generate_dummy_init()
generate_init()

211
libsrc/core/mpi_wrapper.hpp
View File

@ -5,7 +5,7 @@
#ifdef PARALLEL
#define OMPI_SKIP_MPICXX
#include <mpi.h>
// #include <mpi.h>
#endif
#include "array.hpp"
@ -13,6 +13,7 @@
#include "exception.hpp"
#include "profiler.hpp"
#include "ngstream.hpp"
#include "ng_mpi.hpp"
namespace ngcore
{
@ -22,67 +23,67 @@ namespace ngcore
template <class T> struct MPI_typetrait { };
template <> struct MPI_typetrait<int> {
static MPI_Datatype MPIType () { return MPI_INT; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_INT; } };
template <> struct MPI_typetrait<short> {
static MPI_Datatype MPIType () { return MPI_SHORT; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_SHORT; } };
template <> struct MPI_typetrait<char> {
static MPI_Datatype MPIType () { return MPI_CHAR; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
template <> struct MPI_typetrait<signed char> {
static MPI_Datatype MPIType () { return MPI_CHAR; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
template <> struct MPI_typetrait<unsigned char> {
static MPI_Datatype MPIType () { return MPI_CHAR; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_CHAR; } };
template <> struct MPI_typetrait<size_t> {
static MPI_Datatype MPIType () { return MPI_UINT64_T; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_UINT64_T; } };
template <> struct MPI_typetrait<double> {
static MPI_Datatype MPIType () { return MPI_DOUBLE; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_DOUBLE; } };
template <> struct MPI_typetrait<bool> {
static MPI_Datatype MPIType () { return MPI_C_BOOL; } };
static NG_MPI_Datatype MPIType () { return NG_MPI_C_BOOL; } };
template<typename T, size_t S>
struct MPI_typetrait<std::array<T,S>>
{
static MPI_Datatype MPIType ()
static NG_MPI_Datatype MPIType ()
{
static MPI_Datatype MPI_T = 0;
if (!MPI_T)
static NG_MPI_Datatype NG_MPI_T = 0;
if (!NG_MPI_T)
{
MPI_Type_contiguous ( S, MPI_typetrait<T>::MPIType(), &MPI_T);
MPI_Type_commit ( &MPI_T );
NG_MPI_Type_contiguous ( S, MPI_typetrait<T>::MPIType(), &NG_MPI_T);
NG_MPI_Type_commit ( &NG_MPI_T );
}
return MPI_T;
return NG_MPI_T;
}
};
template <class T, class T2 = decltype(MPI_typetrait<T>::MPIType())>
inline MPI_Datatype GetMPIType () {
inline NG_MPI_Datatype GetMPIType () {
return MPI_typetrait<T>::MPIType();
}
template <class T>
inline MPI_Datatype GetMPIType (T &) {
inline NG_MPI_Datatype GetMPIType (T &) {
return GetMPIType<T>();
}
inline void MyMPI_WaitAll (FlatArray<MPI_Request> requests)
inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests)
{
static Timer t("MPI - WaitAll"); RegionTimer reg(t);
if (!requests.Size()) return;
MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE);
NG_MPI_Waitall (requests.Size(), requests.Data(), NG_MPI_STATUSES_IGNORE);
}
inline int MyMPI_WaitAny (FlatArray<MPI_Request> requests)
inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests)
{
int nr;
MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE);
NG_MPI_Waitany (requests.Size(), requests.Data(), &nr, NG_MPI_STATUS_IGNORE);
return nr;
}
@ -91,7 +92,7 @@ namespace ngcore
class NgMPI_Comm
{
protected:
MPI_Comm comm;
NG_MPI_Comm comm;
bool valid_comm;
int * refcount;
int rank, size;
@ -100,11 +101,11 @@ namespace ngcore
: valid_comm(false), refcount(nullptr), rank(0), size(1)
{ ; }
NgMPI_Comm (MPI_Comm _comm, bool owns = false)
NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false)
: comm(_comm), valid_comm(true)
{
int flag;
MPI_Initialized (&flag);
NG_MPI_Initialized (&flag);
if (!flag)
{
valid_comm = false;
@ -119,8 +120,8 @@ namespace ngcore
else
refcount = new int{1};
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
NG_MPI_Comm_rank(comm, &rank);
NG_MPI_Comm_size(comm, &size);
}
NgMPI_Comm (const NgMPI_Comm & c)
@ -141,7 +142,7 @@ namespace ngcore
{
if (refcount)
if (--(*refcount) == 0)
MPI_Comm_free(&comm);
NG_MPI_Comm_free(&comm);
}
bool ValidCommunicator() const
@ -153,7 +154,7 @@ namespace ngcore
{
if (refcount)
if (--(*refcount) == 0)
MPI_Comm_free(&comm);
NG_MPI_Comm_free(&comm);
refcount = c.refcount;
if (refcount) (*refcount)++;
@ -169,7 +170,7 @@ namespace ngcore
InvalidCommException() : Exception("Do not have a valid communicator") { ; }
};
operator MPI_Comm() const {
operator NG_MPI_Comm() const {
if (!valid_comm) throw InvalidCommException();
return comm;
}
@ -178,7 +179,7 @@ namespace ngcore
int Size() const { return size; }
void Barrier() const {
static Timer t("MPI - Barrier"); RegionTimer reg(t);
if (size > 1) MPI_Barrier (comm);
if (size > 1) NG_MPI_Barrier (comm);
}
@ -186,82 +187,82 @@ namespace ngcore
template<typename T, typename T2 = decltype(GetMPIType<T>())>
void Send (T & val, int dest, int tag) const {
MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
NG_MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
}
void Send (const std::string & s, int dest, int tag) const {
MPI_Send( const_cast<char*> (&s[0]), s.length(), MPI_CHAR, dest, tag, comm);
NG_MPI_Send( const_cast<char*> (&s[0]), s.length(), NG_MPI_CHAR, dest, tag, comm);
}
template<typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
void Send(FlatArray<T,TI> s, int dest, int tag) const {
MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
NG_MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
void Recv (T & val, int src, int tag) const {
MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
NG_MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, NG_MPI_STATUS_IGNORE);
}
void Recv (std::string & s, int src, int tag) const {
MPI_Status status;
NG_MPI_Status status;
int len;
MPI_Probe (src, tag, comm, &status);
MPI_Get_count (&status, MPI_CHAR, &len);
NG_MPI_Probe (src, tag, comm, &status);
NG_MPI_Get_count (&status, NG_MPI_CHAR, &len);
// s.assign (len, ' ');
s.resize (len);
MPI_Recv( &s[0], len, MPI_CHAR, src, tag, comm, MPI_STATUS_IGNORE);
NG_MPI_Recv( &s[0], len, NG_MPI_CHAR, src, tag, comm, NG_MPI_STATUS_IGNORE);
}
template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
void Recv (FlatArray <T,TI> s, int src, int tag) const {
MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, MPI_STATUS_IGNORE);
NG_MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, NG_MPI_STATUS_IGNORE);
}
template <typename T, typename TI, typename T2 = decltype(GetMPIType<T>())>
void Recv (Array <T,TI> & s, int src, int tag) const
{
MPI_Status status;
NG_MPI_Status status;
int len;
const MPI_Datatype MPI_T = GetMPIType<T> ();
MPI_Probe (src, tag, comm, &status);
MPI_Get_count (&status, MPI_T, &len);
const NG_MPI_Datatype NG_MPI_T = GetMPIType<T> ();
NG_MPI_Probe (src, tag, comm, &status);
NG_MPI_Get_count (&status, NG_MPI_T, &len);
s.SetSize (len);
MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE);
NG_MPI_Recv (s.Data(), len, NG_MPI_T, src, tag, comm, NG_MPI_STATUS_IGNORE);
}
/** --- non-blocking P2P --- **/
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request ISend (T & val, int dest, int tag) const
NG_MPI_Request ISend (T & val, int dest, int tag) const
{
MPI_Request request;
MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
NG_MPI_Request request;
NG_MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
return request;
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request ISend (FlatArray<T> s, int dest, int tag) const
NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const
{
MPI_Request request;
MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
NG_MPI_Request request;
NG_MPI_Isend (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm, &request);
return request;
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request IRecv (T & val, int dest, int tag) const
NG_MPI_Request IRecv (T & val, int dest, int tag) const
{
MPI_Request request;
MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
NG_MPI_Request request;
NG_MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
return request;
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request IRecv (FlatArray<T> s, int src, int tag) const
NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const
{
MPI_Request request;
MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
NG_MPI_Request request;
NG_MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
return request;
}
@ -269,41 +270,41 @@ namespace ngcore
/** --- collectives --- **/
template <typename T, typename T2 = decltype(GetMPIType<T>())>
T Reduce (T d, const MPI_Op & op, int root = 0) const
T Reduce (T d, const NG_MPI_Op & op, int root = 0) const
{
static Timer t("MPI - Reduce"); RegionTimer reg(t);
if (size == 1) return d;
T global_d;
MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
NG_MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
return global_d;
}
template <typename T, typename T2 = decltype(GetMPIType<T>())>
T AllReduce (T d, const MPI_Op & op) const
T AllReduce (T d, const NG_MPI_Op & op) const
{
static Timer t("MPI - AllReduce"); RegionTimer reg(t);
if (size == 1) return d;
T global_d;
MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
NG_MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
return global_d;
}
template <typename T, typename T2 = decltype(GetMPIType<T>())>
void AllReduce (FlatArray<T> d, const MPI_Op & op) const
void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const
{
static Timer t("MPI - AllReduce Array"); RegionTimer reg(t);
if (size == 1) return;
MPI_Allreduce (MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType<T>(), op, comm);
NG_MPI_Allreduce (NG_MPI_IN_PLACE, d.Data(), d.Size(), GetMPIType<T>(), op, comm);
}
template <typename T, typename T2 = decltype(GetMPIType<T>())>
void Bcast (T & s, int root = 0) const {
if (size == 1) return;
static Timer t("MPI - Bcast"); RegionTimer reg(t);
MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
NG_MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
}
@ -316,7 +317,7 @@ namespace ngcore
Bcast (ds, root);
if (Rank() != root) d.SetSize (ds);
if (ds != 0)
MPI_Bcast (d.Data(), ds, GetMPIType<T>(), root, comm);
NG_MPI_Bcast (d.Data(), ds, GetMPIType<T>(), root, comm);
}
@ -326,13 +327,13 @@ namespace ngcore
int len = s.length();
Bcast (len, root);
if (rank != 0) s.resize (len);
MPI_Bcast (&s[0], len, MPI_CHAR, root, comm);
NG_MPI_Bcast (&s[0], len, NG_MPI_CHAR, root, comm);
}
template <typename T>
void AllToAll (FlatArray<T> send, FlatArray<T> recv) const
{
MPI_Alltoall (send.Data(), 1, GetMPIType<T>(),
NG_MPI_Alltoall (send.Data(), 1, GetMPIType<T>(),
recv.Data(), 1, GetMPIType<T>(), comm);
}
@ -341,15 +342,15 @@ namespace ngcore
void ScatterRoot (FlatArray<T> send) const
{
if (size == 1) return;
MPI_Scatter (send.Data(), 1, GetMPIType<T>(),
MPI_IN_PLACE, -1, GetMPIType<T>(), 0, comm);
NG_MPI_Scatter (send.Data(), 1, GetMPIType<T>(),
NG_MPI_IN_PLACE, -1, GetMPIType<T>(), 0, comm);
}
template <typename T>
void Scatter (T & recv) const
{
if (size == 1) return;
MPI_Scatter (NULL, 0, GetMPIType<T>(),
NG_MPI_Scatter (NULL, 0, GetMPIType<T>(),
&recv, 1, GetMPIType<T>(), 0, comm);
}
@ -358,7 +359,7 @@ namespace ngcore
{
recv[0] = T(0);
if (size == 1) return;
MPI_Gather (MPI_IN_PLACE, 1, GetMPIType<T>(),
NG_MPI_Gather (NG_MPI_IN_PLACE, 1, GetMPIType<T>(),
recv.Data(), 1, GetMPIType<T>(), 0, comm);
}
@ -366,7 +367,7 @@ namespace ngcore
void Gather (T send) const
{
if (size == 1) return;
MPI_Gather (&send, 1, GetMPIType<T>(),
NG_MPI_Gather (&send, 1, GetMPIType<T>(),
NULL, 1, GetMPIType<T>(), 0, comm);
}
@ -379,7 +380,7 @@ namespace ngcore
recv[0] = val;
return;
}
MPI_Allgather (&val, 1, GetMPIType<T>(),
NG_MPI_Allgather (&val, 1, GetMPIType<T>(),
recv.Data(), 1, GetMPIType<T>(),
comm);
}
@ -400,7 +401,7 @@ namespace ngcore
recv_data = DynamicTable<T> (recv_sizes, true);
Array<MPI_Request> requests;
Array<NG_MPI_Request> requests;
for (int dest = 0; dest < size; dest++)
if (dest != rank && send_data[dest].Size())
requests.Append (ISend (FlatArray<T>(send_data[dest]), dest, tag));
@ -418,11 +419,11 @@ namespace ngcore
NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
{
MPI_Comm subcomm;
MPI_Group gcomm, gsubcomm;
MPI_Comm_group(comm, &gcomm);
MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
NG_MPI_Comm subcomm;
NG_MPI_Group gcomm, gsubcomm;
NG_MPI_Comm_group(comm, &gcomm);
NG_MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm);
NG_MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm);
return NgMPI_Comm(subcomm, true);
}
@ -440,16 +441,16 @@ namespace ngcore
MyMPI(int argc, char ** argv)
{
int is_init = -1;
MPI_Initialized(&is_init);
NG_MPI_Initialized(&is_init);
if (!is_init)
{
MPI_Init (&argc, &argv);
NG_MPI_Init (&argc, &argv);
initialized_by_me = true;
}
else
initialized_by_me = false;
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
NGSOStream::SetGlobalActive (comm.Rank() == 0);
if (comm.Size() > 1)
@ -459,7 +460,7 @@ namespace ngcore
~MyMPI()
{
if (initialized_by_me)
MPI_Finalize ();
NG_MPI_Finalize ();
}
};
@ -468,42 +469,42 @@ namespace ngcore
#else // PARALLEL
class MPI_Comm {
class NG_MPI_Comm {
int nr;
public:
MPI_Comm (int _nr = 0) : nr(_nr) { ; }
NG_MPI_Comm (int _nr = 0) : nr(_nr) { ; }
operator int() const { return nr; }
bool operator== (MPI_Comm c2) const { return nr == c2.nr; }
bool operator== (NG_MPI_Comm c2) const { return nr == c2.nr; }
};
static MPI_Comm MPI_COMM_WORLD = 12345, MPI_COMM_NULL = 10000;
static NG_MPI_Comm NG_MPI_COMM_WORLD = 12345, NG_MPI_COMM_NULL = 10000;
typedef int MPI_Op;
typedef int MPI_Datatype;
typedef int MPI_Request;
typedef int NG_MPI_Op;
typedef int NG_MPI_Datatype;
typedef int NG_MPI_Request;
enum { MPI_SUM = 0, MPI_MIN = 1, MPI_MAX = 2, MPI_LOR = 4711 };
enum { NG_MPI_SUM = 0, NG_MPI_MIN = 1, NG_MPI_MAX = 2, NG_MPI_LOR = 4711 };
inline void MPI_Type_contiguous ( int, MPI_Datatype, MPI_Datatype*) { ; }
inline void MPI_Type_commit ( MPI_Datatype * ) { ; }
inline void NG_MPI_Type_contiguous ( int, NG_MPI_Datatype, NG_MPI_Datatype*) { ; }
inline void NG_MPI_Type_commit ( NG_MPI_Datatype * ) { ; }
template <class T> struct MPI_typetrait {
static MPI_Datatype MPIType () { return -1; }
static NG_MPI_Datatype MPIType () { return -1; }
};
template <class T, class T2=void>
inline MPI_Datatype GetMPIType () { return -1; }
inline NG_MPI_Datatype GetMPIType () { return -1; }
class NgMPI_Comm
{
public:
NgMPI_Comm () { ; }
NgMPI_Comm (MPI_Comm _comm, bool owns = false) { ; }
NgMPI_Comm (NG_MPI_Comm _comm, bool owns = false) { ; }
size_t Rank() const { return 0; }
size_t Size() const { return 1; }
bool ValidCommunicator() const { return false; }
void Barrier() const { ; }
operator MPI_Comm() const { return MPI_Comm(); }
operator NG_MPI_Comm() const { return NG_MPI_Comm(); }
template<typename T>
void Send( T & val, int dest, int tag) const { ; }
@ -521,25 +522,25 @@ namespace ngcore
void Recv (Array <T> & s, int src, int tag) const { ; }
template<typename T>
MPI_Request ISend (T & val, int dest, int tag) const { return 0; }
NG_MPI_Request ISend (T & val, int dest, int tag) const { return 0; }
template<typename T>
MPI_Request ISend (FlatArray<T> s, int dest, int tag) const { return 0; }
NG_MPI_Request ISend (FlatArray<T> s, int dest, int tag) const { return 0; }
template<typename T>
MPI_Request IRecv (T & val, int dest, int tag) const { return 0; }
NG_MPI_Request IRecv (T & val, int dest, int tag) const { return 0; }
template<typename T>
MPI_Request IRecv (FlatArray<T> s, int src, int tag) const { return 0; }
NG_MPI_Request IRecv (FlatArray<T> s, int src, int tag) const { return 0; }
template <typename T>
T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; }
T Reduce (T d, const NG_MPI_Op & op, int root = 0) const { return d; }
template <typename T>
T AllReduce (T d, const MPI_Op & op) const { return d; }
T AllReduce (T d, const NG_MPI_Op & op) const { return d; }
template <typename T>
void AllReduce (FlatArray<T> d, const MPI_Op & op) const { ; }
void AllReduce (FlatArray<T> d, const NG_MPI_Op & op) const { ; }
template <typename T>
void Bcast (T & s, int root = 0) const { ; }
@ -562,8 +563,8 @@ namespace ngcore
{ return *this; }
};
inline void MyMPI_WaitAll (FlatArray<MPI_Request> requests) { ; }
inline int MyMPI_WaitAny (FlatArray<MPI_Request> requests) { return 0; }
inline void MyMPI_WaitAll (FlatArray<NG_MPI_Request> requests) { ; }
inline int MyMPI_WaitAny (FlatArray<NG_MPI_Request> requests) { return 0; }
class MyMPI
{

62
libsrc/core/ng_mpi.cpp Normal file
View File

@ -0,0 +1,62 @@
#include "ng_mpi.hpp"
#include <mpi.h>
#include <type_traits>
#include "ngcore_api.hpp"
namespace ngcore {
template <typename T>
uintptr_t mpi2ng(T t) {
if constexpr (std::is_pointer_v<T>)
return reinterpret_cast<uintptr_t>(t);
else
return static_cast<uintptr_t>(t);
}
template <typename T>
T cast_ng2mpi(uintptr_t t) {
if constexpr (std::is_pointer_v<T>)
return reinterpret_cast<T>(t);
else
return static_cast<T>(t);
}
template <typename T>
T cast_ng2mpi(uintptr_t* t) {
if constexpr (std::is_pointer_v<T>)
return reinterpret_cast<T>(t);
else
return static_cast<T>(t);
}
MPI_Comm ng2mpi(NG_MPI_Comm c) {
static_assert(sizeof(MPI_Comm) <= sizeof(c.value), "Size mismatch");
return cast_ng2mpi<MPI_Comm>(c.value);
}
MPI_Comm* ng2mpi(NG_MPI_Comm* c) { return cast_ng2mpi<MPI_Comm*>(&c->value); }
MPI_Datatype ng2mpi(NG_MPI_Datatype c) {
static_assert(sizeof(MPI_Datatype) <= sizeof(c.value), "Size mismatch");
return cast_ng2mpi<MPI_Datatype>(c.value);
}
MPI_Request ng2mpi(NG_MPI_Request c) {
static_assert(sizeof(MPI_Request) <= sizeof(c.value), "Size mismatch");
return cast_ng2mpi<MPI_Request>(c.value);
}
int* ng2mpi(int* c) { return c; }
} // namespace ngcore
using namespace ngcore;
NGCORE_API_EXPORT extern "C" void ng_init_mpi();
void ng_init_mpi() {
#include "ng_mpi_generated_init.hpp"
}

55
libsrc/core/ng_mpi.hpp Normal file
View File

@ -0,0 +1,55 @@
#ifndef NG_MPI_HPP_INCLUDED
#define NG_MPI_HPP_INCLUDED
#include <cstdint>
#include <stdexcept>
#include <string>
#include "ngcore_api.hpp"
namespace ngcore {
NGCORE_API void InitMPI();
NGCORE_API extern std::string mpi_library_version;
inline void not_implemented() { throw std::runtime_error("Not implemented"); }
struct NG_MPI_Status {
uint64_t data[4];
};
struct NG_MPI_Comm {
uintptr_t value;
NG_MPI_Comm() { value = 0;}
NG_MPI_Comm(uintptr_t v) : value(v) {}
};
struct NG_MPI_Datatype {
uintptr_t value;
NG_MPI_Datatype(uintptr_t v) : value(v) {}
operator bool() const { return value != 0; }
};
struct NG_MPI_Request {
uintptr_t value = 0;
NG_MPI_Request(uintptr_t v) : value(v) {}
NG_MPI_Request() = default;
};
struct NG_MPI_Op {
uintptr_t value;
NG_MPI_Op(uintptr_t v) : value(v) {}
};
struct NG_MPI_Group {
uintptr_t value = 0;
NG_MPI_Group(uintptr_t v) : value(v) {}
NG_MPI_Group() = default;
};
#include "ng_mpi_generated_declarations.hpp"
} // namespace ngcore
#endif // NG_MPI_HPP_INCLUDED

31
libsrc/core/ng_mpi_generated_declarations.hpp Normal file
View File

@ -0,0 +1,31 @@
NGCORE_API extern int (*NG_MPI_Init)(int*, char***);
NGCORE_API extern int (*NG_MPI_Initialized)(int*);
NGCORE_API extern int (*NG_MPI_Finalize)();
NGCORE_API extern int (*NG_MPI_Barrier)(NG_MPI_Comm);
NGCORE_API extern int (*NG_MPI_Comm_free)(NG_MPI_Comm*);
NGCORE_API extern int (*NG_MPI_Comm_rank)(NG_MPI_Comm, int*);
NGCORE_API extern int (*NG_MPI_Comm_size)(NG_MPI_Comm, int*);
NGCORE_API extern int (*NG_MPI_Type_commit)(NG_MPI_Datatype*);
NGCORE_API extern int (*NG_MPI_Waitall)(int, NG_MPI_Request*, NG_MPI_Status*);
NGCORE_API extern int (*NG_MPI_Waitany)(int, NG_MPI_Request*, int*, NG_MPI_Status*);
NGCORE_API extern int (*NG_MPI_Wait)(NG_MPI_Request*, NG_MPI_Status*);
NGCORE_API extern int (*NG_MPI_Send)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm);
NGCORE_API extern int (*NG_MPI_Probe)(int, int, NG_MPI_Comm, NG_MPI_Status*);
NGCORE_API extern int (*NG_MPI_Recv)(void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*);
NGCORE_API extern int (*NG_MPI_Get_count)(NG_MPI_Status*, NG_MPI_Datatype, int*);
NGCORE_API extern int (*NG_MPI_Bcast)(void*, int, NG_MPI_Datatype, int, NG_MPI_Comm);
NGCORE_API extern int (*NG_MPI_Comm_group)(NG_MPI_Comm, NG_MPI_Group*);
NGCORE_API extern int (*NG_MPI_Group_incl)(NG_MPI_Group, int, int*, NG_MPI_Group*);
NGCORE_API extern int (*NG_MPI_Comm_create_group)(NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*);
NGCORE_API extern int (*NG_MPI_Get_processor_name)(char*, int*);
NGCORE_API extern NG_MPI_Comm NG_MPI_COMM_WORLD;
NGCORE_API extern NG_MPI_Status* NG_MPI_STATUS_IGNORE;
NGCORE_API extern NG_MPI_Status* NG_MPI_STATUSES_IGNORE;
NGCORE_API extern NG_MPI_Datatype NG_MPI_INT;
NGCORE_API extern NG_MPI_Datatype NG_MPI_SHORT;
NGCORE_API extern NG_MPI_Datatype NG_MPI_CHAR;
NGCORE_API extern NG_MPI_Datatype NG_MPI_UINT64_T;
NGCORE_API extern NG_MPI_Datatype NG_MPI_DOUBLE;
NGCORE_API extern NG_MPI_Datatype NG_MPI_C_BOOL;
NGCORE_API extern void* NG_MPI_IN_PLACE;
NGCORE_API extern int NG_MPI_MAX_PROCESSOR_NAME;

31
libsrc/core/ng_mpi_generated_dummy_init.hpp Normal file
View File

@ -0,0 +1,31 @@
decltype(NG_MPI_Init) NG_MPI_Init = [](int*, char***)->int { throw no_mpi(); };
decltype(NG_MPI_Initialized) NG_MPI_Initialized = [](int*)->int { throw no_mpi(); };
decltype(NG_MPI_Finalize) NG_MPI_Finalize = []()->int { throw no_mpi(); };
decltype(NG_MPI_Barrier) NG_MPI_Barrier = [](NG_MPI_Comm)->int { throw no_mpi(); };
decltype(NG_MPI_Comm_free) NG_MPI_Comm_free = [](NG_MPI_Comm*)->int { throw no_mpi(); };
decltype(NG_MPI_Comm_rank) NG_MPI_Comm_rank = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
decltype(NG_MPI_Comm_size) NG_MPI_Comm_size = [](NG_MPI_Comm, int*)->int { throw no_mpi(); };
decltype(NG_MPI_Type_commit) NG_MPI_Type_commit = [](NG_MPI_Datatype*)->int { throw no_mpi(); };
decltype(NG_MPI_Waitall) NG_MPI_Waitall = [](int, NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
decltype(NG_MPI_Waitany) NG_MPI_Waitany = [](int, NG_MPI_Request*, int*, NG_MPI_Status*)->int { throw no_mpi(); };
decltype(NG_MPI_Wait) NG_MPI_Wait = [](NG_MPI_Request*, NG_MPI_Status*)->int { throw no_mpi(); };
decltype(NG_MPI_Send) NG_MPI_Send = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm)->int { throw no_mpi(); };
decltype(NG_MPI_Probe) NG_MPI_Probe = [](int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
decltype(NG_MPI_Recv) NG_MPI_Recv = [](void*, int, NG_MPI_Datatype, int, int, NG_MPI_Comm, NG_MPI_Status*)->int { throw no_mpi(); };
decltype(NG_MPI_Get_count) NG_MPI_Get_count = [](NG_MPI_Status*, NG_MPI_Datatype, int*)->int { throw no_mpi(); };
decltype(NG_MPI_Bcast) NG_MPI_Bcast = [](void*, int, NG_MPI_Datatype, int, NG_MPI_Comm)->int { throw no_mpi(); };
decltype(NG_MPI_Comm_group) NG_MPI_Comm_group = [](NG_MPI_Comm, NG_MPI_Group*)->int { throw no_mpi(); };
decltype(NG_MPI_Group_incl) NG_MPI_Group_incl = [](NG_MPI_Group, int, int*, NG_MPI_Group*)->int { throw no_mpi(); };
decltype(NG_MPI_Comm_create_group) NG_MPI_Comm_create_group = [](NG_MPI_Comm, NG_MPI_Group, int, NG_MPI_Comm*)->int { throw no_mpi(); };
decltype(NG_MPI_Get_processor_name) NG_MPI_Get_processor_name = [](char*, int*)->int { throw no_mpi(); };
NG_MPI_Comm NG_MPI_COMM_WORLD = 0;
NG_MPI_Status* NG_MPI_STATUS_IGNORE = 0;
NG_MPI_Status* NG_MPI_STATUSES_IGNORE = 0;
NG_MPI_Datatype NG_MPI_INT = 0;
NG_MPI_Datatype NG_MPI_SHORT = 0;
NG_MPI_Datatype NG_MPI_CHAR = 0;
NG_MPI_Datatype NG_MPI_UINT64_T = 0;
NG_MPI_Datatype NG_MPI_DOUBLE = 0;
NG_MPI_Datatype NG_MPI_C_BOOL = 0;
void* NG_MPI_IN_PLACE = 0;
int NG_MPI_MAX_PROCESSOR_NAME = 0;

31
libsrc/core/ng_mpi_generated_init.hpp Normal file
View File

@ -0,0 +1,31 @@
NG_MPI_Init = [](int* arg0, char*** arg1)->int { return MPI_Init( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_Initialized = [](int* arg0)->int { return MPI_Initialized( ng2mpi(arg0)); };
NG_MPI_Finalize = []()->int { return MPI_Finalize(); };
NG_MPI_Barrier = [](NG_MPI_Comm arg0)->int { return MPI_Barrier( ng2mpi(arg0)); };
NG_MPI_Comm_free = [](NG_MPI_Comm* arg0)->int { return MPI_Comm_free( ng2mpi(arg0)); };
NG_MPI_Comm_rank = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_rank( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_Comm_size = [](NG_MPI_Comm arg0, int* arg1)->int { return MPI_Comm_size( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_Type_commit = [](NG_MPI_Datatype* arg0)->int { return MPI_Type_commit( ng2mpi(arg0)); };
NG_MPI_Waitall = [](int arg0, NG_MPI_Request* arg1, NG_MPI_Status* arg2)->int { return MPI_Waitall( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2)); };
NG_MPI_Waitany = [](int arg0, NG_MPI_Request* arg1, int* arg2, NG_MPI_Status* arg3)->int { return MPI_Waitany( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); };
NG_MPI_Wait = [](NG_MPI_Request* arg0, NG_MPI_Status* arg1)->int { return MPI_Wait( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_Send = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5)->int { return MPI_Send( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3), ng2mpi(arg4), ng2mpi(arg5)); };
NG_MPI_Probe = [](int arg0, int arg1, NG_MPI_Comm arg2, NG_MPI_Status* arg3)->int { return MPI_Probe( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); };
NG_MPI_Recv = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, int arg4, NG_MPI_Comm arg5, NG_MPI_Status* arg6)->int { return MPI_Recv( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3), ng2mpi(arg4), ng2mpi(arg5), ng2mpi(arg6)); };
NG_MPI_Get_count = [](NG_MPI_Status* arg0, NG_MPI_Datatype arg1, int* arg2)->int { return MPI_Get_count( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2)); };
NG_MPI_Bcast = [](void* arg0, int arg1, NG_MPI_Datatype arg2, int arg3, NG_MPI_Comm arg4)->int { return MPI_Bcast( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3), ng2mpi(arg4)); };
NG_MPI_Comm_group = [](NG_MPI_Comm arg0, NG_MPI_Group* arg1)->int { return MPI_Comm_group( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_Group_incl = [](NG_MPI_Group arg0, int arg1, int* arg2, NG_MPI_Group* arg3)->int { return MPI_Group_incl( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); };
NG_MPI_Comm_create_group = [](NG_MPI_Comm arg0, NG_MPI_Group arg1, int arg2, NG_MPI_Comm* arg3)->int { return MPI_Comm_create_group( ng2mpi(arg0), ng2mpi(arg1), ng2mpi(arg2), ng2mpi(arg3)); };
NG_MPI_Get_processor_name = [](char* arg0, int* arg1)->int { return MPI_Get_processor_name( ng2mpi(arg0), ng2mpi(arg1)); };
NG_MPI_COMM_WORLD = mpi2ng(MPI_COMM_WORLD);
NG_MPI_STATUS_IGNORE = mpi2ng(MPI_STATUS_IGNORE);
NG_MPI_STATUSES_IGNORE = mpi2ng(MPI_STATUSES_IGNORE);
NG_MPI_INT = mpi2ng(MPI_INT);
NG_MPI_SHORT = mpi2ng(MPI_SHORT);
NG_MPI_CHAR = mpi2ng(MPI_CHAR);
NG_MPI_UINT64_T = mpi2ng(MPI_UINT64_T);
NG_MPI_DOUBLE = mpi2ng(MPI_DOUBLE);
NG_MPI_C_BOOL = mpi2ng(MPI_C_BOOL);
NG_MPI_IN_PLACE = mpi2ng(MPI_IN_PLACE);
NG_MPI_MAX_PROCESSOR_NAME = mpi2ng(MPI_MAX_PROCESSOR_NAME);

94
libsrc/core/ng_mpi_wrapper.cpp Normal file
View File

@ -0,0 +1,94 @@
#include <mpi.h>
#include <iostream>
#include <stdexcept>
#include "ng_mpi.hpp"
#include "utils.hpp"
using std::cout;
using std::endl;
namespace ngcore {
static std::unique_ptr<SharedLibrary> mpi_lib, ng_mpi_lib;
void InitMPI() {
if (ng_mpi_lib) return;
cout << "InitMPI" << endl;
typedef void (*get_version_handle)(char *, int *);
typedef int (*init_handle)(int *, char ***);
typedef int (*mpi_initialized_handle)(int *);
typedef void (*ng_init_handle)();
init_handle mpi_init;
mpi_initialized_handle mpi_initialized;
get_version_handle get_version;
try {
mpi_init = GetSymbol<init_handle>("MPI_Init");
cout << "MPI already loaded " << mpi_init << endl;
mpi_initialized = GetSymbol<mpi_initialized_handle>("MPI_Initialized");
get_version = GetSymbol<get_version_handle>("MPI_Get_library_version");
} catch (std::runtime_error &e) {
cout << "MPI not loaded" << endl;
mpi_lib = std::make_unique<SharedLibrary>("libmpi.so", std::nullopt, true);
mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
mpi_initialized =
mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
get_version =
mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
}
int flag = 0;
mpi_initialized(&flag);
if (!flag) {
cout << "Calling MPI_Init" << endl;
mpi_init(nullptr, nullptr);
}
char version_string[65536];
int resultlen = 0;
get_version(version_string, &resultlen);
mpi_library_version = version_string;
cout << "MPI version: " << version_string << endl;
std::string libname = "";
if (mpi_library_version.substr(0, 8) == "Open MPI") {
cout << "Have Open MPI" << endl;
libname = "/opt/netgen/lib/libng_openmpi.so";
} else if (mpi_library_version.substr(0, 5) == "MPICH") {
cout << "Have MPICH" << endl;
libname = "/opt/netgen/lib/libng_mpich.so";
} else
cout << "Unknown MPI" << endl;
if (libname.size()) {
cout << "loading " << libname << endl;
cout << "NG_MPI_INT before " << NG_MPI_INT.value << endl;
ng_mpi_lib = std::make_unique<SharedLibrary>(libname);
auto ng_init = ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi");
cout << "have ng_init " << ng_init << endl;
ng_init();
cout << "NG_MPI_INT after " << NG_MPI_INT.value << endl;
}
}
// NG_MPI_Comm NG_MPI_COMM_WORLD = 0;
// NG_MPI_Datatype NG_MPI_INT = 0;
// NG_MPI_Datatype NG_MPI_SHORT = 0;
// NG_MPI_Datatype NG_MPI_CHAR = 0;
// NG_MPI_Datatype NG_MPI_UINT64_T = 0;
// NG_MPI_Datatype NG_MPI_DOUBLE = 0;
// NG_MPI_Datatype NG_MPI_C_BOOL = 0;
static std::runtime_error no_mpi() {
return std::runtime_error("MPI not enabled");
}
std::string mpi_library_version = "";
#include "ng_mpi_generated_dummy_init.hpp"
} // namespace ngcore

19
libsrc/core/paje_trace.cpp
View File

@ -13,6 +13,7 @@
extern const char *header;
constexpr int MPI_PAJE_WRITER = 1;
constexpr int ASSUMED_MPI_MAX_PROCESSOR_NAME = 256;
namespace ngcore
{
@ -24,7 +25,7 @@ namespace ngcore
if(id<NgProfiler::SIZE)
return NgProfiler::GetName(id);
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
return NgProfiler::GetName(id-NgProfiler::SIZE*comm.Rank());
#endif // PARALLEL
}
@ -70,7 +71,7 @@ namespace ngcore
// sync start time when running in parallel
#ifdef PARALLEL
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
for([[maybe_unused]] auto i : Range(5))
comm.Barrier();
#endif // PARALLEL
@ -112,7 +113,7 @@ namespace ngcore
for(auto i : IntRange(n_memory_events_at_start, memory_events.size()))
memory_events[i].time -= start_time;
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
if(comm.Size()==1)
{
@ -488,7 +489,7 @@ namespace ngcore
#ifdef PARALLEL
// Hostnames
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
// auto rank = comm.Rank();
auto nranks = comm.Size();
if(nranks>1)
@ -496,9 +497,9 @@ namespace ngcore
nthreads = nranks;
thread_aliases.reserve(nthreads);
std::array<char, MPI_MAX_PROCESSOR_NAME+1> ahostname;
std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
int len;
MPI_Get_processor_name(ahostname.data(), &len);
NG_MPI_Get_processor_name(ahostname.data(), &len);
std::string hostname = ahostname.data();
std::map<std::string, int> host_map;
@ -854,15 +855,15 @@ namespace ngcore
{
#ifdef PARALLEL
// Hostname
NgMPI_Comm comm(MPI_COMM_WORLD);
NgMPI_Comm comm(NG_MPI_COMM_WORLD);
// auto rank = comm.Rank();
// auto nranks = comm.Size();
std::string hostname;
{
std::array<char, MPI_MAX_PROCESSOR_NAME+1> ahostname;
std::array<char, ASSUMED_MPI_MAX_PROCESSOR_NAME+1> ahostname;
int len;
MPI_Get_processor_name(ahostname.data(), &len);
NG_MPI_Get_processor_name(ahostname.data(), &len);
hostname = ahostname.data();
}

6
libsrc/core/python_ngcore_export.cpp
View File

@ -6,6 +6,11 @@ using namespace ngcore;
using namespace std;
using namespace pybind11::literals;
namespace ngcore {
NGCORE_API void InitMPI();
}
PYBIND11_MODULE(pyngcore, m) // NOLINT
{
try
@ -328,4 +333,5 @@ threads : int
}, "Returns list of timers"
);
m.def("ResetTimers", &NgProfiler::Reset);
m.def("InitMPI", &InitMPI);
}

97
libsrc/core/utils.cpp
View File

@ -3,16 +3,24 @@
#include "logging.hpp"
#include "simd_generic.hpp"
#ifndef WIN32
#ifdef WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#undef WIN32_LEAN_AND_MEAN
#else // WIN32
#include <cxxabi.h>
#endif
#include <dlfcn.h>
#endif //WIN32
//
#include <array>
#include <filesystem>
#include <iostream>
#include <regex>
#include <thread>
#include "ngstream.hpp"
namespace ngcore
{
namespace detail
@ -134,5 +142,90 @@ namespace ngcore
return path;
}
SharedLibrary :: SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_, bool global )
: lib_name(lib_name_),directory_to_delete(directory_to_delete_)
{
Load(lib_name, global);
}
SharedLibrary :: ~SharedLibrary()
{
Unload();
if(directory_to_delete)
for([[maybe_unused]] auto i : Range(5))
{
// on Windows, a (detached?) child process of the compiler/linker might still block the directory
// wait for it to finish (up to a second)
try
{
std::filesystem::remove_all(*directory_to_delete);
directory_to_delete = std::nullopt;
break;
}
catch(const std::exception &e)
{
std::this_thread::sleep_for(std::chrono::milliseconds(200));
}
}
if(directory_to_delete)
std::cerr << "Could not delete " << directory_to_delete->string() << std::endl;
}
void SharedLibrary :: Load( const std::filesystem::path & lib_name_, bool global )
{
Unload();
lib_name = lib_name_;
#ifdef WIN32
lib = LoadLibrary(lib_name.string().c_str());
if (!lib) throw std::runtime_error(string("Could not load library ") + lib_name.string());
#else // WIN32
auto flags = RTLD_NOW;
if (global) flags |= RTLD_GLOBAL;
lib = dlopen(lib_name.c_str(), flags);
if(lib == nullptr) throw std::runtime_error(dlerror());
#endif // WIN32
}
void SharedLibrary :: Unload() {
if(lib)
{
#ifdef WIN32
FreeLibrary((HMODULE)lib);
#else // WIN32
int rc = dlclose(lib);
if(rc != 0) std::cerr << "Failed to close library " << lib_name << std::endl;
#endif // WIN32
}
}
void* SharedLibrary :: GetRawSymbol( std::string func_name )
{
#ifdef WIN32
void* func = GetProcAddress((HMODULE)lib, func_name.c_str());
if(func == nullptr)
throw std::runtime_error(string("Could not find function ") + func_name + " in library " + lib_name.string());
#else // WIN32
void* func = dlsym(lib, func_name.c_str());
if(func == nullptr)
throw std::runtime_error(dlerror());
#endif // WIN32
return func;
}
void* GetRawSymbol( std::string func_name )
{
#ifdef WIN32
throw std::runtime_error("GetRawSymbol not implemented on WIN32");
#else // WIN32
void* func = dlsym(RTLD_DEFAULT, func_name.c_str());
if(func == nullptr)
throw std::runtime_error(dlerror());
#endif // WIN32
return func;
}
} // namespace ngcore

41
libsrc/core/utils.hpp
View File

@ -6,6 +6,7 @@
#include <filesystem>
#include <map>
#include <ostream>
#include <optional>
#include <sstream>
#include <string>
@ -347,6 +348,46 @@ namespace ngcore
NGCORE_API std::filesystem::path GetTempFilename();
NGCORE_API void* GetRawSymbol( std::string func_name );
template <typename TFunc>
TFunc GetSymbol( std::string func_name )
{
return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
}
// Class to handle/load shared libraries
class NGCORE_API SharedLibrary
{
std::filesystem::path lib_name;
std::optional<std::filesystem::path> directory_to_delete = std::nullopt;
// #ifdef WIN32
// HINSTANCE lib = nullptr;
// #else // WIN32
void *lib = nullptr;
// #endif // WIN32
public:
SharedLibrary() = default;
SharedLibrary(const std::filesystem::path & lib_name_, std::optional<std::filesystem::path> directory_to_delete_ = std::nullopt, bool global = false );
SharedLibrary(const SharedLibrary &) = delete;
SharedLibrary & operator =(const SharedLibrary &) = delete;
~SharedLibrary();
template <typename TFunc>
TFunc GetSymbol( std::string func_name )
{
return reinterpret_cast<TFunc>(GetRawSymbol(func_name));
}
void Load( const std::filesystem::path & lib_name_, bool global);
void Unload();
void* GetRawSymbol( std::string func_name );
};
} // namespace ngcore
#endif // NETGEN_CORE_UTILS_HPP