netgen/libsrc/core/mpi_wrapper.hpp

258 lines
6.2 KiB
C++
Raw Normal View History

2019-02-12 00:01:07 +05:00
#ifndef NGCORE_MPIWRAPPER_HPP
#define NGCORE_MPIWRAPPER_HPP
#ifdef PARALLEL
#define OMPI_SKIP_MPICXX
#include <mpi.h>
#endif
2019-07-10 13:56:55 +05:00
#include "exception.hpp"
2019-02-12 00:01:07 +05:00
namespace ngcore
{
#ifdef PARALLEL
template <class T> struct MPI_typetrait { };
template <> struct MPI_typetrait<int> {
static MPI_Datatype MPIType () { return MPI_INT; } };
template <> struct MPI_typetrait<short> {
static MPI_Datatype MPIType () { return MPI_SHORT; } };
template <> struct MPI_typetrait<char> {
static MPI_Datatype MPIType () { return MPI_CHAR; } };
2019-02-12 01:37:00 +05:00
template <> struct MPI_typetrait<unsigned char> {
static MPI_Datatype MPIType () { return MPI_CHAR; } };
2019-02-12 00:01:07 +05:00
template <> struct MPI_typetrait<size_t> {
2019-02-12 01:37:00 +05:00
static MPI_Datatype MPIType () { return MPI_UINT64_T; } };
2019-02-12 00:01:07 +05:00
template <> struct MPI_typetrait<double> {
static MPI_Datatype MPIType () { return MPI_DOUBLE; } };
template <> struct MPI_typetrait<bool> {
static MPI_Datatype MPIType () { return MPI_C_BOOL; } };
template <class T, class T2 = decltype(MPI_typetrait<T>::MPIType())>
inline MPI_Datatype GetMPIType () {
return MPI_typetrait<T>::MPIType();
}
2019-02-12 02:12:29 +05:00
2019-02-12 00:01:07 +05:00
class NgMPI_Comm
{
2019-02-13 04:11:35 +05:00
protected:
2019-02-12 00:01:07 +05:00
MPI_Comm comm;
2019-02-13 16:28:24 +05:00
bool valid_comm;
2019-02-12 00:01:07 +05:00
int * refcount;
2019-02-12 01:37:00 +05:00
int rank, size;
2019-02-12 00:01:07 +05:00
public:
2019-02-12 12:03:20 +05:00
NgMPI_Comm ()
2019-02-13 16:28:24 +05:00
: valid_comm(false), refcount(nullptr), rank(0), size(1)
2019-02-12 12:03:20 +05:00
{ ; }
2019-02-12 00:01:07 +05:00
NgMPI_Comm (MPI_Comm _comm, bool owns = false)
2019-02-13 16:28:24 +05:00
: comm(_comm), valid_comm(true)
2019-02-12 00:01:07 +05:00
{
if (!owns)
refcount = nullptr;
else
refcount = new int{1};
2019-02-12 01:37:00 +05:00
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
2019-02-12 00:01:07 +05:00
}
NgMPI_Comm (const NgMPI_Comm & c)
2019-02-13 16:28:24 +05:00
: comm(c.comm), valid_comm(c.valid_comm), refcount(c.refcount),
rank(c.rank), size(c.size)
2019-02-12 00:01:07 +05:00
{
if (refcount) (*refcount)++;
}
NgMPI_Comm (NgMPI_Comm && c)
2019-02-13 16:28:24 +05:00
: comm(c.comm), valid_comm(c.valid_comm), refcount(c.refcount),
rank(c.rank), size(c.size)
2019-02-12 00:01:07 +05:00
{
c.refcount = nullptr;
}
~NgMPI_Comm()
{
if (refcount)
if (--(*refcount) == 0)
MPI_Comm_free(&comm);
}
2019-02-12 05:21:56 +05:00
NgMPI_Comm & operator= (const NgMPI_Comm & c)
{
if (refcount)
if (--(*refcount) == 0)
MPI_Comm_free(&comm);
refcount = c.refcount;
if (refcount) (*refcount)++;
2019-02-12 16:41:08 +05:00
comm = c.comm;
2019-02-13 16:28:24 +05:00
valid_comm = c.valid_comm;
2019-02-12 05:21:56 +05:00
size = c.size;
rank = c.rank;
return *this;
}
2019-02-13 16:28:24 +05:00
class InvalidCommException : public Exception {
public:
InvalidCommException() : Exception("Do not have a valid communicator") { ; }
};
2019-02-12 05:21:56 +05:00
2019-02-13 16:28:24 +05:00
operator MPI_Comm() const {
if (!valid_comm) throw InvalidCommException();
return comm;
}
2019-02-12 00:01:07 +05:00
2019-02-13 02:11:55 +05:00
int Rank() const { return rank; }
int Size() const { return size; }
void Barrier() const {
if (size > 1) MPI_Barrier (comm);
}
2019-02-12 01:37:00 +05:00
2019-02-13 04:11:35 +05:00
/** --- blocking P2P --- **/
2019-02-12 01:37:00 +05:00
template<typename T, typename T2 = decltype(GetMPIType<T>())>
2019-02-13 02:11:55 +05:00
void Send (T & val, int dest, int tag) const {
2019-02-12 01:37:00 +05:00
MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
2019-02-13 04:11:35 +05:00
void Recv (T & val, int src, int tag) const {
2019-02-12 01:37:00 +05:00
MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
}
2019-02-13 02:11:55 +05:00
2019-02-13 04:11:35 +05:00
/** --- non-blocking P2P --- **/
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request ISend (T & val, int dest, int tag) const
{
MPI_Request request;
MPI_Isend (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
return request;
}
template<typename T, typename T2 = decltype(GetMPIType<T>())>
MPI_Request IRecv (T & val, int dest, int tag) const
{
MPI_Request request;
MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
return request;
}
2019-02-13 02:11:55 +05:00
/** --- collectives --- **/
2019-02-13 04:11:35 +05:00
template <typename T, typename T2 = decltype(GetMPIType<T>())>
2019-03-04 18:24:18 +05:00
T Reduce (T d, const MPI_Op & op, int root = 0) const
2019-02-13 04:11:35 +05:00
{
if (size == 1) return d;
T global_d;
MPI_Reduce (&d, &global_d, 1, GetMPIType<T>(), op, root, comm);
return global_d;
}
2019-02-13 02:11:55 +05:00
template <typename T, typename T2 = decltype(GetMPIType<T>())>
T AllReduce (T d, const MPI_Op & op) const
{
if (size == 1) return d;
T global_d;
MPI_Allreduce ( &d, &global_d, 1, GetMPIType<T>(), op, comm);
return global_d;
}
template <typename T, typename T2 = decltype(GetMPIType<T>())>
void Bcast (T & s, int root = 0) const {
if (size == 1) return ;
MPI_Bcast (&s, 1, GetMPIType<T>(), root, comm);
}
void Bcast (std::string & s, int root = 0) const
{
if (size == 1) return;
int len = s.length();
Bcast (len, root);
if (rank != 0) s.resize (len);
MPI_Bcast (&s[0], len, MPI_CHAR, root, comm);
}
2019-02-12 01:37:00 +05:00
2019-02-12 00:01:07 +05:00
};
2019-07-10 13:56:55 +05:00
#else // PARALLEL
2019-02-12 03:13:12 +05:00
class MPI_Comm {
int nr;
public:
MPI_Comm (int _nr = 0) : nr(_nr) { ; }
operator int() const { return nr; }
bool operator== (MPI_Comm c2) const { return nr == c2.nr; }
};
static MPI_Comm MPI_COMM_WORLD = 12345, MPI_COMM_NULL = 10000;
2019-02-13 02:25:22 +05:00
typedef int MPI_Op;
2019-02-13 04:11:35 +05:00
typedef int MPI_Request;
2019-02-13 02:25:22 +05:00
enum { MPI_SUM = 0, MPI_MIN = 1, MPI_MAX = 2 };
2019-02-12 00:01:07 +05:00
class NgMPI_Comm
{
public:
2019-02-12 12:03:20 +05:00
NgMPI_Comm () { ; }
NgMPI_Comm (MPI_Comm _comm, bool owns = false) { ; }
2019-02-12 00:01:07 +05:00
size_t Rank() const { return 0; }
size_t Size() const { return 1; }
2019-02-13 02:11:55 +05:00
void Barrier() const { ; }
2019-02-12 02:12:29 +05:00
operator MPI_Comm() const { return MPI_Comm(); }
2019-02-12 01:37:00 +05:00
template<typename T>
2019-02-13 02:11:55 +05:00
void Send( T & val, int dest, int tag) const { ; }
2019-02-12 01:37:00 +05:00
template<typename T>
2019-02-13 02:11:55 +05:00
void MyMPI_Recv (T & val, int src, int tag) const { ; }
2019-02-13 04:11:35 +05:00
template<typename T>
MPI_Request ISend (T & val, int dest, int tag) const { return 0; }
template<typename T>
MPI_Request IRecv (T & val, int dest, int tag) const { return 0; }
template <typename T>
T Reduce (T d, const MPI_Op & op, int root = 0) { return d; }
2019-02-13 02:16:53 +05:00
template <typename T>
2019-02-13 02:11:55 +05:00
T AllReduce (T d, const MPI_Op & op) const { return d; }
2019-02-13 02:16:53 +05:00
template <typename T>
2019-02-13 02:21:44 +05:00
void Bcast (T & s, int root = 0) const { ; }
2019-02-12 00:01:07 +05:00
};
2019-07-10 13:56:55 +05:00
#endif // PARALLEL
2019-02-12 00:01:07 +05:00
}
2019-07-10 13:56:55 +05:00
#endif // NGCORE_MPIWRAPPER_HPP
2019-02-12 00:01:07 +05:00