From cb2c5d63232cb9fb90477e8da91c67ebd57ec68a Mon Sep 17 00:00:00 2001
From: Lukas <lukas.kogler@tuwien.ac.at>
Date: Mon, 26 Aug 2019 13:02:13 +0200
Subject: [PATCH] move array-send and recv to ngcore

---
 libsrc/core/mpi_wrapper.hpp | 75 ++++++++++++++++++++++++++++++++++---
 1 file changed, 70 insertions(+), 5 deletions(-)

diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp
index b42c551e..2117426b 100644
--- a/libsrc/core/mpi_wrapper.hpp
+++ b/libsrc/core/mpi_wrapper.hpp
@@ -6,6 +6,7 @@
 #include <mpi.h>
 #endif
 
+#include "array.hpp"
 #include "exception.hpp"
 
 namespace ngcore
@@ -127,11 +128,32 @@ namespace ngcore
       MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
     }
     
+    template<typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Send(FlatArray<T> s, int dest, int tag) const {
+      MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
+    }
+    
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
     void Recv (T & val, int src, int tag) const {
       MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
     }
 
+    template <typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (FlatArray <T> s, int src, int tag) const {
+      MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, MPI_STATUS_IGNORE);
+    }
+    
+    template <typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (Array <T> & s, int src, int tag) const
+    {
+      MPI_Status status;
+      int len;
+      const MPI_Datatype MPI_T  = GetMPIType<T> ();
+      MPI_Probe (src, tag, comm, &status);
+      MPI_Get_count (&status, MPI_T, &len);
+      s.SetSize (len);
+      MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE);
+    }
 
     /** --- non-blocking P2P --- **/
     
@@ -144,13 +166,22 @@ namespace ngcore
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request IRecv (T & val, int dest, int tag) const
+    MPI_Request IRecv (T & val, int src, int tag) const
     {
       MPI_Request request;
-      MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      MPI_Irecv (&val, 1, GetMPIType<T>(), src, tag, comm, &request);
       return request;
     }
     
+    template<typename T, typename T2 = decltype(GetMPIType<T>())>
+    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const
+    { 
+      MPI_Request request;
+      MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
+      return request;
+    }
+
+    
     /** --- collectives --- **/
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
@@ -191,7 +222,21 @@ namespace ngcore
     
   };
 
+  }; // class NgMPI_Comm
+
+  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests)
+  {
+    if (!requests.Size()) return;
+    MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE);
+  }
   
+  NETGEN_INLINE int MyMPI_WaitAny (FlatArray<MPI_Request> requests)
+  {
+    int nr;
+    MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE);
+    return nr;
+  }
+
 #else // PARALLEL
   class MPI_Comm {
     int nr;
@@ -223,24 +268,44 @@ namespace ngcore
     void Send( T & val, int dest, int tag) const { ; }
     
     template<typename T>
-    void MyMPI_Recv (T & val, int src, int tag) const { ; }
+    void Send(FlatArray<T> s, int dest, int tag) const { ; }
+
+    template<typename T>
+    void Recv (T & val, int src, int tag) const { ; }
+
+    template <typename T>
+    void Recv (FlatArray <T> s, int src, int tag) const { ; }
+
+    template <typename T>
+    void Recv (Array <T> & s, int src, int tag) const { ; }
 
     template<typename T>
     MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
     
+    template<typename T>
+    MPI_Request ISend (const FlatArray<T> & s, int dest, int tag) const { return 0; }
+
     template<typename T>
     MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
     
+    template<typename T>
+    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const { return 0; }
+
     template <typename T>
-    T Reduce (T d, const MPI_Op & op, int root = 0) { return d; }
+    T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; }
     
     template <typename T>
     T AllReduce (T d, const MPI_Op & op) const { return d; }
 
     template <typename T>
     void Bcast (T & s, int root = 0) const { ; } 
+
+    NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
+    { return *this; }
   };  
-  
+
+  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests) { ; }
+
 #endif // PARALLEL