From cb2c5d63232cb9fb90477e8da91c67ebd57ec68a Mon Sep 17 00:00:00 2001
From: Lukas <lukas.kogler@tuwien.ac.at>
Date: Mon, 26 Aug 2019 13:02:13 +0200
Subject: [PATCH 1/3] move array-send and recv to ngcore

---
 libsrc/core/mpi_wrapper.hpp | 75 ++++++++++++++++++++++++++++++++++---
 1 file changed, 70 insertions(+), 5 deletions(-)

diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp
index b42c551e..2117426b 100644
--- a/libsrc/core/mpi_wrapper.hpp
+++ b/libsrc/core/mpi_wrapper.hpp
@@ -6,6 +6,7 @@
 #include <mpi.h>
 #endif
 
+#include "array.hpp"
 #include "exception.hpp"
 
 namespace ngcore
@@ -127,11 +128,32 @@ namespace ngcore
       MPI_Send (&val, 1, GetMPIType<T>(), dest, tag, comm);
     }
     
+    template<typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Send(FlatArray<T> s, int dest, int tag) const {
+      MPI_Send (s.Data(), s.Size(), GetMPIType<T>(), dest, tag, comm);
+    }
+    
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
     void Recv (T & val, int src, int tag) const {
       MPI_Recv (&val, 1, GetMPIType<T>(), src, tag, comm, MPI_STATUS_IGNORE);
     }
 
+    template <typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (FlatArray <T> s, int src, int tag) const {
+      MPI_Recv (s.Data(), s.Size(), GetMPIType<T> (), src, tag, comm, MPI_STATUS_IGNORE);
+    }
+    
+    template <typename T, typename T2 = decltype(GetMPIType<T>())>
+    void Recv (Array <T> & s, int src, int tag) const
+    {
+      MPI_Status status;
+      int len;
+      const MPI_Datatype MPI_T  = GetMPIType<T> ();
+      MPI_Probe (src, tag, comm, &status);
+      MPI_Get_count (&status, MPI_T, &len);
+      s.SetSize (len);
+      MPI_Recv (s.Data(), len, MPI_T, src, tag, comm, MPI_STATUS_IGNORE);
+    }
 
     /** --- non-blocking P2P --- **/
     
@@ -144,13 +166,22 @@ namespace ngcore
     }
     
     template<typename T, typename T2 = decltype(GetMPIType<T>())> 
-    MPI_Request IRecv (T & val, int dest, int tag) const
+    MPI_Request IRecv (T & val, int src, int tag) const
     {
       MPI_Request request;
-      MPI_Irecv (&val, 1, GetMPIType<T>(), dest, tag, comm, &request);
+      MPI_Irecv (&val, 1, GetMPIType<T>(), src, tag, comm, &request);
       return request;
     }
     
+    template<typename T, typename T2 = decltype(GetMPIType<T>())>
+    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const
+    { 
+      MPI_Request request;
+      MPI_Irecv (s.Data(), s.Size(), GetMPIType<T>(), src, tag, comm, &request);
+      return request;
+    }
+
+    
     /** --- collectives --- **/
 
     template <typename T, typename T2 = decltype(GetMPIType<T>())> 
@@ -191,7 +222,21 @@ namespace ngcore
     
   };
 
+  }; // class NgMPI_Comm
+
+  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests)
+  {
+    if (!requests.Size()) return;
+    MPI_Waitall (requests.Size(), requests.Data(), MPI_STATUSES_IGNORE);
+  }
   
+  NETGEN_INLINE int MyMPI_WaitAny (FlatArray<MPI_Request> requests)
+  {
+    int nr;
+    MPI_Waitany (requests.Size(), requests.Data(), &nr, MPI_STATUS_IGNORE);
+    return nr;
+  }
+
 #else // PARALLEL
   class MPI_Comm {
     int nr;
@@ -223,24 +268,44 @@ namespace ngcore
     void Send( T & val, int dest, int tag) const { ; }
     
     template<typename T>
-    void MyMPI_Recv (T & val, int src, int tag) const { ; }
+    void Send(FlatArray<T> s, int dest, int tag) const { ; }
+
+    template<typename T>
+    void Recv (T & val, int src, int tag) const { ; }
+
+    template <typename T>
+    void Recv (FlatArray <T> s, int src, int tag) const { ; }
+
+    template <typename T>
+    void Recv (Array <T> & s, int src, int tag) const { ; }
 
     template<typename T>
     MPI_Request ISend (T & val, int dest, int tag) const { return 0; } 
     
+    template<typename T>
+    MPI_Request ISend (const FlatArray<T> & s, int dest, int tag) const { return 0; }
+
     template<typename T>
     MPI_Request IRecv (T & val, int dest, int tag) const { return 0; } 
     
+    template<typename T>
+    MPI_Request IRecv (const FlatArray<T> & s, int src, int tag) const { return 0; }
+
     template <typename T>
-    T Reduce (T d, const MPI_Op & op, int root = 0) { return d; }
+    T Reduce (T d, const MPI_Op & op, int root = 0) const { return d; }
     
     template <typename T>
     T AllReduce (T d, const MPI_Op & op) const { return d; }
 
     template <typename T>
     void Bcast (T & s, int root = 0) const { ; } 
+
+    NgMPI_Comm SubCommunicator (FlatArray<int> procs) const
+    { return *this; }
   };  
-  
+
+  NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests) { ; }
+
 #endif // PARALLEL
 
 

From 464f4223e7e99aaededb2f51c44f72cc72bc3a6a Mon Sep 17 00:00:00 2001
From: Lukas <lukas.kogler@tuwien.ac.at>
Date: Tue, 27 Aug 2019 11:14:02 +0200
Subject: [PATCH 2/3] fix typo

---
 libsrc/core/mpi_wrapper.hpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp
index 2117426b..4407173d 100644
--- a/libsrc/core/mpi_wrapper.hpp
+++ b/libsrc/core/mpi_wrapper.hpp
@@ -219,9 +219,6 @@ namespace ngcore
       MPI_Bcast (&s[0], len, MPI_CHAR, root, comm);
     }
 
-    
-  };
-
   }; // class NgMPI_Comm
 
   NETGEN_INLINE void MyMPI_WaitAll (FlatArray<MPI_Request> requests)

From 13c17adf8765d3dc3c4f768f001776390b343fa8 Mon Sep 17 00:00:00 2001
From: Christopher Lackner <christopher.lackner@tuwien.ac.at>
Date: Wed, 28 Aug 2019 09:52:51 +0200
Subject: [PATCH 3/3] restricth for occ and stl geometries

---
 libsrc/occ/occgenmesh.cpp      | 3 +++
 libsrc/stlgeom/stlgeommesh.cpp | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/libsrc/occ/occgenmesh.cpp b/libsrc/occ/occgenmesh.cpp
index aa2c4b0d..cbe2aca3 100644
--- a/libsrc/occ/occgenmesh.cpp
+++ b/libsrc/occ/occgenmesh.cpp
@@ -1264,6 +1264,9 @@ namespace netgen
               }
           }
 
+        for (auto mspnt : mparam.meshsize_points)
+          mesh.RestrictLocalH(mspnt.pnt, mspnt.h);
+
         multithread.task = savetask;
 
       }
diff --git a/libsrc/stlgeom/stlgeommesh.cpp b/libsrc/stlgeom/stlgeommesh.cpp
index a5fcc0c2..301799d8 100644
--- a/libsrc/stlgeom/stlgeommesh.cpp
+++ b/libsrc/stlgeom/stlgeommesh.cpp
@@ -1369,6 +1369,10 @@ int STLMeshingDummy (STLGeometry* stlgeometry, shared_ptr<Mesh> & mesh, const Me
 			 stlgeometry->GetBoundingBox().PMax() + Vec3d(10, 10, 10),
 			 mparam.grading);
       mesh -> LoadLocalMeshSize (mparam.meshsizefilename);
+
+      if (mparam.uselocalh)
+        for (auto mspnt : mparam.meshsize_points)
+          mesh->RestrictLocalH(mspnt.pnt, mspnt.h);
       
       success = 0;