littel parallel polishing

libsrc/include/nginterface_v2.hpp

@@ -368,7 +368,8 @@ namespace netgen
     
 
     // for MPI-parallel
-    std::tuple<int,int*> GetDistantProcs (int nodetype, int locnum) const;
+    FlatArray<int> GetDistantProcs (int nodetype, int locnum) const;
+    size_t GetGlobalVertexNum (int locnum) const;
                                
     shared_ptr<Mesh> GetMesh () const { return mesh; } 
     shared_ptr<Mesh> SelectMesh () const;

libsrc/interface/nginterface.cpp

@@ -905,6 +905,7 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou
 
 // gibt anzahl an distant pnums zurueck
 // * pnums entspricht ARRAY<int[2] >
+[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]                    
 int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
 {
   int size = NgPar_GetNDistantNodeNums (nodetype, locnum);
@@ -931,6 +932,7 @@ int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
   return size;
 }
 
+[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]                    
 int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
 {
   locnum++;
@@ -944,6 +946,7 @@ int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
   return -1;
 }
 
+[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]                    
 int NgPar_GetGlobalNodeNum (int nodetype, int locnum)
 {
   locnum++;

libsrc/interface/nginterface_v2.cpp

@@ -1296,36 +1296,31 @@ void Ngx_Mesh::SetSurfaceElementOrders (int enr, int ox, int oy)
 
   
 
+size_t Ngx_Mesh :: GetGlobalVertexNum (int locnum) const
+{
+  return mesh->GetParallelTopology().GetGlobalPNum (locnum+1)-1;
+}
 
   
-  std::tuple<int,int*>  Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const
+FlatArray<int>  Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const
   {
 #ifdef PARALLEL
     if (mesh->GetCommunicator().Size() == 1)
-      return std::tuple<int,int*>(0,nullptr);
+      return FlatArray<int>(0,nullptr);
     
     switch (nodetype)
       {
       case 0:
-	{
+        return mesh->GetParallelTopology().GetDistantPNums(locnum);
-	  NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantPNums(locnum);
-	  return std::tuple<int,int*>(dn.Size(), &dn[0]);
-	}
       case 1:
-	{
+        return mesh->GetParallelTopology().GetDistantEdgeNums(locnum);
-	  NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantEdgeNums(locnum);
-	  return std::tuple<int,int*>(dn.Size(), &dn[0]);
-	}
       case 2:
-	{
+        return mesh->GetParallelTopology().GetDistantFaceNums(locnum);
-	  NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantFaceNums(locnum);
-	  return std::tuple<int,int*>(dn.Size(), &dn[0]);
-	}
       default:
-	return std::tuple<int,int*>(0,nullptr);
+	return FlatArray<int>(0, nullptr);
       }
 #else
-    return std::tuple<int,int*>(0,nullptr);
+    return FlatArray<int>(0,nullptr);
 #endif
   }
 }

libsrc/meshing/meshclass.cpp

@@ -1298,89 +1298,6 @@ namespace netgen
         // GridFunciton pickling is not compatible, now
         // should go to paralleltopology
         
-        Array<PointIndex, PointIndex> global_pnums(points.Size());
-        global_pnums = -1;
-        int num_master_points = 0;
-        for (PointIndex pi : Range(points))
-          {
-            auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE);
-            // check sorted:
-            for (int j = 0; j+1 < distprocs.Size(); j++)
-              if (distprocs[j+1] < distprocs[j]) cout << "wrong sort" << endl;
-            if (distprocs.Size() == 0 || distprocs[0] > comm.Rank())
-              global_pnums[pi] = PointIndex::BASE+num_master_points++;
-          }
-        Array<int> first_master_point(comm.Size());
-        GetCommunicator().AllGather (num_master_points, first_master_point);
-        
-        size_t num_glob_points = 0;
-        for (int i = 0; i < comm.Size(); i++)
-          {
-            int cur = first_master_point[i];
-            first_master_point[i] = num_glob_points;
-            num_glob_points += cur;
-          }
-    
-        for (PointIndex pi : Range(points))
-          if (global_pnums[pi] != -1)
-            global_pnums[pi] += first_master_point[comm.Rank()];
-    
-        // ScatterDofData (global_nums); 
-
-        Array<int> nsend(comm.Size()), nrecv(comm.Size());
-        nsend = 0;
-        nrecv = 0;
-
-        /** Count send/recv size **/
-        for (PointIndex pi : Range(points))
-          {
-            auto dps = partop.GetDistantPNums(pi-PointIndex::BASE);            
-            if (!dps.Size()) continue;
-            if (rank < dps[0])
-              for(auto p:dps)
-                nsend[p]++;
-            else
-              nrecv[dps[0]]++;
-          }
-        
-        Table<PointIndex> send_data(nsend);
-        Table<PointIndex> recv_data(nrecv);
-
-        /** Fill send_data **/
-        nsend = 0;
-        for (PointIndex pi : Range(points))        
-          {
-            auto dps = partop.GetDistantPNums(pi-PointIndex::BASE);
-            if (dps.Size() && rank < dps[0])
-              for(auto p : dps)
-                send_data[p][nsend[p]++] = global_pnums[pi];
-          }
-
-        Array<MPI_Request> requests;
-        for (int i = 0; i < comm.Size(); i++)
-          {
-            if (nsend[i])
-              requests.Append (comm.ISend (send_data[i], i, 200));
-            if (nrecv[i])
-              requests.Append (comm.IRecv (recv_data[i], i, 200));
-          }
-
-        MyMPI_WaitAll (requests);
-        
-        Array<int> cnt(comm.Size());
-        cnt = 0;
-
-        for (PointIndex pi : Range(points))
-          {
-            auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE);
-            if (distprocs.Size() > 0 && distprocs[0] < comm.Rank())
-              {
-                int master = comm.Size();
-                for (int j = 0; j < distprocs.Size(); j++)
-                  master = min (master, distprocs[j]);
-                global_pnums[pi] = recv_data[master][cnt[master]++];
-              }
-          }
         
         
         // merge points

libsrc/meshing/parallelmesh.cpp

@@ -732,6 +732,7 @@ namespace netgen
     for (auto t : point_types)
       { MPI_Type_free(&t); }
 
+    
     PrintMessage ( 3, "Sending names");
 
     sendrequests.SetSize(3*ntasks);
@@ -815,6 +816,9 @@ namespace netgen
     
     // const_cast<Mesh&>(*this).DeleteMesh();
 
+    // paralleltop -> SetNV (0);
+    // paralleltop->EnumeratePointsGlobally();
+    
     PrintMessage( 3, "send mesh complete");
   }
 
@@ -1016,6 +1020,7 @@ namespace netgen
 	}
     }
 
+
     /** Recv bc-names **/
     ArrayMem<int,4> nnames{0,0,0,0};
     // MPI_Recv(nnames, 4, MPI_INT, 0, MPI_TAG_MESH+6, comm, MPI_STATUS_IGNORE);
@@ -1073,7 +1078,7 @@ namespace netgen
     clusters -> Update();
 
     // paralleltop -> UpdateCoarseGrid();
-    
+    // paralleltop->EnumeratePointsGlobally();
     SetNextMajorTimeStamp();
   }
   

libsrc/meshing/paralleltop.cpp

@@ -55,6 +55,107 @@ namespace netgen
   }
 
 
+  void ParallelMeshTopology :: EnumeratePointsGlobally ()
+  {
+    auto nv = loc2distvert.Size();
+    auto comm = mesh.GetCommunicator();
+    auto rank = comm.Rank();
+
+    // if (rank == 0)
+    // nv = 0;
+    glob_vert.SetSize (nv);
+    glob_vert = -1;
+    int num_master_points = 0;
+    
+    for (auto i : Range(nv))
+      {
+        auto dps = GetDistantPNums(i);
+        // check sorted:
+        for (int j = 0; j+1 < dps.Size(); j++)
+          if (dps[j+1] < dps[j]) cout << "wrong sort" << endl;
+        
+        if (dps.Size() == 0 || dps[0] > comm.Rank())
+          glob_vert[i] = num_master_points++;
+      }
+    
+    Array<int> first_master_point(comm.Size());
+    comm.AllGather (num_master_points, first_master_point);
+        
+    size_t num_glob_points = 0;
+    for (int i = 0; i < comm.Size(); i++)
+      {
+        int cur = first_master_point[i];
+        first_master_point[i] = num_glob_points;
+        num_glob_points += cur;
+      }
+    
+    for (auto i : Range(nv))
+      if (glob_vert[i] != -1)
+        glob_vert[i] += first_master_point[comm.Rank()];
+    
+    // ScatterDofData (global_nums); 
+    
+    Array<int> nsend(comm.Size()), nrecv(comm.Size());
+    nsend = 0;
+    nrecv = 0;
+
+        /** Count send/recv size **/
+    for (auto i : Range(nv))
+      {
+        auto dps = GetDistantPNums(i);
+        if (!dps.Size()) continue;
+        if (rank < dps[0])
+          for(auto p:dps)
+            nsend[p]++;
+        else
+          nrecv[dps[0]]++;
+      }
+    
+    Table<PointIndex> send_data(nsend);
+    Table<PointIndex> recv_data(nrecv);
+    
+    /** Fill send_data **/
+    nsend = 0;
+    for (auto i : Range(nv))
+      {
+        auto dps = GetDistantPNums(i);
+        if (dps.Size() && rank < dps[0])
+          for(auto p : dps)
+            send_data[p][nsend[p]++] = glob_vert[i];
+      }
+    
+    Array<MPI_Request> requests;
+    for (int i = 0; i < comm.Size(); i++)
+      {
+        if (nsend[i])
+          requests.Append (comm.ISend (send_data[i], i, 200));
+        if (nrecv[i])
+          requests.Append (comm.IRecv (recv_data[i], i, 200));
+      }
+    
+    MyMPI_WaitAll (requests);
+    
+    Array<int> cnt(comm.Size());
+    cnt = 0;
+    
+    for (auto i : Range(nv))
+      {
+        auto dps = GetDistantPNums(i);
+        if (dps.Size() > 0 && dps[0] < comm.Rank())
+          {
+            int master = comm.Size();
+            for (int j = 0; j < dps.Size(); j++)
+              master = min (master, dps[j]);
+            glob_vert[i] = recv_data[master][cnt[master]++];
+          }
+      }
+
+    if (PointIndex::BASE==1)
+      for (auto & i : glob_vert)
+        i++;
+  }
+  
+
   void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
   {
     for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
@@ -242,7 +343,12 @@ namespace netgen
     MPI_Group_excl (MPI_GROUP_comm, 1, process_ranks, &MPI_LocalGroup);
     MPI_Comm_create (comm, MPI_LocalGroup, &MPI_LocalComm);
 
-    if (id == 0) return;
+    if (id == 0)
+      {
+        // SetNV(0);
+        // EnumeratePointsGlobally();
+        return;
+      }
     
     const MeshTopology & topology = mesh.GetTopology();
 
@@ -573,7 +679,7 @@ namespace netgen
 	NgProfiler::StopTimer (timerf);
       }
     // cout << "UpdateCoarseGrid - done" << endl;
-    
+    // EnumeratePointsGlobally();
     is_updated = true;
 
     MPI_Group_free(&MPI_LocalGroup);

libsrc/meshing/paralleltop.hpp

@@ -42,60 +42,80 @@ namespace netgen
     void SetNSE (int anse);
     void SetNSegm (int anseg);
 
-
     void SetLoc2Glob_Vert   (int locnum, int globnum) { glob_vert[locnum-1] = globnum; }
+    [[deprecated("Try to avoid global enumration!")]]                
     void SetLoc2Glob_Edge   (int locnum, int globnum) { glob_edge[locnum-1] = globnum; }
+    [[deprecated("Try to avoid global enumration!")]]                
     void SetLoc2Glob_Face   (int locnum, int globnum) { glob_face[locnum-1] = globnum; }
+    [[deprecated("Try to avoid global enumration!")]]                
     void SetLoc2Glob_VolEl  (int locnum, int globnum) { glob_el[locnum-1] = globnum; }
+    [[deprecated("Try to avoid global enumration!")]]                
     void SetLoc2Glob_SurfEl (int locnum, int globnum) { glob_surfel[locnum-1] = globnum; }
+    [[deprecated("Try to avoid global enumration!")]]                
     void SetLoc2Glob_Segm   (int locnum, int globnum) { glob_segm[locnum-1] = globnum; }
 
     int GetGlobalPNum    (int locnum) const { return glob_vert[locnum-1]; }
+    [[deprecated("Try to avoid global enumration!")]]                
     int GetGlobalEdgeNum (int locnum) const { return glob_edge[locnum-1]; }
+    [[deprecated("Try to avoid global enumration!")]]                
     int GetGlobalFaceNum (int locnum) const { return glob_face[locnum-1]; }
+    [[deprecated("Try to avoid global enumration!")]]                
     int GetGlobalElNum   (int locnum) const { return glob_el[locnum-1]; }
+    [[deprecated("Try to avoid global enumration!")]]                
     int GetGlobalSElNum  (int locnum) const { return glob_surfel[locnum-1]; }
 
 
+    void EnumeratePointsGlobally ();
+    
     void SetDistantFaceNum (int dest, int locnum);
     void SetDistantPNum    (int dest, int locnum);
     void SetDistantEdgeNum (int dest, int locnum);
 
+    [[deprecated("Use GetDistantPNums(locnum).Size() instead!")]]            
     int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
+
+    [[deprecated("Use GetDistantFaceNums(locnum).Size() instead!")]]                
     int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); }
+
+    [[deprecated("Use GetDistantEdgeNums(locnum).Size() instead!")]]                    
     int GetNDistantEdgeNums ( int locedgenum) const { return loc2distedge[locedgenum-1].Size(); }
 
+    [[deprecated("Use GetDistantPNums(locnum) -> FlatArray instead!")]]                
     void GetDistantPNums (int locpnum, int * distpnums ) const
     {
       for (int i = 0; i < loc2distvert[locpnum-1].Size(); i++ )
 	distpnums[i] = loc2distvert[locpnum-1][i];
     } 
 
+    [[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]]                    
     void GetDistantFaceNums (int locfacenum, int * distfacenums ) const
     {
       for ( int i = 0; i < loc2distface[locfacenum-1].Size(); i++ )
 	distfacenums[i] = loc2distface[locfacenum-1][i];
     } 
 
+    [[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]]                        
     void GetDistantFaceNums (int locfacenum, NgArray<int> & distfacenums ) const
     {
       distfacenums = loc2distface[locfacenum-1];
     }
 
+    [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]                            
     void GetDistantEdgeNums (int locedgenum, int * distedgenums ) const
     {
       for (int i = 0; i < loc2distedge[locedgenum-1].Size(); i++ )
 	distedgenums[i] = loc2distedge[locedgenum-1][i];
     } 
 
+    [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]                                
     void GetDistantEdgeNums (int locedgenum, NgArray<int> & distedgenums ) const
     {
       distedgenums = loc2distedge[locedgenum-1];
     } 
 
-    NgFlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
+    FlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
-    NgFlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
+    FlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
-    NgFlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
+    FlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
 
     bool IsExchangeVert (int dest, int vnum) const
     {