From 895280a2443bd0d75595e96451014fcd85f13b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Sch=C3=B6berl?= Date: Fri, 21 Aug 2020 16:29:33 +0200 Subject: [PATCH] littel parallel polishing --- libsrc/include/nginterface_v2.hpp | 5 +- libsrc/interface/nginterface.cpp | 3 + libsrc/interface/nginterface_v2.cpp | 27 +++---- libsrc/meshing/meshclass.cpp | 83 --------------------- libsrc/meshing/parallelmesh.cpp | 7 +- libsrc/meshing/paralleltop.cpp | 112 +++++++++++++++++++++++++++- libsrc/meshing/paralleltop.hpp | 38 +++++++--- 7 files changed, 161 insertions(+), 114 deletions(-) diff --git a/libsrc/include/nginterface_v2.hpp b/libsrc/include/nginterface_v2.hpp index 653f7ead..fc1b860f 100644 --- a/libsrc/include/nginterface_v2.hpp +++ b/libsrc/include/nginterface_v2.hpp @@ -368,8 +368,9 @@ namespace netgen // for MPI-parallel - std::tuple GetDistantProcs (int nodetype, int locnum) const; - + FlatArray GetDistantProcs (int nodetype, int locnum) const; + size_t GetGlobalVertexNum (int locnum) const; + shared_ptr GetMesh () const { return mesh; } shared_ptr SelectMesh () const; inline auto GetTimeStamp() const; diff --git a/libsrc/interface/nginterface.cpp b/libsrc/interface/nginterface.cpp index 42d26094..c6bdd3fc 100644 --- a/libsrc/interface/nginterface.cpp +++ b/libsrc/interface/nginterface.cpp @@ -905,6 +905,7 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou // gibt anzahl an distant pnums zurueck // * pnums entspricht ARRAY +[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]] int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums ) { int size = NgPar_GetNDistantNodeNums (nodetype, locnum); @@ -931,6 +932,7 @@ int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums ) return size; } +[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]] int NgPar_GetNDistantNodeNums ( int nodetype, int locnum ) { locnum++; @@ -944,6 +946,7 @@ int NgPar_GetNDistantNodeNums ( int nodetype, int locnum ) return -1; } +[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]] int NgPar_GetGlobalNodeNum (int nodetype, int locnum) { locnum++; diff --git a/libsrc/interface/nginterface_v2.cpp b/libsrc/interface/nginterface_v2.cpp index e2501ab4..b82868ee 100644 --- a/libsrc/interface/nginterface_v2.cpp +++ b/libsrc/interface/nginterface_v2.cpp @@ -1296,36 +1296,31 @@ void Ngx_Mesh::SetSurfaceElementOrders (int enr, int ox, int oy) +size_t Ngx_Mesh :: GetGlobalVertexNum (int locnum) const +{ + return mesh->GetParallelTopology().GetGlobalPNum (locnum+1)-1; +} - std::tuple Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const +FlatArray Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const { #ifdef PARALLEL if (mesh->GetCommunicator().Size() == 1) - return std::tuple(0,nullptr); + return FlatArray(0,nullptr); switch (nodetype) { case 0: - { - NgFlatArray dn = mesh->GetParallelTopology().GetDistantPNums(locnum); - return std::tuple(dn.Size(), &dn[0]); - } + return mesh->GetParallelTopology().GetDistantPNums(locnum); case 1: - { - NgFlatArray dn = mesh->GetParallelTopology().GetDistantEdgeNums(locnum); - return std::tuple(dn.Size(), &dn[0]); - } + return mesh->GetParallelTopology().GetDistantEdgeNums(locnum); case 2: - { - NgFlatArray dn = mesh->GetParallelTopology().GetDistantFaceNums(locnum); - return std::tuple(dn.Size(), &dn[0]); - } + return mesh->GetParallelTopology().GetDistantFaceNums(locnum); default: - return std::tuple(0,nullptr); + return FlatArray(0, nullptr); } #else - return std::tuple(0,nullptr); + return FlatArray(0,nullptr); #endif } } diff --git a/libsrc/meshing/meshclass.cpp b/libsrc/meshing/meshclass.cpp index 5b859931..7006ea07 100644 --- a/libsrc/meshing/meshclass.cpp +++ b/libsrc/meshing/meshclass.cpp @@ -1298,89 +1298,6 @@ namespace netgen // GridFunciton pickling is not compatible, now // should go to paralleltopology - Array global_pnums(points.Size()); - global_pnums = -1; - int num_master_points = 0; - for (PointIndex pi : Range(points)) - { - auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE); - // check sorted: - for (int j = 0; j+1 < distprocs.Size(); j++) - if (distprocs[j+1] < distprocs[j]) cout << "wrong sort" << endl; - if (distprocs.Size() == 0 || distprocs[0] > comm.Rank()) - global_pnums[pi] = PointIndex::BASE+num_master_points++; - } - Array first_master_point(comm.Size()); - GetCommunicator().AllGather (num_master_points, first_master_point); - - size_t num_glob_points = 0; - for (int i = 0; i < comm.Size(); i++) - { - int cur = first_master_point[i]; - first_master_point[i] = num_glob_points; - num_glob_points += cur; - } - - for (PointIndex pi : Range(points)) - if (global_pnums[pi] != -1) - global_pnums[pi] += first_master_point[comm.Rank()]; - - // ScatterDofData (global_nums); - - Array nsend(comm.Size()), nrecv(comm.Size()); - nsend = 0; - nrecv = 0; - - /** Count send/recv size **/ - for (PointIndex pi : Range(points)) - { - auto dps = partop.GetDistantPNums(pi-PointIndex::BASE); - if (!dps.Size()) continue; - if (rank < dps[0]) - for(auto p:dps) - nsend[p]++; - else - nrecv[dps[0]]++; - } - - Table send_data(nsend); - Table recv_data(nrecv); - - /** Fill send_data **/ - nsend = 0; - for (PointIndex pi : Range(points)) - { - auto dps = partop.GetDistantPNums(pi-PointIndex::BASE); - if (dps.Size() && rank < dps[0]) - for(auto p : dps) - send_data[p][nsend[p]++] = global_pnums[pi]; - } - - Array requests; - for (int i = 0; i < comm.Size(); i++) - { - if (nsend[i]) - requests.Append (comm.ISend (send_data[i], i, 200)); - if (nrecv[i]) - requests.Append (comm.IRecv (recv_data[i], i, 200)); - } - - MyMPI_WaitAll (requests); - - Array cnt(comm.Size()); - cnt = 0; - - for (PointIndex pi : Range(points)) - { - auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE); - if (distprocs.Size() > 0 && distprocs[0] < comm.Rank()) - { - int master = comm.Size(); - for (int j = 0; j < distprocs.Size(); j++) - master = min (master, distprocs[j]); - global_pnums[pi] = recv_data[master][cnt[master]++]; - } - } // merge points diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index a7c13d23..b17490ae 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -732,6 +732,7 @@ namespace netgen for (auto t : point_types) { MPI_Type_free(&t); } + PrintMessage ( 3, "Sending names"); sendrequests.SetSize(3*ntasks); @@ -814,6 +815,9 @@ namespace netgen self.BuildElementSearchTree(); // const_cast(*this).DeleteMesh(); + + // paralleltop -> SetNV (0); + // paralleltop->EnumeratePointsGlobally(); PrintMessage( 3, "send mesh complete"); } @@ -1016,6 +1020,7 @@ namespace netgen } } + /** Recv bc-names **/ ArrayMem nnames{0,0,0,0}; // MPI_Recv(nnames, 4, MPI_INT, 0, MPI_TAG_MESH+6, comm, MPI_STATUS_IGNORE); @@ -1073,7 +1078,7 @@ namespace netgen clusters -> Update(); // paralleltop -> UpdateCoarseGrid(); - + // paralleltop->EnumeratePointsGlobally(); SetNextMajorTimeStamp(); } diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index 9e07fd73..91c5f9ad 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -55,6 +55,107 @@ namespace netgen } + void ParallelMeshTopology :: EnumeratePointsGlobally () + { + auto nv = loc2distvert.Size(); + auto comm = mesh.GetCommunicator(); + auto rank = comm.Rank(); + + // if (rank == 0) + // nv = 0; + glob_vert.SetSize (nv); + glob_vert = -1; + int num_master_points = 0; + + for (auto i : Range(nv)) + { + auto dps = GetDistantPNums(i); + // check sorted: + for (int j = 0; j+1 < dps.Size(); j++) + if (dps[j+1] < dps[j]) cout << "wrong sort" << endl; + + if (dps.Size() == 0 || dps[0] > comm.Rank()) + glob_vert[i] = num_master_points++; + } + + Array first_master_point(comm.Size()); + comm.AllGather (num_master_points, first_master_point); + + size_t num_glob_points = 0; + for (int i = 0; i < comm.Size(); i++) + { + int cur = first_master_point[i]; + first_master_point[i] = num_glob_points; + num_glob_points += cur; + } + + for (auto i : Range(nv)) + if (glob_vert[i] != -1) + glob_vert[i] += first_master_point[comm.Rank()]; + + // ScatterDofData (global_nums); + + Array nsend(comm.Size()), nrecv(comm.Size()); + nsend = 0; + nrecv = 0; + + /** Count send/recv size **/ + for (auto i : Range(nv)) + { + auto dps = GetDistantPNums(i); + if (!dps.Size()) continue; + if (rank < dps[0]) + for(auto p:dps) + nsend[p]++; + else + nrecv[dps[0]]++; + } + + Table send_data(nsend); + Table recv_data(nrecv); + + /** Fill send_data **/ + nsend = 0; + for (auto i : Range(nv)) + { + auto dps = GetDistantPNums(i); + if (dps.Size() && rank < dps[0]) + for(auto p : dps) + send_data[p][nsend[p]++] = glob_vert[i]; + } + + Array requests; + for (int i = 0; i < comm.Size(); i++) + { + if (nsend[i]) + requests.Append (comm.ISend (send_data[i], i, 200)); + if (nrecv[i]) + requests.Append (comm.IRecv (recv_data[i], i, 200)); + } + + MyMPI_WaitAll (requests); + + Array cnt(comm.Size()); + cnt = 0; + + for (auto i : Range(nv)) + { + auto dps = GetDistantPNums(i); + if (dps.Size() > 0 && dps[0] < comm.Rank()) + { + int master = comm.Size(); + for (int j = 0; j < dps.Size(); j++) + master = min (master, dps[j]); + glob_vert[i] = recv_data[master][cnt[master]++]; + } + } + + if (PointIndex::BASE==1) + for (auto & i : glob_vert) + i++; + } + + void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum) { for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 ) @@ -242,8 +343,13 @@ namespace netgen MPI_Group_excl (MPI_GROUP_comm, 1, process_ranks, &MPI_LocalGroup); MPI_Comm_create (comm, MPI_LocalGroup, &MPI_LocalComm); - if (id == 0) return; - + if (id == 0) + { + // SetNV(0); + // EnumeratePointsGlobally(); + return; + } + const MeshTopology & topology = mesh.GetTopology(); NgArray cnt_send(ntasks-1); @@ -573,7 +679,7 @@ namespace netgen NgProfiler::StopTimer (timerf); } // cout << "UpdateCoarseGrid - done" << endl; - + // EnumeratePointsGlobally(); is_updated = true; MPI_Group_free(&MPI_LocalGroup); diff --git a/libsrc/meshing/paralleltop.hpp b/libsrc/meshing/paralleltop.hpp index 197ba62e..6f223928 100644 --- a/libsrc/meshing/paralleltop.hpp +++ b/libsrc/meshing/paralleltop.hpp @@ -42,60 +42,80 @@ namespace netgen void SetNSE (int anse); void SetNSegm (int anseg); - void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; } + [[deprecated("Try to avoid global enumration!")]] void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; } + [[deprecated("Try to avoid global enumration!")]] void SetLoc2Glob_Face (int locnum, int globnum) { glob_face[locnum-1] = globnum; } + [[deprecated("Try to avoid global enumration!")]] void SetLoc2Glob_VolEl (int locnum, int globnum) { glob_el[locnum-1] = globnum; } + [[deprecated("Try to avoid global enumration!")]] void SetLoc2Glob_SurfEl (int locnum, int globnum) { glob_surfel[locnum-1] = globnum; } + [[deprecated("Try to avoid global enumration!")]] void SetLoc2Glob_Segm (int locnum, int globnum) { glob_segm[locnum-1] = globnum; } int GetGlobalPNum (int locnum) const { return glob_vert[locnum-1]; } + [[deprecated("Try to avoid global enumration!")]] int GetGlobalEdgeNum (int locnum) const { return glob_edge[locnum-1]; } + [[deprecated("Try to avoid global enumration!")]] int GetGlobalFaceNum (int locnum) const { return glob_face[locnum-1]; } + [[deprecated("Try to avoid global enumration!")]] int GetGlobalElNum (int locnum) const { return glob_el[locnum-1]; } + [[deprecated("Try to avoid global enumration!")]] int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; } + void EnumeratePointsGlobally (); + void SetDistantFaceNum (int dest, int locnum); void SetDistantPNum (int dest, int locnum); void SetDistantEdgeNum (int dest, int locnum); - int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); } - int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); } + [[deprecated("Use GetDistantPNums(locnum).Size() instead!")]] + int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); } + + [[deprecated("Use GetDistantFaceNums(locnum).Size() instead!")]] + int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); } + + [[deprecated("Use GetDistantEdgeNums(locnum).Size() instead!")]] int GetNDistantEdgeNums ( int locedgenum) const { return loc2distedge[locedgenum-1].Size(); } - + + [[deprecated("Use GetDistantPNums(locnum) -> FlatArray instead!")]] void GetDistantPNums (int locpnum, int * distpnums ) const { for (int i = 0; i < loc2distvert[locpnum-1].Size(); i++ ) distpnums[i] = loc2distvert[locpnum-1][i]; } - + + [[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]] void GetDistantFaceNums (int locfacenum, int * distfacenums ) const { for ( int i = 0; i < loc2distface[locfacenum-1].Size(); i++ ) distfacenums[i] = loc2distface[locfacenum-1][i]; } + [[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]] void GetDistantFaceNums (int locfacenum, NgArray & distfacenums ) const { distfacenums = loc2distface[locfacenum-1]; } - + + [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]] void GetDistantEdgeNums (int locedgenum, int * distedgenums ) const { for (int i = 0; i < loc2distedge[locedgenum-1].Size(); i++ ) distedgenums[i] = loc2distedge[locedgenum-1][i]; } + [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]] void GetDistantEdgeNums (int locedgenum, NgArray & distedgenums ) const { distedgenums = loc2distedge[locedgenum-1]; } - NgFlatArray GetDistantPNums (int locnum) const { return loc2distvert[locnum]; } - NgFlatArray GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; } - NgFlatArray GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; } + FlatArray GetDistantPNums (int locnum) const { return loc2distvert[locnum]; } + FlatArray GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; } + FlatArray GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; } bool IsExchangeVert (int dest, int vnum) const {