From abb2e43ccb2217340c92252f08942f6f3f21a937 Mon Sep 17 00:00:00 2001 From: Joachim Schoeberl Date: Tue, 8 Jun 2021 19:08:07 +0200 Subject: [PATCH] optimize parallel load --- libsrc/core/mpi_wrapper.hpp | 1 + libsrc/meshing/parallelmesh.cpp | 58 +++++++++++++++++++++++++-------- libsrc/meshing/topology.cpp | 39 ++++++++++++---------- libsrc/meshing/topology.hpp | 10 +++--- 4 files changed, 71 insertions(+), 37 deletions(-) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index 2e046d18..3675db7e 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -257,6 +257,7 @@ namespace ngcore template ())> void Bcast (T & s, int root = 0) const { if (size == 1) return ; + static Timer t("MPI - Bcast"); RegionTimer reg(t); MPI_Bcast (&s, 1, GetMPIType(), root, comm); } diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 70a40f65..56ba420e 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -208,7 +208,13 @@ namespace netgen void Mesh :: SendMesh () const { - + static Timer tsend("SendMesh"); RegionTimer reg(tsend); + static Timer tbuildvertex("SendMesh::BuildVertex"); + static Timer tbuildvertexa("SendMesh::BuildVertex a"); + static Timer tbuildvertexb("SendMesh::BuildVertex b"); + static Timer tbuilddistpnums("SendMesh::Build_distpnums"); + static Timer tbuildelementtable("SendMesh::Build_elementtable"); + NgMPI_Comm comm = GetCommunicator(); int id = comm.Rank(); int ntasks = comm.Size(); @@ -223,6 +229,7 @@ namespace netgen // build edges/faces. auto & top = const_cast(GetTopology()); if(top.NeedsUpdate()) { + top.SetBuildVertex2Element(false); top.SetBuildEdges(false); top.SetBuildFaces(false); top.Update(); @@ -357,6 +364,7 @@ namespace netgen } /** Now we build the vertex-data to send to the workers. **/ + tbuildvertex.Start(); NgArray vert_flag (GetNV()); NgArray num_procs_on_vert (GetNV()); NgArray num_verts_on_proc (ntasks); @@ -401,6 +409,7 @@ namespace netgen } }; /** count vertices per proc and procs per vertex **/ + tbuildvertexa.Start(); iterate_vertices([&](auto vertex, auto dest){ auto countit = [&] (auto vertex, auto dest) { if (vert_flag[vertex] < dest) @@ -409,14 +418,23 @@ namespace netgen num_verts_on_proc[dest]++; num_procs_on_vert[vertex]++; // GetParallelTopology().SetDistantPNum (dest, vertex); - GetParallelTopology().AddDistantProc (PointIndex(vertex), dest); + // GetParallelTopology().AddDistantProc (PointIndex(vertex), dest); } }; countit(vertex, dest); + /* auto pers = per_verts_trans[vertex]; for(int j = 0; j < pers.Size(); j++) countit(pers[j], dest); + */ + for (auto v : per_verts_trans[vertex]) + countit(v, dest); }); + tbuildvertexa.Stop(); + + + tbuildvertexb.Start(); + TABLE verts_of_proc (num_verts_on_proc); TABLE procs_of_vert (num_procs_on_vert); TABLE loc_num_of_vert (num_procs_on_vert); @@ -430,10 +448,16 @@ namespace netgen } }; addit(vertex, dest); + /* auto pers = per_verts_trans[vertex]; for(int j = 0; j < pers.Size(); j++) addit(pers[j], dest); + */ + for (auto v : per_verts_trans[vertex]) + addit(v, dest); + }); + tbuildvertexb.Stop(); /** local vertex numbers on distant procs (I think this was only used for debugging??) @@ -449,6 +473,7 @@ namespace netgen loc_num_of_vert.Add (vert, verts_of_proc[dest].Size()); } } + tbuildvertex.Stop(); PrintMessage ( 3, "Sending Vertices - vertices"); Array point_types(ntasks-1); @@ -540,6 +565,7 @@ namespace netgen PrintMessage ( 3, "Sending Vertices - distprocs"); + tbuilddistpnums.Start(); Array num_distpnums(ntasks); num_distpnums = 0; @@ -564,7 +590,9 @@ namespace netgen distpnums.Add (procs[j], loc_num_of_vert[vert][k]); } } - + + tbuilddistpnums.Stop(); + for ( int dest = 1; dest < ntasks; dest ++ ) sendrequests.Append (comm.ISend (distpnums[dest], dest, MPI_TAG_MESH+1)); @@ -572,6 +600,7 @@ namespace netgen PrintMessage ( 3, "Sending elements" ); + tbuildelementtable.Start(); Array elarraysize (ntasks); elarraysize = 0; for ( int ei = 1; ei <= GetNE(); ei++) @@ -596,7 +625,8 @@ namespace netgen for (int i = 0; i < el.GetNP(); i++) elementarrays.Add (dest, el[i]); } - + tbuildelementtable.Stop(); + for (int dest = 1; dest < ntasks; dest ++ ) // sendrequests.Append (MyMPI_ISend (elementarrays[dest], dest, MPI_TAG_MESH+2, comm)); sendrequests.Append (comm.ISend (elementarrays[dest], dest, MPI_TAG_MESH+2)); @@ -978,11 +1008,11 @@ namespace netgen // workers receive the mesh from the master void Mesh :: ReceiveParallelMesh ( ) { - int timer = NgProfiler::CreateTimer ("ReceiveParallelMesh"); - int timer_pts = NgProfiler::CreateTimer ("Receive points"); - int timer_els = NgProfiler::CreateTimer ("Receive elements"); - int timer_sels = NgProfiler::CreateTimer ("Receive surface elements"); - NgProfiler::RegionTimer reg(timer); + Timer timer("ReceiveParallelMesh"); + Timer timer_pts("Receive points"); + Timer timer_els("Receive elements"); + Timer timer_sels("Receive surface elements"); + RegionTimer reg(timer); NgMPI_Comm comm = GetCommunicator(); int id = comm.Rank(); @@ -998,7 +1028,7 @@ namespace netgen paralleltop -> SetNE (nelloc); // receive vertices - NgProfiler::StartTimer (timer_pts); + timer_pts.Start(); Array verts; comm.Recv (verts, 0, MPI_TAG_MESH+1); @@ -1054,15 +1084,15 @@ namespace netgen // SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi]); // , dist_pnums[hi+2]); AddDistantProc (PointIndex(dist_pnums[hi]), dist_pnums[hi+1]); - NgProfiler::StopTimer (timer_pts); + timer_pts.Stop(); *testout << "got " << numvert << " vertices" << endl; { + RegionTimer reg(timer_els); + Array elarray; comm.Recv (elarray, 0, MPI_TAG_MESH+2); - - NgProfiler::RegionTimer reg(timer_els); for (int ind = 0, elnum = 1; ind < elarray.Size(); elnum++) { @@ -1093,7 +1123,7 @@ namespace netgen } { - NgProfiler::RegionTimer reg(timer_sels); + RegionTimer reg(timer_sels); Array selbuf; comm.Recv ( selbuf, 0, MPI_TAG_MESH+4); diff --git a/libsrc/meshing/topology.cpp b/libsrc/meshing/topology.cpp index 95122a11..2935463e 100644 --- a/libsrc/meshing/topology.cpp +++ b/libsrc/meshing/topology.cpp @@ -407,25 +407,28 @@ namespace netgen vertex to segment */ - timer_tables.Start(); - vert2element = mesh->CreatePoint2ElementTable(); - vert2surfelement = mesh->CreatePoint2SurfaceElementTable(0); + if (buildvertex2element) + { + timer_tables.Start(); + vert2element = mesh->CreatePoint2ElementTable(); + vert2surfelement = mesh->CreatePoint2SurfaceElementTable(0); - vert2segment = ngcore::CreateSortedTable( mesh->LineSegments().Range(), - [&](auto & table, SegmentIndex segi) - { - const Segment & seg = (*mesh)[segi]; - table.Add (seg[0], segi); - table.Add (seg[1], segi); - }, np); - - vert2pointelement = ngcore::CreateSortedTable( mesh->pointelements.Range(), - [&](auto & table, int pei) - { - const Element0d & pointel = mesh->pointelements[pei]; - table.Add(pointel.pnum, pei); - }, np); - timer_tables.Stop(); + vert2segment = ngcore::CreateSortedTable( mesh->LineSegments().Range(), + [&](auto & table, SegmentIndex segi) + { + const Segment & seg = (*mesh)[segi]; + table.Add (seg[0], segi); + table.Add (seg[1], segi); + }, np); + + vert2pointelement = ngcore::CreateSortedTable( mesh->pointelements.Range(), + [&](auto & table, int pei) + { + const Element0d & pointel = mesh->pointelements[pei]; + table.Add(pointel.pnum, pei); + }, np); + timer_tables.Stop(); + } (*tracer) ("Topology::Update setup tables", true); diff --git a/libsrc/meshing/topology.hpp b/libsrc/meshing/topology.hpp index f8ff1a19..eb537a81 100644 --- a/libsrc/meshing/topology.hpp +++ b/libsrc/meshing/topology.hpp @@ -43,6 +43,7 @@ struct T_FACE class MeshTopology { const Mesh * mesh; + bool buildvertex2element = true; bool buildedges; bool buildfaces; bool build_parent_edges = false; // may be changed to default = false @@ -78,11 +79,10 @@ public: MeshTopology (const Mesh & amesh); ~MeshTopology (); MeshTopology & operator= (MeshTopology && top) = default; - - void SetBuildEdges (bool be) - { buildedges = be; } - void SetBuildFaces (bool bf) - { buildfaces = bf; } + + void SetBuildVertex2Element (bool bv2e) { buildvertex2element = bv2e; } + void SetBuildEdges (bool be) { buildedges = be; } + void SetBuildFaces (bool bf) { buildfaces = bf; } void SetBuildParentEdges (bool bh) { build_parent_edges = bh; } void SetBuildParentFaces (bool bh) { build_parent_faces = bh; }