From fcee13be59490fc570bed1e06acf27cdb03b37f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Sch=C3=B6berl?= Date: Fri, 28 Aug 2020 21:28:18 +0200 Subject: [PATCH] modernize paralleltop --- libsrc/meshing/curvedelems.cpp | 219 ++++++++++++++------------------- libsrc/meshing/meshtype.hpp | 16 ++- libsrc/meshing/paralleltop.cpp | 70 +++++++++-- libsrc/meshing/paralleltop.hpp | 5 +- 4 files changed, 167 insertions(+), 143 deletions(-) diff --git a/libsrc/meshing/curvedelems.cpp b/libsrc/meshing/curvedelems.cpp index 7ea1086b..43628d3e 100644 --- a/libsrc/meshing/curvedelems.cpp +++ b/libsrc/meshing/curvedelems.cpp @@ -560,21 +560,13 @@ namespace netgen ishighorder = 0; order = 1; - // MPI_Comm curve_comm; - const auto & curve_comm = mesh.GetCommunicator(); + auto comm = mesh.GetCommunicator(); #ifdef PARALLEL enum { MPI_TAG_CURVE = MPI_TAG_MESH+20 }; - const ParallelMeshTopology & partop = mesh.GetParallelTopology (); - // MPI_Comm_dup (mesh.GetCommunicator(), &curve_comm); - NgArray procs; -#else - // curve_comm = mesh.GetCommunicator(); #endif - int id = curve_comm.Rank(); - int ntasks = curve_comm.Size(); - - bool working = (ntasks == 1) || (id > 0); + int ntasks = comm.Size(); + bool working = (ntasks == 1) || (comm.Rank() > 0); if (working) order = aorder; @@ -653,38 +645,26 @@ namespace netgen if (ntasks > 1 && working) { for (int e = 0; e < edgeorder.Size(); e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - send_orders.Add (procs[j], edgeorder[e]); - } + for (int proc : partop.GetDistantEdgeNums(e)) + send_orders.Add (proc, edgeorder[e]); for (int f = 0; f < faceorder.Size(); f++) - { - partop.GetDistantFaceNums (f+1, procs); - for (int j = 0; j < procs.Size(); j++) - send_orders.Add (procs[j], faceorder[f]); - } + for (int proc : partop.GetDistantFaceNums(f)) + send_orders.Add (proc, faceorder[f]); } if (ntasks > 1) - MyMPI_ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE, curve_comm); + MyMPI_ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE, comm); if (ntasks > 1 && working) { NgArray cnt(ntasks); cnt = 0; for (int e = 0; e < edgeorder.Size(); e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - edgeorder[e] = max(edgeorder[e], recv_orders[procs[j]][cnt[procs[j]]++]); - } + for (auto proc : partop.GetDistantEdgeNums(e)) + edgeorder[e] = max(edgeorder[e], recv_orders[proc][cnt[proc]++]); for (int f = 0; f < faceorder.Size(); f++) - { - partop.GetDistantFaceNums (f+1, procs); - for (int j = 0; j < procs.Size(); j++) - faceorder[f] = max(faceorder[f], recv_orders[procs[j]][cnt[procs[j]]++]); - } + for (auto proc : partop.GetDistantFaceNums(f)) + faceorder[f] = max(faceorder[f], recv_orders[proc][cnt[proc]++]); } #endif @@ -771,48 +751,40 @@ namespace netgen TABLE senddata(ntasks), recvdata(ntasks); if (working) for (int e = 0; e < nedges; e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - { - senddata.Add (procs[j], surfnr[e]); - if (surfnr[e] != -1) - { - senddata.Add (procs[j], gi0[e].trignum); - senddata.Add (procs[j], gi0[e].u); - senddata.Add (procs[j], gi0[e].v); - senddata.Add (procs[j], gi1[e].trignum); - senddata.Add (procs[j], gi1[e].u); - senddata.Add (procs[j], gi1[e].v); - } - } - } - - MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, curve_comm); + for (int proc : partop.GetDistantEdgeNums(e)) + { + senddata.Add (proc, surfnr[e]); + if (surfnr[e] != -1) + { + senddata.Add (proc, gi0[e].trignum); + senddata.Add (proc, gi0[e].u); + senddata.Add (proc, gi0[e].v); + senddata.Add (proc, gi1[e].trignum); + senddata.Add (proc, gi1[e].u); + senddata.Add (proc, gi1[e].v); + } + } + MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm); NgArray cnt(ntasks); cnt = 0; if (working) for (int e = 0; e < nedges; e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - { - int surfnr1 = recvdata[procs[j]][cnt[procs[j]]++]; - if (surfnr1 != -1) - { - surfnr[e] = surfnr1; - gi0[e].trignum = int (recvdata[procs[j]][cnt[procs[j]]++]); - gi0[e].u = recvdata[procs[j]][cnt[procs[j]]++]; - gi0[e].v = recvdata[procs[j]][cnt[procs[j]]++]; - gi1[e].trignum = int (recvdata[procs[j]][cnt[procs[j]]++]); - gi1[e].u = recvdata[procs[j]][cnt[procs[j]]++]; - gi1[e].v = recvdata[procs[j]][cnt[procs[j]]++]; - } - } - } - + for (int proc : partop.GetDistantEdgeNums(e)) + { + int surfnr1 = recvdata[proc][cnt[proc]++]; + if (surfnr1 != -1) + { + surfnr[e] = surfnr1; + gi0[e].trignum = int (recvdata[proc][cnt[proc]++]); + gi0[e].u = recvdata[proc][cnt[proc]++]; + gi0[e].v = recvdata[proc][cnt[proc]++]; + gi1[e].trignum = int (recvdata[proc][cnt[proc]++]); + gi1[e].u = recvdata[proc][cnt[proc]++]; + gi1[e].v = recvdata[proc][cnt[proc]++]; + } + } } #endif @@ -974,59 +946,53 @@ namespace netgen TABLE senddata(ntasks), recvdata(ntasks); if (working) for (int e = 0; e < nedges; e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - { - senddata.Add (procs[j], use_edge[e]); - if (use_edge[e]) - { - senddata.Add (procs[j], edge_surfnr1[e]); - senddata.Add (procs[j], edge_surfnr2[e]); - senddata.Add (procs[j], edge_gi0[e].edgenr); - senddata.Add (procs[j], edge_gi0[e].body); - senddata.Add (procs[j], edge_gi0[e].dist); - senddata.Add (procs[j], edge_gi0[e].u); - senddata.Add (procs[j], edge_gi0[e].v); - senddata.Add (procs[j], edge_gi1[e].edgenr); - senddata.Add (procs[j], edge_gi1[e].body); - senddata.Add (procs[j], edge_gi1[e].dist); - senddata.Add (procs[j], edge_gi1[e].u); - senddata.Add (procs[j], edge_gi1[e].v); - senddata.Add (procs[j], swap_edge[e]); - } - } - } - MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, curve_comm); + for (int proc : partop.GetDistantEdgeNums(e)) + { + senddata.Add (proc, use_edge[e]); + if (use_edge[e]) + { + senddata.Add (proc, edge_surfnr1[e]); + senddata.Add (proc, edge_surfnr2[e]); + senddata.Add (proc, edge_gi0[e].edgenr); + senddata.Add (proc, edge_gi0[e].body); + senddata.Add (proc, edge_gi0[e].dist); + senddata.Add (proc, edge_gi0[e].u); + senddata.Add (proc, edge_gi0[e].v); + senddata.Add (proc, edge_gi1[e].edgenr); + senddata.Add (proc, edge_gi1[e].body); + senddata.Add (proc, edge_gi1[e].dist); + senddata.Add (proc, edge_gi1[e].u); + senddata.Add (proc, edge_gi1[e].v); + senddata.Add (proc, swap_edge[e]); + } + } + + MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm); NgArray cnt(ntasks); cnt = 0; if (working) for (int e = 0; e < edge_surfnr1.Size(); e++) - { - partop.GetDistantEdgeNums (e+1, procs); - for (int j = 0; j < procs.Size(); j++) - { - int get_edge = int(recvdata[procs[j]][cnt[procs[j]]++]); - if (get_edge) - { - use_edge[e] = 1; - edge_surfnr1[e] = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_surfnr2[e] = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_gi0[e].edgenr = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_gi0[e].body = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_gi0[e].dist = recvdata[procs[j]][cnt[procs[j]]++]; - edge_gi0[e].u = recvdata[procs[j]][cnt[procs[j]]++]; - edge_gi0[e].v = recvdata[procs[j]][cnt[procs[j]]++]; - edge_gi1[e].edgenr = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_gi1[e].body = int (recvdata[procs[j]][cnt[procs[j]]++]); - edge_gi1[e].dist = recvdata[procs[j]][cnt[procs[j]]++]; - edge_gi1[e].u = recvdata[procs[j]][cnt[procs[j]]++]; - edge_gi1[e].v = recvdata[procs[j]][cnt[procs[j]]++]; - swap_edge[e] = recvdata[procs[j]][cnt[procs[j]]++]; - } - } - } - + for (int proc : partop.GetDistantEdgeNums(e)) + { + int get_edge = int(recvdata[proc][cnt[proc]++]); + if (get_edge) + { + use_edge[e] = 1; + edge_surfnr1[e] = int (recvdata[proc][cnt[proc]++]); + edge_surfnr2[e] = int (recvdata[proc][cnt[proc]++]); + edge_gi0[e].edgenr = int (recvdata[proc][cnt[proc]++]); + edge_gi0[e].body = int (recvdata[proc][cnt[proc]++]); + edge_gi0[e].dist = recvdata[proc][cnt[proc]++]; + edge_gi0[e].u = recvdata[proc][cnt[proc]++]; + edge_gi0[e].v = recvdata[proc][cnt[proc]++]; + edge_gi1[e].edgenr = int (recvdata[proc][cnt[proc]++]); + edge_gi1[e].body = int (recvdata[proc][cnt[proc]++]); + edge_gi1[e].dist = recvdata[proc][cnt[proc]++]; + edge_gi1[e].u = recvdata[proc][cnt[proc]++]; + edge_gi1[e].v = recvdata[proc][cnt[proc]++]; + swap_edge[e] = recvdata[proc][cnt[proc]++]; + } + } } #endif @@ -1182,26 +1148,20 @@ namespace netgen if (ntasks > 1 && working) { for (int f = 0; f < nfaces; f++) - { - partop.GetDistantFaceNums (f+1, procs); - for (int j = 0; j < procs.Size(); j++) - send_surfnr.Add (procs[j], surfnr[f]); - } + for (int proc : partop.GetDistantFaceNums(f)) + send_surfnr.Add (proc, surfnr[f]); } if (ntasks > 1) - MyMPI_ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE, curve_comm); + MyMPI_ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE, comm); if (ntasks > 1 && working) { NgArray cnt(ntasks); cnt = 0; for (int f = 0; f < nfaces; f++) - { - partop.GetDistantFaceNums (f+1, procs); - for (int j = 0; j < procs.Size(); j++) - surfnr[f] = max(surfnr[f], recv_surfnr[procs[j]][cnt[procs[j]]++]); - } + for (int proc : partop.GetDistantFaceNums(f)) + surfnr[f] = max(surfnr[f], recv_surfnr[proc][cnt[proc]++]); } #endif @@ -1386,8 +1346,7 @@ namespace netgen #ifdef PARALLEL - curve_comm.Barrier(); - // MPI_Comm_free (&curve_comm); + comm.Barrier(); #endif } diff --git a/libsrc/meshing/meshtype.hpp b/libsrc/meshing/meshtype.hpp index 30c269e6..be4482d6 100644 --- a/libsrc/meshing/meshtype.hpp +++ b/libsrc/meshing/meshtype.hpp @@ -220,10 +220,22 @@ namespace netgen PointIndices (PointIndex i1, PointIndex i2) : INDEX_2(i1,i2) { ; } PointIndex operator[] (int i) const { return PointIndex(INDEX_2::operator[](i)); } PointIndex & operator[] (int i) { return reinterpret_cast(INDEX_2::operator[](i)); } - static PointIndices Sort(PointIndex i1, PointIndex i2) { return INDEX_2::Sort(i1, i2); } + static PointIndices Sort(PointIndex i1, PointIndex i2) { return INDEX_2::Sort(i1, i2); } + template + PointIndex get() const { return PointIndex(INDEX_2::operator[](J)); } }; - +} +namespace std +{ + // structured binding support + template + struct tuple_size> : std::integral_constant {}; + template struct tuple_element> { using type = netgen::PointIndex; }; +} + +namespace netgen +{ class ElementIndex { diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index f2337b18..418913e3 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -15,6 +15,7 @@ namespace ngcore if (v > max) max = v; return max; } + /* template auto Max (FlatArray array, TB initial) -> T @@ -546,11 +547,18 @@ namespace netgen { PointIndex v1 = mesh.mlbetweennodes[pi][0]; PointIndex v2 = mesh.mlbetweennodes[pi][1]; + /* if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) - // for (int dest = 1; dest < ntasks; dest++) for (int dest : GetDistantPNums(v1-PointIndex::BASE)) if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) cnt_send[dest-1]++; + */ + auto procs1 = GetDistantProcs(v1); + auto procs2 = GetDistantProcs(v2); + if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) + for (int p : procs1) + if (procs2.Contains(p)) + cnt_send[p-1]++; } TABLE dest2pair(cnt_send); @@ -559,22 +567,39 @@ namespace netgen { PointIndex v1 = mesh.mlbetweennodes[pi][0]; PointIndex v2 = mesh.mlbetweennodes[pi][1]; + auto procs1 = GetDistantProcs(v1); + auto procs2 = GetDistantProcs(v2); + /* if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) for (int dest : GetDistantPNums(v1-PointIndex::BASE)) if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) dest2pair.Add (dest-1, pi); + */ + if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) + for (int p : procs1) + if (procs2.Contains(p)) + dest2pair.Add (p-1, pi); } cnt_send = 0; int v1, v2; for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++) { - PointIndex v1 = mesh.mlbetweennodes[pi][0]; - PointIndex v2 = mesh.mlbetweennodes[pi][1]; + // PointIndex v1 = mesh.mlbetweennodes[pi][0]; + // PointIndex v2 = mesh.mlbetweennodes[pi][1]; + auto [v1,v2] = mesh.mlbetweennodes[pi]; + auto procs1 = GetDistantProcs(v1); + auto procs2 = GetDistantProcs(v2); + /* if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) for (int dest : GetDistantPNums(v1-PointIndex::BASE)) if (IsExchangeVert(dest, v2)) cnt_send[dest-1]+=2; + */ + if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) + for (int p : procs1) + if (procs2.Contains(p)) + cnt_send[p-1]+=2; } TABLE send_verts(cnt_send); @@ -598,8 +623,11 @@ namespace netgen { PointIndex v1 = mesh.mlbetweennodes[pi][0]; PointIndex v2 = mesh.mlbetweennodes[pi][1]; + auto procs1 = GetDistantProcs(v1); + auto procs2 = GetDistantProcs(v2); if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1) - if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + // if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + if (procs1.Contains(dest) && procs2.Contains(dest)) { send_verts.Add (dest-1, loc2exchange[v1]); send_verts.Add (dest-1, loc2exchange[v2]); @@ -634,7 +662,8 @@ namespace netgen { INDEX_2 re(recvarray[ii], recvarray[ii+1]); INDEX_2 es(loc2exchange[v1], loc2exchange[v2]); - if (es == re && !IsExchangeVert(dest, pi)) + // if (es == re && !IsExchangeVert(dest, pi)) + if (es == re && !GetDistantProcs(pi).Contains(dest)) { SetDistantPNum(dest, pi); changed = true; @@ -870,7 +899,7 @@ namespace netgen #endif - NgArray sendarray, recvarray; + // NgArray sendarray, recvarray; // cout << "UpdateCoarseGrid - edges" << endl; // static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); @@ -900,9 +929,15 @@ namespace netgen for (int edge = 1; edge <= ned; edge++) { topology.GetEdgeVertices (edge, v1, v2); + /* for (int dest = 1; dest < ntasks; dest++) - if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + // if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest)) cnt_send[dest-1]+=1; + */ + for (auto p : GetDistantProcs(v1)) + if (GetDistantProcs(v2).Contains(p)) + cnt_send[p-1]+=1; } TABLE dest2edge(cnt_send); @@ -913,7 +948,8 @@ namespace netgen { topology.GetEdgeVertices (edge, v1, v2); for (int dest = 1; dest < ntasks; dest++) - if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + // if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest)) dest2edge.Add (dest-1, edge); } @@ -929,7 +965,8 @@ namespace netgen for (int edge : dest2edge[dest-1]) { topology.GetEdgeVertices (edge, v1, v2); - if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + // if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2)) + if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest)) { send_edges.Add (dest-1, loc2exchange[v1]); send_edges.Add (dest-1, loc2exchange[v2]); @@ -981,9 +1018,14 @@ namespace netgen topology.GetFaceVertices (face, verts); for (int dest = 1; dest < ntasks; dest++) if (dest != id) + /* if (IsExchangeVert (dest, verts[0]) && IsExchangeVert (dest, verts[1]) && IsExchangeVert (dest, verts[2])) + */ + if (GetDistantProcs (verts[0]).Contains(dest) && + GetDistantProcs (verts[1]).Contains(dest) && + GetDistantProcs (verts[2]).Contains(dest)) cnt_send[dest-1]++; } @@ -993,9 +1035,14 @@ namespace netgen topology.GetFaceVertices (face, verts); for (int dest = 1; dest < ntasks; dest++) if (dest != id) + /* if (IsExchangeVert (dest, verts[0]) && IsExchangeVert (dest, verts[1]) && IsExchangeVert (dest, verts[2])) + */ + if (GetDistantProcs (verts[0]).Contains(dest) && + GetDistantProcs (verts[1]).Contains(dest) && + GetDistantProcs (verts[2]).Contains(dest)) dest2face.Add(dest-1, face); } @@ -1015,9 +1062,14 @@ namespace netgen for (int face : dest2face[dest-1]) { topology.GetFaceVertices (face, verts); + /* if (IsExchangeVert (dest, verts[0]) && IsExchangeVert (dest, verts[1]) && IsExchangeVert (dest, verts[2])) + */ + if (GetDistantProcs (verts[0]).Contains(dest) && + GetDistantProcs (verts[1]).Contains(dest) && + GetDistantProcs (verts[2]).Contains(dest)) { send_faces.Add (dest-1, loc2exchange[verts[0]]); send_faces.Add (dest-1, loc2exchange[verts[1]]); diff --git a/libsrc/meshing/paralleltop.hpp b/libsrc/meshing/paralleltop.hpp index 8589801b..26f3f143 100644 --- a/libsrc/meshing/paralleltop.hpp +++ b/libsrc/meshing/paralleltop.hpp @@ -117,17 +117,18 @@ namespace netgen distedgenums = loc2distedge[locedgenum-1]; } + [[deprecated("Use GetDistantProcs(..)!")]] FlatArray GetDistantPNums (int locnum) const { return loc2distvert[locnum]; } FlatArray GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; } FlatArray GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; } - [[deprecated("Use GetDistantPNums(..).Contains instead!")]] + FlatArray GetDistantProcs (PointIndex pi) const { return loc2distvert[pi-PointIndex::BASE]; } + [[deprecated("Use GetDistantProcs(..).Contains instead!")]] bool IsExchangeVert (int dest, int vnum) const { return loc2distvert[vnum-1].Contains (dest); } }; - }