ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-24 21:10:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

modernize paralleltop

Browse Source
This commit is contained in:
Joachim Schöberl 2020-08-28 21:28:18 +02:00
parent ab34a185ce
commit fcee13be59
4 changed files with 167 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
libsrc/meshing/curvedelems.cpp
View File

@ -560,21 +560,13 @@ namespace netgen
ishighorder = 0;
order = 1;
// MPI_Comm curve_comm;
const auto & curve_comm = mesh.GetCommunicator();
auto comm = mesh.GetCommunicator();
#ifdef PARALLEL
enum { MPI_TAG_CURVE = MPI_TAG_MESH+20 };
const ParallelMeshTopology & partop = mesh.GetParallelTopology ();
// MPI_Comm_dup (mesh.GetCommunicator(), &curve_comm);
NgArray<int> procs;
#else
// curve_comm = mesh.GetCommunicator();
#endif
int id = curve_comm.Rank();
int ntasks = curve_comm.Size();
bool working = (ntasks == 1) || (id > 0);
int ntasks = comm.Size();
bool working = (ntasks == 1) || (comm.Rank() > 0);
if (working)
order = aorder;
@ -653,38 +645,26 @@ namespace netgen
if (ntasks > 1 && working)
{
for (int e = 0; e < edgeorder.Size(); e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
send_orders.Add (procs[j], edgeorder[e]);
}
for (int proc : partop.GetDistantEdgeNums(e))
send_orders.Add (proc, edgeorder[e]);
for (int f = 0; f < faceorder.Size(); f++)
{
partop.GetDistantFaceNums (f+1, procs);
for (int j = 0; j < procs.Size(); j++)
send_orders.Add (procs[j], faceorder[f]);
}
for (int proc : partop.GetDistantFaceNums(f))
send_orders.Add (proc, faceorder[f]);
}
if (ntasks > 1)
MyMPI_ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE, curve_comm);
MyMPI_ExchangeTable (send_orders, recv_orders, MPI_TAG_CURVE, comm);
if (ntasks > 1 && working)
{
NgArray<int> cnt(ntasks);
cnt = 0;
for (int e = 0; e < edgeorder.Size(); e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
edgeorder[e] = max(edgeorder[e], recv_orders[procs[j]][cnt[procs[j]]++]);
}
for (auto proc : partop.GetDistantEdgeNums(e))
edgeorder[e] = max(edgeorder[e], recv_orders[proc][cnt[proc]++]);
for (int f = 0; f < faceorder.Size(); f++)
{
partop.GetDistantFaceNums (f+1, procs);
for (int j = 0; j < procs.Size(); j++)
faceorder[f] = max(faceorder[f], recv_orders[procs[j]][cnt[procs[j]]++]);
}
for (auto proc : partop.GetDistantFaceNums(f))
faceorder[f] = max(faceorder[f], recv_orders[proc][cnt[proc]++]);
}
#endif
@ -771,48 +751,40 @@ namespace netgen
TABLE<double> senddata(ntasks), recvdata(ntasks);
if (working)
for (int e = 0; e < nedges; e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
{
senddata.Add (procs[j], surfnr[e]);
if (surfnr[e] != -1)
{
senddata.Add (procs[j], gi0[e].trignum);
senddata.Add (procs[j], gi0[e].u);
senddata.Add (procs[j], gi0[e].v);
senddata.Add (procs[j], gi1[e].trignum);
senddata.Add (procs[j], gi1[e].u);
senddata.Add (procs[j], gi1[e].v);
}
}
}
MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, curve_comm);
for (int proc : partop.GetDistantEdgeNums(e))
{
senddata.Add (proc, surfnr[e]);
if (surfnr[e] != -1)
{
senddata.Add (proc, gi0[e].trignum);
senddata.Add (proc, gi0[e].u);
senddata.Add (proc, gi0[e].v);
senddata.Add (proc, gi1[e].trignum);
senddata.Add (proc, gi1[e].u);
senddata.Add (proc, gi1[e].v);
}
}
MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm);
NgArray<int> cnt(ntasks);
cnt = 0;
if (working)
for (int e = 0; e < nedges; e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
{
int surfnr1 = recvdata[procs[j]][cnt[procs[j]]++];
if (surfnr1 != -1)
{
surfnr[e] = surfnr1;
gi0[e].trignum = int (recvdata[procs[j]][cnt[procs[j]]++]);
gi0[e].u = recvdata[procs[j]][cnt[procs[j]]++];
gi0[e].v = recvdata[procs[j]][cnt[procs[j]]++];
gi1[e].trignum = int (recvdata[procs[j]][cnt[procs[j]]++]);
gi1[e].u = recvdata[procs[j]][cnt[procs[j]]++];
gi1[e].v = recvdata[procs[j]][cnt[procs[j]]++];
}
}
}
for (int proc : partop.GetDistantEdgeNums(e))
{
int surfnr1 = recvdata[proc][cnt[proc]++];
if (surfnr1 != -1)
{
surfnr[e] = surfnr1;
gi0[e].trignum = int (recvdata[proc][cnt[proc]++]);
gi0[e].u = recvdata[proc][cnt[proc]++];
gi0[e].v = recvdata[proc][cnt[proc]++];
gi1[e].trignum = int (recvdata[proc][cnt[proc]++]);
gi1[e].u = recvdata[proc][cnt[proc]++];
gi1[e].v = recvdata[proc][cnt[proc]++];
}
}
}
#endif
@ -974,59 +946,53 @@ namespace netgen
TABLE<double> senddata(ntasks), recvdata(ntasks);
if (working)
for (int e = 0; e < nedges; e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
{
senddata.Add (procs[j], use_edge[e]);
if (use_edge[e])
{
senddata.Add (procs[j], edge_surfnr1[e]);
senddata.Add (procs[j], edge_surfnr2[e]);
senddata.Add (procs[j], edge_gi0[e].edgenr);
senddata.Add (procs[j], edge_gi0[e].body);
senddata.Add (procs[j], edge_gi0[e].dist);
senddata.Add (procs[j], edge_gi0[e].u);
senddata.Add (procs[j], edge_gi0[e].v);
senddata.Add (procs[j], edge_gi1[e].edgenr);
senddata.Add (procs[j], edge_gi1[e].body);
senddata.Add (procs[j], edge_gi1[e].dist);
senddata.Add (procs[j], edge_gi1[e].u);
senddata.Add (procs[j], edge_gi1[e].v);
senddata.Add (procs[j], swap_edge[e]);
}
}
}
MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, curve_comm);
for (int proc : partop.GetDistantEdgeNums(e))
{
senddata.Add (proc, use_edge[e]);
if (use_edge[e])
{
senddata.Add (proc, edge_surfnr1[e]);
senddata.Add (proc, edge_surfnr2[e]);
senddata.Add (proc, edge_gi0[e].edgenr);
senddata.Add (proc, edge_gi0[e].body);
senddata.Add (proc, edge_gi0[e].dist);
senddata.Add (proc, edge_gi0[e].u);
senddata.Add (proc, edge_gi0[e].v);
senddata.Add (proc, edge_gi1[e].edgenr);
senddata.Add (proc, edge_gi1[e].body);
senddata.Add (proc, edge_gi1[e].dist);
senddata.Add (proc, edge_gi1[e].u);
senddata.Add (proc, edge_gi1[e].v);
senddata.Add (proc, swap_edge[e]);
}
}
MyMPI_ExchangeTable (senddata, recvdata, MPI_TAG_CURVE, comm);
NgArray<int> cnt(ntasks);
cnt = 0;
if (working)
for (int e = 0; e < edge_surfnr1.Size(); e++)
{
partop.GetDistantEdgeNums (e+1, procs);
for (int j = 0; j < procs.Size(); j++)
{
int get_edge = int(recvdata[procs[j]][cnt[procs[j]]++]);
if (get_edge)
{
use_edge[e] = 1;
edge_surfnr1[e] = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_surfnr2[e] = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_gi0[e].edgenr = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_gi0[e].body = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_gi0[e].dist = recvdata[procs[j]][cnt[procs[j]]++];
edge_gi0[e].u = recvdata[procs[j]][cnt[procs[j]]++];
edge_gi0[e].v = recvdata[procs[j]][cnt[procs[j]]++];
edge_gi1[e].edgenr = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_gi1[e].body = int (recvdata[procs[j]][cnt[procs[j]]++]);
edge_gi1[e].dist = recvdata[procs[j]][cnt[procs[j]]++];
edge_gi1[e].u = recvdata[procs[j]][cnt[procs[j]]++];
edge_gi1[e].v = recvdata[procs[j]][cnt[procs[j]]++];
swap_edge[e] = recvdata[procs[j]][cnt[procs[j]]++];
}
}
}
for (int proc : partop.GetDistantEdgeNums(e))
{
int get_edge = int(recvdata[proc][cnt[proc]++]);
if (get_edge)
{
use_edge[e] = 1;
edge_surfnr1[e] = int (recvdata[proc][cnt[proc]++]);
edge_surfnr2[e] = int (recvdata[proc][cnt[proc]++]);
edge_gi0[e].edgenr = int (recvdata[proc][cnt[proc]++]);
edge_gi0[e].body = int (recvdata[proc][cnt[proc]++]);
edge_gi0[e].dist = recvdata[proc][cnt[proc]++];
edge_gi0[e].u = recvdata[proc][cnt[proc]++];
edge_gi0[e].v = recvdata[proc][cnt[proc]++];
edge_gi1[e].edgenr = int (recvdata[proc][cnt[proc]++]);
edge_gi1[e].body = int (recvdata[proc][cnt[proc]++]);
edge_gi1[e].dist = recvdata[proc][cnt[proc]++];
edge_gi1[e].u = recvdata[proc][cnt[proc]++];
edge_gi1[e].v = recvdata[proc][cnt[proc]++];
swap_edge[e] = recvdata[proc][cnt[proc]++];
}
}
}
#endif
@ -1182,26 +1148,20 @@ namespace netgen
if (ntasks > 1 && working)
{
for (int f = 0; f < nfaces; f++)
{
partop.GetDistantFaceNums (f+1, procs);
for (int j = 0; j < procs.Size(); j++)
send_surfnr.Add (procs[j], surfnr[f]);
}
for (int proc : partop.GetDistantFaceNums(f))
send_surfnr.Add (proc, surfnr[f]);
}
if (ntasks > 1)
MyMPI_ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE, curve_comm);
MyMPI_ExchangeTable (send_surfnr, recv_surfnr, MPI_TAG_CURVE, comm);
if (ntasks > 1 && working)
{
NgArray<int> cnt(ntasks);
cnt = 0;
for (int f = 0; f < nfaces; f++)
{
partop.GetDistantFaceNums (f+1, procs);
for (int j = 0; j < procs.Size(); j++)
surfnr[f] = max(surfnr[f], recv_surfnr[procs[j]][cnt[procs[j]]++]);
}
for (int proc : partop.GetDistantFaceNums(f))
surfnr[f] = max(surfnr[f], recv_surfnr[proc][cnt[proc]++]);
}
#endif
@ -1386,8 +1346,7 @@ namespace netgen
#ifdef PARALLEL
curve_comm.Barrier();
// MPI_Comm_free (&curve_comm);
comm.Barrier();
#endif
}

12
libsrc/meshing/meshtype.hpp
View File

@ -221,9 +221,21 @@ namespace netgen
PointIndex operator[] (int i) const { return PointIndex(INDEX_2::operator[](i)); }
PointIndex & operator[] (int i) { return reinterpret_cast<PointIndex&>(INDEX_2::operator[](i)); }
static PointIndices Sort(PointIndex i1, PointIndex i2) { return INDEX_2::Sort(i1, i2); }
template <size_t J>
PointIndex get() const { return PointIndex(INDEX_2::operator[](J)); }
};
}
namespace std
{
// structured binding support
template <auto N>
struct tuple_size<netgen::PointIndices<N>> : std::integral_constant<std::size_t, N> {};
template<size_t N, auto M> struct tuple_element<N,netgen::PointIndices<M>> { using type = netgen::PointIndex; };
}
namespace netgen
{
class ElementIndex
{

70
libsrc/meshing/paralleltop.cpp
View File

@ -15,6 +15,7 @@ namespace ngcore
if (v > max) max = v;
return max;
}
/*
template <typename T, typename TI, typename TB>
auto Max (FlatArray<T,TI> array, TB initial) -> T
@ -546,11 +547,18 @@ namespace netgen
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
/*
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
// for (int dest = 1; dest < ntasks; dest++)
for (int dest : GetDistantPNums(v1-PointIndex::BASE))
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
cnt_send[dest-1]++;
*/
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
for (int p : procs1)
if (procs2.Contains(p))
cnt_send[p-1]++;
}
TABLE<int> dest2pair(cnt_send);
@ -559,22 +567,39 @@ namespace netgen
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
/*
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
for (int dest : GetDistantPNums(v1-PointIndex::BASE))
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
dest2pair.Add (dest-1, pi);
*/
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
for (int p : procs1)
if (procs2.Contains(p))
dest2pair.Add (p-1, pi);
}
cnt_send = 0;
int v1, v2;
for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++)
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
// PointIndex v1 = mesh.mlbetweennodes[pi][0];
// PointIndex v2 = mesh.mlbetweennodes[pi][1];
auto [v1,v2] = mesh.mlbetweennodes[pi];
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
/*
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
for (int dest : GetDistantPNums(v1-PointIndex::BASE))
if (IsExchangeVert(dest, v2))
cnt_send[dest-1]+=2;
*/
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
for (int p : procs1)
if (procs2.Contains(p))
cnt_send[p-1]+=2;
}
TABLE<int> send_verts(cnt_send);
@ -598,8 +623,11 @@ namespace netgen
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (procs1.Contains(dest) && procs2.Contains(dest))
{
send_verts.Add (dest-1, loc2exchange[v1]);
send_verts.Add (dest-1, loc2exchange[v2]);
@ -634,7 +662,8 @@ namespace netgen
{
INDEX_2 re(recvarray[ii], recvarray[ii+1]);
INDEX_2 es(loc2exchange[v1], loc2exchange[v2]);
if (es == re && !IsExchangeVert(dest, pi))
// if (es == re && !IsExchangeVert(dest, pi))
if (es == re && !GetDistantProcs(pi).Contains(dest))
{
SetDistantPNum(dest, pi);
changed = true;
@ -870,7 +899,7 @@ namespace netgen
#endif
NgArray<int> sendarray, recvarray;
// NgArray<int> sendarray, recvarray;
// cout << "UpdateCoarseGrid - edges" << endl;
// static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
@ -900,9 +929,15 @@ namespace netgen
for (int edge = 1; edge <= ned; edge++)
{
topology.GetEdgeVertices (edge, v1, v2);
/*
for (int dest = 1; dest < ntasks; dest++)
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
cnt_send[dest-1]+=1;
*/
for (auto p : GetDistantProcs(v1))
if (GetDistantProcs(v2).Contains(p))
cnt_send[p-1]+=1;
}
TABLE<int> dest2edge(cnt_send);
@ -913,7 +948,8 @@ namespace netgen
{
topology.GetEdgeVertices (edge, v1, v2);
for (int dest = 1; dest < ntasks; dest++)
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
dest2edge.Add (dest-1, edge);
}
@ -929,7 +965,8 @@ namespace netgen
for (int edge : dest2edge[dest-1])
{
topology.GetEdgeVertices (edge, v1, v2);
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
{
send_edges.Add (dest-1, loc2exchange[v1]);
send_edges.Add (dest-1, loc2exchange[v2]);
@ -981,9 +1018,14 @@ namespace netgen
topology.GetFaceVertices (face, verts);
for (int dest = 1; dest < ntasks; dest++)
if (dest != id)
/*
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
cnt_send[dest-1]++;
}
@ -993,9 +1035,14 @@ namespace netgen
topology.GetFaceVertices (face, verts);
for (int dest = 1; dest < ntasks; dest++)
if (dest != id)
/*
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
dest2face.Add(dest-1, face);
}
@ -1015,9 +1062,14 @@ namespace netgen
for (int face : dest2face[dest-1])
{
topology.GetFaceVertices (face, verts);
/*
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
{
send_faces.Add (dest-1, loc2exchange[verts[0]]);
send_faces.Add (dest-1, loc2exchange[verts[1]]);

5
libsrc/meshing/paralleltop.hpp
View File

@ -117,18 +117,19 @@ namespace netgen
distedgenums = loc2distedge[locedgenum-1];
}
[[deprecated("Use GetDistantProcs(..)!")]]
FlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
FlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
FlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
[[deprecated("Use GetDistantPNums(..).Contains instead!")]]
FlatArray<int> GetDistantProcs (PointIndex pi) const { return loc2distvert[pi-PointIndex::BASE]; }
[[deprecated("Use GetDistantProcs(..).Contains instead!")]]
bool IsExchangeVert (int dest, int vnum) const
{
return loc2distvert[vnum-1].Contains (dest);
}
};
}