set ParallelTop functions

This commit is contained in:
Joachim Schöberl 2020-08-29 20:56:29 +02:00
parent 0b74e3cbdc
commit 0fe20c9dee
4 changed files with 74 additions and 33 deletions

View File

@ -645,10 +645,10 @@ namespace netgen
if (ntasks > 1 && working) if (ntasks > 1 && working)
{ {
for (int e = 0; e < edgeorder.Size(); e++) for (int e = 0; e < edgeorder.Size(); e++)
for (int proc : partop.GetDistantEdgeNums(e)) for (int proc : partop.GetDistantEdgeProcs(e))
send_orders.Add (proc, edgeorder[e]); send_orders.Add (proc, edgeorder[e]);
for (int f = 0; f < faceorder.Size(); f++) for (int f = 0; f < faceorder.Size(); f++)
for (int proc : partop.GetDistantFaceNums(f)) for (int proc : partop.GetDistantFaceProcs(f))
send_orders.Add (proc, faceorder[f]); send_orders.Add (proc, faceorder[f]);
} }
@ -660,10 +660,10 @@ namespace netgen
NgArray<int> cnt(ntasks); NgArray<int> cnt(ntasks);
cnt = 0; cnt = 0;
for (int e = 0; e < edgeorder.Size(); e++) for (int e = 0; e < edgeorder.Size(); e++)
for (auto proc : partop.GetDistantEdgeNums(e)) for (auto proc : partop.GetDistantEdgeProcs(e))
edgeorder[e] = max(edgeorder[e], recv_orders[proc][cnt[proc]++]); edgeorder[e] = max(edgeorder[e], recv_orders[proc][cnt[proc]++]);
for (int f = 0; f < faceorder.Size(); f++) for (int f = 0; f < faceorder.Size(); f++)
for (auto proc : partop.GetDistantFaceNums(f)) for (auto proc : partop.GetDistantFaceProcs(f))
faceorder[f] = max(faceorder[f], recv_orders[proc][cnt[proc]++]); faceorder[f] = max(faceorder[f], recv_orders[proc][cnt[proc]++]);
} }
#endif #endif
@ -751,7 +751,7 @@ namespace netgen
TABLE<double> senddata(ntasks), recvdata(ntasks); TABLE<double> senddata(ntasks), recvdata(ntasks);
if (working) if (working)
for (int e = 0; e < nedges; e++) for (int e = 0; e < nedges; e++)
for (int proc : partop.GetDistantEdgeNums(e)) for (int proc : partop.GetDistantEdgeProcs(e))
{ {
senddata.Add (proc, surfnr[e]); senddata.Add (proc, surfnr[e]);
if (surfnr[e] != -1) if (surfnr[e] != -1)
@ -771,7 +771,7 @@ namespace netgen
cnt = 0; cnt = 0;
if (working) if (working)
for (int e = 0; e < nedges; e++) for (int e = 0; e < nedges; e++)
for (int proc : partop.GetDistantEdgeNums(e)) for (int proc : partop.GetDistantEdgeProcs(e))
{ {
int surfnr1 = recvdata[proc][cnt[proc]++]; int surfnr1 = recvdata[proc][cnt[proc]++];
if (surfnr1 != -1) if (surfnr1 != -1)
@ -946,7 +946,7 @@ namespace netgen
TABLE<double> senddata(ntasks), recvdata(ntasks); TABLE<double> senddata(ntasks), recvdata(ntasks);
if (working) if (working)
for (int e = 0; e < nedges; e++) for (int e = 0; e < nedges; e++)
for (int proc : partop.GetDistantEdgeNums(e)) for (int proc : partop.GetDistantEdgeProcs(e))
{ {
senddata.Add (proc, use_edge[e]); senddata.Add (proc, use_edge[e]);
if (use_edge[e]) if (use_edge[e])
@ -972,7 +972,7 @@ namespace netgen
cnt = 0; cnt = 0;
if (working) if (working)
for (int e = 0; e < edge_surfnr1.Size(); e++) for (int e = 0; e < edge_surfnr1.Size(); e++)
for (int proc : partop.GetDistantEdgeNums(e)) for (int proc : partop.GetDistantEdgeProcs(e))
{ {
int get_edge = int(recvdata[proc][cnt[proc]++]); int get_edge = int(recvdata[proc][cnt[proc]++]);
if (get_edge) if (get_edge)
@ -1148,7 +1148,7 @@ namespace netgen
if (ntasks > 1 && working) if (ntasks > 1 && working)
{ {
for (int f = 0; f < nfaces; f++) for (int f = 0; f < nfaces; f++)
for (int proc : partop.GetDistantFaceNums(f)) for (int proc : partop.GetDistantFaceProcs(f))
send_surfnr.Add (proc, surfnr[f]); send_surfnr.Add (proc, surfnr[f]);
} }
@ -1160,7 +1160,7 @@ namespace netgen
NgArray<int> cnt(ntasks); NgArray<int> cnt(ntasks);
cnt = 0; cnt = 0;
for (int f = 0; f < nfaces; f++) for (int f = 0; f < nfaces; f++)
for (int proc : partop.GetDistantFaceNums(f)) for (int proc : partop.GetDistantFaceProcs(f))
surfnr[f] = max(surfnr[f], recv_surfnr[proc][cnt[proc]++]); surfnr[f] = max(surfnr[f], recv_surfnr[proc][cnt[proc]++]);
} }
#endif #endif

View File

@ -278,7 +278,8 @@ namespace netgen
vert_flag[vertex] = dest; vert_flag[vertex] = dest;
num_verts_on_proc[dest]++; num_verts_on_proc[dest]++;
num_procs_on_vert[vertex]++; num_procs_on_vert[vertex]++;
GetParallelTopology().SetDistantPNum (dest, vertex); // GetParallelTopology().SetDistantPNum (dest, vertex);
GetParallelTopology().AddDistantProc (PointIndex(vertex), dest);
} }
}; };
countit(vertex, dest); countit(vertex, dest);
@ -870,7 +871,8 @@ namespace netgen
for (int vert = 0; vert < numvert; vert++) for (int vert = 0; vert < numvert; vert++)
{ {
int globvert = verts[vert] + IndexBASE<T_POINTS::index_type>(); int globvert = verts[vert] + IndexBASE<T_POINTS::index_type>();
paralleltop->SetLoc2Glob_Vert ( vert+1, globvert ); // paralleltop->SetLoc2Glob_Vert ( vert+1, globvert );
paralleltop->L2G (PointIndex(vert+PointIndex::BASE)) = globvert;
glob2loc_vert_ht.Set (globvert, vert+1); glob2loc_vert_ht.Set (globvert, vert+1);
} }
@ -902,16 +904,18 @@ namespace netgen
} }
} }
Array<int> dist_pnums; Array<int> dist_pnums;
comm.Recv (dist_pnums, 0, MPI_TAG_MESH+1); comm.Recv (dist_pnums, 0, MPI_TAG_MESH+1);
for (int hi = 0; hi < dist_pnums.Size(); hi += 3) for (int hi = 0; hi < dist_pnums.Size(); hi += 3)
paralleltop -> paralleltop ->
SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi]); // , dist_pnums[hi+2]); // SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi]); // , dist_pnums[hi+2]);
AddDistantProc (PointIndex(dist_pnums[hi]), dist_pnums[hi+1]);
NgProfiler::StopTimer (timer_pts); NgProfiler::StopTimer (timer_pts);
*testout << "got " << numvert << " vertices" << endl; *testout << "got " << numvert << " vertices" << endl;
{ {
Array<int> elarray; Array<int> elarray;
comm.Recv (elarray, 0, MPI_TAG_MESH+2); comm.Recv (elarray, 0, MPI_TAG_MESH+2);

View File

@ -211,12 +211,18 @@ namespace netgen
// *testout << "l " << i << " globi "<< glob_vert[i] << " dist = " << loc2distvert[i] << endl; // *testout << "l " << i << " globi "<< glob_vert[i] << " dist = " << loc2distvert[i] << endl;
} }
/*
for (size_t i = 0; i+1 < glob_vert.Size(); i++) for (size_t i = 0; i+1 < glob_vert.Size(); i++)
if (glob_vert[i] > glob_vert[i+1]) if (glob_vert[i] > glob_vert[i+1])
cout << "wrong ordering of globvert" << endl; cout << "wrong ordering of globvert" << endl;
*/
if (glob_vert.Size() > 1)
for (auto i : Range(glob_vert).Modify(0,-1))
if (glob_vert[i] > glob_vert[i+1])
cout << "wrong ordering of globvert" << endl;
} }
/*
void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum) void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
{ {
for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 ) for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
@ -241,7 +247,8 @@ namespace netgen
return; return;
loc2distedge.Add (locnum-1, dest); loc2distedge.Add (locnum-1, dest);
} }
*/
void ParallelMeshTopology :: SetNV_Loc2Glob (int anv) void ParallelMeshTopology :: SetNV_Loc2Glob (int anv)
{ {
glob_vert.SetSize(anv); glob_vert.SetSize(anv);
@ -379,6 +386,7 @@ namespace netgen
is_updated = true; is_updated = true;
} }
void ParallelMeshTopology :: IdentifyVerticesAfterRefinement() void ParallelMeshTopology :: IdentifyVerticesAfterRefinement()
{ {
static Timer t("ParallelTopology::UpdateCoarseGrid"); RegionTimer r(t); static Timer t("ParallelTopology::UpdateCoarseGrid"); RegionTimer r(t);
@ -511,7 +519,8 @@ namespace netgen
// if (es == re && !IsExchangeVert(dest, pi)) // if (es == re && !IsExchangeVert(dest, pi))
if (es == re && !GetDistantProcs(pi).Contains(dest)) if (es == re && !GetDistantProcs(pi).Contains(dest))
{ {
SetDistantPNum(dest, pi); // SetDistantPNum(dest, pi);
AddDistantProc (pi, dest);
changed = true; changed = true;
} }
} }
@ -694,7 +703,8 @@ namespace netgen
INDEX_2 re(ex2loc[recvarray[ii]], INDEX_2 re(ex2loc[recvarray[ii]],
ex2loc[recvarray[ii+1]]); ex2loc[recvarray[ii+1]]);
if (vert2edge.Used(re)) if (vert2edge.Used(re))
SetDistantEdgeNum(dest, vert2edge.Get(re)); // SetDistantEdgeNum(dest, vert2edge.Get(re));
AddDistantEdgeProc(vert2edge.Get(re)-1, dest);
} }
} }
@ -799,7 +809,7 @@ namespace netgen
ex2loc[recvarray[ii+1]], ex2loc[recvarray[ii+1]],
ex2loc[recvarray[ii+2]]); ex2loc[recvarray[ii+2]]);
if (vert2face.Used(re)) if (vert2face.Used(re))
SetDistantFaceNum(dest, vert2face.Get(re)); AddDistantFaceProc(vert2face.Get(re)-1, dest);
} }
} }

View File

@ -16,9 +16,11 @@ namespace netgen
*/ */
DynamicTable<int> loc2distvert; DynamicTable<int> loc2distvert;
TABLE<int> loc2distedge, loc2distface; DynamicTable<int> loc2distedge, loc2distface;
Array<int> glob_vert; Array<int> glob_vert;
// will get rid of them
NgArray<int> glob_edge, glob_face; NgArray<int> glob_edge, glob_face;
NgArray<int> glob_el, glob_surfel, glob_segm; NgArray<int> glob_el, glob_surfel, glob_segm;
@ -35,9 +37,21 @@ namespace netgen
void UpdateCoarseGrid(); void UpdateCoarseGrid();
void UpdateCoarseGridGlobal(); void UpdateCoarseGridGlobal();
void IdentifyVerticesAfterRefinement(); void IdentifyVerticesAfterRefinement();
// bool DoCoarseUpdate() const { return !coarseupdate; } void EnumeratePointsGlobally ();
void AddDistantProc (PointIndex pi, int proc) { loc2distvert.AddUnique (pi-PointIndex::BASE, proc); }
void AddDistantFaceProc (int edge, int proc) { loc2distface.AddUnique (edge, proc); }
void AddDistantEdgeProc (int face, int proc) { loc2distedge.AddUnique (face, proc); }
FlatArray<int> GetDistantProcs (PointIndex pi) const { return loc2distvert[pi-PointIndex::BASE]; }
FlatArray<int> GetDistantFaceProcs (int locnum) const { return loc2distface[locnum]; }
FlatArray<int> GetDistantEdgeProcs (int locnum) const { return loc2distedge[locnum]; }
auto & L2G (PointIndex pi) { return glob_vert[pi-PointIndex::BASE]; }
auto L2G (PointIndex pi) const { return glob_vert[pi-PointIndex::BASE]; }
/// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert... /// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert...
void SetNV (int anv); void SetNV (int anv);
@ -46,6 +60,21 @@ namespace netgen
void SetNSE (int anse); void SetNSE (int anse);
void SetNSegm (int anseg); void SetNSegm (int anseg);
[[deprecated("Use AddDistantFaceProc instead!")]]
void SetDistantFaceNum (int dest, int locnum) { loc2distface.AddUnique (locnum-1, dest); }
[[deprecated("Use AddDistantProc instead!")]]
void SetDistantPNum (int dest, int locnum) { loc2distvert.AddUnique (locnum-1, dest); }
[[deprecated("Use AddDistantEdgeProc instead!")]]
void SetDistantEdgeNum (int dest, int locnum) { loc2distedge.AddUnique (locnum-1, dest); }
[[deprecated("Use GetDistantFaceProcx instead!")]]
FlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
[[deprecated("Use GetDistantEdgeProcx instead!")]]
FlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
[[deprecated("Use L2G(pi) instead!")]]
void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; } void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]] [[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; } void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; }
@ -68,12 +97,7 @@ namespace netgen
[[deprecated("Try to avoid global enumration!")]] [[deprecated("Try to avoid global enumration!")]]
int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; } int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; }
void EnumeratePointsGlobally ();
void SetDistantFaceNum (int dest, int locnum);
void SetDistantPNum (int dest, int locnum);
void SetDistantEdgeNum (int dest, int locnum);
[[deprecated("Use GetDistantPNums(locnum).Size() instead!")]] [[deprecated("Use GetDistantPNums(locnum).Size() instead!")]]
int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); } int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
@ -101,7 +125,11 @@ namespace netgen
[[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]] [[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]]
void GetDistantFaceNums (int locfacenum, NgArray<int> & distfacenums ) const void GetDistantFaceNums (int locfacenum, NgArray<int> & distfacenums ) const
{ {
distfacenums = loc2distface[locfacenum-1]; // distfacenums = loc2distface[locfacenum-1];
auto loc = loc2distface[locfacenum-1];
distfacenums.SetSize (loc.Size());
for (int i = 0; i < loc.Size(); i++)
distfacenums[i] = loc[i];
} }
[[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]] [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]
@ -114,17 +142,16 @@ namespace netgen
[[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]] [[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]
void GetDistantEdgeNums (int locedgenum, NgArray<int> & distedgenums ) const void GetDistantEdgeNums (int locedgenum, NgArray<int> & distedgenums ) const
{ {
distedgenums = loc2distedge[locedgenum-1]; // distedgenums = loc2distedge[locedgenum-1];
auto loc = loc2distedge[locedgenum-1];
distedgenums.SetSize (loc.Size());
for (int i = 0; i < loc.Size(); i++)
distedgenums[i] = loc[i];
} }
[[deprecated("Use GetDistantProcs(..)!")]] [[deprecated("Use GetDistantProcs(..)!")]]
FlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; } FlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
FlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
FlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
FlatArray<int> GetDistantProcs (PointIndex pi) const { return loc2distvert[pi-PointIndex::BASE]; }
auto & L2G (PointIndex pi) { return glob_vert[pi-PointIndex::BASE]; }
auto L2G (PointIndex pi) const { return glob_vert[pi-PointIndex::BASE]; }
[[deprecated("Use GetDistantProcs(..).Contains instead!")]] [[deprecated("Use GetDistantProcs(..).Contains instead!")]]