netgen/libsrc/meshing/paralleltop.cpp

829 lines
24 KiB
C++
Raw Normal View History

2009-01-26 00:37:14 +05:00
#ifdef PARALLEL
#include <meshing.hpp>
#include "paralleltop.hpp"
namespace netgen
{
2012-06-21 20:19:25 +06:00
ParallelMeshTopology :: ParallelMeshTopology (const Mesh & amesh)
: mesh(amesh)
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
is_updated = false;
2011-02-15 21:22:05 +05:00
}
2012-06-21 20:19:25 +06:00
2009-01-26 00:37:14 +05:00
2011-02-15 21:22:05 +05:00
ParallelMeshTopology :: ~ParallelMeshTopology ()
{
2011-07-15 03:36:19 +06:00
;
2011-02-15 21:22:05 +05:00
}
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: Reset ()
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
*testout << "ParallelMeshTopology::Reset" << endl;
if ( mesh.GetCommunicator().Size() == 1 ) return;
2009-01-26 00:37:14 +05:00
2020-08-29 12:36:46 +05:00
size_t ned = mesh.GetTopology().GetNEdges();
size_t nfa = mesh.GetTopology().GetNFaces();
2012-06-16 18:04:04 +06:00
2012-06-21 20:19:25 +06:00
if (glob_edge.Size() != ned)
{
glob_edge.SetSize(ned);
glob_face.SetSize(nfa);
glob_edge = -1;
glob_face = -1;
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
loc2distedge.ChangeSize (ned);
loc2distface.ChangeSize (nfa);
}
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
if (glob_vert.Size() != mesh.GetNV())
2011-02-15 21:22:05 +05:00
{
2012-06-21 20:19:25 +06:00
SetNV(mesh.GetNV());
SetNE(mesh.GetNE());
2011-02-15 21:22:05 +05:00
}
2009-01-26 00:37:14 +05:00
}
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: Print() const
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
;
}
2009-01-26 00:37:14 +05:00
2020-08-21 19:29:33 +05:00
void ParallelMeshTopology :: EnumeratePointsGlobally ()
{
auto comm = mesh.GetCommunicator();
auto rank = comm.Rank();
2020-08-28 11:57:30 +05:00
size_t oldnv = glob_vert.Size();
size_t nv = loc2distvert.Size();
*testout << "enumerate globally, loc2distvert.size = " << loc2distvert.Size()
<< ", glob_vert.size = " << glob_vert.Size() << endl;
if (rank == 0)
nv = 0;
2020-08-29 12:36:46 +05:00
// IntRange newvr(oldnv, nv); // new vertex range
auto new_pir = Range(PointIndex(oldnv+PointIndex::BASE),
PointIndex(nv+PointIndex::BASE));
2020-08-28 11:57:30 +05:00
2020-08-21 19:29:33 +05:00
glob_vert.SetSize (nv);
2020-08-29 14:05:02 +05:00
for (auto pi : new_pir)
L2G(pi) = -1;
2020-08-28 11:57:30 +05:00
2020-08-21 19:29:33 +05:00
int num_master_points = 0;
2020-08-28 11:57:30 +05:00
2020-08-29 12:36:46 +05:00
for (auto pi : new_pir)
2020-08-21 19:29:33 +05:00
{
2020-08-29 12:36:46 +05:00
auto dps = GetDistantProcs(pi);
2020-08-21 19:29:33 +05:00
// check sorted:
for (int j = 0; j+1 < dps.Size(); j++)
if (dps[j+1] < dps[j]) cout << "wrong sort" << endl;
if (dps.Size() == 0 || dps[0] > comm.Rank())
2020-08-29 12:36:46 +05:00
L2G(pi) = num_master_points++;
2020-08-21 19:29:33 +05:00
}
2020-08-29 12:36:46 +05:00
2020-08-28 11:57:30 +05:00
*testout << "nummaster = " << num_master_points << endl;
2020-08-21 19:29:33 +05:00
Array<int> first_master_point(comm.Size());
comm.AllGather (num_master_points, first_master_point);
2020-08-28 11:57:30 +05:00
auto max_oldv = comm.AllReduce (Max (glob_vert.Range(0, oldnv)), MPI_MAX);
if (comm.AllReduce (oldnv, MPI_SUM) == 0)
max_oldv = PointIndex::BASE-1;
2020-08-29 12:36:46 +05:00
size_t num_glob_points = max_oldv+1;
2020-08-21 19:29:33 +05:00
for (int i = 0; i < comm.Size(); i++)
{
int cur = first_master_point[i];
first_master_point[i] = num_glob_points;
num_glob_points += cur;
}
2020-08-29 12:36:46 +05:00
for (auto pi : new_pir)
if (L2G(pi) != -1)
L2G(pi) += first_master_point[comm.Rank()];
2020-08-21 19:29:33 +05:00
// ScatterDofData (global_nums);
Array<int> nsend(comm.Size()), nrecv(comm.Size());
nsend = 0;
nrecv = 0;
2020-08-29 14:05:02 +05:00
/** Count send/recv size **/
2020-08-29 12:36:46 +05:00
for (auto pi : new_pir)
2020-08-29 14:05:02 +05:00
if (auto dps = GetDistantProcs(pi); dps.Size())
{
if (rank < dps[0])
for (auto p : dps)
nsend[p]++;
else
nrecv[dps[0]]++;
}
2020-08-21 19:29:33 +05:00
Table<PointIndex> send_data(nsend);
Table<PointIndex> recv_data(nrecv);
/** Fill send_data **/
nsend = 0;
2020-08-29 12:36:46 +05:00
for (auto pi : new_pir)
if (auto dps = GetDistantProcs(pi); dps.Size())
if (rank < dps[0])
for (auto p : dps)
send_data[p][nsend[p]++] = L2G(pi);
2020-08-21 19:29:33 +05:00
Array<MPI_Request> requests;
for (int i = 0; i < comm.Size(); i++)
{
if (nsend[i])
requests.Append (comm.ISend (send_data[i], i, 200));
if (nrecv[i])
requests.Append (comm.IRecv (recv_data[i], i, 200));
}
MyMPI_WaitAll (requests);
Array<int> cnt(comm.Size());
cnt = 0;
2020-08-29 12:36:46 +05:00
for (auto pi : new_pir)
if (auto dps = GetDistantProcs(pi); dps.Size())
if (int master = dps[0]; master < comm.Rank())
L2G(pi) = recv_data[master][cnt[master]++];
2020-08-28 11:57:30 +05:00
// reorder following global ordering:
Array<int> index0(glob_vert.Size());
for (int pi : Range(index0))
index0[pi] = pi;
2020-08-29 12:36:46 +05:00
QuickSortI (glob_vert, index0);
2020-08-28 11:57:30 +05:00
if (rank != 0)
{
Array<PointIndex, PointIndex> inv_index(index0.Size());
for (int i = 0; i < index0.Size(); i++)
inv_index[index0[i]+PointIndex::BASE] = i+PointIndex::BASE;
for (auto & el : mesh.VolumeElements())
for (PointIndex & pi : el.PNums())
pi = inv_index[pi];
for (auto & el : mesh.SurfaceElements())
for (PointIndex & pi : el.PNums())
pi = inv_index[pi];
for (auto & el : mesh.LineSegments())
for (PointIndex & pi : el.PNums())
pi = inv_index[pi];
// auto hpoints (mesh.Points());
Array<MeshPoint, PointIndex> hpoints { mesh.Points() };
for (PointIndex pi : Range(mesh.Points()))
mesh.Points()[inv_index[pi]] = hpoints[pi];
if (mesh.mlbetweennodes.Size() == mesh.Points().Size())
{
NgArray<PointIndices<2>,PointIndex::BASE> hml { mesh.mlbetweennodes };
for (PointIndex pi : Range(mesh.Points()))
mesh.mlbetweennodes[inv_index[pi]] = hml[pi];
}
// *testout << "index0 = " << endl << index0 << endl;
// *testout << "loc2distvertold = " << endl;
// for (auto i : Range(index0))
// *testout << "l " << i << " globi "<< glob_vert[i] << " dist = " << loc2distvert[i] << endl;
2020-08-29 14:05:02 +05:00
DynamicTable<int> oldtable = std::move(loc2distvert);
2020-08-28 11:57:30 +05:00
loc2distvert = DynamicTable<int> (oldtable.Size());
for (size_t i = 0; i < oldtable.Size(); i++)
for (auto val : oldtable[index0[i]])
loc2distvert.Add (i, val);
Array<int> hglob_vert(glob_vert);
for (int i = 0; i < index0.Size(); i++)
glob_vert[i] = hglob_vert[index0[i]];
// *testout << "loc2distvertnew = " << endl;
// for (auto i : Range(index0))
// *testout << "l " << i << " globi "<< glob_vert[i] << " dist = " << loc2distvert[i] << endl;
}
2020-08-29 23:56:29 +05:00
/*
2020-08-29 12:36:46 +05:00
for (size_t i = 0; i+1 < glob_vert.Size(); i++)
2020-08-28 11:57:30 +05:00
if (glob_vert[i] > glob_vert[i+1])
cout << "wrong ordering of globvert" << endl;
2020-08-29 23:56:29 +05:00
*/
if (glob_vert.Size() > 1)
for (auto i : Range(glob_vert).Modify(0,-1))
if (glob_vert[i] > glob_vert[i+1])
cout << "wrong ordering of globvert" << endl;
2020-08-21 19:29:33 +05:00
}
2020-08-29 23:56:29 +05:00
/*
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
2009-01-26 00:37:14 +05:00
if ( loc2distface[locnum-1][i] == dest )
2012-06-21 20:19:25 +06:00
return;
2009-01-26 00:37:14 +05:00
loc2distface.Add(locnum-1, dest);
}
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: SetDistantPNum (int dest, int locnum)
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
for ( int i = 0; i < loc2distvert[locnum-1].Size(); i+=1 )
if ( loc2distvert[locnum-1][i] == dest )
2009-01-26 00:37:14 +05:00
return;
2012-06-21 20:19:25 +06:00
loc2distvert.Add (locnum-1, dest);
2009-01-26 00:37:14 +05:00
}
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: SetDistantEdgeNum (int dest, int locnum)
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
for ( int i = 0; i < loc2distedge[locnum-1].Size(); i+=1 )
2009-01-26 00:37:14 +05:00
if ( loc2distedge[locnum-1][i] == dest )
return;
2012-06-21 20:19:25 +06:00
loc2distedge.Add (locnum-1, dest);
2009-01-26 00:37:14 +05:00
}
2020-08-29 23:56:29 +05:00
*/
2020-08-28 11:57:30 +05:00
void ParallelMeshTopology :: SetNV_Loc2Glob (int anv)
2012-06-16 18:04:04 +06:00
{
2012-06-21 20:19:25 +06:00
glob_vert.SetSize(anv);
glob_vert = -1;
2020-08-28 11:57:30 +05:00
}
void ParallelMeshTopology :: SetNV (int anv)
{
// glob_vert.SetSize(anv);
// glob_vert = -1;
// loc2distvert.ChangeSize (anv);
DynamicTable<int> oldtable(loc2distvert.Size());
for (size_t i = 0; i < loc2distvert.Size(); i++)
for (auto val : loc2distvert[i])
oldtable.Add (i, val);
loc2distvert = DynamicTable<int> (anv);
2020-08-29 14:05:02 +05:00
for (size_t i = 0; i < min(size_t(anv), oldtable.Size()); i++)
2020-08-28 11:57:30 +05:00
for (auto val : oldtable[i])
loc2distvert.Add (i, val);
2012-06-16 18:04:04 +06:00
}
2012-06-21 20:19:25 +06:00
void ParallelMeshTopology :: SetNE ( int ane )
2012-06-16 18:04:04 +06:00
{
2012-06-21 20:19:25 +06:00
glob_el.SetSize (ane);
glob_el = -1;
2012-06-16 18:04:04 +06:00
}
void ParallelMeshTopology :: SetNSE ( int anse )
{
2012-06-21 20:19:25 +06:00
glob_surfel.SetSize(anse);
glob_surfel = -1;
2012-06-16 18:04:04 +06:00
}
void ParallelMeshTopology :: SetNSegm ( int anseg )
{
2012-06-21 20:19:25 +06:00
glob_segm.SetSize (anseg);
glob_segm = -1;
2012-06-16 18:04:04 +06:00
}
2011-02-15 21:22:05 +05:00
void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
{
2017-06-06 12:55:40 +05:00
// cout << "updatecoarsegridglobal called" << endl;
2011-07-15 03:36:19 +06:00
if (id == 0)
2012-08-20 20:10:23 +06:00
PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" );
2009-01-26 00:37:14 +05:00
2011-02-15 21:22:05 +05:00
int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal");
NgProfiler::RegionTimer reg(timer);
2009-01-26 00:37:14 +05:00
2011-02-15 21:22:05 +05:00
*testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl;
2011-07-17 05:13:26 +06:00
2011-02-15 21:22:05 +05:00
const MeshTopology & topology = mesh.GetTopology();
2020-08-05 04:11:26 +05:00
auto comm = mesh.GetCommunicator();
2022-04-21 20:16:26 +05:00
2011-07-17 05:13:26 +06:00
if ( id == 0 )
2009-01-26 00:37:14 +05:00
{
2019-07-09 13:39:16 +05:00
NgArray<NgArray<int>*> sendarrays(ntasks);
2012-06-16 18:04:04 +06:00
for (int dest = 1; dest < ntasks; dest++)
2019-07-09 13:39:16 +05:00
sendarrays[dest] = new NgArray<int>;
2009-01-26 00:37:14 +05:00
2019-07-09 13:39:16 +05:00
NgArray<int> edges, faces;
2012-06-21 20:19:25 +06:00
for (int el = 1; el <= mesh.GetNE(); el++)
2011-07-17 05:13:26 +06:00
{
topology.GetElementFaces (el, faces);
2012-06-16 18:04:04 +06:00
topology.GetElementEdges (el, edges);
2011-07-17 05:13:26 +06:00
const Element & volel = mesh.VolumeElement (el);
2019-07-09 13:39:16 +05:00
// NgArray<int> & sendarray = *sendarrays[volel.GetPartition()];
NgArray<int> & sendarray = *sendarrays[mesh.vol_partition[el-1]];
2011-07-17 05:13:26 +06:00
for ( int i = 0; i < edges.Size(); i++ )
2012-06-21 20:19:25 +06:00
sendarray.Append (edges[i]);
for ( int i = 0; i < faces.Size(); i++ )
sendarray.Append (faces[i]);
2011-07-17 05:13:26 +06:00
}
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
for (int el = 1; el <= mesh.GetNSE(); el++)
2011-07-17 05:13:26 +06:00
{
2012-06-16 18:04:04 +06:00
topology.GetSurfaceElementEdges (el, edges);
2011-07-17 05:13:26 +06:00
const Element2d & surfel = mesh.SurfaceElement (el);
2019-07-09 13:39:16 +05:00
// NgArray<int> & sendarray = *sendarrays[surfel.GetPartition()];
NgArray<int> & sendarray = *sendarrays[mesh.surf_partition[el-1]];
2009-01-26 00:37:14 +05:00
2011-07-17 05:13:26 +06:00
for ( int i = 0; i < edges.Size(); i++ )
2012-06-16 18:04:04 +06:00
sendarray.Append (edges[i]);
2012-08-20 20:10:23 +06:00
sendarray.Append (topology.GetSurfaceElementFace (el));
2011-07-17 05:13:26 +06:00
}
2009-01-26 00:37:14 +05:00
Array<MPI_Request> sendrequests;
2012-06-16 18:04:04 +06:00
for (int dest = 1; dest < ntasks; dest++)
2020-08-05 04:11:26 +05:00
// sendrequests.Append (MyMPI_ISend (*sendarrays[dest], dest, MPI_TAG_MESH+10, comm));
sendrequests.Append (comm.ISend (FlatArray<int>(*sendarrays[dest]), dest, MPI_TAG_MESH+10));
MyMPI_WaitAll (sendrequests);
2009-01-26 00:37:14 +05:00
2012-06-16 18:04:04 +06:00
for (int dest = 1; dest < ntasks; dest++)
delete sendarrays[dest];
}
2009-01-26 00:37:14 +05:00
else
{
2019-07-09 13:39:16 +05:00
NgArray<int> recvarray;
2019-02-13 02:11:55 +05:00
MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10, comm);
2012-06-16 18:04:04 +06:00
2011-07-17 05:13:26 +06:00
int ii = 0;
2009-01-26 00:37:14 +05:00
2019-07-09 13:39:16 +05:00
NgArray<int> faces, edges;
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
for (int volel = 1; volel <= mesh.GetNE(); volel++)
2009-01-26 00:37:14 +05:00
{
2012-06-21 20:19:25 +06:00
topology.GetElementEdges ( volel, edges);
for ( int i = 0; i < edges.Size(); i++)
SetLoc2Glob_Edge ( edges[i], recvarray[ii++]);
2012-06-16 18:04:04 +06:00
2012-06-21 20:19:25 +06:00
topology.GetElementFaces( volel, faces);
for ( int i = 0; i < faces.Size(); i++)
SetLoc2Glob_Face ( faces[i], recvarray[ii++]);
2009-01-26 00:37:14 +05:00
}
2012-06-21 20:19:25 +06:00
for (int surfel = 1; surfel <= mesh.GetNSE(); surfel++)
2011-07-17 05:13:26 +06:00
{
2012-06-21 20:19:25 +06:00
topology.GetSurfaceElementEdges (surfel, edges);
for (int i = 0; i < edges.Size(); i++)
SetLoc2Glob_Edge (edges[i], recvarray[ii++]);
2012-08-20 20:10:23 +06:00
int face = topology.GetSurfaceElementFace (surfel);
SetLoc2Glob_Face ( face, recvarray[ii++]);
2011-07-17 05:13:26 +06:00
}
}
2009-01-26 00:37:14 +05:00
2012-06-21 20:19:25 +06:00
is_updated = true;
2011-02-15 21:22:05 +05:00
}
2009-01-26 00:37:14 +05:00
2020-08-29 23:56:29 +05:00
2020-08-28 11:57:30 +05:00
void ParallelMeshTopology :: IdentifyVerticesAfterRefinement()
{
static Timer t("ParallelTopology::UpdateCoarseGrid"); RegionTimer r(t);
NgMPI_Comm comm = mesh.GetCommunicator();
int id = comm.Rank();
int ntasks = comm.Size();
if (ntasks == 1) return;
Reset();
static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid");
NgProfiler::RegionTimer reg(timer);
(*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl;
if (id == 0)
PrintMessage (1, "update parallel topology");
const MeshTopology & topology = mesh.GetTopology();
2020-08-29 21:16:25 +05:00
NgArray<int> cnt_send(ntasks);
2020-08-28 11:57:30 +05:00
2020-08-29 21:16:25 +05:00
int maxsize = comm.AllReduce (mesh.mlbetweennodes.Size(), MPI_MAX);
2020-08-28 11:57:30 +05:00
// update new vertices after mesh-refinement
2020-08-29 21:16:25 +05:00
if (maxsize > 0)
2020-08-28 11:57:30 +05:00
{
int newnv = mesh.mlbetweennodes.Size();
2020-08-29 14:05:02 +05:00
loc2distvert.ChangeSize(mesh.mlbetweennodes.Size());
2020-08-28 11:57:30 +05:00
bool changed = true;
while (changed)
{
changed = false;
// build exchange vertices
cnt_send = 0;
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:16:25 +05:00
cnt_send[dist]++;
2020-08-28 11:57:30 +05:00
TABLE<int> dest2vert(cnt_send);
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:16:25 +05:00
dest2vert.Add (dist, pi);
2020-08-28 11:57:30 +05:00
for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++)
2020-08-29 21:16:25 +05:00
if (auto [v1,v2] = mesh.mlbetweennodes[pi]; v1.IsValid())
{
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
for (int p : procs1)
if (procs2.Contains(p))
cnt_send[p]++;
}
2020-08-28 11:57:30 +05:00
TABLE<int> dest2pair(cnt_send);
2020-08-29 21:16:25 +05:00
for (PointIndex pi : mesh.mlbetweennodes.Range())
if (auto [v1,v2] = mesh.mlbetweennodes[pi]; v1.IsValid())
{
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
for (int p : procs1)
if (procs2.Contains(p))
dest2pair.Add (p, pi);
2020-08-29 14:05:02 +05:00
}
2020-08-28 11:57:30 +05:00
cnt_send = 0;
2020-08-29 14:05:02 +05:00
for (PointIndex pi : mesh.mlbetweennodes.Range())
if (auto [v1,v2] = mesh.mlbetweennodes[pi]; v1.IsValid())
{
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
for (int p : procs1)
if (procs2.Contains(p))
2020-08-29 21:16:25 +05:00
cnt_send[p]+=2;
2020-08-29 14:05:02 +05:00
}
2020-08-28 11:57:30 +05:00
TABLE<int> send_verts(cnt_send);
2020-08-29 21:16:25 +05:00
2020-08-28 11:57:30 +05:00
NgArray<int, PointIndex::BASE> loc2exchange(mesh.GetNV());
2020-08-29 21:16:25 +05:00
for (int dest = 0; dest < ntasks; dest++)
2020-08-28 11:57:30 +05:00
if (dest != id)
{
loc2exchange = -1;
int cnt = 0;
2020-08-29 21:16:25 +05:00
for (PointIndex pi : dest2vert[dest])
2020-08-28 11:57:30 +05:00
loc2exchange[pi] = cnt++;
2020-08-29 21:16:25 +05:00
for (PointIndex pi : dest2pair[dest])
if (auto [v1,v2] = mesh.mlbetweennodes[pi]; v1.IsValid())
{
auto procs1 = GetDistantProcs(v1);
auto procs2 = GetDistantProcs(v2);
if (procs1.Contains(dest) && procs2.Contains(dest))
{
send_verts.Add (dest, loc2exchange[v1]);
send_verts.Add (dest, loc2exchange[v2]);
}
}
2020-08-28 11:57:30 +05:00
}
2009-01-26 00:37:14 +05:00
2020-08-29 21:16:25 +05:00
TABLE<int> recv_verts(ntasks);
MyMPI_ExchangeTable (send_verts, recv_verts, MPI_TAG_MESH+9, comm);
for (int dest = 0; dest < ntasks; dest++)
2020-08-28 11:57:30 +05:00
if (dest != id)
{
loc2exchange = -1;
int cnt = 0;
2020-08-29 21:16:25 +05:00
for (PointIndex pi : dest2vert[dest])
2020-08-28 11:57:30 +05:00
loc2exchange[pi] = cnt++;
2020-08-29 21:16:25 +05:00
NgFlatArray<int> recvarray = recv_verts[dest];
2020-08-28 11:57:30 +05:00
for (int ii = 0; ii < recvarray.Size(); ii+=2)
2020-08-29 21:16:25 +05:00
for (PointIndex pi : dest2pair[dest])
2020-08-28 11:57:30 +05:00
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
2020-08-29 14:05:02 +05:00
if (v1.IsValid())
2020-08-28 11:57:30 +05:00
{
INDEX_2 re(recvarray[ii], recvarray[ii+1]);
INDEX_2 es(loc2exchange[v1], loc2exchange[v2]);
2020-08-29 00:28:18 +05:00
// if (es == re && !IsExchangeVert(dest, pi))
if (es == re && !GetDistantProcs(pi).Contains(dest))
2020-08-28 11:57:30 +05:00
{
2020-08-29 23:56:29 +05:00
// SetDistantPNum(dest, pi);
AddDistantProc (pi, dest);
2020-08-28 11:57:30 +05:00
changed = true;
}
}
}
}
2020-08-29 21:16:25 +05:00
changed = comm.AllReduce (changed, MPI_LOR);
2020-08-28 11:57:30 +05:00
}
}
NgArray<int> sendarray, recvarray;
// cout << "UpdateCoarseGrid - edges" << endl;
// static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges");
static int timerf = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex faces");
NgProfiler::StartTimer (timere);
// build exchange vertices
cnt_send = 0;
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:16:25 +05:00
cnt_send[dist]++;
2020-08-28 11:57:30 +05:00
TABLE<int> dest2vert(cnt_send);
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:16:25 +05:00
dest2vert.Add (dist, pi);
2020-08-28 11:57:30 +05:00
2020-08-29 21:16:25 +05:00
// MPI_Group_free(&MPI_LocalGroup);
2020-08-28 11:57:30 +05:00
// MPI_Comm_free(&MPI_LocalComm);
}
2009-01-26 00:37:14 +05:00
void ParallelMeshTopology :: UpdateCoarseGrid ()
{
2020-08-10 19:44:14 +05:00
static Timer t("ParallelTopology::UpdateCoarseGrid"); RegionTimer r(t);
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid" << endl;
2016-02-28 12:11:34 +05:00
// if (is_updated) return;
2012-06-21 20:19:25 +06:00
2019-02-13 02:11:55 +05:00
NgMPI_Comm comm = mesh.GetCommunicator();
int id = comm.Rank();
int ntasks = comm.Size();
2019-02-13 02:11:55 +05:00
if (ntasks == 1) return;
2012-06-21 20:19:25 +06:00
Reset();
2011-07-15 03:36:19 +06:00
static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid");
2009-01-26 00:37:14 +05:00
NgProfiler::RegionTimer reg(timer);
2011-02-15 21:22:05 +05:00
2009-01-26 00:37:14 +05:00
(*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl;
2011-07-15 03:36:19 +06:00
if (id == 0)
2012-09-03 15:49:43 +06:00
PrintMessage (1, "update parallel topology");
2009-01-26 00:37:14 +05:00
2011-07-17 05:13:26 +06:00
2012-08-20 20:10:23 +06:00
// UpdateCoarseGridGlobal();
2012-06-21 20:19:25 +06:00
2020-08-29 21:24:01 +05:00
/*
2012-09-28 18:39:41 +06:00
// MPI_Barrier (MPI_COMM_WORLD);
2009-01-26 00:37:14 +05:00
MPI_Group MPI_GROUP_comm;
2012-06-16 18:04:04 +06:00
MPI_Group MPI_LocalGroup;
MPI_Comm MPI_LocalComm;
int process_ranks[] = { 0 };
MPI_Comm_group (comm, &MPI_GROUP_comm);
MPI_Group_excl (MPI_GROUP_comm, 1, process_ranks, &MPI_LocalGroup);
MPI_Comm_create (comm, MPI_LocalGroup, &MPI_LocalComm);
2012-06-16 18:04:04 +06:00
2020-08-21 19:29:33 +05:00
if (id == 0)
{
// SetNV(0);
// EnumeratePointsGlobally();
return;
}
2020-08-29 21:24:01 +05:00
*/
2020-08-21 19:29:33 +05:00
2016-02-28 09:50:15 +05:00
const MeshTopology & topology = mesh.GetTopology();
2009-01-26 00:37:14 +05:00
2020-08-29 21:24:01 +05:00
NgArray<int> cnt_send(ntasks);
2016-02-29 16:25:18 +05:00
2020-08-29 00:28:18 +05:00
// NgArray<int> sendarray, recvarray;
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - edges" << endl;
2009-01-26 00:37:14 +05:00
2014-09-15 14:09:39 +06:00
// static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
2011-07-15 03:36:19 +06:00
static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges");
static int timerf = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex faces");
NgProfiler::StartTimer (timere);
2012-08-20 20:10:23 +06:00
2012-06-21 20:19:25 +06:00
int nfa = topology . GetNFaces();
int ned = topology . GetNEdges();
2011-07-15 14:04:25 +06:00
2016-03-01 01:33:29 +05:00
// build exchange vertices
cnt_send = 0;
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:24:01 +05:00
cnt_send[dist]++;
2016-03-01 01:33:29 +05:00
TABLE<int> dest2vert(cnt_send);
for (PointIndex pi : mesh.Points().Range())
2020-08-29 12:36:46 +05:00
for (int dist : GetDistantProcs(pi))
2020-08-29 21:24:01 +05:00
dest2vert.Add (dist, pi);
2011-07-15 14:04:25 +06:00
2012-06-21 20:19:25 +06:00
// exchange edges
2011-07-15 14:04:25 +06:00
cnt_send = 0;
int v1, v2;
for (int edge = 1; edge <= ned; edge++)
{
topology.GetEdgeVertices (edge, v1, v2);
2020-08-29 00:28:18 +05:00
/*
2011-07-15 14:04:25 +06:00
for (int dest = 1; dest < ntasks; dest++)
2020-08-29 00:28:18 +05:00
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
2016-03-01 01:33:29 +05:00
cnt_send[dest-1]+=1;
2020-08-29 00:28:18 +05:00
*/
for (auto p : GetDistantProcs(v1))
if (GetDistantProcs(v2).Contains(p))
2020-08-29 21:24:01 +05:00
cnt_send[p]+=1;
2011-07-15 14:04:25 +06:00
}
2016-03-01 01:33:29 +05:00
TABLE<int> dest2edge(cnt_send);
for (int & v : cnt_send) v *= 2;
2011-07-15 14:04:25 +06:00
TABLE<int> send_edges(cnt_send);
2016-03-01 01:33:29 +05:00
2011-07-15 14:04:25 +06:00
for (int edge = 1; edge <= ned; edge++)
{
topology.GetEdgeVertices (edge, v1, v2);
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2020-08-29 00:28:18 +05:00
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
2020-08-29 21:24:01 +05:00
dest2edge.Add (dest, edge);
2011-07-15 14:04:25 +06:00
}
2016-03-01 01:33:29 +05:00
2019-07-09 13:39:16 +05:00
NgArray<int, PointIndex::BASE> loc2exchange(mesh.GetNV());
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-02-28 18:00:41 +05:00
{
loc2exchange = -1;
2016-02-29 16:25:18 +05:00
int cnt = 0;
2020-08-29 21:24:01 +05:00
for (PointIndex pi : dest2vert[dest])
2016-03-01 01:33:29 +05:00
loc2exchange[pi] = cnt++;
2020-08-29 21:24:01 +05:00
for (int edge : dest2edge[dest])
2016-02-28 18:00:41 +05:00
{
topology.GetEdgeVertices (edge, v1, v2);
2020-08-29 00:28:18 +05:00
// if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
if (GetDistantProcs(v1).Contains(dest) && GetDistantProcs(v2).Contains(dest))
2016-02-28 18:00:41 +05:00
{
2020-08-29 21:24:01 +05:00
send_edges.Add (dest, loc2exchange[v1]);
send_edges.Add (dest, loc2exchange[v2]);
2016-02-28 18:00:41 +05:00
}
}
}
2011-07-15 22:26:32 +06:00
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - edges mpi-exchange" << endl;
2020-08-29 21:24:01 +05:00
TABLE<int> recv_edges(ntasks);
MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, comm);
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - edges mpi-exchange done" << endl;
2016-03-08 02:25:40 +05:00
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-03-08 02:25:40 +05:00
{
2020-08-29 21:24:01 +05:00
auto ex2loc = dest2vert[dest];
2016-03-08 02:25:40 +05:00
if (ex2loc.Size() == 0) continue;
2020-08-29 21:24:01 +05:00
INDEX_2_CLOSED_HASHTABLE<int> vert2edge(2*dest2edge[dest].Size()+10);
for (int edge : dest2edge[dest])
2016-03-08 02:25:40 +05:00
{
topology.GetEdgeVertices (edge, v1, v2);
vert2edge.Set(INDEX_2(v1,v2), edge);
}
2020-08-29 21:24:01 +05:00
NgFlatArray<int> recvarray = recv_edges[dest];
2016-03-08 02:25:40 +05:00
for (int ii = 0; ii < recvarray.Size(); ii+=2)
{
INDEX_2 re(ex2loc[recvarray[ii]],
ex2loc[recvarray[ii+1]]);
if (vert2edge.Used(re))
2020-08-29 23:56:29 +05:00
// SetDistantEdgeNum(dest, vert2edge.Get(re));
AddDistantEdgeProc(vert2edge.Get(re)-1, dest);
2016-03-08 02:25:40 +05:00
}
}
2016-03-01 01:33:29 +05:00
2011-07-15 03:36:19 +06:00
NgProfiler::StopTimer (timere);
2011-07-17 05:13:26 +06:00
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - faces" << endl;
2011-07-17 05:13:26 +06:00
if (mesh.GetDimension() == 3)
{
2012-06-21 20:19:25 +06:00
NgProfiler::StartTimer (timerf);
2019-07-09 13:39:16 +05:00
NgArray<int> verts;
2012-08-20 20:10:23 +06:00
// exchange faces
cnt_send = 0;
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices (face, verts);
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-03-01 01:33:29 +05:00
if (dest != id)
2020-08-29 00:28:18 +05:00
/*
2016-03-01 01:33:29 +05:00
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
2020-08-29 00:28:18 +05:00
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
2020-08-29 21:24:01 +05:00
cnt_send[dest]++;
2012-08-20 20:10:23 +06:00
}
2016-03-01 01:33:29 +05:00
TABLE<int> dest2face(cnt_send);
2016-02-29 16:25:18 +05:00
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices (face, verts);
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-02-29 16:25:18 +05:00
if (dest != id)
2020-08-29 00:28:18 +05:00
/*
2016-02-29 16:25:18 +05:00
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
2020-08-29 00:28:18 +05:00
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
2020-08-29 21:24:01 +05:00
dest2face.Add(dest, face);
2016-02-29 16:25:18 +05:00
}
2016-03-01 01:33:29 +05:00
for (int & c : cnt_send) c*=3;
2016-02-29 16:25:18 +05:00
TABLE<int> send_faces(cnt_send);
2019-07-09 13:39:16 +05:00
NgArray<int, PointIndex::BASE> loc2exchange(mesh.GetNV());
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-02-29 16:25:18 +05:00
if (dest != id)
{
2020-08-29 21:24:01 +05:00
if (dest2vert[dest].Size() == 0) continue;
2016-03-08 02:25:40 +05:00
loc2exchange = -1;
int cnt = 0;
2020-08-29 21:24:01 +05:00
for (PointIndex pi : dest2vert[dest])
2016-03-08 02:25:40 +05:00
loc2exchange[pi] = cnt++;
2016-02-29 16:25:18 +05:00
2020-08-29 21:24:01 +05:00
for (int face : dest2face[dest])
2016-02-29 16:25:18 +05:00
{
topology.GetFaceVertices (face, verts);
2020-08-29 00:28:18 +05:00
/*
2016-02-29 16:25:18 +05:00
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
2020-08-29 00:28:18 +05:00
*/
if (GetDistantProcs (verts[0]).Contains(dest) &&
GetDistantProcs (verts[1]).Contains(dest) &&
GetDistantProcs (verts[2]).Contains(dest))
2016-02-29 16:25:18 +05:00
{
2020-08-29 21:24:01 +05:00
send_faces.Add (dest, loc2exchange[verts[0]]);
send_faces.Add (dest, loc2exchange[verts[1]]);
send_faces.Add (dest, loc2exchange[verts[2]]);
2016-02-29 16:25:18 +05:00
}
}
}
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - faces mpi-exchange" << endl;
2020-08-29 21:24:01 +05:00
TABLE<int> recv_faces(ntasks);
MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+9, comm);
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - faces mpi-exchange done" << endl;
2016-03-08 02:25:40 +05:00
2020-08-29 21:24:01 +05:00
for (int dest = 0; dest < ntasks; dest++)
2016-03-08 02:25:40 +05:00
{
2020-08-29 21:24:01 +05:00
auto ex2loc = dest2vert[dest];
2016-03-08 02:25:40 +05:00
if (ex2loc.Size() == 0) continue;
2020-08-29 21:24:01 +05:00
INDEX_3_CLOSED_HASHTABLE<int> vert2face(2*dest2face[dest].Size()+10);
for (int face : dest2face[dest])
2016-03-08 02:25:40 +05:00
{
topology.GetFaceVertices (face, verts);
vert2face.Set(INDEX_3(verts[0], verts[1], verts[2]), face);
}
2020-08-29 21:24:01 +05:00
NgFlatArray<int> recvarray = recv_faces[dest];
2016-03-08 02:25:40 +05:00
for (int ii = 0; ii < recvarray.Size(); ii+=3)
{
INDEX_3 re(ex2loc[recvarray[ii]],
ex2loc[recvarray[ii+1]],
ex2loc[recvarray[ii+2]]);
if (vert2face.Used(re))
2020-08-29 23:56:29 +05:00
AddDistantFaceProc(vert2face.Get(re)-1, dest);
2016-03-08 02:25:40 +05:00
}
}
2016-02-29 16:25:18 +05:00
2012-06-21 20:19:25 +06:00
NgProfiler::StopTimer (timerf);
2011-07-17 05:13:26 +06:00
}
2017-06-06 12:55:40 +05:00
// cout << "UpdateCoarseGrid - done" << endl;
2020-08-21 19:29:33 +05:00
// EnumeratePointsGlobally();
2012-06-21 20:19:25 +06:00
is_updated = true;
2020-08-29 21:24:01 +05:00
// MPI_Group_free(&MPI_LocalGroup);
// MPI_Comm_free(&MPI_LocalComm);
2009-01-26 00:37:14 +05:00
}
}
#endif