ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-24 04:50:34 +05:00
Code Issues Packages Projects Releases Wiki Activity

littel parallel polishing

Browse Source
This commit is contained in:
Joachim Schöberl 2020-08-21 16:29:33 +02:00
parent c4f2146519
commit 895280a244
7 changed files with 161 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libsrc/include/nginterface_v2.hpp
View File

@ -368,8 +368,9 @@ namespace netgen
// for MPI-parallel
std::tuple<int,int*> GetDistantProcs (int nodetype, int locnum) const;
FlatArray<int> GetDistantProcs (int nodetype, int locnum) const;
size_t GetGlobalVertexNum (int locnum) const;
shared_ptr<Mesh> GetMesh () const { return mesh; }
shared_ptr<Mesh> SelectMesh () const;
inline auto GetTimeStamp() const;

3
libsrc/interface/nginterface.cpp
View File

@ -905,6 +905,7 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou
// gibt anzahl an distant pnums zurueck
// * pnums entspricht ARRAY<int[2] >
[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]
int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
{
int size = NgPar_GetNDistantNodeNums (nodetype, locnum);
@ -931,6 +932,7 @@ int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
return size;
}
[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]
int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
{
locnum++;
@ -944,6 +946,7 @@ int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
return -1;
}
[[deprecated("Use GetDistantNodeNums(locnum) -> FlatArray instead!")]]
int NgPar_GetGlobalNodeNum (int nodetype, int locnum)
{
locnum++;

27
libsrc/interface/nginterface_v2.cpp
View File

@ -1296,36 +1296,31 @@ void Ngx_Mesh::SetSurfaceElementOrders (int enr, int ox, int oy)
size_t Ngx_Mesh :: GetGlobalVertexNum (int locnum) const
{
return mesh->GetParallelTopology().GetGlobalPNum (locnum+1)-1;
}
std::tuple<int,int*> Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const
FlatArray<int> Ngx_Mesh :: GetDistantProcs (int nodetype, int locnum) const
{
#ifdef PARALLEL
if (mesh->GetCommunicator().Size() == 1)
return std::tuple<int,int*>(0,nullptr);
return FlatArray<int>(0,nullptr);
switch (nodetype)
{
case 0:
{
NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantPNums(locnum);
return std::tuple<int,int*>(dn.Size(), &dn[0]);
}
return mesh->GetParallelTopology().GetDistantPNums(locnum);
case 1:
{
NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantEdgeNums(locnum);
return std::tuple<int,int*>(dn.Size(), &dn[0]);
}
return mesh->GetParallelTopology().GetDistantEdgeNums(locnum);
case 2:
{
NgFlatArray<int> dn = mesh->GetParallelTopology().GetDistantFaceNums(locnum);
return std::tuple<int,int*>(dn.Size(), &dn[0]);
}
return mesh->GetParallelTopology().GetDistantFaceNums(locnum);
default:
return std::tuple<int,int*>(0,nullptr);
return FlatArray<int>(0, nullptr);
}
#else
return std::tuple<int,int*>(0,nullptr);
return FlatArray<int>(0,nullptr);
#endif
}
}

83
libsrc/meshing/meshclass.cpp
View File

@ -1298,89 +1298,6 @@ namespace netgen
// GridFunciton pickling is not compatible, now
// should go to paralleltopology
Array<PointIndex, PointIndex> global_pnums(points.Size());
global_pnums = -1;
int num_master_points = 0;
for (PointIndex pi : Range(points))
{
auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE);
// check sorted:
for (int j = 0; j+1 < distprocs.Size(); j++)
if (distprocs[j+1] < distprocs[j]) cout << "wrong sort" << endl;
if (distprocs.Size() == 0 || distprocs[0] > comm.Rank())
global_pnums[pi] = PointIndex::BASE+num_master_points++;
}
Array<int> first_master_point(comm.Size());
GetCommunicator().AllGather (num_master_points, first_master_point);
size_t num_glob_points = 0;
for (int i = 0; i < comm.Size(); i++)
{
int cur = first_master_point[i];
first_master_point[i] = num_glob_points;
num_glob_points += cur;
}
for (PointIndex pi : Range(points))
if (global_pnums[pi] != -1)
global_pnums[pi] += first_master_point[comm.Rank()];
// ScatterDofData (global_nums);
Array<int> nsend(comm.Size()), nrecv(comm.Size());
nsend = 0;
nrecv = 0;
/** Count send/recv size **/
for (PointIndex pi : Range(points))
{
auto dps = partop.GetDistantPNums(pi-PointIndex::BASE);
if (!dps.Size()) continue;
if (rank < dps[0])
for(auto p:dps)
nsend[p]++;
else
nrecv[dps[0]]++;
}
Table<PointIndex> send_data(nsend);
Table<PointIndex> recv_data(nrecv);
/** Fill send_data **/
nsend = 0;
for (PointIndex pi : Range(points))
{
auto dps = partop.GetDistantPNums(pi-PointIndex::BASE);
if (dps.Size() && rank < dps[0])
for(auto p : dps)
send_data[p][nsend[p]++] = global_pnums[pi];
}
Array<MPI_Request> requests;
for (int i = 0; i < comm.Size(); i++)
{
if (nsend[i])
requests.Append (comm.ISend (send_data[i], i, 200));
if (nrecv[i])
requests.Append (comm.IRecv (recv_data[i], i, 200));
}
MyMPI_WaitAll (requests);
Array<int> cnt(comm.Size());
cnt = 0;
for (PointIndex pi : Range(points))
{
auto distprocs = partop.GetDistantPNums(pi-PointIndex::BASE);
if (distprocs.Size() > 0 && distprocs[0] < comm.Rank())
{
int master = comm.Size();
for (int j = 0; j < distprocs.Size(); j++)
master = min (master, distprocs[j]);
global_pnums[pi] = recv_data[master][cnt[master]++];
}
}
// merge points

7
libsrc/meshing/parallelmesh.cpp
View File

@ -732,6 +732,7 @@ namespace netgen
for (auto t : point_types)
{ MPI_Type_free(&t); }
PrintMessage ( 3, "Sending names");
sendrequests.SetSize(3*ntasks);
@ -814,6 +815,9 @@ namespace netgen
self.BuildElementSearchTree();
// const_cast<Mesh&>(*this).DeleteMesh();
// paralleltop -> SetNV (0);
// paralleltop->EnumeratePointsGlobally();
PrintMessage( 3, "send mesh complete");
}
@ -1016,6 +1020,7 @@ namespace netgen
}
}
/** Recv bc-names **/
ArrayMem<int,4> nnames{0,0,0,0};
// MPI_Recv(nnames, 4, MPI_INT, 0, MPI_TAG_MESH+6, comm, MPI_STATUS_IGNORE);
@ -1073,7 +1078,7 @@ namespace netgen
clusters -> Update();
// paralleltop -> UpdateCoarseGrid();
// paralleltop->EnumeratePointsGlobally();
SetNextMajorTimeStamp();
}

112
libsrc/meshing/paralleltop.cpp
View File

@ -55,6 +55,107 @@ namespace netgen
}
void ParallelMeshTopology :: EnumeratePointsGlobally ()
{
auto nv = loc2distvert.Size();
auto comm = mesh.GetCommunicator();
auto rank = comm.Rank();
// if (rank == 0)
// nv = 0;
glob_vert.SetSize (nv);
glob_vert = -1;
int num_master_points = 0;
for (auto i : Range(nv))
{
auto dps = GetDistantPNums(i);
// check sorted:
for (int j = 0; j+1 < dps.Size(); j++)
if (dps[j+1] < dps[j]) cout << "wrong sort" << endl;
if (dps.Size() == 0 || dps[0] > comm.Rank())
glob_vert[i] = num_master_points++;
}
Array<int> first_master_point(comm.Size());
comm.AllGather (num_master_points, first_master_point);
size_t num_glob_points = 0;
for (int i = 0; i < comm.Size(); i++)
{
int cur = first_master_point[i];
first_master_point[i] = num_glob_points;
num_glob_points += cur;
}
for (auto i : Range(nv))
if (glob_vert[i] != -1)
glob_vert[i] += first_master_point[comm.Rank()];
// ScatterDofData (global_nums);
Array<int> nsend(comm.Size()), nrecv(comm.Size());
nsend = 0;
nrecv = 0;
/** Count send/recv size **/
for (auto i : Range(nv))
{
auto dps = GetDistantPNums(i);
if (!dps.Size()) continue;
if (rank < dps[0])
for(auto p:dps)
nsend[p]++;
else
nrecv[dps[0]]++;
}
Table<PointIndex> send_data(nsend);
Table<PointIndex> recv_data(nrecv);
/** Fill send_data **/
nsend = 0;
for (auto i : Range(nv))
{
auto dps = GetDistantPNums(i);
if (dps.Size() && rank < dps[0])
for(auto p : dps)
send_data[p][nsend[p]++] = glob_vert[i];
}
Array<MPI_Request> requests;
for (int i = 0; i < comm.Size(); i++)
{
if (nsend[i])
requests.Append (comm.ISend (send_data[i], i, 200));
if (nrecv[i])
requests.Append (comm.IRecv (recv_data[i], i, 200));
}
MyMPI_WaitAll (requests);
Array<int> cnt(comm.Size());
cnt = 0;
for (auto i : Range(nv))
{
auto dps = GetDistantPNums(i);
if (dps.Size() > 0 && dps[0] < comm.Rank())
{
int master = comm.Size();
for (int j = 0; j < dps.Size(); j++)
master = min (master, dps[j]);
glob_vert[i] = recv_data[master][cnt[master]++];
}
}
if (PointIndex::BASE==1)
for (auto & i : glob_vert)
i++;
}
void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
{
for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
@ -242,8 +343,13 @@ namespace netgen
MPI_Group_excl (MPI_GROUP_comm, 1, process_ranks, &MPI_LocalGroup);
MPI_Comm_create (comm, MPI_LocalGroup, &MPI_LocalComm);
if (id == 0) return;
if (id == 0)
{
// SetNV(0);
// EnumeratePointsGlobally();
return;
}
const MeshTopology & topology = mesh.GetTopology();
NgArray<int> cnt_send(ntasks-1);
@ -573,7 +679,7 @@ namespace netgen
NgProfiler::StopTimer (timerf);
}
// cout << "UpdateCoarseGrid - done" << endl;
// EnumeratePointsGlobally();
is_updated = true;
MPI_Group_free(&MPI_LocalGroup);

38
libsrc/meshing/paralleltop.hpp
View File

@ -42,60 +42,80 @@ namespace netgen
void SetNSE (int anse);
void SetNSegm (int anseg);
void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_Face (int locnum, int globnum) { glob_face[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_VolEl (int locnum, int globnum) { glob_el[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_SurfEl (int locnum, int globnum) { glob_surfel[locnum-1] = globnum; }
[[deprecated("Try to avoid global enumration!")]]
void SetLoc2Glob_Segm (int locnum, int globnum) { glob_segm[locnum-1] = globnum; }
int GetGlobalPNum (int locnum) const { return glob_vert[locnum-1]; }
[[deprecated("Try to avoid global enumration!")]]
int GetGlobalEdgeNum (int locnum) const { return glob_edge[locnum-1]; }
[[deprecated("Try to avoid global enumration!")]]
int GetGlobalFaceNum (int locnum) const { return glob_face[locnum-1]; }
[[deprecated("Try to avoid global enumration!")]]
int GetGlobalElNum (int locnum) const { return glob_el[locnum-1]; }
[[deprecated("Try to avoid global enumration!")]]
int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; }
void EnumeratePointsGlobally ();
void SetDistantFaceNum (int dest, int locnum);
void SetDistantPNum (int dest, int locnum);
void SetDistantEdgeNum (int dest, int locnum);
int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); }
[[deprecated("Use GetDistantPNums(locnum).Size() instead!")]]
int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
[[deprecated("Use GetDistantFaceNums(locnum).Size() instead!")]]
int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); }
[[deprecated("Use GetDistantEdgeNums(locnum).Size() instead!")]]
int GetNDistantEdgeNums ( int locedgenum) const { return loc2distedge[locedgenum-1].Size(); }
[[deprecated("Use GetDistantPNums(locnum) -> FlatArray instead!")]]
void GetDistantPNums (int locpnum, int * distpnums ) const
{
for (int i = 0; i < loc2distvert[locpnum-1].Size(); i++ )
distpnums[i] = loc2distvert[locpnum-1][i];
}
[[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]]
void GetDistantFaceNums (int locfacenum, int * distfacenums ) const
{
for ( int i = 0; i < loc2distface[locfacenum-1].Size(); i++ )
distfacenums[i] = loc2distface[locfacenum-1][i];
}
[[deprecated("Use GetDistantFaceNums(locnum) -> FlatArray instead!")]]
void GetDistantFaceNums (int locfacenum, NgArray<int> & distfacenums ) const
{
distfacenums = loc2distface[locfacenum-1];
}
[[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]
void GetDistantEdgeNums (int locedgenum, int * distedgenums ) const
{
for (int i = 0; i < loc2distedge[locedgenum-1].Size(); i++ )
distedgenums[i] = loc2distedge[locedgenum-1][i];
}
[[deprecated("Use GetDistantEdgeNums(locnum) -> FlatArray instead!")]]
void GetDistantEdgeNums (int locedgenum, NgArray<int> & distedgenums ) const
{
distedgenums = loc2distedge[locedgenum-1];
}
NgFlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
NgFlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
NgFlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
FlatArray<int> GetDistantPNums (int locnum) const { return loc2distvert[locnum]; }
FlatArray<int> GetDistantFaceNums (int locnum) const { return loc2distface[locnum]; }
FlatArray<int> GetDistantEdgeNums (int locnum) const { return loc2distedge[locnum]; }
bool IsExchangeVert (int dest, int vnum) const
{