ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2025-01-13 14:40:35 +05:00
Code Issues Packages Projects Releases Wiki Activity

parallel tuning

Browse Source
This commit is contained in:
Joachim Schoeberl 2012-06-21 14:19:25 +00:00
parent 094f52ee3c
commit fc81f592f5
10 changed files with 320 additions and 940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
libsrc/interface/nginterface.cpp
View File

@ -938,55 +938,51 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou
int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums ) int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
{ {
int size = NgPar_GetNDistantNodeNums (nodetype, locnum); int size = NgPar_GetNDistantNodeNums (nodetype, locnum);
locnum++;
switch ( nodetype ) switch ( nodetype )
{ {
case 0: case 0:
mesh->GetParallelTopology().GetDistantPNums( locnum+1, distnums ); mesh->GetParallelTopology().GetDistantPNums( locnum, distnums );
break; break;
case 1: case 1:
mesh->GetParallelTopology().GetDistantEdgeNums( locnum+1, distnums ); mesh->GetParallelTopology().GetDistantEdgeNums( locnum, distnums );
break; break;
case 2: case 2:
mesh->GetParallelTopology().GetDistantFaceNums( locnum+1, distnums ); mesh->GetParallelTopology().GetDistantFaceNums( locnum, distnums );
break; break;
case 3: case 3:
mesh->GetParallelTopology().GetDistantElNums( locnum+1, distnums ); // mesh->GetParallelTopology().GetDistantElNums( locnum, distnums );
break; break;
default: default:
cerr << "NgPar_GetDistantNodeNums() Unknown nodetype " << nodetype << endl; cerr << "NgPar_GetDistantNodeNums() Unknown nodetype " << nodetype << endl;
size = -1; size = -1;
} }
// 0 - based
for ( int i = 0; i < size; i++ )
distnums[2*i+1]--;
return size; return size;
} }
int NgPar_GetNDistantNodeNums ( int nodetype, int locnum ) int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
{ {
locnum++;
switch ( nodetype ) switch ( nodetype )
{ {
case 0: case 0: return mesh->GetParallelTopology().GetNDistantPNums (locnum);
return mesh->GetParallelTopology().GetNDistantPNums( locnum+1 ); case 1: return mesh->GetParallelTopology().GetNDistantEdgeNums (locnum);
case 1: case 2: return mesh->GetParallelTopology().GetNDistantFaceNums(locnum );
return mesh->GetParallelTopology().GetNDistantEdgeNums( locnum+1 ); case 3: return 0;
case 2:
return mesh->GetParallelTopology().GetNDistantFaceNums( locnum+1 );
case 3:
return mesh->GetParallelTopology().GetNDistantElNums( locnum+1 );
} }
return -1; return -1;
} }
int NgPar_GetGlobalNodeNum (int nodetype, int locnum) int NgPar_GetGlobalNodeNum (int nodetype, int locnum)
{ {
locnum++;
switch (nodetype) switch (nodetype)
{ {
case 0: return mesh->GetParallelTopology().GetGlobalPNum (locnum+1)-1; case 0: return mesh->GetParallelTopology().GetGlobalPNum (locnum)-1;
case 1: return mesh->GetParallelTopology().GetGlobalEdgeNum (locnum+1)-1; case 1: return mesh->GetParallelTopology().GetGlobalEdgeNum (locnum)-1;
case 2: return mesh->GetParallelTopology().GetGlobalFaceNum (locnum+1)-1; case 2: return mesh->GetParallelTopology().GetGlobalFaceNum (locnum)-1;
case 3: return mesh->GetParallelTopology().GetGlobalElNum (locnum+1)-1; case 3: return mesh->GetParallelTopology().GetGlobalElNum (locnum)-1;
} }
return -1; return -1;
} }

36
libsrc/meshing/meshclass.cpp
View File

@ -157,10 +157,6 @@ namespace netgen
PointIndex pi = points.Size() + PointIndex::BASE; PointIndex pi = points.Size() + PointIndex::BASE;
points.Append ( MeshPoint (p, layer, INNERPOINT) ); points.Append ( MeshPoint (p, layer, INNERPOINT) );
#ifdef PARALLEL
points.Last().SetGhost(0);
#endif
lock.UnLock(); lock.UnLock();
return pi; return pi;
@ -176,10 +172,6 @@ namespace netgen
PointIndex pi = points.Size() + PointIndex::BASE; PointIndex pi = points.Size() + PointIndex::BASE;
points.Append ( MeshPoint (p, layer, type) ); points.Append ( MeshPoint (p, layer, type) );
#ifdef PARALLEL
points.Last().SetGhost(0);
#endif
lock.UnLock(); lock.UnLock();
return pi; return pi;
@ -197,8 +189,6 @@ namespace netgen
PointIndex pi = points.Size() + PointIndex::BASE; PointIndex pi = points.Size() + PointIndex::BASE;
points.Append ( MeshPoint (p, layer, INNERPOINT) ); points.Append ( MeshPoint (p, layer, INNERPOINT) );
points.Last().SetGhost(isghost);
lock.UnLock(); lock.UnLock();
return pi; return pi;
@ -214,8 +204,6 @@ namespace netgen
PointIndex pi = points.Size() + PointIndex::BASE; PointIndex pi = points.Size() + PointIndex::BASE;
points.Append ( MeshPoint (p, layer, type) ); points.Append ( MeshPoint (p, layer, type) );
points.Last().SetGhost(isghost);
lock.UnLock(); lock.UnLock();
return pi; return pi;
@ -313,10 +301,6 @@ namespace netgen
surfelements.Last().next = facedecoding[el.index-1].firstelement; surfelements.Last().next = facedecoding[el.index-1].firstelement;
facedecoding[el.index-1].firstelement = si; facedecoding[el.index-1].firstelement = si;
#ifdef PARALLEL
surfelements.Last().SetGhost ( el.IsGhost() );
#endif
lock.UnLock(); lock.UnLock();
return si; return si;
} }
@ -355,10 +339,6 @@ namespace netgen
volelements.Append (el); volelements.Append (el);
volelements.Last().flags.illegal_valid = 0; volelements.Last().flags.illegal_valid = 0;
#ifdef PARALLEL
volelements.Last().SetGhost ( el.IsGhost() );
#endif
// while (volelements.Size() > eltyps.Size()) // while (volelements.Size() > eltyps.Size())
// eltyps.Append (FREEELEMENT); // eltyps.Append (FREEELEMENT);
@ -1209,23 +1189,23 @@ namespace netgen
strcpy (str, ""); strcpy (str, "");
} }
CalcSurfacesOfNode (); CalcSurfacesOfNode ();
// BuildConnectedNodes (); // BuildConnectedNodes ();
topology -> Update(); topology -> Update();
clusters -> Update(); clusters -> Update();
#ifdef PARALLEL
if ( ntasks > 1 ) Distribute ();
#endif
SetNextMajorTimeStamp(); SetNextMajorTimeStamp();
// PrintMemInfo (cout); // PrintMemInfo (cout);
#ifdef PARALLEL
if ( ntasks > 1 )
{
// for parallel processing
Distribute ();
return;
}
#endif
} }

28
libsrc/meshing/meshtype.cpp
View File

@ -172,9 +172,6 @@ namespace netgen
orderx = ordery = 1; orderx = ordery = 1;
refflag = 1; refflag = 1;
strongrefflag = false; strongrefflag = false;
#ifdef PARALLEL
isghost = 0;
#endif
} }
@ -200,9 +197,6 @@ namespace netgen
orderx = ordery = 1; orderx = ordery = 1;
refflag = 1; refflag = 1;
strongrefflag = false; strongrefflag = false;
#ifdef PARALLEL
isghost = 0;
#endif
} }
Element2d :: Element2d (ELEMENT_TYPE atyp) Element2d :: Element2d (ELEMENT_TYPE atyp)
@ -222,10 +216,6 @@ namespace netgen
orderx = ordery = 1; orderx = ordery = 1;
refflag = 1; refflag = 1;
strongrefflag = false; strongrefflag = false;
#ifdef PARALLEL
isghost = 0;
#endif
} }
@ -250,11 +240,6 @@ namespace netgen
deleted = 0; deleted = 0;
visible = 1; visible = 1;
orderx = ordery = 1; orderx = ordery = 1;
#ifdef PARALLEL
isghost = 0;
#endif
} }
Element2d :: Element2d (int pi1, int pi2, int pi3, int pi4) Element2d :: Element2d (int pi1, int pi2, int pi3, int pi4)
@ -278,10 +263,6 @@ namespace netgen
deleted = 0; deleted = 0;
visible = 1; visible = 1;
orderx = ordery = 1; orderx = ordery = 1;
#ifdef PARALLEL
isghost = 0;
#endif
} }
@ -966,7 +947,6 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
partitionNumber = -1; partitionNumber = -1;
isghost = 0;
#endif #endif
} }
@ -1000,10 +980,6 @@ namespace netgen
default: cerr << "Element::Element: unknown element with " << np << " points" << endl; default: cerr << "Element::Element: unknown element with " << np << " points" << endl;
} }
orderx = ordery = orderz = 1; orderx = ordery = orderz = 1;
#ifdef PARALLEL
isghost = 0;
#endif
} }
void Element :: SetOrder (const int aorder) void Element :: SetOrder (const int aorder)
@ -1041,10 +1017,6 @@ namespace netgen
flags.deleted = 0; flags.deleted = 0;
flags.fixed = 0; flags.fixed = 0;
orderx = ordery = orderz = 1; orderx = ordery = orderz = 1;
#ifdef PARALLEL
isghost = 0;
#endif
} }

29
libsrc/meshing/meshtype.hpp
View File

@ -232,25 +232,16 @@ namespace netgen
double singular; // singular factor for hp-refinement double singular; // singular factor for hp-refinement
POINTTYPE type; POINTTYPE type;
#ifdef PARALLEL
bool isghost;
#endif
public: public:
MeshPoint () MeshPoint ()
{ {
#ifdef PARALLEL
isghost = 0;
#endif
; ;
} }
MeshPoint (const Point<3> & ap, int alayer = 1, POINTTYPE apt = INNERPOINT) MeshPoint (const Point<3> & ap, int alayer = 1, POINTTYPE apt = INNERPOINT)
: Point<3> (ap), layer(alayer), singular(0.),type(apt) : Point<3> (ap), layer(alayer), singular(0.),type(apt)
{ {
#ifdef PARALLEL
isghost = 0;
#endif
; ;
} }
@ -259,9 +250,6 @@ namespace netgen
Point<3>::operator= (ap); Point<3>::operator= (ap);
layer = 0; layer = 0;
singular = 0; singular = 0;
#ifdef PARALLEL
isghost = 0;
#endif
} }
int GetLayer() const { return layer; } int GetLayer() const { return layer; }
@ -274,9 +262,6 @@ namespace netgen
bool IsSingular() const { return (singular != 0.0); } bool IsSingular() const { return (singular != 0.0); }
#ifdef PARALLEL #ifdef PARALLEL
bool IsGhost () const { return isghost; }
void SetGhost ( bool aisghost ) { isghost = aisghost; }
static MPI_Datatype MyGetMPIType ( ); static MPI_Datatype MyGetMPIType ( );
#endif #endif
@ -325,7 +310,6 @@ namespace netgen
unsigned int ordery:6; unsigned int ordery:6;
#ifdef PARALLEL #ifdef PARALLEL
bool isghost;
int partitionNumber; int partitionNumber;
#endif #endif
@ -506,15 +490,8 @@ namespace netgen
int hp_elnr; int hp_elnr;
#ifdef PARALLEL #ifdef PARALLEL
bool IsGhost () const { return isghost; }
void SetGhost ( bool aisghost ) { isghost = aisghost; }
// by JS, only for 2D meshes ????
int GetPartition () const { return partitionNumber; } int GetPartition () const { return partitionNumber; }
void SetPartition (int nr) { partitionNumber = nr; }; void SetPartition (int nr) { partitionNumber = nr; };
#else
bool IsGhost () const { return false; }
#endif #endif
}; };
@ -583,7 +560,7 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
/// number of partition for parallel computation /// number of partition for parallel computation
int partitionNumber; int partitionNumber;
bool isghost;
#endif #endif
public: public:
@ -778,11 +755,7 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
int GetPartition () const { return partitionNumber; } int GetPartition () const { return partitionNumber; }
void SetPartition (int nr) { partitionNumber = nr; }; void SetPartition (int nr) { partitionNumber = nr; };
bool IsGhost () const { return isghost; }
void SetGhost ( const bool aisghost ) { isghost = aisghost; }
#else #else
bool IsGhost () const { return false; }
int GetPartition () const { return 0; } int GetPartition () const { return 0; }
#endif #endif

161
libsrc/meshing/parallelmesh.cpp
View File

@ -29,7 +29,7 @@ namespace netgen
{ {
template <> template <>
inline MPI_Datatype MyGetMPIType<netgen::PointIndex> ( ) inline MPI_Datatype MyGetMPIType<PointIndex> ( )
{ return MPI_INT; } { return MPI_INT; }
@ -50,7 +50,7 @@ namespace netgen
paralleltop -> SetNE (GetNE()); paralleltop -> SetNE (GetNE());
paralleltop -> SetNSegm (GetNSeg()); paralleltop -> SetNSegm (GetNSeg());
paralleltop -> SetNSE (GetNSE()); paralleltop -> SetNSE (GetNSE());
nelglob = GetNE(); nelglob = GetNE();
nseglob = GetNSE(); nseglob = GetNSE();
nvglob = GetNV(); nvglob = GetNV();
@ -60,17 +60,13 @@ namespace netgen
MyMPI_Bcast (nelglob); MyMPI_Bcast (nelglob);
MyMPI_Bcast (nseglob); MyMPI_Bcast (nseglob);
MyMPI_Bcast (nvglob); MyMPI_Bcast (nvglob);
if (id > 0)
{
paralleltop -> SetNEGlob (nelglob);
paralleltop -> SetNSEGlob (nseglob);
paralleltop -> SetNVGlob (nvglob);
}
} }
{ {
// distribute number of local elements // distribute number of local elements
if (id == 0) if (id == 0)
@ -97,6 +93,10 @@ namespace netgen
SendMesh (); SendMesh ();
else else
ReceiveParallelMesh(); ReceiveParallelMesh();
paralleltop -> UpdateCoarseGrid();
} }
@ -165,11 +165,11 @@ namespace netgen
num_verts_on_proc[dest]++; num_verts_on_proc[dest]++;
num_procs_on_vert[epi]++; num_procs_on_vert[epi]++;
paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]); paralleltop -> SetDistantPNum (dest, epi); // , num_verts_on_proc[dest]);
} }
} }
nelloc[dest] ++; nelloc[dest] ++;
paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] ); // paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] );
} }
@ -188,11 +188,11 @@ namespace netgen
num_verts_on_proc[dest]++; num_verts_on_proc[dest]++;
num_procs_on_vert[epi]++; num_procs_on_vert[epi]++;
paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]); paralleltop -> SetDistantPNum ( dest, epi); // num_verts_on_proc[dest]);
} }
} }
nselloc[dest] ++; nselloc[dest] ++;
paralleltop -> SetDistantSurfEl ( dest, sels[hi]+1, nselloc[dest] ); // paralleltop -> SetDistantSurfEl ( dest, sels[hi]+1, nselloc[dest] );
} }
} }
@ -445,7 +445,7 @@ namespace netgen
segmbuf.Add (dest, seg.singedge_left); segmbuf.Add (dest, seg.singedge_left);
segi[dest] += 14; segi[dest] += 14;
} }
paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) ); // paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) );
} }
} }
} }
@ -508,7 +508,7 @@ namespace netgen
for (int hi = 0; hi < dist_pnums.Size(); hi += 3) for (int hi = 0; hi < dist_pnums.Size(); hi += 3)
paralleltop -> paralleltop ->
SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi], dist_pnums[hi+2]); SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi]); // , dist_pnums[hi+2]);
NgProfiler::StopTimer (timer_pts); NgProfiler::StopTimer (timer_pts);
*testout << "got " << numvert << " vertices" << endl; *testout << "got " << numvert << " vertices" << endl;
@ -574,8 +574,6 @@ namespace netgen
tri.GeomInfoPi(j).trignum = selbuf[ii++]; tri.GeomInfoPi(j).trignum = selbuf[ii++];
} }
tri.SetGhost(isghost);
paralleltop->SetLoc2Glob_SurfEl ( sel+1, globsel ); paralleltop->SetLoc2Glob_SurfEl ( sel+1, globsel );
AddSurfaceElement (tri); AddSurfaceElement (tri);
sel ++; sel ++;
@ -645,9 +643,10 @@ namespace netgen
topology -> Update(); topology -> Update();
clusters -> Update(); clusters -> Update();
// paralleltop -> UpdateCoarseGrid();
SetNextMajorTimeStamp(); SetNextMajorTimeStamp();
// paralleltop->Print();
} }
@ -676,7 +675,7 @@ namespace netgen
MyMPI_SendCmd ("mesh"); MyMPI_SendCmd ("mesh");
SendRecvMesh (); SendRecvMesh ();
paralleltop -> UpdateCoarseGrid(); // paralleltop -> UpdateCoarseGrid();
// paralleltop -> Print(); // paralleltop -> Print();
} }
@ -719,8 +718,8 @@ namespace netgen
bool uniform_els = true; bool uniform_els = true;
ELEMENT_TYPE elementtype = TET; ELEMENT_TYPE elementtype = TET;
for ( int el = 1; el <= GetNE(); el++ ) for (int el = 1; el <= GetNE(); el++)
if ( VolumeElement(el).GetType() != elementtype ) if (VolumeElement(el).GetType() != elementtype)
{ {
uniform_els = false; uniform_els = false;
break; break;
@ -745,72 +744,69 @@ namespace netgen
etype = 3; etype = 3;
for (int i=1; i<=ne; i++) for (int i=1; i<=ne; i++)
for (int j=1; j<=npe; j++) for (int j=1; j<=npe; j++)
elmnts[(i-1)*npe+(j-1)] = VolumeElement(i).PNum(j)-1; elmnts[(i-1)*npe+(j-1)] = VolumeElement(i).PNum(j)-1;
int numflag = 0;
int nparts = ntasks-1;
int ncommon = 3;
int edgecut;
Array<idxtype> epart(ne), npart(nn);
// if ( ntasks == 1 )
// {
// (*this) = *mastermesh;
// nparts = 4;
// metis :: METIS_PartMeshDual (&ne, &nn, elmnts, &etype, &numflag, &nparts,
// &edgecut, epart, npart);
// cout << "done" << endl;
// cout << "edge-cut: " << edgecut << ", balance: " << metis :: ComputeElementBalance(ne, nparts, epart) << endl; int numflag = 0;
int nparts = ntasks-1;
int ncommon = 3;
int edgecut;
Array<idxtype> epart(ne), npart(nn);
// for (int i=1; i<=ne; i++) // if ( ntasks == 1 )
// { // {
// mastermesh->VolumeElement(i).SetPartition(epart[i-1]); // (*this) = *mastermesh;
// } // nparts = 4;
// metis :: METIS_PartMeshDual (&ne, &nn, elmnts, &etype, &numflag, &nparts,
// &edgecut, epart, npart);
// cout << "done" << endl;
// cout << "edge-cut: " << edgecut << ", balance: " << metis :: ComputeElementBalance(ne, nparts, epart) << endl;
// for (int i=1; i<=ne; i++)
// {
// mastermesh->VolumeElement(i).SetPartition(epart[i-1]);
// }
// return;
// }
int timermetis = NgProfiler::CreateTimer ("Metis itself");
NgProfiler::StartTimer (timermetis);
// return;
// }
int timermetis = NgProfiler::CreateTimer ("Metis itself");
NgProfiler::StartTimer (timermetis);
#ifdef METIS4 #ifdef METIS4
cout << "call metis ... " << flush; cout << "call metis(4)_PartMeshDual ... " << flush;
METIS_PartMeshDual (&ne, &nn, &elmnts[0], &etype, &numflag, &nparts, METIS_PartMeshDual (&ne, &nn, &elmnts[0], &etype, &numflag, &nparts,
&edgecut, &epart[0], &npart[0]); &edgecut, &epart[0], &npart[0]);
#else #else
cout << "call metis-5 ... " << endl; cout << "call metis(5)_PartMeshDual ... " << endl;
idx_t options[METIS_NOPTIONS]; idx_t options[METIS_NOPTIONS];
Array<idx_t> eptr(ne+1); Array<idx_t> eptr(ne+1);
for (int j = 0; j < ne+1; j++) for (int j = 0; j < ne+1; j++)
eptr[j] = 4*j; eptr[j] = 4*j;
METIS_PartMeshDual (&ne, &nn, &eptr[0], &elmnts[0], NULL, NULL, &ncommon, &nparts, METIS_PartMeshDual (&ne, &nn, &eptr[0], &elmnts[0], NULL, NULL, &ncommon, &nparts,
NULL, NULL, NULL, NULL,
&edgecut, &epart[0], &npart[0]); &edgecut, &epart[0], &npart[0]);
#endif #endif
NgProfiler::StopTimer (timermetis); NgProfiler::StopTimer (timermetis);
cout << "complete" << endl; cout << "complete" << endl;
#ifdef METIS4 #ifdef METIS4
cout << "edge-cut: " << edgecut << ", balance: " cout << "edge-cut: " << edgecut << ", balance: "
<< ComputeElementBalance(ne, nparts, &epart[0]) << endl; << ComputeElementBalance(ne, nparts, &epart[0]) << endl;
#endif #endif
// partition numbering by metis : 0 ... ntasks - 1 // partition numbering by metis : 0 ... ntasks - 1
// we want: 1 ... ntasks // we want: 1 ... ntasks
for (int i=1; i<=ne; i++) for (int i=1; i<=ne; i++)
VolumeElement(i).SetPartition(epart[i-1] + 1); VolumeElement(i).SetPartition(epart[i-1] + 1);
} }
for (int sei = 1; sei <= GetNSE(); sei++ ) for (int sei = 1; sei <= GetNSE(); sei++ )
{ {
@ -905,7 +901,6 @@ namespace netgen
Element & volel = VolumeElement(el); Element & volel = VolumeElement(el);
nodesinpart = 0; nodesinpart = 0;
// VolumeElement(el).SetPartition(part[ volel[1] ] + 1);
int el_np = volel.GetNP(); int el_np = volel.GetNP();
int partition = 0; int partition = 0;
@ -917,16 +912,8 @@ namespace netgen
partition = i; partition = i;
volel.SetPartition(partition); volel.SetPartition(partition);
} }
/*
for ( int i=1; i<=ne; i++)
{
neloc[ VolumeElement(i).GetPartition() ] ++;
}
*/
delete [] xadj; delete [] xadj;
delete [] part; delete [] part;
delete [] adjacency; delete [] adjacency;

751
libsrc/meshing/paralleltop.cpp
View File

@ -8,39 +8,12 @@
namespace netgen namespace netgen
{ {
ParallelMeshTopology :: ParallelMeshTopology (const Mesh & amesh)
void ParallelMeshTopology :: Reset () : mesh(amesh)
{ {
*testout << "ParallelMeshTopology::Reset" << endl; is_updated = false;
if ( ntasks == 1 ) return;
int nvold = nv;
ne = mesh.GetNE();
nv = mesh.GetNV();
nseg = mesh.GetNSeg();
nsurfel = mesh.GetNSE();
ned = mesh.GetTopology().GetNEdges();
nfa = mesh.GetTopology().GetNFaces();
loc2distedge.ChangeSize (ned);
for (int i = 0; i < ned; i++)
if (loc2distedge[i].Size() == 0)
loc2distedge.Add (i, -1); // will be the global nr
loc2distface.ChangeSize (nfa);
for (int i = 0; i < nfa; i++)
if (loc2distface[i].Size() == 0)
loc2distface.Add (i, -1); // will be the global nr
if ( nvold == nv ) return;
SetNV(nv);
SetNE(ne);
} }
ParallelMeshTopology :: ~ParallelMeshTopology () ParallelMeshTopology :: ~ParallelMeshTopology ()
{ {
@ -48,323 +21,88 @@ namespace netgen
} }
void ParallelMeshTopology :: Reset ()
ParallelMeshTopology :: ParallelMeshTopology ( const netgen::Mesh & amesh )
: mesh(amesh)
{ {
ned = 0; *testout << "ParallelMeshTopology::Reset" << endl;
nfa = 0;
nv = 0; if ( ntasks == 1 ) return;
ne = 0;
np = 0;
nseg = 0;
nsurfel = 0;
neglob = 0;
nvglob = 0;
nparel = 0; int ned = mesh.GetTopology().GetNEdges();
int nfa = mesh.GetTopology().GetNFaces();
coarseupdate = 0; if (glob_edge.Size() != ned)
{
glob_edge.SetSize(ned);
glob_face.SetSize(nfa);
glob_edge = -1;
glob_face = -1;
loc2distedge.ChangeSize (ned);
loc2distface.ChangeSize (nfa);
}
if (glob_vert.Size() != mesh.GetNV())
{
SetNV(mesh.GetNV());
SetNE(mesh.GetNE());
}
} }
void ParallelMeshTopology :: Print() const void ParallelMeshTopology :: Print() const
{ {
/* ;
(*testout) << endl << "TOPOLOGY FOR PARALLEL MESHES" << endl << endl;
for ( int i = 1; i <= nv; i++ )
if ( IsExchangeVert (i) )
{
(*testout) << "exchange point " << i << ": global " << GetLoc2Glob_Vert(i) << endl;
for ( int dest = 0; dest < ntasks; dest ++)
if ( dest != id )
if ( GetDistantPNum( dest, i ) > 0 )
(*testout) << " p" << dest << ": " << GetDistantPNum ( dest, i ) << endl;
}
for ( int i = 1; i <= ned; i++ )
if ( IsExchangeEdge ( i ) )
{
int v1, v2;
mesh . GetTopology().GetEdgeVertices(i, v1, v2);
(*testout) << "exchange edge " << i << ": global vertices " << GetLoc2Glob_Vert(v1) << " "
<< GetLoc2Glob_Vert(v2) << endl;
for ( int dest = 0; dest < ntasks; dest++)
if ( GetDistantEdgeNum ( dest, i ) > 0 )
if ( dest != id )
(*testout) << " p" << dest << ": " << GetDistantEdgeNum ( dest, i ) << endl;
}
for ( int i = 1; i <= nfa; i++ )
if ( IsExchangeFace(i) )
{
Array<int> facevert;
mesh . GetTopology().GetFaceVertices(i, facevert);
(*testout) << "exchange face " << i << ": global vertices " ;
for ( int fi=0; fi < facevert.Size(); fi++)
(*testout) << GetLoc2Glob_Vert(facevert[fi]) << " ";
(*testout) << endl;
for ( int dest = 0; dest < ntasks; dest++)
if ( dest != id )
{
if ( GetDistantFaceNum ( dest, i ) >= 0 )
(*testout) << " p" << dest << ": " << GetDistantFaceNum ( dest, i ) << endl;
}
}
*/
/*
for ( int i = 1; i < mesh.GetNE(); i++)
{
if ( !IsExchangeElement(i) ) continue;
Array<int> vert;
const Element & el = mesh.VolumeElement(i);
(*testout) << "parallel local element " << i << endl;
(*testout) << "vertices " ;
for ( int j = 0; j < el.GetNV(); j++)
(*testout) << el.PNum(j+1) << " ";
(*testout) << "is ghost " << IsGhostEl(i) << endl;
(*testout) << endl;
}
*/
} }
void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
// gibt anzahl an distant pnums zurueck
// * pnums entspricht Array<int[2] >
void ParallelMeshTopology :: GetDistantPNums ( int locpnum, int * distpnums ) const
{ {
distpnums[0] = 0; for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
distpnums[1] = loc2distvert[locpnum][0];
for ( int i = 1; i < loc2distvert[locpnum].Size(); i++ )
distpnums[i+1] = loc2distvert[locpnum][i];
// int size = loc2distvert[locpnum].Size() / 2 + 1;
// return size;
}
void ParallelMeshTopology :: GetDistantFaceNums ( int locfacenum, int * distfacenums ) const
{
distfacenums[0] = 0;
distfacenums[1] = loc2distface[locfacenum-1][0];
for ( int i = 1; i < loc2distface[locfacenum-1].Size(); i++ )
distfacenums[i+1] = loc2distface[locfacenum-1][i];
// int size = loc2distface[locfacenum-1].Size() / 2 + 1;
// return size;
}
void ParallelMeshTopology :: GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const
{
distedgenums[0] = 0;
distedgenums[1] = loc2distedge[locedgenum-1][0];
for ( int i = 1; i < loc2distedge[locedgenum-1].Size(); i++ )
distedgenums[i+1] = loc2distedge[locedgenum-1][i];
// int size = loc2distedge[locedgenum-1].Size() / 2 + 1;
// return size;
}
void ParallelMeshTopology :: GetDistantElNums ( int locelnum, int * distelnums ) const
{
distelnums[0] = 0;
distelnums[1] = loc2distel[locelnum-1][0];
for ( int i = 1; i < loc2distel[locelnum-1].Size(); i++ )
distelnums[i+1] = loc2distel[locelnum-1][i];
// int size = loc2distel[locelnum-1].Size() / 2 + 1;
// return size;
}
void ParallelMeshTopology :: SetDistantFaceNum ( int dest, int locnum, int distnum )
{
if ( dest == 0 )
{
loc2distface[locnum-1][0] = distnum;
return;
}
for ( int i = 1; i < loc2distface[locnum-1].Size(); i+=2 )
if ( loc2distface[locnum-1][i] == dest ) if ( loc2distface[locnum-1][i] == dest )
{ return;
loc2distface[locnum-1][i+1] = distnum;
return;
}
loc2distface.Add(locnum-1, dest); loc2distface.Add(locnum-1, dest);
loc2distface.Add(locnum-1, distnum);
} }
void ParallelMeshTopology :: SetDistantPNum ( int dest, int locnum, int distnum ) void ParallelMeshTopology :: SetDistantPNum (int dest, int locnum)
{ {
if ( dest == 0 ) for ( int i = 0; i < loc2distvert[locnum-1].Size(); i+=1 )
{ if ( loc2distvert[locnum-1][i] == dest )
loc2distvert[locnum][0] = distnum; // HERE
return; return;
} loc2distvert.Add (locnum-1, dest);
for ( int i = 1; i < loc2distvert[locnum].Size(); i+=2 )
if ( loc2distvert[locnum][i] == dest )
{
loc2distvert[locnum][i+1] = distnum;
return;
}
loc2distvert.Add (locnum, dest);
loc2distvert.Add (locnum, distnum);
} }
void ParallelMeshTopology :: SetDistantEdgeNum ( int dest, int locnum, int distnum ) void ParallelMeshTopology :: SetDistantEdgeNum (int dest, int locnum)
{ {
if ( dest == 0 ) for ( int i = 0; i < loc2distedge[locnum-1].Size(); i+=1 )
{
loc2distedge[locnum-1][0] = distnum;
return;
}
for ( int i = 1; i < loc2distedge[locnum-1].Size(); i+=2 )
if ( loc2distedge[locnum-1][i] == dest ) if ( loc2distedge[locnum-1][i] == dest )
{ return;
loc2distedge[locnum-1][i+1] = distnum;
return;
}
loc2distedge.Add (locnum-1, dest); loc2distedge.Add (locnum-1, dest);
loc2distedge.Add (locnum-1, distnum);
} }
void ParallelMeshTopology :: SetDistantEl ( int dest, int locnum, int distnum )
{
if ( dest == 0 )
{
loc2distel[locnum-1][0] = distnum;
return;
}
for ( int i = 1; i < loc2distel[locnum-1].Size(); i+=2 )
if ( loc2distel[locnum-1][i] == dest )
{
loc2distel[locnum-1][i+1] = distnum;
return;
}
loc2distel.Add (locnum-1, dest);
loc2distel.Add (locnum-1, distnum);
}
void ParallelMeshTopology :: SetDistantSurfEl ( int dest, int locnum, int distnum )
{
if ( dest == 0 )
{
loc2distsurfel[locnum-1][0] = distnum;
return;
}
for ( int i = 1; i < loc2distsurfel[locnum-1].Size(); i+=2 )
if ( loc2distsurfel[locnum-1][i] == dest )
{
loc2distsurfel[locnum-1][i+1] = distnum;
return;
}
loc2distsurfel.Add (locnum-1, dest);
loc2distsurfel.Add (locnum-1, distnum);
}
void ParallelMeshTopology :: SetDistantSegm ( int dest, int locnum, int distnum )
{
if ( dest == 0 )
{
loc2distsegm[locnum-1][0] = distnum;
return;
}
for (int i = 1; i < loc2distsegm[locnum-1].Size(); i+=2 )
if ( loc2distsegm[locnum-1][i] == dest )
{
loc2distsegm[locnum-1][i+1] = distnum;
return;
}
loc2distsegm.Add (locnum-1, dest);
loc2distsegm.Add (locnum-1, distnum);
}
void ParallelMeshTopology :: GetVertNeighbours ( int vnum, Array<int> & dests ) const
{
dests.SetSize(0);
int i = 1;
while ( i < loc2distvert[vnum].Size() )
{
dests.Append ( loc2distvert[vnum][i] );
i+=2;
}
}
void ParallelMeshTopology :: SetNV (int anv) void ParallelMeshTopology :: SetNV (int anv)
{ {
*testout << "called setnv" << endl glob_vert.SetSize(anv);
<< "old size: " << loc2distvert.Size() << endl glob_vert = -1;
<< "new size: " << anv << endl;
loc2distvert.ChangeSize (anv); loc2distvert.ChangeSize (anv);
for (int i = 1; i <= anv; i++)
if (loc2distvert.EntrySize(i) == 0)
loc2distvert.Add (i, -1); // will be the global nr
nv = anv;
} }
void ParallelMeshTopology :: SetNE ( const int ane ) void ParallelMeshTopology :: SetNE ( int ane )
{ {
loc2distel.ChangeSize (ane); glob_el.SetSize (ane);
for (int i = 0; i < ane; i++) glob_el = -1;
if (loc2distel[i].Size() == 0)
loc2distel.Add (i, -1); // will be the global nr
ne = ane;
} }
void ParallelMeshTopology :: SetNSE ( int anse ) void ParallelMeshTopology :: SetNSE ( int anse )
{ {
loc2distsurfel.ChangeSize (anse); glob_surfel.SetSize(anse);
for (int i = 0; i < anse; i++) glob_surfel = -1;
if (loc2distsurfel[i].Size() == 0)
loc2distsurfel.Add (i, -1); // will be the global nr
nsurfel = anse;
} }
void ParallelMeshTopology :: SetNSegm ( int anseg ) void ParallelMeshTopology :: SetNSegm ( int anseg )
{ {
loc2distsegm.ChangeSize (anseg); glob_segm.SetSize (anseg);
for (int i = 0; i < anseg; i++) glob_segm = -1;
if (loc2distsegm[i].Size() == 0)
loc2distsegm.Add (i, -1); // will be the global nr
nseg = anseg;
} }
@ -372,126 +110,48 @@ namespace netgen
void ParallelMeshTopology :: Update ()
{
ne = mesh.GetNE();
nv = mesh.GetNV();
nseg = mesh.GetNSeg();
nsurfel = mesh.GetNSE();
ned = mesh.GetTopology().GetNEdges();
nfa = mesh.GetTopology().GetNFaces();
}
void ParallelMeshTopology :: UpdateCoarseGridGlobal () void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
{ {
if (id == 0) if (id == 0)
PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS
int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal"); int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal");
NgProfiler::RegionTimer reg(timer); NgProfiler::RegionTimer reg(timer);
*testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl; *testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl;
const MeshTopology & topology = mesh.GetTopology(); const MeshTopology & topology = mesh.GetTopology();
nfa = topology . GetNFaces();
ned = topology . GetNEdges();
np = mesh . GetNP();
nv = mesh . GetNV();
ne = mesh . GetNE();
nseg = mesh.GetNSeg();
nsurfel = mesh.GetNSE();
// low order processor - save mesh partition
if ( id == 0 )
{
for ( int eli = 1; eli <= ne; eli++ )
loc2distel[eli-1][0] = eli;
for ( int i = 1; i <= mesh .GetNV(); i++)
loc2distvert[i][0] = i;
for ( int i = 0; i < mesh . GetNSeg(); i++)
loc2distsegm[i][0] = i+1;
for ( int i = 0; i < mesh . GetNSE(); i++)
loc2distsurfel[i][0] = i+1;
for ( int i = 0; i < topology .GetNEdges(); i++)
loc2distedge[i][0] = i+1;
for ( int i = 0; i < topology .GetNFaces(); i++)
loc2distface[i][0] = i+1;
}
// Array<int> sendarray, recvarray;
if ( id == 0 ) if ( id == 0 )
{ {
Array<Array<int>*> sendarrays(ntasks); Array<Array<int>*> sendarrays(ntasks);
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
{ sendarrays[dest] = new Array<int>;
sendarrays[dest] = new Array<int>;
sendarrays[dest]->Append (nfa);
sendarrays[dest]->Append (ned);
}
Array<int> edges, pnums, faces, elpnums; Array<int> edges, faces;
for (int el = 1; el <= ne; el++) for (int el = 1; el <= mesh.GetNE(); el++)
{ {
topology.GetElementFaces (el, faces); topology.GetElementFaces (el, faces);
topology.GetElementEdges (el, edges); topology.GetElementEdges (el, edges);
const Element & volel = mesh.VolumeElement (el); const Element & volel = mesh.VolumeElement (el);
Array<int> & sendarray = *sendarrays[volel.GetPartition()]; Array<int> & sendarray = *sendarrays[volel.GetPartition()];
sendarray. Append ( el );
sendarray. Append ( faces.Size() );
sendarray. Append ( edges.Size() );
sendarray. Append ( volel.GetNP() );
for ( int i = 0; i < faces.Size(); i++ )
sendarray. Append(faces[i] );
for ( int i = 0; i < edges.Size(); i++ ) for ( int i = 0; i < edges.Size(); i++ )
sendarray. Append(edges[i] ); sendarray.Append (edges[i]);
for ( int i = 0; i < volel.GetNP(); i++ ) for ( int i = 0; i < faces.Size(); i++ )
sendarray. Append(volel[i] ); sendarray.Append (faces[i]);
} }
for (int dest = 1; dest < ntasks; dest++) for (int el = 1; el <= mesh.GetNSE(); el++)
sendarrays[dest]->Append (-1);
for (int el = 1; el <= nsurfel; el++)
{ {
topology.GetSurfaceElementEdges (el, edges); topology.GetSurfaceElementEdges (el, edges);
const Element2d & surfel = mesh.SurfaceElement (el); const Element2d & surfel = mesh.SurfaceElement (el);
Array<int> & sendarray = *sendarrays[surfel.GetPartition()]; Array<int> & sendarray = *sendarrays[surfel.GetPartition()];
sendarray.Append (el);
sendarray.Append (edges.Size());
sendarray.Append (surfel.GetNP());
for ( int i = 0; i < edges.Size(); i++ ) for ( int i = 0; i < edges.Size(); i++ )
sendarray.Append (edges[i]); sendarray.Append (edges[i]);
for ( int i = 0; i < surfel.GetNP(); i++ )
sendarray.Append (surfel[i]);
} }
for (int dest = 1; dest < ntasks; dest++)
sendarrays[dest]->Append (-1);
Array<MPI_Request> sendrequests; Array<MPI_Request> sendrequests;
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
@ -506,113 +166,44 @@ namespace netgen
{ {
Array<int> recvarray; Array<int> recvarray;
// MyMPI_Bcast ( recvarray );
MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10); MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10);
Array<int,1> glob2loc_el;
glob2loc_el.SetSize (neglob);
glob2loc_el = -1;
for ( int locel = 1; locel <= mesh.GetNE(); locel++)
glob2loc_el[GetLoc2Glob_VolEl(locel)] = locel;
int ii = 0; int ii = 0;
nfaglob = recvarray[ii++];
nedglob = recvarray[ii++];
Array<int> faces, edges; Array<int> faces, edges;
Array<int> pnums, globalpnums;
// int recv_ne = recvarray[ii++]; for (int volel = 1; volel <= mesh.GetNE(); volel++)
// for (int hi = 0; hi < recv_ne; hi++)
while (1)
{ {
int globvolel = recvarray[ii++]; topology.GetElementEdges ( volel, edges);
if (globvolel == -1) break; for ( int i = 0; i < edges.Size(); i++)
SetLoc2Glob_Edge ( edges[i], recvarray[ii++]);
int distnfa = recvarray[ii++]; topology.GetElementFaces( volel, faces);
int distned = recvarray[ii++]; for ( int i = 0; i < faces.Size(); i++)
int distnp = recvarray[ii++]; SetLoc2Glob_Face ( faces[i], recvarray[ii++]);
int volel = glob2loc_el[globvolel];
if (volel != -1)
{
topology.GetElementFaces( volel, faces);
topology.GetElementEdges ( volel, edges);
const Element & volelement = mesh.VolumeElement (volel);
for ( int i = 0; i < faces.Size(); i++)
SetDistantFaceNum ( 0, faces[i], recvarray[ii++]);
for ( int i = 0; i < edges.Size(); i++)
SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]);
for ( int i = 0; i < volelement.GetNP(); i++)
SetDistantPNum ( 0, volelement[i], recvarray[ii++]);
}
else
ii += distnfa + distned + distnp;
} }
for (int surfel = 1; surfel <= mesh.GetNSE(); surfel++)
Array<int,1> glob2loc_sel;
// int recv_nse = recvarray[ii++];
// nseglob = recv_nse;
glob2loc_sel.SetSize (nseglob);
glob2loc_sel = -1;
for ( int locel = 1; locel <= mesh.GetNSE(); locel++)
glob2loc_sel[GetLoc2Glob_SurfEl(locel)] = locel;
// for (int hi = 0; hi < recv_nse; hi++)
while (1)
{ {
int globvolel = recvarray[ii++]; topology.GetSurfaceElementEdges (surfel, edges);
if (globvolel == -1) break; for (int i = 0; i < edges.Size(); i++)
SetLoc2Glob_Edge (edges[i], recvarray[ii++]);
int distned = recvarray[ii++];
int distnp = recvarray[ii++];
int surfel = glob2loc_sel[globvolel];
if (surfel != -1)
{
topology.GetSurfaceElementEdges ( surfel, edges);
const Element2d & element = mesh.SurfaceElement (surfel);
for ( int i = 0; i < edges.Size(); i++)
SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]);
for ( int i = 0; i < element.GetNP(); i++)
SetDistantPNum ( 0, element[i], recvarray[ii++]);
}
else
ii += distned + distnp;
} }
} }
is_updated = true;
if (id != 0)
{
*testout << "l2d - vert = " << loc2distvert << endl;
*testout << "l2d - edge = " << loc2distedge << endl;
*testout << "l2d - el = " << loc2distel << endl;
*testout << "l2d - sel = " << loc2distsurfel << endl;
}
coarseupdate = 1;
} }
void ParallelMeshTopology :: UpdateCoarseGrid () void ParallelMeshTopology :: UpdateCoarseGrid ()
{ {
if (is_updated) return;
Reset();
static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid"); static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid");
NgProfiler::RegionTimer reg(timer); NgProfiler::RegionTimer reg(timer);
@ -628,10 +219,12 @@ namespace netgen
const MeshTopology & topology = mesh.GetTopology(); const MeshTopology & topology = mesh.GetTopology();
UpdateCoarseGridGlobal(); UpdateCoarseGridGlobal();
MPI_Barrier (MPI_COMM_WORLD); MPI_Barrier (MPI_COMM_WORLD);
MPI_Group MPI_GROUP_WORLD; MPI_Group MPI_GROUP_WORLD;
MPI_Group MPI_LocalGroup; MPI_Group MPI_LocalGroup;
MPI_Comm MPI_LocalComm; MPI_Comm MPI_LocalComm;
@ -647,14 +240,6 @@ namespace netgen
Array<int> sendarray, recvarray; Array<int> sendarray, recvarray;
nfa = topology . GetNFaces();
ned = topology . GetNEdges();
np = mesh . GetNP();
nv = mesh . GetNV();
ne = mesh . GetNE();
nseg = mesh.GetNSeg();
nsurfel = mesh.GetNSE();
static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges"); static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges");
@ -666,72 +251,56 @@ namespace netgen
NgProfiler::StartTimer (timere); NgProfiler::StartTimer (timere);
// exchange edges int nfa = topology . GetNFaces();
int maxedge = 0; int ned = topology . GetNEdges();
for (int edge = 1; edge <= ned; edge++)
maxedge = max (maxedge, GetGlobalEdgeNum (edge));
glob2loc.SetSize (maxedge);
glob2loc = -1;
for (int edge = 1; edge <= ned; edge++)
glob2loc[GetGlobalEdgeNum(edge)] = edge;
// exchange edges
cnt_send = 0; cnt_send = 0;
int v1, v2; int v1, v2;
for (int edge = 1; edge <= ned; edge++) for (int edge = 1; edge <= ned; edge++)
{ {
topology.GetEdgeVertices (edge, v1, v2); topology.GetEdgeVertices (edge, v1, v2);
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
if (IsExchangeVert (dest, v1) && if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
IsExchangeVert (dest, v2)) cnt_send[dest-1]+=2;
{
cnt_send[dest-1]+=2;
}
} }
TABLE<int> send_edges(cnt_send); TABLE<int> send_edges(cnt_send);
INDEX_2_HASHTABLE<int> gv2e(2*ned);
for (int edge = 1; edge <= ned; edge++) for (int edge = 1; edge <= ned; edge++)
{ {
topology.GetEdgeVertices (edge, v1, v2); topology.GetEdgeVertices (edge, v1, v2);
INDEX_2 es(GetGlobalPNum(v1), GetGlobalPNum(v2));
es.Sort();
gv2e.Set (es, edge);
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
{ {
if (IsExchangeVert (dest, v1) && if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
IsExchangeVert (dest, v2))
{ {
send_edges.Add (dest-1, GetGlobalEdgeNum(edge)); send_edges.Add (dest-1, es[0]);
send_edges.Add (dest-1, edge); send_edges.Add (dest-1, es[1]);
} }
} }
} }
// *testout << "send exchange edges: " << send_edges << endl;
TABLE<int> recv_edges(ntasks-1); TABLE<int> recv_edges(ntasks-1);
MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_LocalComm); MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_LocalComm);
// *testout << "recv exchange edges: " << recv_edges << endl;
for (int sender = 1; sender < ntasks; sender ++) for (int sender = 1; sender < ntasks; sender ++)
if (id != sender) if (id != sender)
{ {
FlatArray<int> recvarray = recv_edges[sender-1]; FlatArray<int> recvarray = recv_edges[sender-1];
for (int ii = 0; ii < recvarray.Size(); ii+=2)
for (int ii = 0; ii < recvarray.Size(); )
{ {
int globe = recvarray[ii++]; INDEX_2 gv12 (recvarray[ii],recvarray[ii+1]);
int diste = recvarray[ii++]; if (gv2e.Used (gv12))
SetDistantEdgeNum (sender, gv2e.Get(gv12));
if (globe <= maxedge)
{
int loce = glob2loc[globe];
if (loce != -1)
SetDistantEdgeNum (sender, loce, diste);
}
} }
} }
NgProfiler::StopTimer (timere); NgProfiler::StopTimer (timere);
@ -741,85 +310,77 @@ namespace netgen
if (mesh.GetDimension() == 3) if (mesh.GetDimension() == 3)
{ {
NgProfiler::StartTimer (timerf);
NgProfiler::StartTimer (timerf);
int maxface = 0;
glob2loc.SetSize (nfaglob); for (int face = 1; face <= nfa; face++)
glob2loc = -1; maxface = max (maxface, GetGlobalFaceNum (face));
for (int loc = 1; loc <= nfa; loc++) // glob2loc.SetSize (nfaglob);
glob2loc[GetGlobalFaceNum(loc)] = loc; glob2loc.SetSize (maxface);
glob2loc = -1;
cnt_send = 0;
Array<int> verts; for (int loc = 1; loc <= nfa; loc++)
for (int face = 1; face <= nfa; face++) glob2loc[GetGlobalFaceNum(loc)] = loc;
{
topology.GetFaceVertices (face, verts); cnt_send = 0;
for (int dest = 1; dest < ntasks; dest++) Array<int> verts;
if (IsExchangeVert (dest, verts[0]) && for (int face = 1; face <= nfa; face++)
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
{
cnt_send[dest-1]+=2;
}
}
TABLE<int> send_faces(cnt_send);
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices (face, verts);
for (int dest = 1; dest < ntasks; dest++)
{ {
if (IsExchangeVert (dest, verts[0]) && topology.GetFaceVertices (face, verts);
IsExchangeVert (dest, verts[1]) && for (int dest = 1; dest < ntasks; dest++)
IsExchangeVert (dest, verts[2])) if (IsExchangeVert (dest, verts[0]) &&
{ IsExchangeVert (dest, verts[1]) &&
send_faces.Add (dest-1, GetGlobalFaceNum(face)); IsExchangeVert (dest, verts[2]))
send_faces.Add (dest-1, face); {
} cnt_send[dest-1]+=2;
}
} }
TABLE<int> send_faces(cnt_send);
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices (face, verts);
for (int dest = 1; dest < ntasks; dest++)
{
if (IsExchangeVert (dest, verts[0]) &&
IsExchangeVert (dest, verts[1]) &&
IsExchangeVert (dest, verts[2]))
{
send_faces.Add (dest-1, GetGlobalFaceNum(face));
send_faces.Add (dest-1, face);
}
}
} }
TABLE<int> recv_faces(ntasks-1); TABLE<int> recv_faces(ntasks-1);
MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_LocalComm); MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_LocalComm);
// *testout << "send exchange faces: " << send_faces << endl; for (int sender = 1; sender < ntasks; sender ++)
// *testout << "recv exchange faces: " << recv_faces << endl; if (id != sender)
{
for (int sender = 1; sender < ntasks; sender ++) FlatArray<int> recvarray = recv_faces[sender-1];
if (id != sender)
{ for (int ii = 0; ii < recvarray.Size(); )
FlatArray<int> recvarray = recv_faces[sender-1]; {
int globf = recvarray[ii++];
for (int ii = 0; ii < recvarray.Size(); ) int distf = recvarray[ii++];
{
int globf = recvarray[ii++]; if (globf <= maxface)
int distf = recvarray[ii++]; {
int locf = glob2loc[globf]; int locf = glob2loc[globf];
if (locf != -1)
// *testout << "set distant face, sender = " << sender << ", locf = " << locf << "; distf = " << distf << endl; SetDistantFaceNum (sender, locf);
if (locf != -1) }
SetDistantFaceNum (sender, locf, distf); }
} }
}
NgProfiler::StopTimer (timerf);
NgProfiler::StopTimer (timerf);
} }
is_updated = true;
// set which elements are where for the master processor
coarseupdate = 1;
} }
} }
#endif #endif

179
libsrc/meshing/paralleltop.hpp
View File

@ -4,158 +4,95 @@
namespace netgen namespace netgen
{ {
// extern int ntasks;
class ParallelMeshTopology class ParallelMeshTopology
{ {
const Mesh & mesh; const Mesh & mesh;
// number of local elements, vertices, points (?), edges, faces
int ne, nv, np, ned, nfa;
// number of local segments and surface elements
int nseg, nsurfel;
// number of global elements, vertices, ???, faces
int neglob, nseglob, nvglob;
int nparel;
int nfaglob;
int nedglob;
/** /**
mapping from local to distant vertex number mapping from local to distant vertex number
each row of the table corresponds to one vertex each row of the table corresponds to one vertex
each row contains a list of pairs (procnr, dist_vnum) each row contains a list of pairs (procnr, dist_vnum)
*/ */
TABLE<int,PointIndex::BASE> loc2distvert;
TABLE<int,0> loc2distedge, loc2distface;
TABLE<int,0> loc2distel, loc2distsegm, loc2distsurfel;
bool coarseupdate; TABLE<int> loc2distvert, loc2distedge, loc2distface;
Array<int> glob_vert, glob_edge, glob_face;
Array<int> glob_el, glob_surfel, glob_segm;
bool is_updated;
public: public:
ParallelMeshTopology (const Mesh & amesh); ParallelMeshTopology (const Mesh & amesh);
~ParallelMeshTopology (); ~ParallelMeshTopology ();
void Reset ();
void Print() const;
void UpdateCoarseGrid();
void UpdateCoarseGridGlobal();
// bool DoCoarseUpdate() const { return !coarseupdate; }
/// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert... /// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert...
void SetNV (int anv); void SetNV (int anv);
void SetNE (int ane); void SetNE (int ane);
void SetNSE (int anse); void SetNSE (int anse);
void SetNSegm (int anseg); void SetNSegm (int anseg);
void SetNVGlob ( int anvglob ) { nvglob = anvglob; }
void SetNEGlob ( int aneglob ) { neglob = aneglob; }
void SetNSEGlob ( int anseglob ) { nseglob = anseglob; }
int GetNVGlob () { return nvglob; } void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; }
int GetNEGlob () { return neglob; } void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; }
void SetLoc2Glob_Face (int locnum, int globnum) { glob_face[locnum-1] = globnum; }
void SetLoc2Glob_VolEl (int locnum, int globnum) { glob_el[locnum-1] = globnum; }
void SetLoc2Glob_SurfEl (int locnum, int globnum) { glob_surfel[locnum-1] = globnum; }
void SetLoc2Glob_Segm (int locnum, int globnum) { glob_segm[locnum-1] = globnum; }
int GetGlobalPNum (int locnum) const { return glob_vert[locnum-1]; }
int GetGlobalEdgeNum (int locnum) const { return glob_edge[locnum-1]; }
int GetGlobalFaceNum (int locnum) const { return glob_face[locnum-1]; }
int GetGlobalElNum (int locnum) const { return glob_el[locnum-1]; }
int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; }
void Reset (); void SetDistantFaceNum (int dest, int locnum);
void SetDistantPNum (int dest, int locnum);
void SetDistantEdgeNum (int dest, int locnum);
void SetLoc2Glob_Vert ( int locnum, int globnum ) { loc2distvert[locnum][0] = globnum; } int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
void SetLoc2Glob_VolEl ( int locnum, int globnum ) { loc2distel[locnum-1][0] = globnum; } int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); }
void SetLoc2Glob_SurfEl ( int locnum, int globnum ) { loc2distsurfel[locnum-1][0] = globnum; } int GetNDistantEdgeNums ( int locedgenum) const { return loc2distedge[locedgenum-1].Size(); }
void SetLoc2Glob_Segm ( int locnum, int globnum ) { loc2distsegm[locnum-1][0] = globnum; }
void GetDistantPNums (int locpnum, int * distpnums ) const
int GetLoc2Glob_Vert ( int locnum ) const { return loc2distvert[locnum][0]; }
int GetLoc2Glob_VolEl ( int locnum ) const { return loc2distel[locnum-1][0]; }
int GetLoc2Glob_SurfEl ( int locnum ) const { return loc2distsurfel[locnum-1][0]; }
void GetVertNeighbours ( int vnum, Array<int> & dests ) const;
int GetNDistantPNums ( int locpnum ) const
{ return loc2distvert[locpnum].Size() / 2 + 1; }
int GetNDistantFaceNums ( int locfacenum ) const
{ return loc2distface[locfacenum-1].Size() / 2 + 1; }
int GetNDistantEdgeNums ( int locedgenum ) const
{ return loc2distedge[locedgenum-1].Size() / 2 + 1; }
int GetNDistantElNums ( int locelnum ) const
{ return loc2distel[locelnum-1].Size() / 2 + 1; }
int GetGlobalPNum ( int locpnum ) const
{ return loc2distvert[locpnum][0]; }
int GetGlobalEdgeNum ( int locedgenum ) const
{ return loc2distedge[locedgenum-1][0]; }
int GetGlobalFaceNum ( int locfacenum ) const
{ return loc2distface[locfacenum-1][0]; }
int GetGlobalElNum ( int locelnum ) const
{ return loc2distel[locelnum-1][0]; }
/*
int GetDistantPNum ( int proc, int locpnum ) const;
int GetDistantEdgeNum ( int proc, int locedgenum ) const;
int GetDistantFaceNum ( int proc, int locedgenum ) const;
int GetDistantElNum ( int proc, int locelnum ) const;
*/
void GetDistantPNums ( int locpnum, int * distpnums ) const;
void GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const;
void GetDistantFaceNums ( int locedgenum, int * distfacenums ) const;
void GetDistantElNums ( int locelnum, int * distfacenums ) const;
void Print() const;
bool IsExchangeVert ( PointIndex vnum ) const { return loc2distvert[vnum].Size() > 1; }
bool IsExchangeEdge ( int ednum ) const { return loc2distedge[ednum-1].Size() > 1; }
bool IsExchangeFace ( int fnum ) const { return loc2distface[fnum-1].Size() > 1; }
bool IsExchangeElement ( int elnum ) const { return false; }
// bool IsExchangeSEl ( int selnum ) const { return loc2distsurfel[selnum-1].Size() > 1; }
bool IsExchangeVert (int dest, int vnum ) const
{ {
FlatArray<int> exchange = loc2distvert[vnum]; for (int i = 0; i < loc2distvert[locpnum-1].Size(); i++ )
for (int i = 1; i < exchange.Size(); i += 2) distpnums[i] = loc2distvert[locpnum-1][i];
}
void GetDistantFaceNums (int locfacenum, int * distfacenums ) const
{
for ( int i = 0; i < loc2distface[locfacenum-1].Size(); i++ )
distfacenums[i] = loc2distface[locfacenum-1][i];
}
void GetDistantEdgeNums (int locedgenum, int * distedgenums ) const
{
for (int i = 0; i < loc2distedge[locedgenum-1].Size(); i++ )
distedgenums[i] = loc2distedge[locedgenum-1][i];
}
bool IsExchangeVert (int dest, int vnum) const
{
return loc2distvert[vnum-1].Contains (dest);
/*
FlatArray<int> exchange = loc2distvert[vnum-1];
for (int i = 0; i < exchange.Size(); i ++)
if (exchange[i] == dest) return true; if (exchange[i] == dest) return true;
return false; return false;
*/
} }
bool IsExchangeEdge (int dest, int ednum ) const
{
FlatArray<int> exchange = loc2distedge[ednum-1];
for (int i = 1; i < exchange.Size(); i += 2)
if (exchange[i] == dest) return true;
return false;
}
bool IsExchangeFace (int dest, int fnum ) const
{
FlatArray<int> exchange = loc2distface[fnum-1];
for (int i = 1; i < exchange.Size(); i += 2)
if (exchange[i] == dest) return true;
return false;
}
// bool IsExchangeElement (int dest, int elnum ) const { return false; }
void Update();
void UpdateCoarseGrid();
void UpdateExchangeElements();
void UpdateCoarseGridGlobal();
bool DoCoarseUpdate() const { return !coarseupdate; }
void SetDistantFaceNum ( int dest, int locnum, int distnum );
void SetDistantPNum ( int dest, int locnum, int distnum );
void SetDistantEdgeNum ( int dest, int locnum, int distnum );
void SetDistantEl ( int dest, int locnum, int distnum );
void SetDistantSurfEl ( int dest, int locnum, int distnum );
void SetDistantSegm ( int dest, int locnum, int distnum );
bool IsGhostEl ( int elnum ) const { return mesh.VolumeElement(elnum).IsGhost(); }
}; };

18
libsrc/meshing/topology.cpp
View File

@ -73,7 +73,7 @@ namespace netgen
NgProfiler::RegionTimer reg (timer); NgProfiler::RegionTimer reg (timer);
#ifdef PARALLEL #ifdef PARALLEL
ParallelMeshTopology & paralleltop = mesh.GetParallelTopology(); // ParallelMeshTopology & paralleltop = mesh.GetParallelTopology();
#endif #endif
@ -399,8 +399,6 @@ namespace netgen
} }
} }
edge2vert.SetSize (ned); edge2vert.SetSize (ned);
for (int i = 1; i <= ne; i++) for (int i = 1; i <= ne; i++)
{ {
@ -484,8 +482,6 @@ namespace netgen
<< surfedges.Elem(i)[1] << ", " << surfedges.Elem(i)[1] << ", "
<< surfedges.Elem(i)[2] << endl; << surfedges.Elem(i)[2] << endl;
*/ */
} }
@ -1077,8 +1073,8 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
(*testout) << " RESET Paralleltop" << endl; // (*testout) << " RESET Paralleltop" << endl;
paralleltop.Reset (); // paralleltop.Reset ();
#endif #endif
Array<short int> face_els(nfa), face_surfels(nfa); Array<short int> face_els(nfa), face_surfels(nfa);
@ -1111,7 +1107,8 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
if ( ntasks > 1 ) if ( ntasks > 1 )
{ {
if ( !paralleltop.DoCoarseUpdate() ) continue; continue;
// if ( !paralleltop.DoCoarseUpdate() ) continue;
} }
else else
#endif #endif
@ -1156,9 +1153,8 @@ namespace netgen
#ifdef PARALLEL #ifdef PARALLEL
if (id != 0) if (id != 0)
{ {
paralleltop.Update(); // if ( paralleltop.DoCoarseUpdate() )
if ( paralleltop.DoCoarseUpdate() ) // paralleltop.UpdateCoarseGrid();
paralleltop.UpdateCoarseGrid();
} }
#endif #endif

5
libsrc/visualization/vsmesh.cpp
View File

@ -1858,10 +1858,7 @@ namespace netgen
if (vispar.drawmetispartition && el.GetPartition()!=-1) if (vispar.drawmetispartition && el.GetPartition()!=-1)
ind = el.GetPartition() % 4; ind = el.GetPartition() % 4;
if ( el.IsGhost() ) glMaterialfv(GL_FRONT, GL_AMBIENT_AND_DIFFUSE, tetcols[ind]);
glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE, tetcols_ghost[ind]);
else
glMaterialfv(GL_FRONT, GL_AMBIENT_AND_DIFFUSE, tetcols[ind]);
if (curv.IsHighOrder()) // && curv.IsElementCurved(ei)) if (curv.IsHighOrder()) // && curv.IsElementCurved(ei))

19
libsrc/visualization/vssolution.cpp
View File

@ -827,10 +827,6 @@ namespace netgen
{ {
const Element2d & el = (*mesh)[sei]; const Element2d & el = (*mesh)[sei];
#ifdef PARALLEL
// parallel visualization --> dont draw ghost elements
if ( el . IsGhost() ) continue;
#endif
bool curved = curv.IsHighOrder(); // && curv.IsSurfaceElementCurved(sei); bool curved = curv.IsHighOrder(); // && curv.IsSurfaceElementCurved(sei);
if (el.GetType() == TRIG || el.GetType() == TRIG6) if (el.GetType() == TRIG || el.GetType() == TRIG6)
@ -1064,8 +1060,6 @@ namespace netgen
{ {
const Element2d & el = (*mesh)[sei]; const Element2d & el = (*mesh)[sei];
if ( el . IsGhost() ) continue;
if (vispar.drawdomainsurf > 0) if (vispar.drawdomainsurf > 0)
{ {
if (mesh->GetDimension() == 3) if (mesh->GetDimension() == 3)
@ -1197,8 +1191,6 @@ namespace netgen
{ {
const Element2d & el = (*mesh)[sei]; const Element2d & el = (*mesh)[sei];
if ( el . IsGhost() ) continue;
if(vispar.drawdomainsurf > 0) if(vispar.drawdomainsurf > 0)
{ {
if (mesh->GetDimension() == 3) if (mesh->GetDimension() == 3)
@ -1431,8 +1423,6 @@ namespace netgen
{ {
Element2d & el = (*mesh)[sei]; Element2d & el = (*mesh)[sei];
if ( el . IsGhost() ) continue;
// bool curved = curv.IsSurfaceElementCurved (sei); // bool curved = curv.IsSurfaceElementCurved (sei);
int nv = (el.GetType() == TRIG || el.GetType() == TRIG6) ? 3 : 4; int nv = (el.GetType() == TRIG || el.GetType() == TRIG6) ? 3 : 4;
@ -1559,8 +1549,6 @@ namespace netgen
// if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue; // if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue;
// if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue; // if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue;
if ( (*mesh)[ei] . IsGhost() ) continue;
ELEMENT_TYPE type = (*mesh)[ei].GetType(); ELEMENT_TYPE type = (*mesh)[ei].GetType();
if (type == HEX || type == PRISM || type == TET || type == PYRAMID) if (type == HEX || type == PRISM || type == TET || type == PYRAMID)
{ {
@ -1905,8 +1893,6 @@ namespace netgen
{ {
const Element2d & el = (*mesh)[sei]; const Element2d & el = (*mesh)[sei];
if ( el . IsGhost() ) continue;
if (el.GetType() == TRIG || el.GetType() == TRIG6) if (el.GetType() == TRIG || el.GetType() == TRIG6)
{ {
@ -3499,11 +3485,6 @@ namespace netgen
{ {
int first_point_of_element = pts.Size(); int first_point_of_element = pts.Size();
#ifdef PARALLEL
// parallel visualization --> dont draw ghost elements
if ( (*mesh)[ei] . IsGhost() ) continue;
#endif
locgrid.SetSize(n3); locgrid.SetSize(n3);
if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue; if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue;
if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue; if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue;