mirror of
https://github.com/NGSolve/netgen.git
synced 2024-12-26 05:50:32 +05:00
parallel tuning
This commit is contained in:
Joachim Schoeberl
parent
094f52ee3c
commit
fc81f592f5
@ -938,55 +938,51 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou
|
||||
int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * distnums )
|
||||
{
|
||||
int size = NgPar_GetNDistantNodeNums (nodetype, locnum);
|
||||
locnum++;
|
||||
switch ( nodetype )
|
||||
{
|
||||
case 0:
|
||||
mesh->GetParallelTopology().GetDistantPNums( locnum+1, distnums );
|
||||
mesh->GetParallelTopology().GetDistantPNums( locnum, distnums );
|
||||
break;
|
||||
case 1:
|
||||
mesh->GetParallelTopology().GetDistantEdgeNums( locnum+1, distnums );
|
||||
mesh->GetParallelTopology().GetDistantEdgeNums( locnum, distnums );
|
||||
break;
|
||||
case 2:
|
||||
mesh->GetParallelTopology().GetDistantFaceNums( locnum+1, distnums );
|
||||
mesh->GetParallelTopology().GetDistantFaceNums( locnum, distnums );
|
||||
break;
|
||||
case 3:
|
||||
mesh->GetParallelTopology().GetDistantElNums( locnum+1, distnums );
|
||||
// mesh->GetParallelTopology().GetDistantElNums( locnum, distnums );
|
||||
break;
|
||||
default:
|
||||
cerr << "NgPar_GetDistantNodeNums() Unknown nodetype " << nodetype << endl;
|
||||
size = -1;
|
||||
}
|
||||
// 0 - based
|
||||
for ( int i = 0; i < size; i++ )
|
||||
distnums[2*i+1]--;
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
int NgPar_GetNDistantNodeNums ( int nodetype, int locnum )
|
||||
{
|
||||
locnum++;
|
||||
switch ( nodetype )
|
||||
{
|
||||
case 0:
|
||||
return mesh->GetParallelTopology().GetNDistantPNums( locnum+1 );
|
||||
case 1:
|
||||
return mesh->GetParallelTopology().GetNDistantEdgeNums( locnum+1 );
|
||||
case 2:
|
||||
return mesh->GetParallelTopology().GetNDistantFaceNums( locnum+1 );
|
||||
case 3:
|
||||
return mesh->GetParallelTopology().GetNDistantElNums( locnum+1 );
|
||||
case 0: return mesh->GetParallelTopology().GetNDistantPNums (locnum);
|
||||
case 1: return mesh->GetParallelTopology().GetNDistantEdgeNums (locnum);
|
||||
case 2: return mesh->GetParallelTopology().GetNDistantFaceNums(locnum );
|
||||
case 3: return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int NgPar_GetGlobalNodeNum (int nodetype, int locnum)
|
||||
{
|
||||
locnum++;
|
||||
switch (nodetype)
|
||||
{
|
||||
case 0: return mesh->GetParallelTopology().GetGlobalPNum (locnum+1)-1;
|
||||
case 1: return mesh->GetParallelTopology().GetGlobalEdgeNum (locnum+1)-1;
|
||||
case 2: return mesh->GetParallelTopology().GetGlobalFaceNum (locnum+1)-1;
|
||||
case 3: return mesh->GetParallelTopology().GetGlobalElNum (locnum+1)-1;
|
||||
case 0: return mesh->GetParallelTopology().GetGlobalPNum (locnum)-1;
|
||||
case 1: return mesh->GetParallelTopology().GetGlobalEdgeNum (locnum)-1;
|
||||
case 2: return mesh->GetParallelTopology().GetGlobalFaceNum (locnum)-1;
|
||||
case 3: return mesh->GetParallelTopology().GetGlobalElNum (locnum)-1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -157,10 +157,6 @@ namespace netgen
|
||||
PointIndex pi = points.Size() + PointIndex::BASE;
|
||||
points.Append ( MeshPoint (p, layer, INNERPOINT) );
|
||||
|
||||
#ifdef PARALLEL
|
||||
points.Last().SetGhost(0);
|
||||
#endif
|
||||
|
||||
lock.UnLock();
|
||||
|
||||
return pi;
|
||||
@ -176,10 +172,6 @@ namespace netgen
|
||||
PointIndex pi = points.Size() + PointIndex::BASE;
|
||||
points.Append ( MeshPoint (p, layer, type) );
|
||||
|
||||
#ifdef PARALLEL
|
||||
points.Last().SetGhost(0);
|
||||
#endif
|
||||
|
||||
lock.UnLock();
|
||||
|
||||
return pi;
|
||||
@ -197,8 +189,6 @@ namespace netgen
|
||||
PointIndex pi = points.Size() + PointIndex::BASE;
|
||||
points.Append ( MeshPoint (p, layer, INNERPOINT) );
|
||||
|
||||
points.Last().SetGhost(isghost);
|
||||
|
||||
lock.UnLock();
|
||||
|
||||
return pi;
|
||||
@ -214,8 +204,6 @@ namespace netgen
|
||||
PointIndex pi = points.Size() + PointIndex::BASE;
|
||||
points.Append ( MeshPoint (p, layer, type) );
|
||||
|
||||
points.Last().SetGhost(isghost);
|
||||
|
||||
lock.UnLock();
|
||||
|
||||
return pi;
|
||||
@ -313,10 +301,6 @@ namespace netgen
|
||||
surfelements.Last().next = facedecoding[el.index-1].firstelement;
|
||||
facedecoding[el.index-1].firstelement = si;
|
||||
|
||||
#ifdef PARALLEL
|
||||
surfelements.Last().SetGhost ( el.IsGhost() );
|
||||
#endif
|
||||
|
||||
lock.UnLock();
|
||||
return si;
|
||||
}
|
||||
@ -355,10 +339,6 @@ namespace netgen
|
||||
volelements.Append (el);
|
||||
volelements.Last().flags.illegal_valid = 0;
|
||||
|
||||
#ifdef PARALLEL
|
||||
volelements.Last().SetGhost ( el.IsGhost() );
|
||||
#endif
|
||||
|
||||
// while (volelements.Size() > eltyps.Size())
|
||||
// eltyps.Append (FREEELEMENT);
|
||||
|
||||
@ -1209,23 +1189,23 @@ namespace netgen
|
||||
strcpy (str, "");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
CalcSurfacesOfNode ();
|
||||
// BuildConnectedNodes ();
|
||||
topology -> Update();
|
||||
clusters -> Update();
|
||||
|
||||
#ifdef PARALLEL
|
||||
if ( ntasks > 1 ) Distribute ();
|
||||
#endif
|
||||
|
||||
|
||||
SetNextMajorTimeStamp();
|
||||
// PrintMemInfo (cout);
|
||||
|
||||
|
||||
#ifdef PARALLEL
|
||||
if ( ntasks > 1 )
|
||||
{
|
||||
// for parallel processing
|
||||
Distribute ();
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -172,9 +172,6 @@ namespace netgen
|
||||
orderx = ordery = 1;
|
||||
refflag = 1;
|
||||
strongrefflag = false;
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -200,9 +197,6 @@ namespace netgen
|
||||
orderx = ordery = 1;
|
||||
refflag = 1;
|
||||
strongrefflag = false;
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
Element2d :: Element2d (ELEMENT_TYPE atyp)
|
||||
@ -222,10 +216,6 @@ namespace netgen
|
||||
orderx = ordery = 1;
|
||||
refflag = 1;
|
||||
strongrefflag = false;
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -250,11 +240,6 @@ namespace netgen
|
||||
deleted = 0;
|
||||
visible = 1;
|
||||
orderx = ordery = 1;
|
||||
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
Element2d :: Element2d (int pi1, int pi2, int pi3, int pi4)
|
||||
@ -278,10 +263,6 @@ namespace netgen
|
||||
deleted = 0;
|
||||
visible = 1;
|
||||
orderx = ordery = 1;
|
||||
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -966,7 +947,6 @@ namespace netgen
|
||||
|
||||
#ifdef PARALLEL
|
||||
partitionNumber = -1;
|
||||
isghost = 0;
|
||||
#endif
|
||||
|
||||
}
|
||||
@ -1000,10 +980,6 @@ namespace netgen
|
||||
default: cerr << "Element::Element: unknown element with " << np << " points" << endl;
|
||||
}
|
||||
orderx = ordery = orderz = 1;
|
||||
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void Element :: SetOrder (const int aorder)
|
||||
@ -1041,10 +1017,6 @@ namespace netgen
|
||||
flags.deleted = 0;
|
||||
flags.fixed = 0;
|
||||
orderx = ordery = orderz = 1;
|
||||
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -232,25 +232,16 @@ namespace netgen
|
||||
double singular; // singular factor for hp-refinement
|
||||
POINTTYPE type;
|
||||
|
||||
#ifdef PARALLEL
|
||||
bool isghost;
|
||||
#endif
|
||||
|
||||
public:
|
||||
MeshPoint ()
|
||||
{
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
;
|
||||
}
|
||||
|
||||
MeshPoint (const Point<3> & ap, int alayer = 1, POINTTYPE apt = INNERPOINT)
|
||||
: Point<3> (ap), layer(alayer), singular(0.),type(apt)
|
||||
{
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
;
|
||||
}
|
||||
|
||||
@ -259,9 +250,6 @@ namespace netgen
|
||||
Point<3>::operator= (ap);
|
||||
layer = 0;
|
||||
singular = 0;
|
||||
#ifdef PARALLEL
|
||||
isghost = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int GetLayer() const { return layer; }
|
||||
@ -274,9 +262,6 @@ namespace netgen
|
||||
bool IsSingular() const { return (singular != 0.0); }
|
||||
|
||||
#ifdef PARALLEL
|
||||
bool IsGhost () const { return isghost; }
|
||||
void SetGhost ( bool aisghost ) { isghost = aisghost; }
|
||||
|
||||
static MPI_Datatype MyGetMPIType ( );
|
||||
#endif
|
||||
|
||||
@ -325,7 +310,6 @@ namespace netgen
|
||||
unsigned int ordery:6;
|
||||
|
||||
#ifdef PARALLEL
|
||||
bool isghost;
|
||||
int partitionNumber;
|
||||
#endif
|
||||
|
||||
@ -506,15 +490,8 @@ namespace netgen
|
||||
int hp_elnr;
|
||||
|
||||
#ifdef PARALLEL
|
||||
bool IsGhost () const { return isghost; }
|
||||
void SetGhost ( bool aisghost ) { isghost = aisghost; }
|
||||
|
||||
// by JS, only for 2D meshes ????
|
||||
int GetPartition () const { return partitionNumber; }
|
||||
void SetPartition (int nr) { partitionNumber = nr; };
|
||||
|
||||
#else
|
||||
bool IsGhost () const { return false; }
|
||||
#endif
|
||||
};
|
||||
|
||||
@ -583,7 +560,7 @@ namespace netgen
|
||||
#ifdef PARALLEL
|
||||
/// number of partition for parallel computation
|
||||
int partitionNumber;
|
||||
bool isghost;
|
||||
|
||||
#endif
|
||||
|
||||
public:
|
||||
@ -778,11 +755,7 @@ namespace netgen
|
||||
#ifdef PARALLEL
|
||||
int GetPartition () const { return partitionNumber; }
|
||||
void SetPartition (int nr) { partitionNumber = nr; };
|
||||
|
||||
bool IsGhost () const { return isghost; }
|
||||
void SetGhost ( const bool aisghost ) { isghost = aisghost; }
|
||||
#else
|
||||
bool IsGhost () const { return false; }
|
||||
int GetPartition () const { return 0; }
|
||||
#endif
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ namespace netgen
|
||||
{
|
||||
|
||||
template <>
|
||||
inline MPI_Datatype MyGetMPIType<netgen::PointIndex> ( )
|
||||
inline MPI_Datatype MyGetMPIType<PointIndex> ( )
|
||||
{ return MPI_INT; }
|
||||
|
||||
|
||||
@ -60,14 +60,10 @@ namespace netgen
|
||||
MyMPI_Bcast (nelglob);
|
||||
MyMPI_Bcast (nseglob);
|
||||
MyMPI_Bcast (nvglob);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (id > 0)
|
||||
{
|
||||
paralleltop -> SetNEGlob (nelglob);
|
||||
paralleltop -> SetNSEGlob (nseglob);
|
||||
paralleltop -> SetNVGlob (nvglob);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -97,6 +93,10 @@ namespace netgen
|
||||
SendMesh ();
|
||||
else
|
||||
ReceiveParallelMesh();
|
||||
|
||||
|
||||
paralleltop -> UpdateCoarseGrid();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -165,11 +165,11 @@ namespace netgen
|
||||
num_verts_on_proc[dest]++;
|
||||
num_procs_on_vert[epi]++;
|
||||
|
||||
paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]);
|
||||
paralleltop -> SetDistantPNum (dest, epi); // , num_verts_on_proc[dest]);
|
||||
}
|
||||
}
|
||||
nelloc[dest] ++;
|
||||
paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] );
|
||||
// paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] );
|
||||
}
|
||||
|
||||
|
||||
@ -188,11 +188,11 @@ namespace netgen
|
||||
num_verts_on_proc[dest]++;
|
||||
num_procs_on_vert[epi]++;
|
||||
|
||||
paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]);
|
||||
paralleltop -> SetDistantPNum ( dest, epi); // num_verts_on_proc[dest]);
|
||||
}
|
||||
}
|
||||
nselloc[dest] ++;
|
||||
paralleltop -> SetDistantSurfEl ( dest, sels[hi]+1, nselloc[dest] );
|
||||
// paralleltop -> SetDistantSurfEl ( dest, sels[hi]+1, nselloc[dest] );
|
||||
}
|
||||
}
|
||||
|
||||
@ -445,7 +445,7 @@ namespace netgen
|
||||
segmbuf.Add (dest, seg.singedge_left);
|
||||
segi[dest] += 14;
|
||||
}
|
||||
paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) );
|
||||
// paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -508,7 +508,7 @@ namespace netgen
|
||||
|
||||
for (int hi = 0; hi < dist_pnums.Size(); hi += 3)
|
||||
paralleltop ->
|
||||
SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi], dist_pnums[hi+2]);
|
||||
SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi]); // , dist_pnums[hi+2]);
|
||||
|
||||
NgProfiler::StopTimer (timer_pts);
|
||||
*testout << "got " << numvert << " vertices" << endl;
|
||||
@ -574,8 +574,6 @@ namespace netgen
|
||||
tri.GeomInfoPi(j).trignum = selbuf[ii++];
|
||||
}
|
||||
|
||||
tri.SetGhost(isghost);
|
||||
|
||||
paralleltop->SetLoc2Glob_SurfEl ( sel+1, globsel );
|
||||
AddSurfaceElement (tri);
|
||||
sel ++;
|
||||
@ -646,8 +644,9 @@ namespace netgen
|
||||
topology -> Update();
|
||||
clusters -> Update();
|
||||
|
||||
// paralleltop -> UpdateCoarseGrid();
|
||||
|
||||
SetNextMajorTimeStamp();
|
||||
// paralleltop->Print();
|
||||
}
|
||||
|
||||
|
||||
@ -676,7 +675,7 @@ namespace netgen
|
||||
MyMPI_SendCmd ("mesh");
|
||||
SendRecvMesh ();
|
||||
|
||||
paralleltop -> UpdateCoarseGrid();
|
||||
// paralleltop -> UpdateCoarseGrid();
|
||||
// paralleltop -> Print();
|
||||
}
|
||||
|
||||
@ -774,17 +773,15 @@ namespace netgen
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
int timermetis = NgProfiler::CreateTimer ("Metis itself");
|
||||
NgProfiler::StartTimer (timermetis);
|
||||
|
||||
#ifdef METIS4
|
||||
cout << "call metis ... " << flush;
|
||||
cout << "call metis(4)_PartMeshDual ... " << flush;
|
||||
METIS_PartMeshDual (&ne, &nn, &elmnts[0], &etype, &numflag, &nparts,
|
||||
&edgecut, &epart[0], &npart[0]);
|
||||
#else
|
||||
cout << "call metis-5 ... " << endl;
|
||||
cout << "call metis(5)_PartMeshDual ... " << endl;
|
||||
idx_t options[METIS_NOPTIONS];
|
||||
|
||||
Array<idx_t> eptr(ne+1);
|
||||
@ -811,7 +808,6 @@ namespace netgen
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (int sei = 1; sei <= GetNSE(); sei++ )
|
||||
{
|
||||
int ei1, ei2;
|
||||
@ -905,7 +901,6 @@ namespace netgen
|
||||
Element & volel = VolumeElement(el);
|
||||
nodesinpart = 0;
|
||||
|
||||
// VolumeElement(el).SetPartition(part[ volel[1] ] + 1);
|
||||
|
||||
int el_np = volel.GetNP();
|
||||
int partition = 0;
|
||||
@ -917,16 +912,8 @@ namespace netgen
|
||||
partition = i;
|
||||
|
||||
volel.SetPartition(partition);
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
for ( int i=1; i<=ne; i++)
|
||||
{
|
||||
neloc[ VolumeElement(i).GetPartition() ] ++;
|
||||
}
|
||||
*/
|
||||
|
||||
delete [] xadj;
|
||||
delete [] part;
|
||||
delete [] adjacency;
|
||||
|
||||
@ -8,37 +8,10 @@
|
||||
namespace netgen
|
||||
{
|
||||
|
||||
|
||||
void ParallelMeshTopology :: Reset ()
|
||||
ParallelMeshTopology :: ParallelMeshTopology (const Mesh & amesh)
|
||||
: mesh(amesh)
|
||||
{
|
||||
*testout << "ParallelMeshTopology::Reset" << endl;
|
||||
|
||||
if ( ntasks == 1 ) return;
|
||||
|
||||
int nvold = nv;
|
||||
|
||||
ne = mesh.GetNE();
|
||||
nv = mesh.GetNV();
|
||||
nseg = mesh.GetNSeg();
|
||||
nsurfel = mesh.GetNSE();
|
||||
|
||||
ned = mesh.GetTopology().GetNEdges();
|
||||
nfa = mesh.GetTopology().GetNFaces();
|
||||
|
||||
loc2distedge.ChangeSize (ned);
|
||||
for (int i = 0; i < ned; i++)
|
||||
if (loc2distedge[i].Size() == 0)
|
||||
loc2distedge.Add (i, -1); // will be the global nr
|
||||
|
||||
loc2distface.ChangeSize (nfa);
|
||||
for (int i = 0; i < nfa; i++)
|
||||
if (loc2distface[i].Size() == 0)
|
||||
loc2distface.Add (i, -1); // will be the global nr
|
||||
|
||||
if ( nvold == nv ) return;
|
||||
|
||||
SetNV(nv);
|
||||
SetNE(ne);
|
||||
is_updated = false;
|
||||
}
|
||||
|
||||
|
||||
@ -48,323 +21,88 @@ namespace netgen
|
||||
}
|
||||
|
||||
|
||||
|
||||
ParallelMeshTopology :: ParallelMeshTopology ( const netgen::Mesh & amesh )
|
||||
: mesh(amesh)
|
||||
void ParallelMeshTopology :: Reset ()
|
||||
{
|
||||
ned = 0;
|
||||
nfa = 0;
|
||||
nv = 0;
|
||||
ne = 0;
|
||||
np = 0;
|
||||
nseg = 0;
|
||||
nsurfel = 0;
|
||||
neglob = 0;
|
||||
nvglob = 0;
|
||||
*testout << "ParallelMeshTopology::Reset" << endl;
|
||||
|
||||
nparel = 0;
|
||||
if ( ntasks == 1 ) return;
|
||||
|
||||
coarseupdate = 0;
|
||||
int ned = mesh.GetTopology().GetNEdges();
|
||||
int nfa = mesh.GetTopology().GetNFaces();
|
||||
|
||||
if (glob_edge.Size() != ned)
|
||||
{
|
||||
glob_edge.SetSize(ned);
|
||||
glob_face.SetSize(nfa);
|
||||
glob_edge = -1;
|
||||
glob_face = -1;
|
||||
|
||||
loc2distedge.ChangeSize (ned);
|
||||
loc2distface.ChangeSize (nfa);
|
||||
}
|
||||
|
||||
if (glob_vert.Size() != mesh.GetNV())
|
||||
{
|
||||
SetNV(mesh.GetNV());
|
||||
SetNE(mesh.GetNE());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ParallelMeshTopology :: Print() const
|
||||
{
|
||||
/*
|
||||
(*testout) << endl << "TOPOLOGY FOR PARALLEL MESHES" << endl << endl;
|
||||
|
||||
for ( int i = 1; i <= nv; i++ )
|
||||
if ( IsExchangeVert (i) )
|
||||
{
|
||||
(*testout) << "exchange point " << i << ": global " << GetLoc2Glob_Vert(i) << endl;
|
||||
for ( int dest = 0; dest < ntasks; dest ++)
|
||||
if ( dest != id )
|
||||
if ( GetDistantPNum( dest, i ) > 0 )
|
||||
(*testout) << " p" << dest << ": " << GetDistantPNum ( dest, i ) << endl;
|
||||
}
|
||||
|
||||
for ( int i = 1; i <= ned; i++ )
|
||||
if ( IsExchangeEdge ( i ) )
|
||||
{
|
||||
int v1, v2;
|
||||
mesh . GetTopology().GetEdgeVertices(i, v1, v2);
|
||||
(*testout) << "exchange edge " << i << ": global vertices " << GetLoc2Glob_Vert(v1) << " "
|
||||
<< GetLoc2Glob_Vert(v2) << endl;
|
||||
for ( int dest = 0; dest < ntasks; dest++)
|
||||
if ( GetDistantEdgeNum ( dest, i ) > 0 )
|
||||
if ( dest != id )
|
||||
(*testout) << " p" << dest << ": " << GetDistantEdgeNum ( dest, i ) << endl;
|
||||
}
|
||||
|
||||
for ( int i = 1; i <= nfa; i++ )
|
||||
if ( IsExchangeFace(i) )
|
||||
{
|
||||
Array<int> facevert;
|
||||
mesh . GetTopology().GetFaceVertices(i, facevert);
|
||||
|
||||
(*testout) << "exchange face " << i << ": global vertices " ;
|
||||
for ( int fi=0; fi < facevert.Size(); fi++)
|
||||
(*testout) << GetLoc2Glob_Vert(facevert[fi]) << " ";
|
||||
(*testout) << endl;
|
||||
for ( int dest = 0; dest < ntasks; dest++)
|
||||
if ( dest != id )
|
||||
{
|
||||
if ( GetDistantFaceNum ( dest, i ) >= 0 )
|
||||
(*testout) << " p" << dest << ": " << GetDistantFaceNum ( dest, i ) << endl;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*
|
||||
for ( int i = 1; i < mesh.GetNE(); i++)
|
||||
{
|
||||
if ( !IsExchangeElement(i) ) continue;
|
||||
Array<int> vert;
|
||||
const Element & el = mesh.VolumeElement(i);
|
||||
|
||||
(*testout) << "parallel local element " << i << endl;
|
||||
|
||||
(*testout) << "vertices " ;
|
||||
for ( int j = 0; j < el.GetNV(); j++)
|
||||
(*testout) << el.PNum(j+1) << " ";
|
||||
(*testout) << "is ghost " << IsGhostEl(i) << endl;
|
||||
(*testout) << endl;
|
||||
}
|
||||
*/
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// gibt anzahl an distant pnums zurueck
|
||||
// * pnums entspricht Array<int[2] >
|
||||
void ParallelMeshTopology :: GetDistantPNums ( int locpnum, int * distpnums ) const
|
||||
void ParallelMeshTopology :: SetDistantFaceNum (int dest, int locnum)
|
||||
{
|
||||
distpnums[0] = 0;
|
||||
distpnums[1] = loc2distvert[locpnum][0];
|
||||
for ( int i = 1; i < loc2distvert[locpnum].Size(); i++ )
|
||||
distpnums[i+1] = loc2distvert[locpnum][i];
|
||||
|
||||
// int size = loc2distvert[locpnum].Size() / 2 + 1;
|
||||
// return size;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: GetDistantFaceNums ( int locfacenum, int * distfacenums ) const
|
||||
{
|
||||
distfacenums[0] = 0;
|
||||
distfacenums[1] = loc2distface[locfacenum-1][0];
|
||||
|
||||
for ( int i = 1; i < loc2distface[locfacenum-1].Size(); i++ )
|
||||
distfacenums[i+1] = loc2distface[locfacenum-1][i];
|
||||
|
||||
// int size = loc2distface[locfacenum-1].Size() / 2 + 1;
|
||||
// return size;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const
|
||||
{
|
||||
distedgenums[0] = 0;
|
||||
distedgenums[1] = loc2distedge[locedgenum-1][0];
|
||||
|
||||
for ( int i = 1; i < loc2distedge[locedgenum-1].Size(); i++ )
|
||||
distedgenums[i+1] = loc2distedge[locedgenum-1][i];
|
||||
|
||||
// int size = loc2distedge[locedgenum-1].Size() / 2 + 1;
|
||||
// return size;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: GetDistantElNums ( int locelnum, int * distelnums ) const
|
||||
{
|
||||
distelnums[0] = 0;
|
||||
distelnums[1] = loc2distel[locelnum-1][0];
|
||||
|
||||
for ( int i = 1; i < loc2distel[locelnum-1].Size(); i++ )
|
||||
distelnums[i+1] = loc2distel[locelnum-1][i];
|
||||
|
||||
// int size = loc2distel[locelnum-1].Size() / 2 + 1;
|
||||
// return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
void ParallelMeshTopology :: SetDistantFaceNum ( int dest, int locnum, int distnum )
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distface[locnum-1][0] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
for ( int i = 1; i < loc2distface[locnum-1].Size(); i+=2 )
|
||||
for ( int i = 0; i < loc2distface[locnum-1].Size(); i+=1 )
|
||||
if ( loc2distface[locnum-1][i] == dest )
|
||||
{
|
||||
loc2distface[locnum-1][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
loc2distface.Add(locnum-1, dest);
|
||||
loc2distface.Add(locnum-1, distnum);
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetDistantPNum ( int dest, int locnum, int distnum )
|
||||
void ParallelMeshTopology :: SetDistantPNum (int dest, int locnum)
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distvert[locnum][0] = distnum; // HERE
|
||||
for ( int i = 0; i < loc2distvert[locnum-1].Size(); i+=1 )
|
||||
if ( loc2distvert[locnum-1][i] == dest )
|
||||
return;
|
||||
loc2distvert.Add (locnum-1, dest);
|
||||
}
|
||||
|
||||
for ( int i = 1; i < loc2distvert[locnum].Size(); i+=2 )
|
||||
if ( loc2distvert[locnum][i] == dest )
|
||||
|
||||
void ParallelMeshTopology :: SetDistantEdgeNum (int dest, int locnum)
|
||||
{
|
||||
loc2distvert[locnum][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
loc2distvert.Add (locnum, dest);
|
||||
loc2distvert.Add (locnum, distnum);
|
||||
}
|
||||
|
||||
|
||||
void ParallelMeshTopology :: SetDistantEdgeNum ( int dest, int locnum, int distnum )
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distedge[locnum-1][0] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
for ( int i = 1; i < loc2distedge[locnum-1].Size(); i+=2 )
|
||||
for ( int i = 0; i < loc2distedge[locnum-1].Size(); i+=1 )
|
||||
if ( loc2distedge[locnum-1][i] == dest )
|
||||
{
|
||||
loc2distedge[locnum-1][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
loc2distedge.Add (locnum-1, dest);
|
||||
loc2distedge.Add (locnum-1, distnum);
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetDistantEl ( int dest, int locnum, int distnum )
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distel[locnum-1][0] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
for ( int i = 1; i < loc2distel[locnum-1].Size(); i+=2 )
|
||||
if ( loc2distel[locnum-1][i] == dest )
|
||||
{
|
||||
loc2distel[locnum-1][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
loc2distel.Add (locnum-1, dest);
|
||||
loc2distel.Add (locnum-1, distnum);
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetDistantSurfEl ( int dest, int locnum, int distnum )
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distsurfel[locnum-1][0] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
for ( int i = 1; i < loc2distsurfel[locnum-1].Size(); i+=2 )
|
||||
if ( loc2distsurfel[locnum-1][i] == dest )
|
||||
{
|
||||
loc2distsurfel[locnum-1][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
loc2distsurfel.Add (locnum-1, dest);
|
||||
loc2distsurfel.Add (locnum-1, distnum);
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetDistantSegm ( int dest, int locnum, int distnum )
|
||||
{
|
||||
if ( dest == 0 )
|
||||
{
|
||||
loc2distsegm[locnum-1][0] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 1; i < loc2distsegm[locnum-1].Size(); i+=2 )
|
||||
if ( loc2distsegm[locnum-1][i] == dest )
|
||||
{
|
||||
loc2distsegm[locnum-1][i+1] = distnum;
|
||||
return;
|
||||
}
|
||||
|
||||
loc2distsegm.Add (locnum-1, dest);
|
||||
loc2distsegm.Add (locnum-1, distnum);
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: GetVertNeighbours ( int vnum, Array<int> & dests ) const
|
||||
{
|
||||
dests.SetSize(0);
|
||||
int i = 1;
|
||||
while ( i < loc2distvert[vnum].Size() )
|
||||
{
|
||||
dests.Append ( loc2distvert[vnum][i] );
|
||||
i+=2;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
void ParallelMeshTopology :: SetNV (int anv)
|
||||
{
|
||||
*testout << "called setnv" << endl
|
||||
<< "old size: " << loc2distvert.Size() << endl
|
||||
<< "new size: " << anv << endl;
|
||||
|
||||
glob_vert.SetSize(anv);
|
||||
glob_vert = -1;
|
||||
loc2distvert.ChangeSize (anv);
|
||||
for (int i = 1; i <= anv; i++)
|
||||
if (loc2distvert.EntrySize(i) == 0)
|
||||
loc2distvert.Add (i, -1); // will be the global nr
|
||||
|
||||
nv = anv;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetNE ( const int ane )
|
||||
void ParallelMeshTopology :: SetNE ( int ane )
|
||||
{
|
||||
loc2distel.ChangeSize (ane);
|
||||
for (int i = 0; i < ane; i++)
|
||||
if (loc2distel[i].Size() == 0)
|
||||
loc2distel.Add (i, -1); // will be the global nr
|
||||
ne = ane;
|
||||
glob_el.SetSize (ane);
|
||||
glob_el = -1;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetNSE ( int anse )
|
||||
{
|
||||
loc2distsurfel.ChangeSize (anse);
|
||||
for (int i = 0; i < anse; i++)
|
||||
if (loc2distsurfel[i].Size() == 0)
|
||||
loc2distsurfel.Add (i, -1); // will be the global nr
|
||||
|
||||
nsurfel = anse;
|
||||
glob_surfel.SetSize(anse);
|
||||
glob_surfel = -1;
|
||||
}
|
||||
|
||||
void ParallelMeshTopology :: SetNSegm ( int anseg )
|
||||
{
|
||||
loc2distsegm.ChangeSize (anseg);
|
||||
for (int i = 0; i < anseg; i++)
|
||||
if (loc2distsegm[i].Size() == 0)
|
||||
loc2distsegm.Add (i, -1); // will be the global nr
|
||||
|
||||
nseg = anseg;
|
||||
glob_segm.SetSize (anseg);
|
||||
glob_segm = -1;
|
||||
}
|
||||
|
||||
|
||||
@ -372,126 +110,48 @@ namespace netgen
|
||||
|
||||
|
||||
|
||||
void ParallelMeshTopology :: Update ()
|
||||
{
|
||||
ne = mesh.GetNE();
|
||||
nv = mesh.GetNV();
|
||||
nseg = mesh.GetNSeg();
|
||||
nsurfel = mesh.GetNSE();
|
||||
|
||||
ned = mesh.GetTopology().GetNEdges();
|
||||
nfa = mesh.GetTopology().GetNFaces();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
|
||||
{
|
||||
if (id == 0)
|
||||
PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS
|
||||
|
||||
|
||||
|
||||
|
||||
int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal");
|
||||
NgProfiler::RegionTimer reg(timer);
|
||||
|
||||
|
||||
*testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl;
|
||||
|
||||
const MeshTopology & topology = mesh.GetTopology();
|
||||
|
||||
|
||||
nfa = topology . GetNFaces();
|
||||
ned = topology . GetNEdges();
|
||||
np = mesh . GetNP();
|
||||
nv = mesh . GetNV();
|
||||
ne = mesh . GetNE();
|
||||
nseg = mesh.GetNSeg();
|
||||
nsurfel = mesh.GetNSE();
|
||||
|
||||
// low order processor - save mesh partition
|
||||
if ( id == 0 )
|
||||
{
|
||||
for ( int eli = 1; eli <= ne; eli++ )
|
||||
loc2distel[eli-1][0] = eli;
|
||||
|
||||
for ( int i = 1; i <= mesh .GetNV(); i++)
|
||||
loc2distvert[i][0] = i;
|
||||
|
||||
for ( int i = 0; i < mesh . GetNSeg(); i++)
|
||||
loc2distsegm[i][0] = i+1;
|
||||
|
||||
for ( int i = 0; i < mesh . GetNSE(); i++)
|
||||
loc2distsurfel[i][0] = i+1;
|
||||
|
||||
for ( int i = 0; i < topology .GetNEdges(); i++)
|
||||
loc2distedge[i][0] = i+1;
|
||||
|
||||
for ( int i = 0; i < topology .GetNFaces(); i++)
|
||||
loc2distface[i][0] = i+1;
|
||||
}
|
||||
|
||||
|
||||
// Array<int> sendarray, recvarray;
|
||||
|
||||
if ( id == 0 )
|
||||
{
|
||||
Array<Array<int>*> sendarrays(ntasks);
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
{
|
||||
sendarrays[dest] = new Array<int>;
|
||||
sendarrays[dest]->Append (nfa);
|
||||
sendarrays[dest]->Append (ned);
|
||||
}
|
||||
|
||||
Array<int> edges, pnums, faces, elpnums;
|
||||
for (int el = 1; el <= ne; el++)
|
||||
Array<int> edges, faces;
|
||||
for (int el = 1; el <= mesh.GetNE(); el++)
|
||||
{
|
||||
topology.GetElementFaces (el, faces);
|
||||
topology.GetElementEdges (el, edges);
|
||||
const Element & volel = mesh.VolumeElement (el);
|
||||
|
||||
Array<int> & sendarray = *sendarrays[volel.GetPartition()];
|
||||
sendarray. Append ( el );
|
||||
|
||||
sendarray. Append ( faces.Size() );
|
||||
sendarray. Append ( edges.Size() );
|
||||
sendarray. Append ( volel.GetNP() );
|
||||
|
||||
for ( int i = 0; i < faces.Size(); i++ )
|
||||
sendarray. Append(faces[i] );
|
||||
for ( int i = 0; i < edges.Size(); i++ )
|
||||
sendarray.Append (edges[i]);
|
||||
for ( int i = 0; i < volel.GetNP(); i++ )
|
||||
sendarray. Append(volel[i] );
|
||||
for ( int i = 0; i < faces.Size(); i++ )
|
||||
sendarray.Append (faces[i]);
|
||||
}
|
||||
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
sendarrays[dest]->Append (-1);
|
||||
|
||||
for (int el = 1; el <= nsurfel; el++)
|
||||
for (int el = 1; el <= mesh.GetNSE(); el++)
|
||||
{
|
||||
topology.GetSurfaceElementEdges (el, edges);
|
||||
const Element2d & surfel = mesh.SurfaceElement (el);
|
||||
Array<int> & sendarray = *sendarrays[surfel.GetPartition()];
|
||||
|
||||
sendarray.Append (el);
|
||||
sendarray.Append (edges.Size());
|
||||
sendarray.Append (surfel.GetNP());
|
||||
|
||||
for ( int i = 0; i < edges.Size(); i++ )
|
||||
sendarray.Append (edges[i]);
|
||||
|
||||
for ( int i = 0; i < surfel.GetNP(); i++ )
|
||||
sendarray.Append (surfel[i]);
|
||||
}
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
sendarrays[dest]->Append (-1);
|
||||
|
||||
Array<MPI_Request> sendrequests;
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
@ -506,113 +166,44 @@ namespace netgen
|
||||
|
||||
{
|
||||
Array<int> recvarray;
|
||||
// MyMPI_Bcast ( recvarray );
|
||||
MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10);
|
||||
|
||||
Array<int,1> glob2loc_el;
|
||||
|
||||
glob2loc_el.SetSize (neglob);
|
||||
glob2loc_el = -1;
|
||||
for ( int locel = 1; locel <= mesh.GetNE(); locel++)
|
||||
glob2loc_el[GetLoc2Glob_VolEl(locel)] = locel;
|
||||
|
||||
int ii = 0;
|
||||
nfaglob = recvarray[ii++];
|
||||
nedglob = recvarray[ii++];
|
||||
|
||||
Array<int> faces, edges;
|
||||
Array<int> pnums, globalpnums;
|
||||
|
||||
// int recv_ne = recvarray[ii++];
|
||||
// for (int hi = 0; hi < recv_ne; hi++)
|
||||
while (1)
|
||||
for (int volel = 1; volel <= mesh.GetNE(); volel++)
|
||||
{
|
||||
int globvolel = recvarray[ii++];
|
||||
if (globvolel == -1) break;
|
||||
|
||||
int distnfa = recvarray[ii++];
|
||||
int distned = recvarray[ii++];
|
||||
int distnp = recvarray[ii++];
|
||||
|
||||
int volel = glob2loc_el[globvolel];
|
||||
if (volel != -1)
|
||||
{
|
||||
topology.GetElementFaces( volel, faces);
|
||||
topology.GetElementEdges ( volel, edges);
|
||||
const Element & volelement = mesh.VolumeElement (volel);
|
||||
|
||||
for ( int i = 0; i < faces.Size(); i++)
|
||||
SetDistantFaceNum ( 0, faces[i], recvarray[ii++]);
|
||||
|
||||
for ( int i = 0; i < edges.Size(); i++)
|
||||
SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]);
|
||||
SetLoc2Glob_Edge ( edges[i], recvarray[ii++]);
|
||||
|
||||
for ( int i = 0; i < volelement.GetNP(); i++)
|
||||
SetDistantPNum ( 0, volelement[i], recvarray[ii++]);
|
||||
}
|
||||
else
|
||||
ii += distnfa + distned + distnp;
|
||||
topology.GetElementFaces( volel, faces);
|
||||
for ( int i = 0; i < faces.Size(); i++)
|
||||
SetLoc2Glob_Face ( faces[i], recvarray[ii++]);
|
||||
}
|
||||
|
||||
|
||||
Array<int,1> glob2loc_sel;
|
||||
|
||||
// int recv_nse = recvarray[ii++];
|
||||
// nseglob = recv_nse;
|
||||
|
||||
glob2loc_sel.SetSize (nseglob);
|
||||
glob2loc_sel = -1;
|
||||
for ( int locel = 1; locel <= mesh.GetNSE(); locel++)
|
||||
glob2loc_sel[GetLoc2Glob_SurfEl(locel)] = locel;
|
||||
|
||||
|
||||
// for (int hi = 0; hi < recv_nse; hi++)
|
||||
while (1)
|
||||
{
|
||||
int globvolel = recvarray[ii++];
|
||||
if (globvolel == -1) break;
|
||||
|
||||
int distned = recvarray[ii++];
|
||||
int distnp = recvarray[ii++];
|
||||
|
||||
int surfel = glob2loc_sel[globvolel];
|
||||
if (surfel != -1)
|
||||
for (int surfel = 1; surfel <= mesh.GetNSE(); surfel++)
|
||||
{
|
||||
topology.GetSurfaceElementEdges (surfel, edges);
|
||||
const Element2d & element = mesh.SurfaceElement (surfel);
|
||||
|
||||
for (int i = 0; i < edges.Size(); i++)
|
||||
SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]);
|
||||
|
||||
for ( int i = 0; i < element.GetNP(); i++)
|
||||
SetDistantPNum ( 0, element[i], recvarray[ii++]);
|
||||
}
|
||||
else
|
||||
ii += distned + distnp;
|
||||
SetLoc2Glob_Edge (edges[i], recvarray[ii++]);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (id != 0)
|
||||
{
|
||||
*testout << "l2d - vert = " << loc2distvert << endl;
|
||||
*testout << "l2d - edge = " << loc2distedge << endl;
|
||||
*testout << "l2d - el = " << loc2distel << endl;
|
||||
*testout << "l2d - sel = " << loc2distsurfel << endl;
|
||||
is_updated = true;
|
||||
}
|
||||
|
||||
coarseupdate = 1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
void ParallelMeshTopology :: UpdateCoarseGrid ()
|
||||
{
|
||||
if (is_updated) return;
|
||||
|
||||
Reset();
|
||||
static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid");
|
||||
NgProfiler::RegionTimer reg(timer);
|
||||
|
||||
@ -629,9 +220,11 @@ namespace netgen
|
||||
|
||||
UpdateCoarseGridGlobal();
|
||||
|
||||
MPI_Barrier (MPI_COMM_WORLD);
|
||||
|
||||
|
||||
|
||||
MPI_Barrier (MPI_COMM_WORLD);
|
||||
|
||||
MPI_Group MPI_GROUP_WORLD;
|
||||
MPI_Group MPI_LocalGroup;
|
||||
MPI_Comm MPI_LocalComm;
|
||||
@ -647,14 +240,6 @@ namespace netgen
|
||||
|
||||
Array<int> sendarray, recvarray;
|
||||
|
||||
nfa = topology . GetNFaces();
|
||||
ned = topology . GetNEdges();
|
||||
np = mesh . GetNP();
|
||||
nv = mesh . GetNV();
|
||||
ne = mesh . GetNE();
|
||||
nseg = mesh.GetNSeg();
|
||||
nsurfel = mesh.GetNSE();
|
||||
|
||||
|
||||
static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
|
||||
static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges");
|
||||
@ -666,72 +251,56 @@ namespace netgen
|
||||
|
||||
NgProfiler::StartTimer (timere);
|
||||
|
||||
int nfa = topology . GetNFaces();
|
||||
int ned = topology . GetNEdges();
|
||||
|
||||
|
||||
// exchange edges
|
||||
int maxedge = 0;
|
||||
for (int edge = 1; edge <= ned; edge++)
|
||||
maxedge = max (maxedge, GetGlobalEdgeNum (edge));
|
||||
|
||||
glob2loc.SetSize (maxedge);
|
||||
glob2loc = -1;
|
||||
|
||||
for (int edge = 1; edge <= ned; edge++)
|
||||
glob2loc[GetGlobalEdgeNum(edge)] = edge;
|
||||
|
||||
cnt_send = 0;
|
||||
int v1, v2;
|
||||
for (int edge = 1; edge <= ned; edge++)
|
||||
{
|
||||
topology.GetEdgeVertices (edge, v1, v2);
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
if (IsExchangeVert (dest, v1) &&
|
||||
IsExchangeVert (dest, v2))
|
||||
{
|
||||
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
|
||||
cnt_send[dest-1]+=2;
|
||||
}
|
||||
}
|
||||
|
||||
TABLE<int> send_edges(cnt_send);
|
||||
INDEX_2_HASHTABLE<int> gv2e(2*ned);
|
||||
|
||||
for (int edge = 1; edge <= ned; edge++)
|
||||
{
|
||||
topology.GetEdgeVertices (edge, v1, v2);
|
||||
INDEX_2 es(GetGlobalPNum(v1), GetGlobalPNum(v2));
|
||||
es.Sort();
|
||||
|
||||
gv2e.Set (es, edge);
|
||||
|
||||
for (int dest = 1; dest < ntasks; dest++)
|
||||
{
|
||||
if (IsExchangeVert (dest, v1) &&
|
||||
IsExchangeVert (dest, v2))
|
||||
if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
|
||||
{
|
||||
send_edges.Add (dest-1, GetGlobalEdgeNum(edge));
|
||||
send_edges.Add (dest-1, edge);
|
||||
send_edges.Add (dest-1, es[0]);
|
||||
send_edges.Add (dest-1, es[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// *testout << "send exchange edges: " << send_edges << endl;
|
||||
|
||||
TABLE<int> recv_edges(ntasks-1);
|
||||
MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_LocalComm);
|
||||
|
||||
// *testout << "recv exchange edges: " << recv_edges << endl;
|
||||
|
||||
for (int sender = 1; sender < ntasks; sender ++)
|
||||
if (id != sender)
|
||||
{
|
||||
FlatArray<int> recvarray = recv_edges[sender-1];
|
||||
|
||||
for (int ii = 0; ii < recvarray.Size(); )
|
||||
for (int ii = 0; ii < recvarray.Size(); ii+=2)
|
||||
{
|
||||
int globe = recvarray[ii++];
|
||||
int diste = recvarray[ii++];
|
||||
|
||||
if (globe <= maxedge)
|
||||
{
|
||||
int loce = glob2loc[globe];
|
||||
if (loce != -1)
|
||||
SetDistantEdgeNum (sender, loce, diste);
|
||||
INDEX_2 gv12 (recvarray[ii],recvarray[ii+1]);
|
||||
if (gv2e.Used (gv12))
|
||||
SetDistantEdgeNum (sender, gv2e.Get(gv12));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
NgProfiler::StopTimer (timere);
|
||||
@ -741,10 +310,14 @@ namespace netgen
|
||||
|
||||
if (mesh.GetDimension() == 3)
|
||||
{
|
||||
|
||||
NgProfiler::StartTimer (timerf);
|
||||
|
||||
glob2loc.SetSize (nfaglob);
|
||||
int maxface = 0;
|
||||
for (int face = 1; face <= nfa; face++)
|
||||
maxface = max (maxface, GetGlobalFaceNum (face));
|
||||
|
||||
// glob2loc.SetSize (nfaglob);
|
||||
glob2loc.SetSize (maxface);
|
||||
glob2loc = -1;
|
||||
|
||||
for (int loc = 1; loc <= nfa; loc++)
|
||||
@ -782,9 +355,6 @@ namespace netgen
|
||||
TABLE<int> recv_faces(ntasks-1);
|
||||
MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_LocalComm);
|
||||
|
||||
// *testout << "send exchange faces: " << send_faces << endl;
|
||||
// *testout << "recv exchange faces: " << recv_faces << endl;
|
||||
|
||||
for (int sender = 1; sender < ntasks; sender ++)
|
||||
if (id != sender)
|
||||
{
|
||||
@ -794,11 +364,13 @@ namespace netgen
|
||||
{
|
||||
int globf = recvarray[ii++];
|
||||
int distf = recvarray[ii++];
|
||||
int locf = glob2loc[globf];
|
||||
|
||||
// *testout << "set distant face, sender = " << sender << ", locf = " << locf << "; distf = " << distf << endl;
|
||||
if (globf <= maxface)
|
||||
{
|
||||
int locf = glob2loc[globf];
|
||||
if (locf != -1)
|
||||
SetDistantFaceNum (sender, locf, distf);
|
||||
SetDistantFaceNum (sender, locf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -806,20 +378,9 @@ namespace netgen
|
||||
NgProfiler::StopTimer (timerf);
|
||||
}
|
||||
|
||||
|
||||
// set which elements are where for the master processor
|
||||
|
||||
coarseupdate = 1;
|
||||
is_updated = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
@ -4,158 +4,95 @@
|
||||
namespace netgen
|
||||
{
|
||||
|
||||
// extern int ntasks;
|
||||
|
||||
class ParallelMeshTopology
|
||||
{
|
||||
const Mesh & mesh;
|
||||
|
||||
// number of local elements, vertices, points (?), edges, faces
|
||||
int ne, nv, np, ned, nfa;
|
||||
|
||||
// number of local segments and surface elements
|
||||
int nseg, nsurfel;
|
||||
|
||||
// number of global elements, vertices, ???, faces
|
||||
int neglob, nseglob, nvglob;
|
||||
int nparel;
|
||||
int nfaglob;
|
||||
int nedglob;
|
||||
|
||||
/**
|
||||
mapping from local to distant vertex number
|
||||
each row of the table corresponds to one vertex
|
||||
each row contains a list of pairs (procnr, dist_vnum)
|
||||
*/
|
||||
TABLE<int,PointIndex::BASE> loc2distvert;
|
||||
TABLE<int,0> loc2distedge, loc2distface;
|
||||
TABLE<int,0> loc2distel, loc2distsegm, loc2distsurfel;
|
||||
|
||||
bool coarseupdate;
|
||||
TABLE<int> loc2distvert, loc2distedge, loc2distface;
|
||||
|
||||
Array<int> glob_vert, glob_edge, glob_face;
|
||||
Array<int> glob_el, glob_surfel, glob_segm;
|
||||
|
||||
bool is_updated;
|
||||
|
||||
public:
|
||||
|
||||
ParallelMeshTopology (const Mesh & amesh);
|
||||
~ParallelMeshTopology ();
|
||||
|
||||
void Reset ();
|
||||
void Print() const;
|
||||
|
||||
void UpdateCoarseGrid();
|
||||
void UpdateCoarseGridGlobal();
|
||||
// bool DoCoarseUpdate() const { return !coarseupdate; }
|
||||
|
||||
|
||||
|
||||
/// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert...
|
||||
void SetNV (int anv);
|
||||
void SetNE (int ane);
|
||||
void SetNSE (int anse);
|
||||
void SetNSegm (int anseg);
|
||||
|
||||
void SetNVGlob ( int anvglob ) { nvglob = anvglob; }
|
||||
void SetNEGlob ( int aneglob ) { neglob = aneglob; }
|
||||
void SetNSEGlob ( int anseglob ) { nseglob = anseglob; }
|
||||
|
||||
int GetNVGlob () { return nvglob; }
|
||||
int GetNEGlob () { return neglob; }
|
||||
void SetLoc2Glob_Vert (int locnum, int globnum) { glob_vert[locnum-1] = globnum; }
|
||||
void SetLoc2Glob_Edge (int locnum, int globnum) { glob_edge[locnum-1] = globnum; }
|
||||
void SetLoc2Glob_Face (int locnum, int globnum) { glob_face[locnum-1] = globnum; }
|
||||
void SetLoc2Glob_VolEl (int locnum, int globnum) { glob_el[locnum-1] = globnum; }
|
||||
void SetLoc2Glob_SurfEl (int locnum, int globnum) { glob_surfel[locnum-1] = globnum; }
|
||||
void SetLoc2Glob_Segm (int locnum, int globnum) { glob_segm[locnum-1] = globnum; }
|
||||
|
||||
int GetGlobalPNum (int locnum) const { return glob_vert[locnum-1]; }
|
||||
int GetGlobalEdgeNum (int locnum) const { return glob_edge[locnum-1]; }
|
||||
int GetGlobalFaceNum (int locnum) const { return glob_face[locnum-1]; }
|
||||
int GetGlobalElNum (int locnum) const { return glob_el[locnum-1]; }
|
||||
int GetGlobalSElNum (int locnum) const { return glob_surfel[locnum-1]; }
|
||||
|
||||
|
||||
void Reset ();
|
||||
void SetDistantFaceNum (int dest, int locnum);
|
||||
void SetDistantPNum (int dest, int locnum);
|
||||
void SetDistantEdgeNum (int dest, int locnum);
|
||||
|
||||
void SetLoc2Glob_Vert ( int locnum, int globnum ) { loc2distvert[locnum][0] = globnum; }
|
||||
void SetLoc2Glob_VolEl ( int locnum, int globnum ) { loc2distel[locnum-1][0] = globnum; }
|
||||
void SetLoc2Glob_SurfEl ( int locnum, int globnum ) { loc2distsurfel[locnum-1][0] = globnum; }
|
||||
void SetLoc2Glob_Segm ( int locnum, int globnum ) { loc2distsegm[locnum-1][0] = globnum; }
|
||||
int GetNDistantPNums (int locpnum) const { return loc2distvert[locpnum-1].Size(); }
|
||||
int GetNDistantFaceNums (int locfacenum) const { return loc2distface[locfacenum-1].Size(); }
|
||||
int GetNDistantEdgeNums ( int locedgenum) const { return loc2distedge[locedgenum-1].Size(); }
|
||||
|
||||
int GetLoc2Glob_Vert ( int locnum ) const { return loc2distvert[locnum][0]; }
|
||||
int GetLoc2Glob_VolEl ( int locnum ) const { return loc2distel[locnum-1][0]; }
|
||||
int GetLoc2Glob_SurfEl ( int locnum ) const { return loc2distsurfel[locnum-1][0]; }
|
||||
void GetDistantPNums (int locpnum, int * distpnums ) const
|
||||
{
|
||||
for (int i = 0; i < loc2distvert[locpnum-1].Size(); i++ )
|
||||
distpnums[i] = loc2distvert[locpnum-1][i];
|
||||
}
|
||||
|
||||
void GetVertNeighbours ( int vnum, Array<int> & dests ) const;
|
||||
void GetDistantFaceNums (int locfacenum, int * distfacenums ) const
|
||||
{
|
||||
for ( int i = 0; i < loc2distface[locfacenum-1].Size(); i++ )
|
||||
distfacenums[i] = loc2distface[locfacenum-1][i];
|
||||
}
|
||||
|
||||
|
||||
int GetNDistantPNums ( int locpnum ) const
|
||||
{ return loc2distvert[locpnum].Size() / 2 + 1; }
|
||||
|
||||
int GetNDistantFaceNums ( int locfacenum ) const
|
||||
{ return loc2distface[locfacenum-1].Size() / 2 + 1; }
|
||||
|
||||
int GetNDistantEdgeNums ( int locedgenum ) const
|
||||
{ return loc2distedge[locedgenum-1].Size() / 2 + 1; }
|
||||
|
||||
int GetNDistantElNums ( int locelnum ) const
|
||||
{ return loc2distel[locelnum-1].Size() / 2 + 1; }
|
||||
|
||||
int GetGlobalPNum ( int locpnum ) const
|
||||
{ return loc2distvert[locpnum][0]; }
|
||||
|
||||
int GetGlobalEdgeNum ( int locedgenum ) const
|
||||
{ return loc2distedge[locedgenum-1][0]; }
|
||||
|
||||
int GetGlobalFaceNum ( int locfacenum ) const
|
||||
{ return loc2distface[locfacenum-1][0]; }
|
||||
|
||||
int GetGlobalElNum ( int locelnum ) const
|
||||
{ return loc2distel[locelnum-1][0]; }
|
||||
|
||||
/*
|
||||
int GetDistantPNum ( int proc, int locpnum ) const;
|
||||
int GetDistantEdgeNum ( int proc, int locedgenum ) const;
|
||||
int GetDistantFaceNum ( int proc, int locedgenum ) const;
|
||||
int GetDistantElNum ( int proc, int locelnum ) const;
|
||||
*/
|
||||
|
||||
void GetDistantPNums ( int locpnum, int * distpnums ) const;
|
||||
void GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const;
|
||||
void GetDistantFaceNums ( int locedgenum, int * distfacenums ) const;
|
||||
void GetDistantElNums ( int locelnum, int * distfacenums ) const;
|
||||
|
||||
void Print() const;
|
||||
|
||||
|
||||
bool IsExchangeVert ( PointIndex vnum ) const { return loc2distvert[vnum].Size() > 1; }
|
||||
bool IsExchangeEdge ( int ednum ) const { return loc2distedge[ednum-1].Size() > 1; }
|
||||
bool IsExchangeFace ( int fnum ) const { return loc2distface[fnum-1].Size() > 1; }
|
||||
bool IsExchangeElement ( int elnum ) const { return false; }
|
||||
|
||||
|
||||
// bool IsExchangeSEl ( int selnum ) const { return loc2distsurfel[selnum-1].Size() > 1; }
|
||||
void GetDistantEdgeNums (int locedgenum, int * distedgenums ) const
|
||||
{
|
||||
for (int i = 0; i < loc2distedge[locedgenum-1].Size(); i++ )
|
||||
distedgenums[i] = loc2distedge[locedgenum-1][i];
|
||||
}
|
||||
|
||||
bool IsExchangeVert (int dest, int vnum) const
|
||||
{
|
||||
FlatArray<int> exchange = loc2distvert[vnum];
|
||||
for (int i = 1; i < exchange.Size(); i += 2)
|
||||
return loc2distvert[vnum-1].Contains (dest);
|
||||
/*
|
||||
FlatArray<int> exchange = loc2distvert[vnum-1];
|
||||
for (int i = 0; i < exchange.Size(); i ++)
|
||||
if (exchange[i] == dest) return true;
|
||||
return false;
|
||||
*/
|
||||
}
|
||||
|
||||
bool IsExchangeEdge (int dest, int ednum ) const
|
||||
{
|
||||
FlatArray<int> exchange = loc2distedge[ednum-1];
|
||||
for (int i = 1; i < exchange.Size(); i += 2)
|
||||
if (exchange[i] == dest) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsExchangeFace (int dest, int fnum ) const
|
||||
{
|
||||
FlatArray<int> exchange = loc2distface[fnum-1];
|
||||
for (int i = 1; i < exchange.Size(); i += 2)
|
||||
if (exchange[i] == dest) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// bool IsExchangeElement (int dest, int elnum ) const { return false; }
|
||||
|
||||
void Update();
|
||||
|
||||
void UpdateCoarseGrid();
|
||||
void UpdateExchangeElements();
|
||||
|
||||
void UpdateCoarseGridGlobal();
|
||||
|
||||
bool DoCoarseUpdate() const { return !coarseupdate; }
|
||||
|
||||
void SetDistantFaceNum ( int dest, int locnum, int distnum );
|
||||
void SetDistantPNum ( int dest, int locnum, int distnum );
|
||||
void SetDistantEdgeNum ( int dest, int locnum, int distnum );
|
||||
void SetDistantEl ( int dest, int locnum, int distnum );
|
||||
void SetDistantSurfEl ( int dest, int locnum, int distnum );
|
||||
void SetDistantSegm ( int dest, int locnum, int distnum );
|
||||
|
||||
bool IsGhostEl ( int elnum ) const { return mesh.VolumeElement(elnum).IsGhost(); }
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -73,7 +73,7 @@ namespace netgen
|
||||
NgProfiler::RegionTimer reg (timer);
|
||||
|
||||
#ifdef PARALLEL
|
||||
ParallelMeshTopology & paralleltop = mesh.GetParallelTopology();
|
||||
// ParallelMeshTopology & paralleltop = mesh.GetParallelTopology();
|
||||
#endif
|
||||
|
||||
|
||||
@ -399,8 +399,6 @@ namespace netgen
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
edge2vert.SetSize (ned);
|
||||
for (int i = 1; i <= ne; i++)
|
||||
{
|
||||
@ -484,8 +482,6 @@ namespace netgen
|
||||
<< surfedges.Elem(i)[1] << ", "
|
||||
<< surfedges.Elem(i)[2] << endl;
|
||||
*/
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -1077,8 +1073,8 @@ namespace netgen
|
||||
|
||||
|
||||
#ifdef PARALLEL
|
||||
(*testout) << " RESET Paralleltop" << endl;
|
||||
paralleltop.Reset ();
|
||||
// (*testout) << " RESET Paralleltop" << endl;
|
||||
// paralleltop.Reset ();
|
||||
#endif
|
||||
|
||||
Array<short int> face_els(nfa), face_surfels(nfa);
|
||||
@ -1111,7 +1107,8 @@ namespace netgen
|
||||
#ifdef PARALLEL
|
||||
if ( ntasks > 1 )
|
||||
{
|
||||
if ( !paralleltop.DoCoarseUpdate() ) continue;
|
||||
continue;
|
||||
// if ( !paralleltop.DoCoarseUpdate() ) continue;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
@ -1156,9 +1153,8 @@ namespace netgen
|
||||
#ifdef PARALLEL
|
||||
if (id != 0)
|
||||
{
|
||||
paralleltop.Update();
|
||||
if ( paralleltop.DoCoarseUpdate() )
|
||||
paralleltop.UpdateCoarseGrid();
|
||||
// if ( paralleltop.DoCoarseUpdate() )
|
||||
// paralleltop.UpdateCoarseGrid();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -1858,9 +1858,6 @@ namespace netgen
|
||||
if (vispar.drawmetispartition && el.GetPartition()!=-1)
|
||||
ind = el.GetPartition() % 4;
|
||||
|
||||
if ( el.IsGhost() )
|
||||
glMaterialfv(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE, tetcols_ghost[ind]);
|
||||
else
|
||||
glMaterialfv(GL_FRONT, GL_AMBIENT_AND_DIFFUSE, tetcols[ind]);
|
||||
|
||||
|
||||
|
||||
@ -827,10 +827,6 @@ namespace netgen
|
||||
{
|
||||
const Element2d & el = (*mesh)[sei];
|
||||
|
||||
#ifdef PARALLEL
|
||||
// parallel visualization --> dont draw ghost elements
|
||||
if ( el . IsGhost() ) continue;
|
||||
#endif
|
||||
bool curved = curv.IsHighOrder(); // && curv.IsSurfaceElementCurved(sei);
|
||||
|
||||
if (el.GetType() == TRIG || el.GetType() == TRIG6)
|
||||
@ -1064,8 +1060,6 @@ namespace netgen
|
||||
{
|
||||
const Element2d & el = (*mesh)[sei];
|
||||
|
||||
if ( el . IsGhost() ) continue;
|
||||
|
||||
if (vispar.drawdomainsurf > 0)
|
||||
{
|
||||
if (mesh->GetDimension() == 3)
|
||||
@ -1197,8 +1191,6 @@ namespace netgen
|
||||
{
|
||||
const Element2d & el = (*mesh)[sei];
|
||||
|
||||
if ( el . IsGhost() ) continue;
|
||||
|
||||
if(vispar.drawdomainsurf > 0)
|
||||
{
|
||||
if (mesh->GetDimension() == 3)
|
||||
@ -1431,8 +1423,6 @@ namespace netgen
|
||||
{
|
||||
Element2d & el = (*mesh)[sei];
|
||||
|
||||
if ( el . IsGhost() ) continue;
|
||||
|
||||
// bool curved = curv.IsSurfaceElementCurved (sei);
|
||||
|
||||
int nv = (el.GetType() == TRIG || el.GetType() == TRIG6) ? 3 : 4;
|
||||
@ -1559,8 +1549,6 @@ namespace netgen
|
||||
// if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue;
|
||||
// if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue;
|
||||
|
||||
if ( (*mesh)[ei] . IsGhost() ) continue;
|
||||
|
||||
ELEMENT_TYPE type = (*mesh)[ei].GetType();
|
||||
if (type == HEX || type == PRISM || type == TET || type == PYRAMID)
|
||||
{
|
||||
@ -1905,8 +1893,6 @@ namespace netgen
|
||||
{
|
||||
const Element2d & el = (*mesh)[sei];
|
||||
|
||||
if ( el . IsGhost() ) continue;
|
||||
|
||||
if (el.GetType() == TRIG || el.GetType() == TRIG6)
|
||||
{
|
||||
|
||||
@ -3499,11 +3485,6 @@ namespace netgen
|
||||
{
|
||||
int first_point_of_element = pts.Size();
|
||||
|
||||
#ifdef PARALLEL
|
||||
// parallel visualization --> dont draw ghost elements
|
||||
if ( (*mesh)[ei] . IsGhost() ) continue;
|
||||
#endif
|
||||
|
||||
locgrid.SetSize(n3);
|
||||
if(vispar.clipdomain > 0 && vispar.clipdomain != (*mesh)[ei].GetIndex()) continue;
|
||||
if(vispar.donotclipdomain > 0 && vispar.donotclipdomain == (*mesh)[ei].GetIndex()) continue;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user