ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-26 22:00:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

improved parallel mesh loading

Browse Source
This commit is contained in:
Joachim Schoeberl 2011-07-13 18:32:11 +00:00
parent 70292000c3
commit 3aeb3d79b9
4 changed files with 239 additions and 397 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libsrc/general/mpi_interface.hpp
View File

@ -237,7 +237,7 @@ namespace netgen
inline void MyMPI_Send ( int *& s, int len, int dest, int tag) inline void MyMPI_Send ( int *& s, int len, int dest, int tag)
{ {
int hlen = len; int hlen = len;
MPI_Send( &hen, 1, MPI_INT, dest, tag, MPI_COMM_WORLD); MPI_Send( &hlen, 1, MPI_INT, dest, tag, MPI_COMM_WORLD);
MPI_Send( s, len, MPI_INT, dest, tag, MPI_COMM_WORLD); MPI_Send( s, len, MPI_INT, dest, tag, MPI_COMM_WORLD);
} }

388
libsrc/meshing/parallelmesh.cpp
View File

@ -25,6 +25,8 @@ namespace netgen
void Mesh :: SendRecvMesh () void Mesh :: SendRecvMesh ()
{ {
PrintMessage (1, "Send/Receive mesh"); PrintMessage (1, "Send/Receive mesh");
@ -55,6 +57,7 @@ namespace netgen
} }
{ {
// distribute number of local elements // distribute number of local elements
if (id == 0) if (id == 0)
@ -81,10 +84,17 @@ namespace netgen
if (id == 0) if (id == 0)
SendMesh (); SendMesh ();
else else
ReceiveParallelMesh(); ReceiveParallelMesh();
MPI_Barrier (MPI_COMM_WORLD);
} }
@ -94,34 +104,21 @@ namespace netgen
int timer = NgProfiler::CreateTimer ("ReceiveParallelMesh"); int timer = NgProfiler::CreateTimer ("ReceiveParallelMesh");
NgProfiler::RegionTimer reg(timer); NgProfiler::RegionTimer reg(timer);
int timer_pts = NgProfiler::CreateTimer ("Receive Points");
int timer_els = NgProfiler::CreateTimer ("Receive Elements");
int timer_sels = NgProfiler::CreateTimer ("Receive Surface Elements");
int timer_edges = NgProfiler::CreateTimer ("Receive Edges");
#ifdef SCALASCA #ifdef SCALASCA
#pragma pomp inst begin(loadmesh) #pragma pomp inst begin(loadmesh)
#endif #endif
string st; string st;
bool endmesh = false;
INDEX_CLOSED_HASHTABLE<int> glob2loc_vert_ht (1); INDEX_CLOSED_HASHTABLE<int> glob2loc_vert_ht (1);
// int ve = 0;
while (!endmesh)
{
MyMPI_Recv (st, 0, MPI_TAG_MESH);
// receive vertices
if (st == "vertex")
{ {
NgProfiler::RegionTimer reg(timer_pts); // receive vertices
Array<int> verts; Array<int> verts;
MyMPI_Recv (verts, 0, MPI_TAG_MESH); MyMPI_Recv (verts, 0);
int numvert = verts.Size(); int numvert = verts.Size();
paralleltop -> SetNV (numvert); paralleltop -> SetNV (numvert);
@ -140,10 +137,10 @@ namespace netgen
MPI_Datatype mptype = MeshPoint::MyGetMPIType(); MPI_Datatype mptype = MeshPoint::MyGetMPIType();
MPI_Status status; MPI_Status status;
MPI_Recv( &points[1], numvert, mptype, 0, MPI_TAG_MESH, MPI_COMM_WORLD, &status); MPI_Recv( &points[1], numvert, mptype, 0, 33, MPI_COMM_WORLD, &status);
Array<int> dist_pnums; Array<int> dist_pnums;
MyMPI_Recv (dist_pnums, 0, MPI_TAG_MESH); MyMPI_Recv (dist_pnums, 0);
for (int hi = 0; hi < dist_pnums.Size(); hi += 3) for (int hi = 0; hi < dist_pnums.Size(); hi += 3)
paralleltop -> paralleltop ->
@ -152,16 +149,13 @@ namespace netgen
*testout << "got " << numvert << " vertices" << endl; *testout << "got " << numvert << " vertices" << endl;
} }
if (strcmp (st.c_str(), "volumeelements" ) == 0 )
{ {
NgProfiler::RegionTimer reg(timer_els);
*testout << "receiving elements" << endl;
Element el; Element el;
Array<int> elarray; Array<int> elarray;
MyMPI_Recv (elarray, 0, MPI_TAG_MESH); MyMPI_Recv (elarray, 0);
for (int ind = 0, elnum = 1; ind < elarray.Size(); elnum++) for (int ind = 0, elnum = 1; ind < elarray.Size(); elnum++)
{ {
@ -178,57 +172,42 @@ namespace netgen
} }
if (strcmp (st.c_str(), "facedescriptor") == 0)
{ {
Array<double> doublebuf; Array<double> fddata;
MyMPI_Recv( doublebuf, 0, MPI_TAG_MESH ); MyMPI_Recv (fddata, 0);
int faceind = AddFaceDescriptor (FaceDescriptor(int(doublebuf[0]), int(doublebuf[1]), int(doublebuf[2]), 0)); for (int i = 0; i < fddata.Size(); i += 6)
GetFaceDescriptor(faceind).SetBCProperty (int(doublebuf[3])); {
GetFaceDescriptor(faceind).domin_singular = doublebuf[4]; int faceind = AddFaceDescriptor (FaceDescriptor(int(fddata[i]), int(fddata[i+1]), int(fddata[i+2]), 0));
GetFaceDescriptor(faceind).domout_singular = doublebuf[5]; GetFaceDescriptor(faceind).SetBCProperty (int(fddata[i+3]));
GetFaceDescriptor(faceind).domin_singular = fddata[i+4];
GetFaceDescriptor(faceind).domout_singular = fddata[i+5];
}
} }
if (strcmp (st.c_str(), "surfaceelementsgi") == 0)
{ {
NgProfiler::RegionTimer reg(timer_sels); // surface elements
int j; Array<int> selbuf;
int nep, faceind = 0;
int globsel; MyMPI_Recv ( selbuf, 0);
int * selbuf;
selbuf = 0;
int bufsize;
// receive:
// faceindex
// nep
// tri.pnum
// tri.geominfopi.trignum
int nlocsel;
MyMPI_Recv ( selbuf, bufsize, 0, MPI_TAG_MESH);
int ii = 0; int ii = 0;
int sel = 0; int sel = 0;
nlocsel = selbuf[ii++]; int nlocsel = selbuf[ii++];
paralleltop -> SetNSE ( nlocsel ); paralleltop -> SetNSE ( nlocsel );
while ( ii < bufsize-1 ) while (ii < selbuf.Size()-1)
{ {
globsel = selbuf[ii++]; int globsel = selbuf[ii++];
faceind = selbuf[ii++]; int faceind = selbuf[ii++];
bool isghost = selbuf[ii++]; bool isghost = selbuf[ii++];
nep = selbuf[ii++]; int nep = selbuf[ii++];
Element2d tri(nep); Element2d tri(nep);
tri.SetIndex(faceind); tri.SetIndex(faceind);
for( j=1; j<=nep; j++) for(int j = 1; j <= nep; j++)
{ {
tri.PNum(j) = glob2loc_vert_ht.Get (selbuf[ii++]); tri.PNum(j) = glob2loc_vert_ht.Get (selbuf[ii++]);
// tri.GeomInfoPi(j).trignum = paralleltop->Glob2Loc_SurfEl(selbuf[ii++]);
tri.GeomInfoPi(j).trignum = selbuf[ii++]; tri.GeomInfoPi(j).trignum = selbuf[ii++];
// Frage JS->AS: Brauchst du die trignum irgendwo ?
// sie sonst nur bei STL - Geometrien benötigt
// die Umrechnung war ein bottleneck !
} }
tri.SetGhost(isghost); tri.SetGhost(isghost);
@ -237,26 +216,21 @@ namespace netgen
AddSurfaceElement (tri); AddSurfaceElement (tri);
sel ++; sel ++;
} }
delete [] selbuf ;
} }
if (strcmp (st.c_str(), "edgesegmentsgi") == 0)
{
NgProfiler::RegionTimer reg(timer_edges);
int endtime, starttime;
starttime = clock();
double * segmbuf = 0;
int bufsize; {
MyMPI_Recv ( segmbuf, bufsize, 0, MPI_TAG_MESH); Array<double> segmbuf;
MyMPI_Recv ( segmbuf, 0);
Segment seg; Segment seg;
int globsegi; int globsegi;
int ii = 0; int ii = 0;
int segi = 1; int segi = 1;
int nsegloc = int ( bufsize / 14 ) ; int nsegloc = int ( segmbuf.Size() / 14 ) ;
paralleltop -> SetNSegm ( nsegloc ); paralleltop -> SetNSegm ( nsegloc );
while ( ii < bufsize ) while ( ii < segmbuf.Size() )
{ {
globsegi = int (segmbuf[ii++]); globsegi = int (segmbuf[ii++]);
seg.si = int (segmbuf[ii++]); seg.si = int (segmbuf[ii++]);
@ -288,16 +262,6 @@ namespace netgen
} }
} }
delete [] segmbuf;
endtime = clock();
(*testout) << "Receiving Time fde = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
}
if (strcmp (st.c_str(), "endmesh") == 0)
{
endmesh = true;
}
} }
@ -365,7 +329,7 @@ namespace netgen
// send partition // send partition
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
MyMPI_Send ("mesh", dest, MPI_TAG_CMD); MyMPI_Send ("mesh", dest, MPI_TAG_MESH);
SendRecvMesh (); SendRecvMesh ();
@ -813,12 +777,6 @@ namespace netgen
void Mesh :: SendMesh () const void Mesh :: SendMesh () const
{ {
const Mesh * mastermesh = this; // the original plan was different
clock_t starttime, endtime;
starttime = clock();
MPI_Request sendrequest[ntasks]; MPI_Request sendrequest[ntasks];
Array<int> num_els_on_proc(ntasks); Array<int> num_els_on_proc(ntasks);
@ -899,7 +857,7 @@ namespace netgen
} }
} }
for (int vert = 1; vert <= mastermesh->GetNP(); vert++ ) for (int vert = 1; vert <= GetNP(); vert++ )
{ {
FlatArray<int> procs = procs_of_vert[vert]; FlatArray<int> procs = procs_of_vert[vert];
for (int j = 0; j < procs.Size(); j++) for (int j = 0; j < procs.Size(); j++)
@ -913,8 +871,6 @@ namespace netgen
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
{ {
FlatArray<PointIndex> verts = verts_of_proc[dest]; FlatArray<PointIndex> verts = verts_of_proc[dest];
MyMPI_Send ("vertex", dest, MPI_TAG_MESH);
MyMPI_ISend (verts, dest, MPI_TAG_MESH); MyMPI_ISend (verts, dest, MPI_TAG_MESH);
MPI_Datatype mptype = MeshPoint::MyGetMPIType(); MPI_Datatype mptype = MeshPoint::MyGetMPIType();
@ -931,7 +887,7 @@ namespace netgen
MPI_Type_commit (&newtype); MPI_Type_commit (&newtype);
MPI_Request request; MPI_Request request;
MPI_Isend( &points[0], 1, newtype, dest, MPI_TAG_MESH, MPI_COMM_WORLD, &request); MPI_Isend( &points[0], 1, newtype, dest, 33, MPI_COMM_WORLD, &request);
MPI_Request_free (&request); MPI_Request_free (&request);
} }
@ -940,7 +896,7 @@ namespace netgen
Array<int> num_distpnums(ntasks); Array<int> num_distpnums(ntasks);
num_distpnums = 0; num_distpnums = 0;
for (int vert = 1; vert <= mastermesh -> GetNP(); vert++) for (int vert = 1; vert <= GetNP(); vert++)
{ {
FlatArray<int> procs = procs_of_vert[vert]; FlatArray<int> procs = procs_of_vert[vert];
for (int j = 0; j < procs.Size(); j++) for (int j = 0; j < procs.Size(); j++)
@ -949,7 +905,7 @@ namespace netgen
TABLE<int> distpnums (num_distpnums); TABLE<int> distpnums (num_distpnums);
for (int vert = 1; vert <= mastermesh -> GetNP(); vert++) for (int vert = 1; vert <= GetNP(); vert++)
{ {
FlatArray<int> procs = procs_of_vert[vert]; FlatArray<int> procs = procs_of_vert[vert];
for (int j = 0; j < procs.Size(); j++) for (int j = 0; j < procs.Size(); j++)
@ -963,35 +919,25 @@ namespace netgen
} }
for ( int dest = 1; dest < ntasks; dest ++ ) for ( int dest = 1; dest < ntasks; dest ++ )
{ MyMPI_ISend (distpnums[dest], dest, MPI_TAG_MESH);
MyMPI_ISend ( distpnums[dest], dest, MPI_TAG_MESH, sendrequest[dest] );
MPI_Request_free (&sendrequest[dest]);
}
endtime = clock();
(*testout) << "Sending Time verts = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
PrintMessage ( 3, "Sending elements" ); PrintMessage ( 3, "Sending elements" );
starttime = clock();
Array<int> elarraysize (ntasks); Array<int> elarraysize (ntasks);
elarraysize = 0; elarraysize = 0;
for ( int ei = 1; ei <= mastermesh->GetNE(); ei++) for ( int ei = 1; ei <= GetNE(); ei++)
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
int dest = el.GetPartition(); int dest = el.GetPartition();
elarraysize[dest] += 3 + el.GetNP(); elarraysize[dest] += 3 + el.GetNP();
} }
TABLE<int> elementarrays(elarraysize); TABLE<int> elementarrays(elarraysize);
for ( int ei = 1; ei <= mastermesh->GetNE(); ei++) for (int ei = 1; ei <= GetNE(); ei++)
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
int dest = el.GetPartition(); int dest = el.GetPartition();
elementarrays.Add (dest, ei); elementarrays.Add (dest, ei);
@ -1002,163 +948,102 @@ namespace netgen
} }
for (int dest = 1; dest < ntasks; dest ++ ) for (int dest = 1; dest < ntasks; dest ++ )
{ MyMPI_ISend (elementarrays[dest], dest, MPI_TAG_MESH);
MyMPI_Send ( "volumeelements", dest, MPI_TAG_MESH);
MyMPI_ISend ( elementarrays[dest], dest, MPI_TAG_MESH, sendrequest[dest] );
}
endtime = clock();
(*testout) << "Sending Time els = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
starttime = clock();
PrintMessage ( 3, "Sending Face Descriptors" ); PrintMessage ( 3, "Sending Face Descriptors" );
Array<double> double6(6); Array<double> fddata (6 * GetNFD());
for ( int dest = 1; dest < ntasks; dest++) for (int fdi = 1; fdi <= GetNFD(); fdi++)
for ( int fdi = 1; fdi <= mastermesh->GetNFD(); fdi++)
{ {
MyMPI_Send("facedescriptor", dest, MPI_TAG_MESH); fddata[6*fdi-6] = GetFaceDescriptor(fdi).SurfNr();
fddata[6*fdi-5] = GetFaceDescriptor(fdi).DomainIn();
fddata[6*fdi-4] = GetFaceDescriptor(fdi).DomainOut();
fddata[6*fdi-3] = GetFaceDescriptor(fdi).BCProperty();
fddata[6*fdi-2] = GetFaceDescriptor(fdi).domin_singular;
fddata[6*fdi-1] = GetFaceDescriptor(fdi).domout_singular;
double6[0] = GetFaceDescriptor(fdi).SurfNr();
double6[1] = GetFaceDescriptor(fdi).DomainIn();
double6[2] = GetFaceDescriptor(fdi).DomainOut();
double6[3] = GetFaceDescriptor(fdi).BCProperty();
double6[4] = GetFaceDescriptor(fdi).domin_singular;
double6[5] = GetFaceDescriptor(fdi).domout_singular;
MyMPI_Send ( double6, dest, MPI_TAG_MESH);
} }
for (int dest = 1; dest < ntasks; dest++)
endtime = clock(); MyMPI_ISend (fddata, dest, MPI_TAG_MESH);
(*testout) << "Sending Time fdi = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
starttime = clock();
PrintMessage ( 3, "Sending Surface elements" ); PrintMessage ( 3, "Sending Surface elements" );
Array <int> nlocsel(ntasks), bufsize ( ntasks), seli(ntasks); Array <int> nlocsel(ntasks), bufsize(ntasks); // , seli(ntasks);
for ( int i = 0; i < ntasks; i++) for ( int i = 0; i < ntasks; i++)
{ {
nlocsel[i] = 0; nlocsel[i] = 0;
bufsize[i] = 1; bufsize[i] = 1;
seli[i] = 1;
} }
for ( int sei = 1; sei <= mastermesh -> GetNSE(); sei ++ ) for (int sei = 1; sei <= GetNSE(); sei++ )
{ {
int ei1, ei2; int ei1, ei2;
mastermesh -> GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); GetTopology().GetSurface2VolumeElement (sei, ei1, ei2);
const Element2d & sel = mastermesh -> SurfaceElement (sei); const Element2d & sel = SurfaceElement (sei);
int dest;
// first element
for (int j = 0; j < 2; j++) for (int j = 0; j < 2; j++)
{ {
int ei = (j == 0) ? ei1 : ei2; int ei = (j == 0) ? ei1 : ei2;
if ( ei > 0 && ei <= GetNE() )
if ( ei > 0 && ei <= mastermesh->GetNE() )
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
dest = el.GetPartition(); int dest = el.GetPartition();
nlocsel[dest] ++; nlocsel[dest] ++;
bufsize[dest] += 4 + 2*sel.GetNP(); bufsize[dest] += 4 + 2*sel.GetNP();
} }
} }
} }
int ** selbuf = 0; TABLE<int> selbuf(bufsize);
selbuf = new int*[ntasks];
for ( int i = 0; i < ntasks; i++)
if ( bufsize[i] > 0 )
{*(selbuf+i) = new int[bufsize[i]];}
else
selbuf[i] = 0;
Array<int> nselloc (ntasks); Array<int> nselloc (ntasks);
nselloc = 0; nselloc = 0;
for (int dest = 1; dest < ntasks; dest++ ) for (int dest = 1; dest < ntasks; dest++ )
{ selbuf.Add (dest, nlocsel[dest]);
MyMPI_Send ( "surfaceelementsgi", dest, MPI_TAG_MESH);
selbuf[dest][0] = nlocsel[dest];
}
for ( int sei = 1; sei <= mastermesh -> GetNSE(); sei ++ ) for (int sei = 1; sei <= GetNSE(); sei ++ )
{ {
int ei1, ei2; int ei1, ei2;
mastermesh -> GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); GetTopology().GetSurface2VolumeElement (sei, ei1, ei2);
const Element2d & sel = mastermesh -> SurfaceElement (sei); const Element2d & sel = SurfaceElement (sei);
int dest;
int isghost = 0; int isghost = 0;
for (int j = 0; j < 2; j++) for (int j = 0; j < 2; j++)
{ {
int ei = (j == 0) ? ei1 : ei2; int ei = (j == 0) ? ei1 : ei2;
if ( ei > 0 && ei <= GetNE() )
if ( ei > 0 && ei <= mastermesh->GetNE() )
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
dest = el.GetPartition(); int dest = el.GetPartition();
if (dest > 0) if (dest > 0)
{ {
// send: selbuf.Add (dest, sei);
// sei selbuf.Add (dest, sel.GetIndex());
// faceind selbuf.Add (dest, isghost);
// nep selbuf.Add (dest, sel.GetNP());
// tri.pnums, tri.geominfopi.trignums
selbuf[dest][seli[dest]++] = sei;
selbuf[dest][seli[dest]++] = sel.GetIndex();
selbuf[dest][seli[dest]++] = isghost;
selbuf[dest][seli[dest]++] = sel.GetNP();
for ( int ii = 1; ii <= sel.GetNP(); ii++) for ( int ii = 1; ii <= sel.GetNP(); ii++)
{ {
selbuf[dest][seli[dest]++] = sel.PNum(ii); selbuf.Add (dest, sel.PNum(ii));
selbuf[dest][seli[dest]++] = sel.GeomInfoPi(ii).trignum; selbuf.Add (dest, sel.GeomInfoPi(ii).trignum);
} }
nselloc[dest] ++; nselloc[dest] ++;
paralleltop -> SetDistantSurfEl ( dest, sei, nselloc[dest] ); paralleltop -> SetDistantSurfEl ( dest, sei, nselloc[dest] );
isghost = 1; isghost = 1;
} }
} }
} }
} }
for (int dest = 1; dest < ntasks; dest++) for (int dest = 1; dest < ntasks; dest++)
MyMPI_Send( selbuf[dest], bufsize[dest], dest, MPI_TAG_MESH); MyMPI_ISend(selbuf[dest], dest, MPI_TAG_MESH);
for ( int dest = 0; dest < ntasks; dest++ )
{
if (selbuf[dest])
delete [] *(selbuf+dest);
}
delete [] selbuf;
endtime = clock();
(*testout) << "Sending Time surfels = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
starttime = clock();
PrintMessage ( 3, "Sending Edge Segments"); PrintMessage ( 3, "Sending Edge Segments");
for ( int dest = 1; dest < ntasks; dest++ )
MyMPI_Send ( "edgesegmentsgi", dest, MPI_TAG_MESH);
Array <int> nlocseg(ntasks), segi(ntasks); Array <int> nlocseg(ntasks), segi(ntasks);
for ( int i = 0; i < ntasks; i++) for ( int i = 0; i < ntasks; i++)
@ -1168,106 +1053,67 @@ namespace netgen
segi[i] = 0; segi[i] = 0;
} }
for ( int segi = 1; segi <= mastermesh -> GetNSeg(); segi ++ ) for (int segi = 1; segi <= GetNSeg(); segi ++ )
{ {
Array<int> volels; Array<int> volels;
const MeshTopology & topol = mastermesh -> GetTopology(); const MeshTopology & topol = GetTopology();
topol . GetSegmentVolumeElements ( segi, volels ); topol . GetSegmentVolumeElements ( segi, volels );
// const Segment & segm = mastermesh -> LineSegment (segi);
// int dest;
for (int j = 0; j < volels.Size(); j++) for (int j = 0; j < volels.Size(); j++)
{ {
int ei = volels[j]; int ei = volels[j];
int dest; if ( ei > 0 && ei <= GetNE() )
if ( ei > 0 && ei <= mastermesh->GetNE() )
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
dest = el.GetPartition(); int dest = el.GetPartition();
nlocseg[dest] ++; nlocseg[dest] ++;
bufsize[dest] += 14; bufsize[dest] += 14;
} }
} }
} }
TABLE<double> segmbuf(bufsize);
double ** segmbuf; for ( int ls=1; ls <= GetNSeg(); ls++)
segmbuf = new double*[ntasks];
for ( int i = 0; i < ntasks; i++)
if ( bufsize[i] > 0 )
segmbuf[i] = new double[bufsize[i]];
else
segmbuf[i] = 0;
// cout << "mastermesh " << mastermesh -> GetNSeg() << " lineseg " << mastermesh -> LineSegment (1) << endl;
for ( int ls=1; ls <= mastermesh -> GetNSeg(); ls++)
{ {
Array<int> volels; Array<int> volels;
mastermesh -> GetTopology().GetSegmentVolumeElements ( ls, volels ); GetTopology().GetSegmentVolumeElements ( ls, volels );
const Segment & seg = mastermesh -> LineSegment (ls); const Segment & seg = LineSegment (ls);
for (int j = 0; j < volels.Size(); j++) for (int j = 0; j < volels.Size(); j++)
{ {
int ei = volels[j]; int ei = volels[j];
int dest; if ( ei > 0 && ei <= GetNE() )
if ( ei > 0 && ei <= mastermesh->GetNE() )
{ {
const Element & el = mastermesh -> VolumeElement (ei); const Element & el = VolumeElement (ei);
dest = el.GetPartition(); int dest = el.GetPartition();
if ( dest > 0 ) if ( dest > 0 )
{ {
segmbuf[dest][segi[dest]++] = ls; segmbuf.Add (dest, ls);
segmbuf[dest][segi[dest]++] = seg.si; segmbuf.Add (dest, seg.si);
segmbuf[dest][segi[dest]++] = seg.pnums[0]; segmbuf.Add (dest, seg.pnums[0]);
segmbuf[dest][segi[dest]++] = seg.pnums[1]; segmbuf.Add (dest, seg.pnums[1]);
segmbuf[dest][segi[dest]++] = seg.geominfo[0].trignum; segmbuf.Add (dest, seg.geominfo[0].trignum);
segmbuf[dest][segi[dest]++] = seg.geominfo[1].trignum; segmbuf.Add (dest, seg.geominfo[1].trignum);
segmbuf[dest][segi[dest]++] = seg.surfnr1; segmbuf.Add (dest, seg.surfnr1);
segmbuf[dest][segi[dest]++] = seg.surfnr2; segmbuf.Add (dest, seg.surfnr2);
segmbuf[dest][segi[dest]++] = seg.edgenr; segmbuf.Add (dest, seg.edgenr);
segmbuf[dest][segi[dest]++] = seg.epgeominfo[0].dist; segmbuf.Add (dest, seg.epgeominfo[0].dist);
segmbuf[dest][segi[dest]++] = seg.epgeominfo[1].edgenr; segmbuf.Add (dest, seg.epgeominfo[1].edgenr);
segmbuf[dest][segi[dest]++] = seg.epgeominfo[1].dist; segmbuf.Add (dest, seg.epgeominfo[1].dist);
segmbuf[dest][segi[dest]++] = seg.singedge_right; segmbuf.Add (dest, seg.singedge_right);
segmbuf[dest][segi[dest]++] = seg.singedge_left; segmbuf.Add (dest, seg.singedge_left);
segi[dest] += 14;
} }
paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) ); paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) );
} }
} }
} }
PrintMessage ( 3, "sending segments" );
for ( int dest = 1; dest < ntasks; dest++) for ( int dest = 1; dest < ntasks; dest++)
{ MyMPI_ISend(segmbuf[dest], dest, MPI_TAG_MESH, sendrequest[dest]);
MyMPI_Send( segmbuf[dest], bufsize[dest], dest, MPI_TAG_MESH);
}
for ( int dest = 0; dest < ntasks; dest++ ) MPI_Waitall (ntasks-1, &sendrequest[1], MPI_STATUS_IGNORE);
{
if ( segmbuf[dest] )
delete [] segmbuf[dest];
}
delete [] segmbuf;
PrintMessage ( 3, "segments sent");
endtime = clock();
(*testout) << "Sending Time segments = " << double(endtime - starttime)/CLOCKS_PER_SEC << endl;
starttime = clock();
for ( int dest = 1; dest < ntasks; dest++ )
MyMPI_Send("endmesh", dest, MPI_TAG_MESH);
for ( int dest = 1; dest < ntasks; dest ++ )
{
MPI_Status status;
MPI_Wait (&sendrequest[dest], &status);
}
} }
@ -1853,7 +1699,7 @@ namespace netgen
{ {
(*globelnums)[i] = paralleltop -> GetLoc2Glob_VolEl ( i+1 ); (*globelnums)[i] = paralleltop -> GetLoc2Glob_VolEl ( i+1 );
} }
MyMPI_Send ( *globelnums, 0 ); MyMPI_Send ( *globelnums, 0, MPI_TAG_MESH );
} }
delete globelnums; delete globelnums;

5
libsrc/meshing/paralleltop.cpp
View File

@ -565,11 +565,6 @@ namespace netgen
{ {
PrintMessage ( 1, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS PrintMessage ( 1, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS
// MPI_Barrier (MPI_COMM_WORLD);
// PrintMessage ( 1, "all friends are here " ); // JS
// MPI_Barrier (MPI_COMM_WORLD);
MPI_Group MPI_GROUP_WORLD; MPI_Group MPI_GROUP_WORLD;

5
libsrc/meshing/topology.cpp
View File

@ -1194,11 +1194,12 @@ namespace netgen
cout << cnt_err << " elements are not matching !!!" << endl; cout << cnt_err << " elements are not matching !!!" << endl;
else if (cnt_err && ntasks > 1) else if (cnt_err && ntasks > 1)
{ {
cout << "p" << id << ": " << cnt_err << " elements are not local" << endl; // cout << "p" << id << ": " << cnt_err << " elements are not local" << endl;
isparallel = 1; isparallel = 1;
} }
else if ( ntasks > 1 ) else if ( ntasks > 1 )
cout << "p" << id << ": " << "Partition " << id << " is totally local" << endl; ;
// cout << "p" << id << ": " << "Partition " << id << " is totally local" << endl;
#endif #endif
} }