From 342983ee75519224142e327b3dd541a7c482f089 Mon Sep 17 00:00:00 2001 From: Joachim Schoeberl Date: Thu, 14 Jul 2011 21:36:19 +0000 Subject: [PATCH] parallel topology simplification --- libsrc/general/hashtabl.cpp | 3 +- libsrc/general/hashtabl.hpp | 24 +- libsrc/general/mpi_interface.hpp | 71 +- libsrc/general/profiler.cpp | 4 +- libsrc/general/table.cpp | 23 +- libsrc/general/table.hpp | 10 + libsrc/interface/nginterface.cpp | 39 +- libsrc/meshing/clusters.cpp | 4 +- libsrc/meshing/parallelmesh.cpp | 1458 ++++++++------------------- libsrc/meshing/paralleltop.cpp | 962 ++++-------------- libsrc/meshing/paralleltop.hpp | 165 +-- libsrc/meshing/topology.cpp | 18 +- libsrc/visualization/mvdraw.cpp | 21 +- libsrc/visualization/vsmesh.cpp | 18 +- libsrc/visualization/vssolution.cpp | 56 +- ng/ng.tcl | 20 +- ng/ngappinit.cpp | 9 +- ng/ngpkg.cpp | 3 +- ng/parallelfunc.cpp | 14 +- ng/parallelinterface.cpp | 5 +- 20 files changed, 907 insertions(+), 2020 deletions(-) diff --git a/libsrc/general/hashtabl.cpp b/libsrc/general/hashtabl.cpp index d5b77f3c..53d7eafb 100644 --- a/libsrc/general/hashtabl.cpp +++ b/libsrc/general/hashtabl.cpp @@ -61,8 +61,7 @@ namespace netgen int BASE_INDEX_HASHTABLE :: Position (int bnr, const INDEX & ind) const { - int i; - for (i = 1; i <= hash.EntrySize (bnr); i++) + for (int i = 1; i <= hash.EntrySize (bnr); i++) if (hash.Get(bnr, i) == ind) return i; return 0; diff --git a/libsrc/general/hashtabl.hpp b/libsrc/general/hashtabl.hpp index a705d399..bd825182 100644 --- a/libsrc/general/hashtabl.hpp +++ b/libsrc/general/hashtabl.hpp @@ -18,7 +18,7 @@ class BASE_INDEX_HASHTABLE { protected: /// keys are stored in this table - TABLE hash; + TABLE hash; public: /// @@ -41,9 +41,9 @@ template class INDEX_HASHTABLE : private BASE_INDEX_HASHTABLE { /// - TABLE cont; + TABLE cont; -public: +public: /// inline INDEX_HASHTABLE (int size); /// @@ -483,7 +483,7 @@ public: /// int HashValue (const INDEX & ind) const { - return ind % hash.Size() + 1; + return (3*ind) % hash.Size() + 1; } @@ -499,6 +499,22 @@ public: } } + int CalcPositionCosts (const INDEX & ind) const + { + int i = HashValue(ind); + int costs = 1; + while (1) + { + if (hash.Get(i) == ind) return costs; + if (hash.Get(i) == invalid) return costs; + i++; + if (i > hash.Size()) i = 1; + costs++; + } + } + + + // returns 1, if new postion is created int PositionCreate (const INDEX & ind, int & apos) { diff --git a/libsrc/general/mpi_interface.hpp b/libsrc/general/mpi_interface.hpp index e7d30d59..fbe02bed 100644 --- a/libsrc/general/mpi_interface.hpp +++ b/libsrc/general/mpi_interface.hpp @@ -36,9 +36,7 @@ namespace netgen { return MPI_DOUBLE; } - - // damit gehen auch echte Konstante ohne Adresse inline void MyMPI_Send (int i, int dest, int tag) { int hi = i; @@ -114,7 +112,7 @@ namespace netgen } - + /* template inline void MyMPI_ISend (FlatArray s, int dest, int tag, MPI_Request & request) { @@ -127,44 +125,26 @@ namespace netgen { MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, & request); } - - /* - template - inline void MyMPI_ISendTag (FlatArray s, int dest, int tag, MPI_Request & request) - { - MPI_Isend( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, & request); - } - - - template - inline void MyMPI_IRecvTag (FlatArray s, int dest, int tag, MPI_Request & request) - { - MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, & request); - } */ + template + inline MPI_Request MyMPI_ISend (FlatArray s, int dest, int tag, MPI_Comm comm = MPI_COMM_WORLD) + { + MPI_Request request; + MPI_Isend( &s.First(), s.Size(), MyGetMPIType(), dest, tag, comm, &request); + return request; + } + + + template + inline MPI_Request MyMPI_IRecv (FlatArray s, int dest, int tag, MPI_Comm comm = MPI_COMM_WORLD) + { + MPI_Request request; + MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, comm, &request); + return request; + } /* - template - inline MPI_Request MyMPI_ISend (FlatArray s, int dest) - { - MPI_Request request; - MPI_Isend( &s.First(), s.Size(), MyGetMPIType(), dest, 1, MPI_COMM_WORLD, &request); - return request; - // MPI_Request_free (&request); - } - - - template - inline MPI_Request MyMPI_IRecv (FlatArray s, int dest) - { - MPI_Request request; - MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, 1, MPI_COMM_WORLD, &request); - return request; - // MPI_Request_free (&request); - } - */ - template inline void MyMPI_ISend (FlatArray s, int dest, int tag) { @@ -181,9 +161,21 @@ namespace netgen MPI_Irecv( &s.First(), s.Size(), MyGetMPIType(), dest, tag, MPI_COMM_WORLD, &request); MPI_Request_free (&request); } + */ + inline void MyMPI_SendCmd (const char * cmd) + { + char buf[100]; + strcpy (buf, cmd); + MPI_Bcast (&buf, 100, MPI_CHAR, 0, MPI_COMM_WORLD); + } - + inline string MyMPI_RecvCmd () + { + char buf[100]; + MPI_Bcast (&buf, 100, MPI_CHAR, 0, MPI_COMM_WORLD); + return string(buf); + } template inline void MyMPI_Bcast (T & s, MPI_Comm comm = MPI_COMM_WORLD) @@ -204,7 +196,6 @@ namespace netgen inline void MyMPI_Bcast (Array & s, int root, MPI_Comm comm = MPI_COMM_WORLD) { int id; - // MPI_Comm_rank(MPI_HIGHORDER_COMM, &id); MPI_Comm_rank(comm, &id); int size = s.Size(); @@ -234,6 +225,7 @@ namespace netgen + /* inline void MyMPI_Send ( int *& s, int len, int dest, int tag) { int hlen = len; @@ -270,6 +262,7 @@ namespace netgen s = new double [len]; MPI_Recv( s, len, MPI_DOUBLE, src, tag, MPI_COMM_WORLD, &status); } + */ #endif // PARALLEL diff --git a/libsrc/general/profiler.cpp b/libsrc/general/profiler.cpp index 4f65d51c..91db4eff 100644 --- a/libsrc/general/profiler.cpp +++ b/libsrc/general/profiler.cpp @@ -32,7 +32,9 @@ namespace netgen NgProfiler :: ~NgProfiler() { +#ifndef PARALLEL StopTimer (total_timer); +#endif //ofstream prof; //prof.open("ng.prof"); @@ -50,7 +52,7 @@ namespace netgen sprintf (filename, "netgen.prof"); #endif - printf ("write profile to file %s\n", filename); + if (id == 0) printf ("write profile to file netgen.prof\n"); FILE *prof = fopen(filename,"w"); Print (prof); fclose(prof); diff --git a/libsrc/general/table.cpp b/libsrc/general/table.cpp index b6168374..d9d6cf13 100644 --- a/libsrc/general/table.cpp +++ b/libsrc/general/table.cpp @@ -103,7 +103,7 @@ namespace netgen return; } #endif - + linestruct & line = data[i]; if (line.size == line.maxsize) { @@ -121,6 +121,27 @@ namespace netgen + + void BASE_TABLE :: SetEntrySize2 (int i, int newsize, int elsize) + { + linestruct & line = data[i]; + if (newsize > line.maxsize) + { + void * p = new char [newsize * elsize]; + + memcpy (p, line.col, min2 (newsize, line.size) * elsize); + delete [] (char*)line.col; + + line.col = p; + } + + line.size = newsize; + } + + + + + /* void BASE_TABLE :: DecSize (int i) { diff --git a/libsrc/general/table.hpp b/libsrc/general/table.hpp index dcc161b8..f63b8074 100644 --- a/libsrc/general/table.hpp +++ b/libsrc/general/table.hpp @@ -52,8 +52,18 @@ public: else IncSize2 (i, elsize); } + + void SetEntrySize (int i, int newsize, int elsize) + { + if (newsize < data[i].maxsize) + data[i].size = newsize; + else + SetEntrySize2 (i, newsize, elsize); + } + /// void IncSize2 (int i, int elsize); + void SetEntrySize2 (int i, int newsize, int elsize); // void DecSize (int i); diff --git a/libsrc/interface/nginterface.cpp b/libsrc/interface/nginterface.cpp index 7851ecce..cf04bb1a 100644 --- a/libsrc/interface/nginterface.cpp +++ b/libsrc/interface/nginterface.cpp @@ -27,7 +27,7 @@ static pthread_t meshingthread; void RunParallel ( void * (*fun)(void *), void * in) { - if (netgen::mparam.parthread && (ntasks == 1) ) + if (netgen::mparam.parthread) // && (ntasks == 1) ) { pthread_attr_t attr; pthread_attr_init (&attr); @@ -70,7 +70,12 @@ static pthread_t meshingthread; void RunParallel ( void * (*fun)(void *), void * in) { - if (netgen::mparam.parthread && (netgen::ntasks == 1)) + bool parthread = netgen::mparam.parthread; + + if (netgen::id > 0) parthread = true; + // if (netgen::ntasks > 1) parthread = false; + + if (parthread) { pthread_attr_t attr; pthread_attr_init (&attr); @@ -926,45 +931,60 @@ void Ng_GetSurfaceElementNeighbouringDomains(const int selnr, int & in, int & ou // Is Element ei an element of this processor ?? bool Ng_IsGhostEl (int ei) { + return false; + /* if ( mesh->GetDimension() == 3 ) return mesh->VolumeElement(ei).IsGhost(); else return false; + */ } void Ng_SetGhostEl(const int ei, const bool aisghost ) { + ; + /* if ( mesh -> GetDimension () == 3 ) mesh -> VolumeElement(ei).SetGhost (aisghost); + */ } bool Ng_IsGhostSEl (int ei) { + return false; + /* if ( mesh -> GetDimension () == 3 ) return mesh->SurfaceElement(ei).IsGhost(); else return false; + */ } void Ng_SetGhostSEl(const int ei, const bool aisghost ) { + ; + /* if ( mesh -> GetDimension () == 3 ) mesh -> SurfaceElement(ei).SetGhost (aisghost); + */ } bool Ng_IsGhostVert ( int pnum ) { - return mesh -> Point ( pnum ).IsGhost() ; + return false; + // return mesh -> Point ( pnum ).IsGhost() ; } bool Ng_IsGhostEdge ( int ednum ) { - return mesh -> GetParallelTopology() . IsGhostEdge ( ednum ); + return false; + // return mesh -> GetParallelTopology() . IsGhostEdge ( ednum ); } bool Ng_IsGhostFace ( int fanum ) { - return mesh -> GetParallelTopology() . IsGhostFace ( fanum ); + return false; + // return mesh -> GetParallelTopology() . IsGhostFace ( fanum ); } // void Ng_SetGhostVert ( const int pnum, const bool aisghost ); @@ -979,10 +999,15 @@ bool Ng_IsExchangeSEl ( int selnum ) { return mesh -> GetParallelTopology() . IsExchangeSEl ( selnum ); } void Ng_UpdateOverlap() -{ mesh->UpdateOverlap(); } +{ + ; // mesh->UpdateOverlap(); +} int Ng_Overlap () -{ return mesh->GetParallelTopology() . Overlap(); } +{ + return 0; + // return mesh->GetParallelTopology() . Overlap(); +} #endif diff --git a/libsrc/meshing/clusters.cpp b/libsrc/meshing/clusters.cpp index 3282613c..e89def6a 100644 --- a/libsrc/meshing/clusters.cpp +++ b/libsrc/meshing/clusters.cpp @@ -27,8 +27,8 @@ namespace netgen if (!hasedges || !hasfaces) return; - - PrintMessage (3, "Update Clusters"); + if (id == 0) + PrintMessage (3, "Update Clusters"); nv = mesh.GetNV(); ned = top.GetNEdges(); diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 7b5e9571..212dbc6c 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -5,17 +5,16 @@ #ifdef METIS -namespace metis -{ +namespace metis { extern "C" { #include } } using namespace metis; - #endif + namespace netgen { @@ -29,7 +28,8 @@ namespace netgen void Mesh :: SendRecvMesh () { - PrintMessage (1, "Send/Receive mesh"); + if (id == 0) + PrintMessage (1, "Send/Receive mesh"); { @@ -82,81 +82,423 @@ namespace netgen - - - - - - if (id == 0) SendMesh (); else ReceiveParallelMesh(); - - - MPI_Barrier (MPI_COMM_WORLD); } + + + + + + void Mesh :: SendMesh () const + { + Array sendrequests; + + Array num_els_on_proc(ntasks); + num_els_on_proc = 0; + for (ElementIndex ei = 0; ei < GetNE(); ei++) + num_els_on_proc[(*this)[ei].GetPartition()]++; + + + // get number of vertices for each processor + Array nelloc (ntasks); + + nelloc = 0; + // elarraysize = 1; + + PrintMessage ( 3, "Sending vertices"); + + TABLE els_of_proc (num_els_on_proc); + for (ElementIndex ei = 0; ei < GetNE(); ei++) + els_of_proc.Add ( (*this)[ei].GetPartition(), ei); + + + Array vert_flag (GetNV()); + Array num_procs_on_vert (GetNV()); + Array num_verts_on_proc (ntasks); + + num_verts_on_proc = 0; + num_procs_on_vert = 0; + vert_flag = -1; + + for (int dest = 1; dest < ntasks; dest++) + { + FlatArray els = els_of_proc[dest]; + + for (int hi = 0; hi < els.Size(); hi++) + { + const Element & el = (*this) [ els[hi] ]; + for (int i = 0; i < el.GetNP(); i++) + { + PointIndex epi = el[i]; + if (vert_flag[epi] < dest) + { + vert_flag[epi] = dest; + + num_verts_on_proc[dest]++; + num_procs_on_vert[epi]++; + + paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]); + } + } + + // elarraysize[dest] += 3 + el.GetNP(); + nelloc[dest] ++; + paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] ); + } + } + + TABLE verts_of_proc (num_verts_on_proc); + TABLE procs_of_vert (num_procs_on_vert); + TABLE loc_num_of_vert (num_procs_on_vert); + + vert_flag = -1; + for (int dest = 1; dest < ntasks; dest++) + { + FlatArray els = els_of_proc[dest]; + for (int hi = 0; hi < els.Size(); hi++) + { + const Element & el = (*this) [ els[hi] ]; + + for (int i = 0; i < el.GetNP(); i++) + { + PointIndex epi = el[i]; + if (vert_flag[epi] < dest) + { + vert_flag[epi] = dest; + procs_of_vert.Add (epi, dest); + } + } + } + } + + for (int vert = 1; vert <= GetNP(); vert++ ) + { + FlatArray procs = procs_of_vert[vert]; + for (int j = 0; j < procs.Size(); j++) + { + int dest = procs[j]; + verts_of_proc.Add (dest, vert); + loc_num_of_vert.Add (vert, verts_of_proc[dest].Size()); + } + } + + for (int dest = 1; dest < ntasks; dest++) + { + FlatArray verts = verts_of_proc[dest]; + + sendrequests.Append (MyMPI_ISend (verts, dest, MPI_TAG_MESH+1)); + + MPI_Datatype mptype = MeshPoint::MyGetMPIType(); + + int numv = verts.Size(); + + MPI_Datatype newtype; + Array blocklen (numv); + blocklen = 1; + + MPI_Type_indexed (numv, &blocklen[0], + reinterpret_cast (&verts[0]), + mptype, &newtype); + MPI_Type_commit (&newtype); + + MPI_Request request; + MPI_Isend( &points[0], 1, newtype, dest, MPI_TAG_MESH+1, MPI_COMM_WORLD, &request); + sendrequests.Append (request); + } + + + + Array num_distpnums(ntasks); + num_distpnums = 0; + + for (int vert = 1; vert <= GetNP(); vert++) + { + FlatArray procs = procs_of_vert[vert]; + for (int j = 0; j < procs.Size(); j++) + num_distpnums[procs[j]] += 3 * (procs.Size()-1); + } + + TABLE distpnums (num_distpnums); + + for (int vert = 1; vert <= GetNP(); vert++) + { + FlatArray procs = procs_of_vert[vert]; + for (int j = 0; j < procs.Size(); j++) + for (int k = 0; k < procs.Size(); k++) + if (j != k) + { + distpnums.Add (procs[j], loc_num_of_vert[vert][j]); + distpnums.Add (procs[j], procs_of_vert[vert][k]); + distpnums.Add (procs[j], loc_num_of_vert[vert][k]); + } + } + + for ( int dest = 1; dest < ntasks; dest ++ ) + sendrequests.Append (MyMPI_ISend (distpnums[dest], dest, MPI_TAG_MESH+1)); + + PrintMessage ( 3, "Sending elements" ); + + Array elarraysize (ntasks); + elarraysize = 0; + for ( int ei = 1; ei <= GetNE(); ei++) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + elarraysize[dest] += 3 + el.GetNP(); + } + + TABLE elementarrays(elarraysize); + + for (int ei = 1; ei <= GetNE(); ei++) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + + elementarrays.Add (dest, ei); + elementarrays.Add (dest, el.GetIndex()); + elementarrays.Add (dest, el.GetNP()); + for (int i = 0; i < el.GetNP(); i++) + elementarrays.Add (dest, el[i]); + } + + for (int dest = 1; dest < ntasks; dest ++ ) + sendrequests.Append (MyMPI_ISend (elementarrays[dest], dest, MPI_TAG_MESH+2)); + + PrintMessage ( 3, "Sending Face Descriptors" ); + + + Array fddata (6 * GetNFD()); + for (int fdi = 1; fdi <= GetNFD(); fdi++) + { + fddata[6*fdi-6] = GetFaceDescriptor(fdi).SurfNr(); + fddata[6*fdi-5] = GetFaceDescriptor(fdi).DomainIn(); + fddata[6*fdi-4] = GetFaceDescriptor(fdi).DomainOut(); + fddata[6*fdi-3] = GetFaceDescriptor(fdi).BCProperty(); + fddata[6*fdi-2] = GetFaceDescriptor(fdi).domin_singular; + fddata[6*fdi-1] = GetFaceDescriptor(fdi).domout_singular; + + } + for (int dest = 1; dest < ntasks; dest++) + sendrequests.Append (MyMPI_ISend (fddata, dest, MPI_TAG_MESH+3)); + + PrintMessage ( 3, "Sending Surface elements" ); + + Array nlocsel(ntasks), bufsize(ntasks); // , seli(ntasks); + for ( int i = 0; i < ntasks; i++) + { + nlocsel[i] = 0; + bufsize[i] = 1; + } + + for (int sei = 1; sei <= GetNSE(); sei++ ) + { + int ei1, ei2; + GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); + const Element2d & sel = SurfaceElement (sei); + + for (int j = 0; j < 2; j++) + { + int ei = (j == 0) ? ei1 : ei2; + if ( ei > 0 && ei <= GetNE() ) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + nlocsel[dest] ++; + bufsize[dest] += 4 + 2*sel.GetNP(); + } + } + } + + TABLE selbuf(bufsize); + + Array nselloc (ntasks); + nselloc = 0; + + for (int dest = 1; dest < ntasks; dest++ ) + selbuf.Add (dest, nlocsel[dest]); + + for (int sei = 1; sei <= GetNSE(); sei ++ ) + { + int ei1, ei2; + GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); + const Element2d & sel = SurfaceElement (sei); + + int isghost = 0; + for (int j = 0; j < 2; j++) + { + int ei = (j == 0) ? ei1 : ei2; + if ( ei > 0 && ei <= GetNE() ) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + if (dest > 0) + { + selbuf.Add (dest, sei); + selbuf.Add (dest, sel.GetIndex()); + selbuf.Add (dest, isghost); + selbuf.Add (dest, sel.GetNP()); + + for ( int ii = 1; ii <= sel.GetNP(); ii++) + { + selbuf.Add (dest, sel.PNum(ii)); + selbuf.Add (dest, sel.GeomInfoPi(ii).trignum); + } + nselloc[dest] ++; + paralleltop -> SetDistantSurfEl ( dest, sei, nselloc[dest] ); + isghost = 1; + } + } + } + } + + for (int dest = 1; dest < ntasks; dest++) + sendrequests.Append (MyMPI_ISend(selbuf[dest], dest, MPI_TAG_MESH+4)); + + PrintMessage ( 3, "Sending Edge Segments"); + + Array nlocseg(ntasks), segi(ntasks); + for ( int i = 0; i < ntasks; i++) + { + nlocseg[i] = 0; + bufsize[i] = 0; + segi[i] = 0; + } + + for (int segi = 1; segi <= GetNSeg(); segi ++ ) + { + Array volels; + const MeshTopology & topol = GetTopology(); + topol . GetSegmentVolumeElements ( segi, volels ); + for (int j = 0; j < volels.Size(); j++) + { + int ei = volels[j]; + if ( ei > 0 && ei <= GetNE() ) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + nlocseg[dest] ++; + bufsize[dest] += 14; + } + } + } + + TABLE segmbuf(bufsize); + + for ( int ls=1; ls <= GetNSeg(); ls++) + { + Array volels; + GetTopology().GetSegmentVolumeElements ( ls, volels ); + const Segment & seg = LineSegment (ls); + + for (int j = 0; j < volels.Size(); j++) + { + int ei = volels[j]; + if ( ei > 0 && ei <= GetNE() ) + { + const Element & el = VolumeElement (ei); + int dest = el.GetPartition(); + + if ( dest > 0 ) + { + segmbuf.Add (dest, ls); + segmbuf.Add (dest, seg.si); + segmbuf.Add (dest, seg.pnums[0]); + segmbuf.Add (dest, seg.pnums[1]); + segmbuf.Add (dest, seg.geominfo[0].trignum); + segmbuf.Add (dest, seg.geominfo[1].trignum); + segmbuf.Add (dest, seg.surfnr1); + segmbuf.Add (dest, seg.surfnr2); + segmbuf.Add (dest, seg.edgenr); + segmbuf.Add (dest, seg.epgeominfo[0].dist); + segmbuf.Add (dest, seg.epgeominfo[1].edgenr); + segmbuf.Add (dest, seg.epgeominfo[1].dist); + segmbuf.Add (dest, seg.singedge_right); + segmbuf.Add (dest, seg.singedge_left); + segi[dest] += 14; + } + paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) ); + } + } + } + + for ( int dest = 1; dest < ntasks; dest++) + sendrequests.Append (MyMPI_ISend(segmbuf[dest], dest, MPI_TAG_MESH+5)); + + MPI_Waitall (sendrequests.Size(), &sendrequests[0], MPI_STATUS_IGNORE); + } + + + + + + + + // slaves receive the mesh from the master void Mesh :: ReceiveParallelMesh ( ) { int timer = NgProfiler::CreateTimer ("ReceiveParallelMesh"); + int timer_pts = NgProfiler::CreateTimer ("Receive points"); + int timer_els = NgProfiler::CreateTimer ("Receive elements"); + int timer_sels = NgProfiler::CreateTimer ("Receive surface elements"); NgProfiler::RegionTimer reg(timer); -#ifdef SCALASCA -#pragma pomp inst begin(loadmesh) -#endif - string st; + // string st; - INDEX_CLOSED_HASHTABLE glob2loc_vert_ht (1); - - - { - // receive vertices - - Array verts; - MyMPI_Recv (verts, 0); - - int numvert = verts.Size(); - paralleltop -> SetNV (numvert); - - glob2loc_vert_ht.SetSize (3*numvert+1); - - for (int vert = 0; vert < numvert; vert++) - { - int globvert = verts[vert]; - paralleltop->SetLoc2Glob_Vert ( vert+1, globvert ); - glob2loc_vert_ht.Set (globvert, vert+1); - } - - for (int i = 0; i < numvert; i++) - AddPoint (netgen::Point<3> (0,0,0)); - - MPI_Datatype mptype = MeshPoint::MyGetMPIType(); - MPI_Status status; - MPI_Recv( &points[1], numvert, mptype, 0, 33, MPI_COMM_WORLD, &status); - - Array dist_pnums; - MyMPI_Recv (dist_pnums, 0); - - for (int hi = 0; hi < dist_pnums.Size(); hi += 3) - paralleltop -> - SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi], dist_pnums[hi+2]); - - *testout << "got " << numvert << " vertices" << endl; - } + // receive vertices + NgProfiler::StartTimer (timer_pts); + + Array verts; + MyMPI_Recv (verts, 0, MPI_TAG_MESH+1); + + int numvert = verts.Size(); + paralleltop -> SetNV (numvert); + + // INDEX_CLOSED_HASHTABLE glob2loc_vert_ht (3*numvert+1); + INDEX_HASHTABLE glob2loc_vert_ht (100); + for (int vert = 0; vert < numvert; vert++) + { + int globvert = verts[vert]; + paralleltop->SetLoc2Glob_Vert ( vert+1, globvert ); + glob2loc_vert_ht.Set (globvert, vert+1); + } + + for (int i = 0; i < numvert; i++) + AddPoint (netgen::Point<3> (0,0,0)); + + MPI_Datatype mptype = MeshPoint::MyGetMPIType(); + MPI_Status status; + MPI_Recv( &points[1], numvert, mptype, 0, MPI_TAG_MESH+1, MPI_COMM_WORLD, &status); + + Array dist_pnums; + MyMPI_Recv (dist_pnums, 0, MPI_TAG_MESH+1); + + for (int hi = 0; hi < dist_pnums.Size(); hi += 3) + paralleltop -> + SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi], dist_pnums[hi+2]); + + NgProfiler::StopTimer (timer_pts); + *testout << "got " << numvert << " vertices" << endl; + { Element el; Array elarray; - MyMPI_Recv (elarray, 0); + MyMPI_Recv (elarray, 0, MPI_TAG_MESH+2); + NgProfiler::RegionTimer reg(timer_els); + + // double sumcosts = 0, sum = 0; for (int ind = 0, elnum = 1; ind < elarray.Size(); elnum++) { paralleltop->SetLoc2Glob_VolEl ( elnum, elarray[ind++]); @@ -164,17 +506,30 @@ namespace netgen el.SetIndex(elarray[ind++]); el.SetNP(elarray[ind++]); + /* + for ( int j = 0; j < el.GetNP(); j++) + { + int costs = 1; // glob2loc_vert_ht.CalcPositionCosts (elarray[ind+j]); + // *testout << "costs to hash " << elarray[ind+j] << ", hashvalue = " + // << glob2loc_vert_ht.HashValue(elarray[ind+j]) << ": " << costs << endl; + sumcosts += costs; + sum++; + } + */ + for ( int j = 0; j < el.GetNP(); j++) el[j] = glob2loc_vert_ht.Get (elarray[ind++]); AddVolumeElement (el); } + // *testout << "average hashing costs: " << sumcosts / sum << endl; + // cout << "average hashing costs: " << sumcosts / sum << endl; } { Array fddata; - MyMPI_Recv (fddata, 0); + MyMPI_Recv (fddata, 0, MPI_TAG_MESH+3); for (int i = 0; i < fddata.Size(); i += 6) { int faceind = AddFaceDescriptor (FaceDescriptor(int(fddata[i]), int(fddata[i+1]), int(fddata[i+2]), 0)); @@ -185,10 +540,11 @@ namespace netgen } { + NgProfiler::RegionTimer reg(timer_sels); // surface elements Array selbuf; - MyMPI_Recv ( selbuf, 0); + MyMPI_Recv ( selbuf, 0, MPI_TAG_MESH+4); int ii = 0; int sel = 0; @@ -222,7 +578,7 @@ namespace netgen { Array segmbuf; - MyMPI_Recv ( segmbuf, 0); + MyMPI_Recv ( segmbuf, 0, MPI_TAG_MESH+5); Segment seg; int globsegi; @@ -277,17 +633,11 @@ namespace netgen NgProfiler::StopTimer (timerloc2); - topology -> Update(); clusters -> Update(); SetNextMajorTimeStamp(); - // paralleltop->Print(); - -#ifdef SCALASCA -#pragma pomp inst end(loadmesh) -#endif } @@ -301,9 +651,6 @@ namespace netgen if ( id != 0 || ntasks == 1 ) return; // metis partition of mesh, only if more than one proc -#ifdef SCALASCA -#pragma pomp inst begin(metis) -#endif // partition mesh @@ -322,23 +669,16 @@ namespace netgen *testout << "sel(" << int(ei) << ") is in part " << (*this)[ei].GetPartition() << endl; -#ifdef SCALASCA -#pragma pomp inst end(metis) -#endif - // send partition - for (int dest = 1; dest < ntasks; dest++) - MyMPI_Send ("mesh", dest, MPI_TAG_CMD); + // for (int dest = 1; dest < ntasks; dest++) + // MyMPI_Send ("mesh", dest, MPI_TAG_CMD); + MyMPI_SendCmd ("mesh"); SendRecvMesh (); paralleltop -> UpdateCoarseGrid(); -#ifdef SCALASCA -#pragma pomp inst end(loadmesh_seq) -#endif - // paralleltop -> Print(); } @@ -452,13 +792,13 @@ namespace netgen int timermetis = NgProfiler::CreateTimer ("Metis itself"); NgProfiler::StartTimer (timermetis); - metis :: METIS_PartMeshDual (&ne, &nn, elmnts, &etype, &numflag, &nparts, - &edgecut, epart, npart); + METIS_PartMeshDual (&ne, &nn, elmnts, &etype, &numflag, &nparts, + &edgecut, epart, npart); NgProfiler::StopTimer (timermetis); cout << "complete" << endl; - cout << "edge-cut: " << edgecut << ", balance: " << metis :: ComputeElementBalance(ne, nparts, epart) << endl; + cout << "edge-cut: " << edgecut << ", balance: " << ComputeElementBalance(ne, nparts, epart) << endl; // partition numbering by metis : 0 ... ntasks - 1 @@ -533,7 +873,7 @@ namespace netgen BubbleSort(array); } - metis :: METIS_PartGraphKway ( &nn, xadj, adjacency, v_weights, e_weights, &weightflag, + METIS_PartGraphKway ( &nn, xadj, adjacency, v_weights, e_weights, &weightflag, &numflag, &nparts, options, &edgecut, part ); Array nodesinpart(ntasks); @@ -652,7 +992,7 @@ namespace netgen int timermetis = NgProfiler::CreateTimer ("Metis itself"); NgProfiler::StartTimer (timermetis); - metis :: METIS_PartGraphKway ( &ne, xadj, adjacency, v_weights, e_weights, &weightflag, + METIS_PartGraphKway ( &ne, xadj, adjacency, v_weights, e_weights, &weightflag, &numflag, &nparts, options, &edgecut, part ); NgProfiler::StopTimer (timermetis); @@ -759,8 +1099,8 @@ namespace netgen for ( int el = 0; el < ne; el++ ) BubbleSort (adjacency.Range (xadj[el], xadj[el+1])); - metis :: METIS_PartGraphKway ( &ne, &xadj[0], &adjacency[0], v_weights, e_weights, &weightflag, - &numflag, &nparts, options, &edgecut, &part[0] ); + METIS_PartGraphKway ( &ne, &xadj[0], &adjacency[0], v_weights, e_weights, &weightflag, + &numflag, &nparts, options, &edgecut, &part[0] ); for (SurfaceElementIndex sei = 0; sei < ne; sei++) (*this) [sei].SetPartition (part[sei]+1); @@ -775,960 +1115,10 @@ namespace netgen - void Mesh :: SendMesh () const - { - MPI_Request sendrequest[ntasks]; - - Array num_els_on_proc(ntasks); - num_els_on_proc = 0; - for (ElementIndex ei = 0; ei < GetNE(); ei++) - num_els_on_proc[(*this)[ei].GetPartition()]++; - - - // get number of vertices for each processor - Array nelloc (ntasks); - - nelloc = 0; - // elarraysize = 1; - - PrintMessage ( 3, "Sending vertices"); - - TABLE els_of_proc (num_els_on_proc); - for (ElementIndex ei = 0; ei < GetNE(); ei++) - els_of_proc.Add ( (*this)[ei].GetPartition(), ei); - - - Array vert_flag (GetNV()); - Array num_procs_on_vert (GetNV()); - Array num_verts_on_proc (ntasks); - - num_verts_on_proc = 0; - num_procs_on_vert = 0; - vert_flag = -1; - - for (int dest = 1; dest < ntasks; dest++) - { - FlatArray els = els_of_proc[dest]; - - for (int hi = 0; hi < els.Size(); hi++) - { - const Element & el = (*this) [ els[hi] ]; - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - - num_verts_on_proc[dest]++; - num_procs_on_vert[epi]++; - - paralleltop -> SetDistantPNum ( dest, epi, num_verts_on_proc[dest]); - } - } - - // elarraysize[dest] += 3 + el.GetNP(); - nelloc[dest] ++; - paralleltop -> SetDistantEl ( dest, els[hi]+1, nelloc[dest] ); - } - } - - TABLE verts_of_proc (num_verts_on_proc); - TABLE procs_of_vert (num_procs_on_vert); - TABLE loc_num_of_vert (num_procs_on_vert); - - vert_flag = -1; - for (int dest = 1; dest < ntasks; dest++) - { - FlatArray els = els_of_proc[dest]; - for (int hi = 0; hi < els.Size(); hi++) - { - const Element & el = (*this) [ els[hi] ]; - - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - procs_of_vert.Add (epi, dest); - } - } - } - } - - for (int vert = 1; vert <= GetNP(); vert++ ) - { - FlatArray procs = procs_of_vert[vert]; - for (int j = 0; j < procs.Size(); j++) - { - int dest = procs[j]; - verts_of_proc.Add (dest, vert); - loc_num_of_vert.Add (vert, verts_of_proc[dest].Size()); - } - } - - for (int dest = 1; dest < ntasks; dest++) - { - FlatArray verts = verts_of_proc[dest]; - MyMPI_ISend (verts, dest, MPI_TAG_MESH); - - MPI_Datatype mptype = MeshPoint::MyGetMPIType(); - - int numv = verts.Size(); - - MPI_Datatype newtype; - Array blocklen (numv); - blocklen = 1; - - MPI_Type_indexed (numv, &blocklen[0], - reinterpret_cast (&verts[0]), - mptype, &newtype); - MPI_Type_commit (&newtype); - - MPI_Request request; - MPI_Isend( &points[0], 1, newtype, dest, 33, MPI_COMM_WORLD, &request); - MPI_Request_free (&request); - } - - - - Array num_distpnums(ntasks); - num_distpnums = 0; - - for (int vert = 1; vert <= GetNP(); vert++) - { - FlatArray procs = procs_of_vert[vert]; - for (int j = 0; j < procs.Size(); j++) - num_distpnums[procs[j]] += 3 * (procs.Size()-1); - } - - TABLE distpnums (num_distpnums); - - for (int vert = 1; vert <= GetNP(); vert++) - { - FlatArray procs = procs_of_vert[vert]; - for (int j = 0; j < procs.Size(); j++) - for (int k = 0; k < procs.Size(); k++) - if (j != k) - { - distpnums.Add (procs[j], loc_num_of_vert[vert][j]); - distpnums.Add (procs[j], procs_of_vert[vert][k]); - distpnums.Add (procs[j], loc_num_of_vert[vert][k]); - } - } - - for ( int dest = 1; dest < ntasks; dest ++ ) - MyMPI_ISend (distpnums[dest], dest, MPI_TAG_MESH); - - - PrintMessage ( 3, "Sending elements" ); - - Array elarraysize (ntasks); - elarraysize = 0; - for ( int ei = 1; ei <= GetNE(); ei++) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - elarraysize[dest] += 3 + el.GetNP(); - } - - TABLE elementarrays(elarraysize); - - for (int ei = 1; ei <= GetNE(); ei++) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - - elementarrays.Add (dest, ei); - elementarrays.Add (dest, el.GetIndex()); - elementarrays.Add (dest, el.GetNP()); - for (int i = 0; i < el.GetNP(); i++) - elementarrays.Add (dest, el[i]); - } - - for (int dest = 1; dest < ntasks; dest ++ ) - MyMPI_ISend (elementarrays[dest], dest, MPI_TAG_MESH); - - - PrintMessage ( 3, "Sending Face Descriptors" ); - - - Array fddata (6 * GetNFD()); - for (int fdi = 1; fdi <= GetNFD(); fdi++) - { - fddata[6*fdi-6] = GetFaceDescriptor(fdi).SurfNr(); - fddata[6*fdi-5] = GetFaceDescriptor(fdi).DomainIn(); - fddata[6*fdi-4] = GetFaceDescriptor(fdi).DomainOut(); - fddata[6*fdi-3] = GetFaceDescriptor(fdi).BCProperty(); - fddata[6*fdi-2] = GetFaceDescriptor(fdi).domin_singular; - fddata[6*fdi-1] = GetFaceDescriptor(fdi).domout_singular; - - } - for (int dest = 1; dest < ntasks; dest++) - MyMPI_ISend (fddata, dest, MPI_TAG_MESH); - - - PrintMessage ( 3, "Sending Surface elements" ); - - Array nlocsel(ntasks), bufsize(ntasks); // , seli(ntasks); - for ( int i = 0; i < ntasks; i++) - { - nlocsel[i] = 0; - bufsize[i] = 1; - } - - for (int sei = 1; sei <= GetNSE(); sei++ ) - { - int ei1, ei2; - GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); - const Element2d & sel = SurfaceElement (sei); - - for (int j = 0; j < 2; j++) - { - int ei = (j == 0) ? ei1 : ei2; - if ( ei > 0 && ei <= GetNE() ) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - nlocsel[dest] ++; - bufsize[dest] += 4 + 2*sel.GetNP(); - } - } - } - - TABLE selbuf(bufsize); - - Array nselloc (ntasks); - nselloc = 0; - - for (int dest = 1; dest < ntasks; dest++ ) - selbuf.Add (dest, nlocsel[dest]); - - for (int sei = 1; sei <= GetNSE(); sei ++ ) - { - int ei1, ei2; - GetTopology().GetSurface2VolumeElement (sei, ei1, ei2); - const Element2d & sel = SurfaceElement (sei); - - int isghost = 0; - for (int j = 0; j < 2; j++) - { - int ei = (j == 0) ? ei1 : ei2; - if ( ei > 0 && ei <= GetNE() ) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - if (dest > 0) - { - selbuf.Add (dest, sei); - selbuf.Add (dest, sel.GetIndex()); - selbuf.Add (dest, isghost); - selbuf.Add (dest, sel.GetNP()); - - for ( int ii = 1; ii <= sel.GetNP(); ii++) - { - selbuf.Add (dest, sel.PNum(ii)); - selbuf.Add (dest, sel.GeomInfoPi(ii).trignum); - } - nselloc[dest] ++; - paralleltop -> SetDistantSurfEl ( dest, sei, nselloc[dest] ); - isghost = 1; - } - } - } - } - - for (int dest = 1; dest < ntasks; dest++) - MyMPI_ISend(selbuf[dest], dest, MPI_TAG_MESH); - - - PrintMessage ( 3, "Sending Edge Segments"); - - Array nlocseg(ntasks), segi(ntasks); - for ( int i = 0; i < ntasks; i++) - { - nlocseg[i] = 0; - bufsize[i] = 0; - segi[i] = 0; - } - - for (int segi = 1; segi <= GetNSeg(); segi ++ ) - { - Array volels; - const MeshTopology & topol = GetTopology(); - topol . GetSegmentVolumeElements ( segi, volels ); - for (int j = 0; j < volels.Size(); j++) - { - int ei = volels[j]; - if ( ei > 0 && ei <= GetNE() ) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - nlocseg[dest] ++; - bufsize[dest] += 14; - } - } - } - - TABLE segmbuf(bufsize); - - for ( int ls=1; ls <= GetNSeg(); ls++) - { - Array volels; - GetTopology().GetSegmentVolumeElements ( ls, volels ); - const Segment & seg = LineSegment (ls); - - for (int j = 0; j < volels.Size(); j++) - { - int ei = volels[j]; - if ( ei > 0 && ei <= GetNE() ) - { - const Element & el = VolumeElement (ei); - int dest = el.GetPartition(); - - if ( dest > 0 ) - { - segmbuf.Add (dest, ls); - segmbuf.Add (dest, seg.si); - segmbuf.Add (dest, seg.pnums[0]); - segmbuf.Add (dest, seg.pnums[1]); - segmbuf.Add (dest, seg.geominfo[0].trignum); - segmbuf.Add (dest, seg.geominfo[1].trignum); - segmbuf.Add (dest, seg.surfnr1); - segmbuf.Add (dest, seg.surfnr2); - segmbuf.Add (dest, seg.edgenr); - segmbuf.Add (dest, seg.epgeominfo[0].dist); - segmbuf.Add (dest, seg.epgeominfo[1].edgenr); - segmbuf.Add (dest, seg.epgeominfo[1].dist); - segmbuf.Add (dest, seg.singedge_right); - segmbuf.Add (dest, seg.singedge_left); - segi[dest] += 14; - } - paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) ); - } - } - } - - for ( int dest = 1; dest < ntasks; dest++) - MyMPI_ISend(segmbuf[dest], dest, MPI_TAG_MESH, sendrequest[dest]); - - MPI_Waitall (ntasks-1, &sendrequest[1], MPI_STATUS_IGNORE); - } - - - - - - - void Mesh :: UpdateOverlap() { cout << "UpdateOverlap depreciated" << endl; - -#ifdef OLDOLD - (*testout) << "UPDATE OVERLAP" << endl; - Array * globelnums; - -#ifdef SCALASCA -#pragma pomp inst begin(updateoverlap) -#endif - - paralleltop->IncreaseOverlap(); - - if ( id > 0 ) - { - int nvglob = paralleltop->GetNVGlob (); - // int nelglob = paralleltop->GetNEGlob(); - - // int nv = GetNV(); - // int ned = topology -> GetNEdges(); - // int nfa = topology -> GetNFaces(); - // int nel = GetNE(); - - Array glob2loc_vert(nvglob); - glob2loc_vert = -1; - - for ( int locv = 1; locv <= GetNV(); locv++) - { - int globv = paralleltop->GetLoc2Glob_Vert(locv); - glob2loc_vert[globv] = locv; - } - - BitArray addedpoint ( paralleltop -> GetNVGlob () ); - BitArray addedel ( paralleltop -> GetNEGlob () ); - addedpoint.Clear(); - addedel.Clear(); - - Array distvert(ntasks), distel(ntasks), nsenddistel(ntasks); - - nsenddistel = 0; - - MyMPI_Allgather (GetNV(), distvert.Range(1, ntasks), MPI_HIGHORDER_COMM); - MyMPI_Allgather (GetNE(), distel.Range(1, ntasks), MPI_HIGHORDER_COMM); - - BitArray appendedpoint ( GetNP() * ntasks ); - appendedpoint.Clear(); - - TABLE sendpoints(ntasks); - TABLE sendelements(ntasks); - TABLE sendsel(ntasks); - - /* - TABLE cluster_neighbors(nv+ned+nfa+nel); - - for ( int node = 1; node <= nv+ned+nfa+nel; node++ ) - { - int cluster_rep; - cluster_rep = clusters->GetVertexRepresentant(node); - - if ( node == cluster_rep ) continue; - - Array dests; - int nneigh = 0; - if ( node - GetNV() <= 0 ) // cluster representant is vertex - { - int vert = node; - if ( paralleltop -> IsExchangeVert( vert ) ) - { - nneigh = paralleltop -> GetNDistantPNums(vert); - dests.SetSize(2*nneigh); - paralleltop -> GetDistantPNums ( vert, &dests[0] ); - } - } - else if ( node - GetNV() - ned <= 0 ) // cluster representant is edge - { - int edge = node - GetNV(); - if ( paralleltop -> IsExchangeEdge( edge ) ) - { - nneigh = paralleltop -> GetNDistantEdgeNums(edge); - dests.SetSize(2*nneigh); - paralleltop -> GetDistantEdgeNums ( edge, &dests[0] ); - } - } - else if ( node - GetNV() - ned - nfa <= 0 ) // cluster representant is face - { - int face = node - GetNV() - ned; - if ( paralleltop -> IsExchangeFace( face ) ) - { - nneigh = paralleltop -> GetNDistantFaceNums(face); - dests.SetSize(2*nneigh); - paralleltop -> GetDistantFaceNums ( face, &dests[0] ); - } - } - else // cluster representant is element - { - int el = node - GetNV() - ned - nfa; - if ( paralleltop -> IsExchangeElement( el ) ) - { - nneigh = paralleltop -> GetNDistantElNums(el); - dests.SetSize(2*nneigh); - paralleltop -> GetDistantElNums ( el, &dests[0] ); - } - } - - for ( int j = 1; j < nneigh; j++ ) - if ( !cluster_neighbors[cluster_rep].Contains ( dests[2*j] ) ) - { - cluster_neighbors.Add( cluster_rep-1, dests[2*j] ); - } - } - */ - - for ( int i = 0; i < ntasks; i++ ) - { - sendelements.Add (i, 0); - sendsel.Add(i, 0); - } - - Array nsentsel (ntasks); - nsentsel = 0; - - for ( int seli = 1; seli <= GetNSE(); seli++ ) - { - const Element2d & sel = SurfaceElement(seli); - int selnp = sel.GetNP(); - Array vert (selnp); - - Array alldests (0), dests; - - bool isparsel = false; - for ( int i = 0; i < selnp; i++ ) - { - vert[i] = sel.PNum(i+1); - if ( paralleltop -> IsExchangeVert ( vert[i] ) ) - { - isparsel = true; - paralleltop -> GetVertNeighbours ( vert[i], dests ); - for ( int j = 0; j < dests.Size(); j++ ) - if ( !alldests.Contains ( dests[j] ) ) - alldests.Append( dests[j] ); - } - } - - /* - int face = topology->GetSurfaceElementFace(seli); - int cluster_rep = clusters->GetFaceRepresentant(face); - if ( cluster_neighbors[cluster_rep-1].Size() > 0 ) - { - isparsel = true; - for ( int j = 0; j < cluster_neighbors[cluster_rep-1].Size(); j++ ) - if ( !alldests.Contains ( cluster_neighbors[cluster_rep-1][j] ) ) - alldests.Append( cluster_neighbors[cluster_rep-1][j] ); - - } - */ - - if ( !isparsel ) continue; - - for ( int i = 0; i < alldests.Size(); i ++ ) - { - // send the surface element to all distant procs: - - // loc number, - // number of points - // global vert numbers - // surface_element_index - - int dest = alldests[i]; - - // ***************** MISSING id = 0 - if ( dest == 0 ) continue; - - sendsel.Add (dest, seli); - sendsel.Add (dest, selnp); - - for ( int ii=0; ii GetLoc2Glob_Vert (vert[ii]) ); - } - sendsel.Add (dest, sel.GetIndex() ); - nsentsel[dest] ++; - } - } - for ( int dest = 1; dest < ntasks; dest++ ) - sendsel[dest][0] = nsentsel[dest]; - - for ( int eli = 1; eli <= GetNE(); eli++ ) - { - const Element & el = VolumeElement(eli); - int elnp = el.GetNP(); - Array vert (elnp); - - Array alldests (0), dests; - - for ( int i = 0; i < elnp; i++ ) - { - vert[i] = el.PNum(i+1); - if ( paralleltop -> IsExchangeVert ( vert[i] ) ) - { - paralleltop -> GetVertNeighbours ( vert[i], dests ); - for ( int j = 0; j < dests.Size(); j++ ) - if ( !alldests.Contains ( dests[j] ) ) - { - alldests.Append( dests[j] ); - paralleltop->SetExchangeElement ( dests[j], eli ); - } - paralleltop->SetExchangeElement ( eli ); - } - } - - /* - int cluster_rep = clusters->GetElementRepresentant(eli); - if ( cluster_neighbors[cluster_rep-1].Size() > 0 ) - { - for ( int j = 0; j < cluster_neighbors[cluster_rep-1].Size(); j++ ) - if ( !alldests.Contains ( cluster_neighbors[cluster_rep-1][j] ) ) - { - alldests.Append( cluster_neighbors[cluster_rep-1][j] ); - paralleltop->SetExchangeElement ( cluster_neighbors[cluster_rep-1][j], eli ); - } - paralleltop->SetExchangeElement ( eli ); - - } - */ - } - - for ( int eli = 1; eli <= GetNE(); eli++ ) - { - const Element & el = VolumeElement(eli); - int elnp = el.GetNP(); - Array vert (elnp); - - // append to point list: - // local pnum - // global pnum - // point coordinates - - Array points(elnp); - for ( int i = 0; i < elnp; i++ ) - { - vert[i] = el.PNum(i+1); - points[i] = Point(vert[i]); - Array knowndests; - // send point to all dests which get the volume element - for ( int dest = 0; dest < ntasks; dest ++ ) - { - // nur die neuen verts - if ( !paralleltop -> IsExchangeElement ( dest, eli ) ) continue; - - // jeder vertex nur ein mal - if ( appendedpoint.Test( (vert[i]-1) * ntasks + dest ) ) continue; - - appendedpoint.Set( (vert[i]-1) * ntasks + dest ); - paralleltop -> SetExchangeVert (dest, vert[i]); - paralleltop -> SetExchangeVert ( vert[i] ); - - - // append vertex to be sent - // loc pnum - // glob pnum - // coords - - // local pnum - sendpoints.Add (dest, vert[i]); - - // global pnum - sendpoints.Add (dest, paralleltop -> GetLoc2Glob_Vert ( vert[i] ) ); - // coordinates - sendpoints.Add (dest, points[i].X() ); - sendpoints.Add (dest, points[i].Y() ); - sendpoints.Add (dest, points[i].Z() ); - } - } - - - for ( int dest = 1; dest < ntasks; dest ++ ) - { - // send the volume element to all distant procs: - - // loc number, - // glob number - // number of points - // glob vertices - // element_index - if ( !paralleltop -> IsExchangeElement ( dest, eli ) ) continue; - - // loc number - sendelements.Add (dest, eli); - // glob number - sendelements.Add (dest, paralleltop -> GetLoc2Glob_VolEl(eli)); - - sendelements.Add (dest, elnp); - - for ( int j = 0; j < elnp; j++ ) - sendelements.Add (dest, paralleltop -> GetLoc2Glob_Vert(vert[j]) ); - sendelements.Add (dest, el.GetIndex() ); - - distel[dest]++; - nsenddistel[dest] ++; - paralleltop -> SetDistantEl ( dest, eli, -1);// distel[dest] ); - } - - - } - for ( int dest = 1; dest < ntasks; dest++ ) - if ( dest != id ) - sendelements[dest][0] = nsenddistel[dest]; - // find parallel surface elements, if there, append to sendsel - list - - - distel = 0; - - // sizes of sendpoints, sendelements, sendsels - Array sendsize_pts(ntasks), recvsize_pts(ntasks); - Array sendsize_els(ntasks), recvsize_els(ntasks); - Array sendsize_sels(ntasks), recvsize_sels(ntasks); - - for (int i = 0; i < ntasks; i++) - { - sendsize_pts[i] = sendpoints[i].Size(); - sendsize_els[i] = sendelements[i].Size(); - sendsize_sels[i] = sendsel[i].Size(); - } - - MyMPI_Alltoall (sendsize_pts.Range(1, ntasks), recvsize_pts.Range(1, ntasks), MPI_HIGHORDER_COMM); - MyMPI_Alltoall (sendsize_els.Range(1, ntasks), recvsize_els.Range(1, ntasks), MPI_HIGHORDER_COMM); - MyMPI_Alltoall (sendsize_sels.Range(1, ntasks), recvsize_sels.Range(1, ntasks), MPI_HIGHORDER_COMM); - recvsize_pts[0] = 0; - recvsize_els[0] = 0; - recvsize_sels[0] = 0; - - TABLE recvpoints(recvsize_pts); - TABLE recvelements(recvsize_els); - TABLE recvsel(recvsize_sels); - - recvpoints.SetElementSizesToMaxSizes (); - recvelements.SetElementSizesToMaxSizes (); - recvsel.SetElementSizesToMaxSizes (); - - /* - Array sendrequest(3*ntasks), recvrequest(3*ntasks); - Array status(3*ntasks); - - for ( int proc = 1; proc < ntasks; proc++) - { - MyMPI_ISend ( sendpoints[proc], proc, sendrequest[proc] ); - MyMPI_IRecv ( recvpoints[proc], proc, recvrequest[proc] ); - } - - for ( int proc = 1; proc < ntasks; proc++) - { - MPI_Wait(&sendrequest[proc], &status[proc]); - MPI_Wait(&recvrequest[proc], &status[proc]); - MyMPI_ISend ( sendelements[proc], proc, sendrequest[proc+1]); - MyMPI_IRecv ( recvelements[proc], proc, recvrequest[proc+1]); - } - for ( int proc = 1; proc < ntasks; proc++) - { - MPI_Wait(&sendrequest[proc+1], &status[proc+1]); - MPI_Wait(&recvrequest[proc+1], &status[proc+1]); - MyMPI_ISend ( sendsel[proc], proc, sendrequest[proc+2] ); - MyMPI_IRecv ( recvsel[proc], proc, recvrequest[proc+2] ); - } - for ( int proc = 1; proc < ntasks; proc++) - { - MPI_Wait(&sendrequest[proc+2], &status[proc+2]); - MPI_Wait(&recvrequest[proc+2], &status[proc+2]); - } - */ - - Array requests; - - for ( int proc = 1; proc < ntasks; proc++) - { - requests.Append (MyMPI_ISend ( sendpoints[proc], proc )); - requests.Append (MyMPI_IRecv ( recvpoints[proc], proc )); - } - - for ( int proc = 1; proc < ntasks; proc++) - { - requests.Append (MyMPI_ISend ( sendelements[proc], proc )); - requests.Append (MyMPI_IRecv ( recvelements[proc], proc )); - } - for ( int proc = 1; proc < ntasks; proc++) - { - requests.Append (MyMPI_ISend ( sendsel[proc], proc )); - requests.Append (MyMPI_IRecv ( recvsel[proc], proc )); - } - - MPI_Status stat; - for (int i = 0; i < requests.Size(); i++) - MPI_Wait (&requests[i], &stat); - - - - - - Array * distpnum2parpnum; - distpnum2parpnum = new Array [2]; - distpnum2parpnum[0].SetSize(0); - distpnum2parpnum[1].SetSize(0); - - Array firstdistpnum (ntasks); - - - for ( int sender = 1; sender < ntasks; sender++) - { - firstdistpnum[sender] = distpnum2parpnum[0].Size(); - - if ( sender == id ) continue; - - int ii = 0; - // receiving points - // dist pnum - // glob pnum - // coords - int numrecvpts = int ( recvpoints[sender].Size() / 5 ); - - paralleltop -> SetNV ( GetNV() + numrecvpts ); - int expectnp = GetNV () + numrecvpts; - - // received points - while ( ii < recvpoints[sender].Size() ) - { - int distpnum = int ( (recvpoints[sender])[ii++] ); - int globpnum = int ( (recvpoints[sender])[ii++] ); - - Point3d point; - point.X() = (recvpoints[sender])[ii++]; - point.Y() = (recvpoints[sender])[ii++]; - point.Z() = (recvpoints[sender])[ii++]; - - // append point as ghost - // if not already there - int pnum= glob2loc_vert[globpnum];//paralleltop -> Glob2Loc_Vert ( globpnum ); - if ( pnum <= 0 ) - { - pnum = AddPoint ( point, true ); - } - paralleltop -> SetDistantPNum ( 0, pnum, globpnum ); - glob2loc_vert[globpnum] = pnum; - paralleltop -> SetDistantPNum ( sender, pnum, distpnum ); - paralleltop -> SetExchangeVert ( pnum ); - } - - ii = 0; - - int recvnel = (recvelements[sender])[ii++]; - - paralleltop -> SetNE ( recvnel + GetNE() ); - - while ( ii < recvelements[sender].Size() ) - { - // receive list: - // distant number, - // glob number - // number of points - // glob vertices - // element_index - - int distelnum = (recvelements[sender])[ii++]; - int globelnum = (recvelements[sender])[ii++] ; - int elnp = (recvelements[sender])[ii++] ; - Array pnums(elnp), globpnums(elnp); - - // append volel - ELEMENT_TYPE eltype; - switch ( elnp ) - { - case 4: eltype = TET; break; - case 5: eltype = PYRAMID; break; - case 6: eltype = PRISM; break; - case 8: eltype = HEX; break; - } - - Element el ( eltype ) ; - - for ( int i = 0; i < elnp; i++ ) - { - globpnums[i] = int ( (recvelements[sender])[ii++] ); - pnums[i] = glob2loc_vert[globpnums[i]]; //paralleltop -> Glob2Loc_Vert(globpnums[i]); - } - - el.SetIndex ( (recvelements[sender])[ii++] ); - el.SetGhost ( 1 ); - - - for ( int i = 0; i < elnp; i++) - { - (int&) el[i] = pnums[i]; - } - - int eli = AddVolumeElement (el) + 1; - - paralleltop -> SetDistantEl ( sender, eli, distelnum); - paralleltop -> SetDistantEl ( 0, eli, globelnum ); - paralleltop -> SetExchangeElement ( eli ); - } - - ii = 0; - int nrecvsendsel = 0; - if ( recvsel[sender].Size() > 0 ) - nrecvsendsel = (recvsel[sender])[ii++]; - - paralleltop -> SetNSE ( nrecvsendsel + GetNSE() ); - - while ( ii < recvsel[sender] . Size() ) - { - // receive list: - // distant number, - // number of points - // global vert numbers - // surface_element_index - - int distselnum = (recvsel[sender])[ii++]; - int selnp = (recvsel[sender])[ii++] ; - - Array globpnums(selnp); - Array pnums(selnp); - - // append volel - ELEMENT_TYPE eltype; - switch ( selnp ) - { - case 4: eltype = QUAD; break; - case 3: eltype = TRIG; break; - } - - Element2d sel ( eltype ) ; - for ( int i = 0; i < selnp; i++ ) - { - globpnums[i] = int ( (recvsel[sender])[ii++] ); - pnums[i] = glob2loc_vert[globpnums[i]]; - } - - sel.SetIndex ( (recvsel[sender])[ii++] ); - sel.SetGhost ( 1 ); - - - for ( int i = 0; i < selnp; i++) - { - (int&) sel[i] = pnums[i]; - } - - int seli = AddSurfaceElement (sel); - - } - - - } - - delete [] distpnum2parpnum; - - } - - globelnums = new Array; - if ( id == 0 ) - { - for ( int dest = 1; dest < ntasks; dest++) - { - MyMPI_Recv ( *globelnums, dest ); - for ( int i = 0; i < globelnums->Size(); i++ ) - { - paralleltop -> SetDistantEl ( dest, (*globelnums)[i],i+1 ); - paralleltop -> SetExchangeElement ( dest, (*globelnums)[i] ); - } - } - } - else - { - globelnums -> SetSize(GetNE()); - for ( int i = 0; i < GetNE(); i++ ) - { - (*globelnums)[i] = paralleltop -> GetLoc2Glob_VolEl ( i+1 ); - } - MyMPI_Send ( *globelnums, 0, MPI_TAG_MESH ); - } - - delete globelnums; - // send which elements are where - - topology -> Update(); - - // edge, facenums have probably changed as elements were added - // paralleltop has to be updated - - - paralleltop -> UpdateExchangeElements(); - - - paralleltop -> UpdateCoarseGridOverlap(); - //paralleltop -> UpdateTopology(); - -// *testout << "############################################" << endl << endl; -// paralleltop -> Print(); - - clusters -> Update(); - ; -#ifdef SCALASCA -#pragma pomp inst end(updateoverlap) -#endif - - -#endif - } diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index 04a3533f..1087eaac 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -8,9 +8,34 @@ namespace netgen { + static MPI_Group MPI_HIGHORDER_WORLD; + static MPI_Comm MPI_HIGHORDER_COMM; + + void MyMPI_ExchangeTable (TABLE & send_verts, + TABLE & recv_verts, int tag, + MPI_Comm comm = MPI_COMM_WORLD) + { + int ntasks, rank; + MPI_Comm_size(comm, &ntasks); + MPI_Comm_rank(comm, &rank); + + Array requests; + for (int dest = 0; dest < ntasks; dest++) + if (dest != rank) + requests.Append (MyMPI_ISend (send_verts[dest], dest, tag, comm)); + + for (int i = 0; i < ntasks-1; i++) + { + MPI_Status status; + MPI_Probe (MPI_ANY_SOURCE, tag, comm, &status); + int size, src = status.MPI_SOURCE; + MPI_Get_count (&status, MPI_INT, &size); + recv_verts.SetEntrySize (src, size, sizeof(int)); + requests.Append (MyMPI_IRecv (recv_verts[src], src, tag, comm)); + } + MPI_Waitall (requests.Size(), &requests[0], MPI_STATUS_IGNORE); + } - MPI_Group MPI_HIGHORDER_WORLD; - MPI_Comm MPI_HIGHORDER_COMM; @@ -19,11 +44,8 @@ namespace netgen *testout << "ParallelMeshTopology::Reset" << endl; if ( ntasks == 1 ) return; - PrintMessage ( 4, "RESET"); int nvold = nv; - // int nedold = ned; - // int nfaold = nfa; ne = mesh.GetNE(); nv = mesh.GetNV(); @@ -33,7 +55,6 @@ namespace netgen ned = mesh.GetTopology().GetNEdges(); nfa = mesh.GetTopology().GetNFaces(); - loc2distedge.ChangeSize (ned); for (int i = 0; i < ned; i++) if (loc2distedge[i].Size() == 0) @@ -44,63 +65,16 @@ namespace netgen if (loc2distface[i].Size() == 0) loc2distface.Add (i, -1); // will be the global nr - - if ( !isexchangevert ) - { - isexchangevert = new BitArray (nv * ( ntasks+1 )); - isexchangevert->Clear(); - } - if ( !isexchangeedge ) - { - isexchangeedge = new BitArray (ned*(ntasks+1) ); - isexchangeedge->Clear(); - } - if ( !isexchangeface ) - { - isexchangeface = new BitArray (nfa*(ntasks+1) ); - isexchangeface->Clear(); - } - if ( !isexchangeel ) - { - isexchangeel = new BitArray (ne*(ntasks+1) ); - isexchangeel->Clear(); - } - - - // if the number of vertices did not change, return if ( nvold == nv ) return; - // faces and edges get new numbers -> delete - isexchangeface -> SetSize(nfa*(ntasks+1) ); - isexchangeedge -> SetSize(ned*(ntasks+1) ); - isexchangeface -> Clear(); - isexchangeedge -> Clear(); - - SetNV(nv); SetNE(ne); - - - if ( !isghostedge.Size() ) - { - isghostedge.SetSize(ned); - isghostedge.Clear(); - } - if ( !isghostface.Size() ) - { - isghostface.SetSize(nfa); - isghostface.Clear(); - } - } ParallelMeshTopology :: ~ParallelMeshTopology () { - delete isexchangeface; - delete isexchangevert; - delete isexchangeedge; - delete isexchangeel; + ; } @@ -108,8 +82,8 @@ namespace netgen ParallelMeshTopology :: ParallelMeshTopology ( const netgen::Mesh & amesh ) : mesh(amesh) { - ned = 0; //mesh.GetTopology().GetNEdges(); - nfa = 0; //mesh.GetTopology().GetNFaces(); + ned = 0; + nfa = 0; nv = 0; ne = 0; np = 0; @@ -120,26 +94,21 @@ namespace netgen nparel = 0; - isexchangeface = 0; - isexchangevert = 0; - isexchangeel = 0; - isexchangeedge = 0; - coarseupdate = 0; + /* isghostedge.SetSize(0); isghostface.SetSize(0); - - overlap = 0; + */ + // overlap = 0; } - int ParallelMeshTopology :: Glob2Loc_Vert (int globnum ) + int ParallelMeshTopology :: Glob2Loc_Vert (int globnum) { for (int i = 1; i <= nv; i++) if ( globnum == loc2distvert[i][0] ) return i; - return -1; } @@ -147,12 +116,8 @@ namespace netgen { int locnum = -1; for (int i = 0; i < ne; i++) - { - if ( globnum == loc2distel[i][0] ) - { - locnum = i+1; - } - } + if ( globnum == loc2distel[i][0] ) + locnum = i+1; return locnum; } @@ -160,12 +125,8 @@ namespace netgen { int locnum = -1; for (int i = 0; i < nsurfel; i++) - { - if ( globnum == loc2distsurfel[i][0] ) - { - locnum = i+1; - } - } + if ( globnum == loc2distsurfel[i][0] ) + locnum = i+1; return locnum; } @@ -173,12 +134,8 @@ namespace netgen { int locnum = -1; for (int i = 0; i < nseg; i++) - { - if ( globnum == loc2distsegm[i][0] ) - { - locnum = i+1; - } - } + if ( globnum == loc2distsegm[i][0] ) + locnum = i+1; return locnum; } @@ -189,7 +146,6 @@ namespace netgen (*testout) << endl << "TOPOLOGY FOR PARALLEL MESHES" << endl << endl; - for ( int i = 1; i <= nv; i++ ) if ( IsExchangeVert (i) ) { @@ -210,12 +166,9 @@ namespace netgen for ( int dest = 0; dest < ntasks; dest++) if ( GetDistantEdgeNum ( dest, i ) > 0 ) if ( dest != id ) - { - (*testout) << " p" << dest << ": " << GetDistantEdgeNum ( dest, i ) << endl; - } + (*testout) << " p" << dest << ": " << GetDistantEdgeNum ( dest, i ) << endl; } - for ( int i = 1; i <= nfa; i++ ) if ( IsExchangeFace(i) ) { @@ -235,6 +188,7 @@ namespace netgen } + /* for ( int i = 1; i < mesh.GetNE(); i++) { if ( !IsExchangeElement(i) ) continue; @@ -248,9 +202,8 @@ namespace netgen (*testout) << el.PNum(j+1) << " "; (*testout) << "is ghost " << IsGhostEl(i) << endl; (*testout) << endl; - - } + */ } @@ -307,42 +260,10 @@ namespace netgen - /* - // - // gibt anzahl an distant pnums zurueck - int ParallelMeshTopology :: GetNDistantPNums ( int locpnum ) const - { - return loc2distvert[locpnum].Size() / 2 + 1; - } - - int ParallelMeshTopology :: GetNDistantFaceNums ( int locfacenum ) const - { - int size = loc2distface[locfacenum-1].Size() / 2 + 1; - return size; - } - - int ParallelMeshTopology :: GetNDistantEdgeNums ( int locedgenum ) const - { - int size = loc2distedge[locedgenum-1].Size() / 2 + 1; - return size; - } - - int ParallelMeshTopology :: GetNDistantElNums ( int locelnum ) const - { - int size = loc2distel[locelnum-1].Size() / 2 + 1; - return size; - } - */ - - // gibt anzahl an distant pnums zurueck // * pnums entspricht Array int ParallelMeshTopology :: GetDistantPNums ( int locpnum, int * distpnums ) const { - // distpnums[0] = loc2distvert[locpnum][0]; - - // for (int i = 1; i < loc2distvert[locpnum].Size(); i += 2) - // distpnums[ loc2distvert[locpnum][i] ] = loc2distvert[locpnum][i+1]; distpnums[0] = 0; distpnums[1] = loc2distvert[locpnum][0]; for ( int i = 1; i < loc2distvert[locpnum].Size(); i++ ) @@ -354,11 +275,6 @@ namespace netgen int ParallelMeshTopology :: GetDistantFaceNums ( int locfacenum, int * distfacenums ) const { - // distfacenums[0] = loc2distface[locfacenum-1][0]; - - // for ( int i = 1; i < loc2distface[locfacenum-1].Size(); i+=2 ) - // distfacenums[loc2distface[locfacenum-1][i]] = loc2distface[locfacenum-1][i+1]; - distfacenums[0] = 0; distfacenums[1] = loc2distface[locfacenum-1][0]; @@ -371,11 +287,6 @@ namespace netgen int ParallelMeshTopology :: GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const { - // distedgenums[0] = loc2distedge[locedgenum-1][0]; - - // for ( int i = 1; i < loc2distedge[locedgenum-1].Size(); i+=2 ) - // distedgenums[loc2distedge[locedgenum-1][i]] = loc2distedge[locedgenum-1][i+1]; - distedgenums[0] = 0; distedgenums[1] = loc2distedge[locedgenum-1][0]; @@ -388,11 +299,6 @@ namespace netgen int ParallelMeshTopology :: GetDistantElNums ( int locelnum, int * distelnums ) const { - // distelnums[0] = loc2distel[locelnum-1][0]; - - // for ( int i = 1; i < loc2distel[locelnum-1].Size(); i+=2 ) - // distelnums[loc2distel[locelnum-1][i]] = loc2distel[locelnum-1][i+1]; - distelnums[0] = 0; distelnums[1] = loc2distel[locelnum-1][0]; @@ -406,7 +312,6 @@ namespace netgen - void ParallelMeshTopology :: SetDistantFaceNum ( int dest, int locnum, int distnum ) { if ( dest == 0 ) @@ -555,7 +460,7 @@ namespace netgen void ParallelMeshTopology :: UpdateRefinement () { - ; + ; } @@ -563,8 +468,8 @@ namespace netgen void ParallelMeshTopology :: UpdateCoarseGridGlobal () { - PrintMessage ( 1, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS - + if (id == 0) + PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS MPI_Group MPI_GROUP_WORLD; @@ -576,16 +481,12 @@ namespace netgen MPI_Comm_group ( MPI_COMM_WORLD, &MPI_GROUP_WORLD); MPI_Group_incl ( MPI_GROUP_WORLD, n_ho, process_ranks, & MPI_HIGHORDER_WORLD); MPI_Comm_create ( MPI_COMM_WORLD, MPI_HIGHORDER_WORLD, & MPI_HIGHORDER_COMM); - - int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal"); NgProfiler::RegionTimer reg(timer); - - *testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl; const MeshTopology & topology = mesh.GetTopology(); @@ -598,47 +499,13 @@ namespace netgen ne = mesh . GetNE(); nseg = mesh.GetNSeg(); nsurfel = mesh.GetNSE(); - // low order processor - save mesh partition if ( id == 0 ) { - if ( !isexchangeel ) - { - isexchangeel = new BitArray ( (ntasks+1) * ne ); - isexchangeel -> Clear(); - } - - for ( int eli = 1; eli <= ne; eli++ ) - { - loc2distel[eli-1][0] = eli; - SetExchangeElement ( eli ); - const Element & el = mesh . VolumeElement ( eli ); - int dest = el . GetPartition ( ); - SetExchangeElement ( dest, eli ); - - for ( int i = 0; i < el.GetNP(); i++ ) - { - SetExchangeVert ( dest, el.PNum(i+1) ); - SetExchangeVert ( el.PNum(i+1) ); - } - Array edges; - topology . GetElementEdges ( eli, edges ); - for ( int i = 0; i < edges.Size(); i++ ) - { - SetExchangeEdge ( dest, edges[i] ); - SetExchangeEdge ( edges[i] ); - } - topology . GetElementFaces ( eli, edges ); - for ( int i = 0; i < edges.Size(); i++ ) - { - SetExchangeFace ( dest, edges[i] ); - SetExchangeFace ( edges[i] ); - } - } + loc2distel[eli-1][0] = eli; - // HERE for ( int i = 1; i <= mesh .GetNV(); i++) loc2distvert[i][0] = i; @@ -768,11 +635,11 @@ namespace netgen if (id == 0) { - PrintMessage (4, "UpdateCoarseGridGlobal : bcast, size = ", int (sendarray.Size()*sizeof(int)) ); + // PrintMessage (4, "UpdateCoarseGridGlobal : bcast, size = ", int (sendarray.Size()*sizeof(int)) ); MyMPI_Bcast ( sendarray ); } else - MyMPI_ISend ( sendarray, 0, MPI_TAG_MESH, sendrequest ); + sendrequest = MyMPI_ISend ( sendarray, 0, MPI_TAG_MESH ); int nloops = (id == 0) ? ntasks-1 : 1; @@ -783,7 +650,7 @@ namespace netgen if (id == 0) { sender = MyMPI_Recv ( recvarray, MPI_TAG_MESH ); - PrintMessage (4, "have received from ", sender); + // PrintMessage (4, "have received from ", sender); } else { @@ -799,7 +666,7 @@ namespace netgen if ( id != 0 ) nfaglob = recvarray[ii++]; - + if (id == 0) continue; Array faces, edges; Array pnums, globalpnums; @@ -841,14 +708,7 @@ namespace netgen coarseupdate = 1; if (id != 0) - { - MPI_Status status; - MPI_Wait (&sendrequest, &status); - } - -#ifdef SCALASCA -#pragma pomp inst end(updatecoarsegrid) -#endif + MPI_Wait (&sendrequest, MPI_STATUS_IGNORE); } @@ -858,15 +718,13 @@ namespace netgen void ParallelMeshTopology :: UpdateCoarseGrid () { - int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid"); + static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid"); NgProfiler::RegionTimer reg(timer); -#ifdef SCALASCA -#pragma pomp inst begin(updatecoarsegrid) -#endif (*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl; - PrintMessage (1, "UPDATE COARSE GRID PARALLEL TOPOLOGY "); + if (id == 0) + PrintMessage (1, "UPDATE COARSE GRID PARALLEL TOPOLOGY "); // find exchange edges - first send exchangeedges locnum, v1, v2 // receive distant distnum, v1, v2 @@ -874,9 +732,14 @@ namespace netgen const MeshTopology & topology = mesh.GetTopology(); UpdateCoarseGridGlobal(); + + MPI_Barrier (MPI_COMM_WORLD); - - if ( id == 0 ) return; + if ( id == 0 ) + { + MPI_Barrier (MPI_COMM_WORLD); + return; + } Array sendarray, recvarray; @@ -890,8 +753,69 @@ namespace netgen nsurfel = mesh.GetNSE(); - // exchange vertices + static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); + static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges"); + static int timerf = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex faces"); + + Array glob2loc; + Array cnt_send(ntasks-1); + + + /* + // exchange vertices + NgProfiler::StartTimer (timerv); + + *testout << "exchange vertices" << endl; + glob2loc.SetSize (nvglob); + glob2loc = -1; + + for (int locv = 1; locv <= nv; locv++) + if (IsExchangeVert (locv) ) + glob2loc[GetDistantPNum(0, locv)] = locv; + + + cnt_send = 0; + for (int vertex = 1; vertex <= nv; vertex++) + for (int dest = 1; dest < ntasks; dest++) + if (IsExchangeVert (dest, vertex)) + cnt_send[dest-1]+=2; + + // *testout << "cnt_send = " << cnt_send << endl; + + TABLE send_verts(cnt_send); + for (int vertex = 1; vertex <= nv; vertex++) + for (int dest = 1; dest < ntasks; dest++) + if (IsExchangeVert (dest, vertex)) + { + send_verts.Add (dest-1, GetLoc2Glob_Vert (vertex)); + send_verts.Add (dest-1, vertex); + } + + TABLE recv_verts (ntasks-1); + MyMPI_ExchangeTable (send_verts, recv_verts, MPI_TAG_MESH+8, MPI_HIGHORDER_COMM); + + for (int sender = 1; sender < ntasks; sender ++) + if (id != sender) + { + FlatArray recvarray = recv_verts[sender-1]; + for (int ii = 0; ii < recvarray.Size(); ) + { + int globv = recvarray[ii++]; + int distv = recvarray[ii++]; + int locv = glob2loc[globv]; + + if (locv != -1) + { + if (GetDistantPNum(sender, locv) != distv) + cerr << "distant pnum inconsistent: sender = " << sender << ", locv = "<< locv << ", distv = " << distv << ", getdist = " << GetDistantPNum(sender, locv) << endl; + // SetDistantPNum (sender, locv, distv); + } + } + } + */ + + /* for (int vertex = 1; vertex <= nv; vertex++) if (IsExchangeVert (vertex) ) { @@ -926,6 +850,12 @@ namespace netgen } } + NgProfiler::StopTimer (timerv); + */ + + + NgProfiler::StartTimer (timere); + sendarray.SetSize (0); recvarray.SetSize (0); @@ -933,7 +863,7 @@ namespace netgen // exchange edges int maxedge = 0; for (int edge = 1; edge <= ned; edge++) - if (IsExchangeEdge (edge) ) + // if (IsExchangeEdge (edge) ) { sendarray.Append (GetDistantEdgeNum (0, edge)); sendarray.Append (edge); @@ -944,7 +874,7 @@ namespace netgen glob2loc = -1; for (int loc = 1; loc <= ned; loc++) - if (IsExchangeEdge (loc) ) + // if (IsExchangeEdge (loc) ) glob2loc[GetDistantEdgeNum(0, loc)] = loc; for (int sender = 1; sender < ntasks; sender ++) @@ -968,9 +898,71 @@ namespace netgen } } + NgProfiler::StopTimer (timere); + NgProfiler::StartTimer (timerf); + + glob2loc.SetSize (nfaglob); + glob2loc = -1; + + for (int loc = 1; loc <= nfa; loc++) + glob2loc[GetDistantFaceNum(0, loc)] = loc; + + cnt_send = 0; + Array verts; + for (int face = 1; face <= nfa; face++) + { + topology.GetFaceVertices (face, verts); + for (int dest = 1; dest < ntasks; dest++) + if (IsExchangeVert (dest, verts[0]) && + IsExchangeVert (dest, verts[1]) && + IsExchangeVert (dest, verts[2])) + { + cnt_send[dest-1]+=2; + } + } + + TABLE send_faces(cnt_send); + for (int face = 1; face <= nfa; face++) + { + topology.GetFaceVertices (face, verts); + for (int dest = 1; dest < ntasks; dest++) + { + if (IsExchangeVert (dest, verts[0]) && + IsExchangeVert (dest, verts[1]) && + IsExchangeVert (dest, verts[2])) + { + send_faces.Add (dest-1, GetDistantFaceNum(0, face)); + send_faces.Add (dest-1, face); + } + } + } + TABLE recv_faces(ntasks-1); + MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_HIGHORDER_COMM); + + // *testout << "send exchange faces: " << send_faces << endl; + // *testout << "recv exchange faces: " << recv_faces << endl; + + for (int sender = 1; sender < ntasks; sender ++) + if (id != sender) + { + FlatArray recvarray = recv_faces[sender-1]; + + for (int ii = 0; ii < recvarray.Size(); ) + { + int globf = recvarray[ii++]; + int distf = recvarray[ii++]; + int locf = glob2loc[globf]; + + // *testout << "set distant face, sender = " << sender << ", locf = " << locf << "; distf = " << distf << endl; + if (locf != -1) + SetDistantFaceNum (sender, locf, distf); + } + } + + + /* sendarray.SetSize (0); recvarray.SetSize (0); - // exchange faces for (int face = 1; face <= nfa; face++) @@ -1005,8 +997,9 @@ namespace netgen } } } + */ - + NgProfiler::StopTimer (timerf); @@ -1155,381 +1148,20 @@ namespace netgen // set which elements are where for the master processor coarseupdate = 1; - -#ifdef SCALASCA -#pragma pomp inst end(updatecoarsegrid) -#endif - + MPI_Barrier (MPI_COMM_WORLD); } - void ParallelMeshTopology :: UpdateCoarseGridOverlap () - { - UpdateCoarseGridGlobal(); - -#ifdef SCALASCA -#pragma pomp inst begin(updatecoarsegrid) -#endif - (*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY, OVERLAP " << endl; - PrintMessage ( 1, "UPDATE COARSE GRID PARALLEL TOPOLOGY, OVERLAP " ); - - const MeshTopology & topology = mesh.GetTopology(); - - nfa = topology . GetNFaces(); - ned = topology . GetNEdges(); - np = mesh . GetNP(); - nv = mesh . GetNV(); - ne = mesh . GetNE(); - nseg = mesh.GetNSeg(); - nsurfel = mesh.GetNSE(); - - - if ( id != 0 ) - { - - // find exchange edges - first send exchangeedges locnum, v1, v2 - // receive distant distnum, v1, v2 - // find matching - - Array * sendarray, *recvarray; - sendarray = new Array (0); - recvarray = new Array; - - sendarray -> SetSize (0); - - BitArray recvface(nfa); - recvface.Clear(); - - for ( int el = 1; el <= ne; el++ ) - { - Array edges, pnums, faces; - topology.GetElementFaces (el, faces); - int globeli = GetLoc2Glob_VolEl(el); - for ( int fai = 0; fai < faces.Size(); fai++) - { - int fa = faces[fai]; - - topology.GetFaceEdges ( fa, edges ); - topology.GetFaceVertices ( fa, pnums ); - - if ( !IsExchangeElement ( el ) ) continue; - - int globfa = GetDistantFaceNum(0, fa) ; - - // send : - // localfacenum - // globalfacenum - // globalelnum - // np - // ned - // globalpnums - // localpnums - - // localedgenums mit globalelnums mit globalv1, globalv2 - // - - sendarray -> Append ( fa ); - sendarray -> Append ( globfa ); - sendarray -> Append ( globeli ); - sendarray -> Append ( pnums.Size() ); - sendarray -> Append ( edges.Size() ); - for ( int i = 0; i < pnums.Size(); i++ ) - { - sendarray -> Append( GetLoc2Glob_Vert(pnums[i]) ); - } - for ( int i = 0; i < pnums.Size(); i++ ) - { - sendarray -> Append(pnums[i] ); - } - for ( int i = 0; i < edges.Size(); i++ ) - { - int globedge = GetDistantEdgeNum(0, edges[i] ); - int v1, v2; - topology . GetEdgeVertices ( edges[i], v1, v2 ); - int dv1 = GetLoc2Glob_Vert ( v1 ); - int dv2 = GetLoc2Glob_Vert ( v2 ); - - sendarray -> Append(edges[i] ); - sendarray -> Append (globedge); - sendarray -> Append ( dv1 ); - sendarray -> Append ( dv2 ); - } - } - } - - BitArray edgeisinit(ned), vertisinit(np); - edgeisinit.Clear(); - vertisinit.Clear(); - - // Array for temporary use, to find local from global element fast - // only for not too big meshes - // seems ok, as low-order space is treated on one proc - Array * glob2loc_el; - - glob2loc_el = new Array ( neglob ); - (*glob2loc_el) = -1; - for ( int locel = 1; locel <= mesh.GetNE(); locel++) - (*glob2loc_el)[GetLoc2Glob_VolEl(locel)] = locel; - - for ( int sender = 1; sender < ntasks; sender ++ ) - { - if ( id == sender ) - MyMPI_Bcast (*sendarray, sender-1, MPI_HIGHORDER_COMM); - - // { - // for ( int dest = 1; dest < ntasks; dest ++ ) - // if ( dest != id) - // { - // MyMPI_Send (*sendarray, dest); - // } - // } - - if ( id != sender ) - { - // MyMPI_Recv ( *recvarray, sender); - MyMPI_Bcast (*recvarray, sender-1, MPI_HIGHORDER_COMM); - // compare received vertices with own ones - int ii = 0; - // int cntel = 0; - int volel = 1; - - while ( ii< recvarray -> Size() ) - { - - // receive list : - // distant facenum - // np - // ned - // globalpnums - // distant pnums - - // distant edgenums mit globalv1, globalv2 - - int distfa = (*recvarray)[ii++]; - int globfa = (*recvarray)[ii++]; - int globvolel = (*recvarray)[ii++]; - int distnp = (*recvarray)[ii++]; - int distned =(*recvarray)[ii++]; - - if ( id > 0 ) // GetLoc2Glob_VolEl ( volel ) != globvolel ) - volel = (*glob2loc_el)[globvolel]; //Glob2Loc_VolEl ( globvolel ); - else - volel = globvolel; - - if ( volel == -1 ) - { - ii += 2*distnp + 4*distned; - volel = 1; - continue; - } - - Array faces, edges; - topology.GetElementFaces( volel, faces); - topology.GetElementEdges ( volel, edges); - for ( int fai= 0; fai < faces.Size(); fai++ ) - { - int fa = faces[fai]; - if ( !IsExchangeFace ( fa ) && sender != 0 ) continue; - // if ( recvface.Test ( fa-1 ) ) continue; - - Array pnums, globalpnums; - //topology.GetFaceEdges ( fa, edges ); - topology.GetFaceVertices ( fa, pnums ); - - - // find exchange faces ... - // have to be of same type - if ( pnums.Size () != distnp ) continue; - - - globalpnums.SetSize ( distnp ); - for ( int i = 0; i < distnp; i++) - globalpnums[i] = GetLoc2Glob_Vert ( pnums[i] ); - - - - - - // test if 3 vertices match - bool match = 1; - for ( int i = 0; i < distnp; i++) - if ( !globalpnums.Contains ( (*recvarray)[ii+i] ) ) - match = 0; - - if ( !match ) continue; - - // recvface.Set(fa-1); - - SetDistantFaceNum ( sender, fa, distfa ); - - SetDistantFaceNum ( 0, fa, globfa ); - - // find exchange points - for ( int i = 0; i < distnp; i++) - { - int distglobalpnum = (*recvarray)[ii+i]; - for ( int j = 0; j < distnp; j++ ) - if ( globalpnums[j] == distglobalpnum ) - { - // set sender -- distpnum ---- locpnum - int distpnum = (*recvarray)[ii + i +distnp]; - SetDistantPNum ( sender, pnums[j], distpnum ); - } - - } - - int * distedgenums = new int [distned]; - // find exchange edges - for ( int i = 0; i < edges.Size(); i++) - { - int v1, v2; - topology . GetEdgeVertices ( edges[i], v1, v2 ); - int dv1 = GetLoc2Glob_Vert ( v1 ); - int dv2 = GetLoc2Glob_Vert ( v2 ); - if ( dv1 > dv2 ) swap ( dv1, dv2 ); - for ( int ed = 0; ed < distned; ed++) - { - distedgenums[ed] = (*recvarray)[ii + 2*distnp + 4*ed]; - int globedgenum = (*recvarray)[ii + 2*distnp + 4*ed + 1]; - int ddv1 = (*recvarray)[ii + 2*distnp + 4*ed + 2]; - int ddv2 = (*recvarray)[ii + 2*distnp + 4*ed + 3]; - if ( ddv1 > ddv2 ) swap ( ddv1, ddv2 ); - if ( dv1 == ddv1 && dv2 == ddv2 ) - { - // set sender -- distednum -- locednum - SetDistantEdgeNum ( sender, edges[i], distedgenums[ed] ); - SetDistantEdgeNum ( 0, edges[i], globedgenum ); - } - } - - - } - delete [] distedgenums; - - - } - ii += 2*distnp + 4*distned; - } - - - - - } - - } - - // set which elements are where for the master processor - - delete sendarray; delete recvarray; - if ( id > 0 ) - delete glob2loc_el; - coarseupdate = 1; - - } - - - // send global-local el/face/edge/vert-info to id 0 - - - // nfa = topology . GetNFaces(); - // ned = topology . GetNEdges(); - // np = mesh . GetNP(); - // nv = mesh . GetNV(); - // ne = mesh . GetNE(); - // nseg = mesh.GetNSeg(); - // nsurfel = mesh.GetNSE(); - if ( id != 0 ) - { - Array * sendarray; - sendarray = new Array (4); - - int sendnfa = 0, sendned = 0; - - (*sendarray)[0] = ne; - (*sendarray)[1] = nfa; - (*sendarray)[2] = ned; - (*sendarray)[3] = np; - - // int ii = 4; - for ( int el = 1; el <= ne; el++ ) - (*sendarray).Append ( GetLoc2Glob_VolEl (el ) ); - - for ( int fa = 1; fa <= nfa; fa++ ) - { - if ( !IsExchangeFace (fa) ) continue; - sendnfa++; - (*sendarray).Append ( fa ); - (*sendarray).Append ( GetDistantFaceNum (0, fa) ); - } - - for ( int ed = 1; ed <= ned; ed++ ) - { - if ( !IsExchangeEdge (ed) ) continue; - sendned++; - sendarray->Append ( ed ); - sendarray->Append ( GetDistantEdgeNum(0, ed) ); - } - - for ( int vnum = 1; vnum <= np; vnum++ ) - sendarray->Append ( GetLoc2Glob_Vert(vnum) ); - - (*sendarray)[1] = sendnfa; - (*sendarray)[2] = sendned; - - MyMPI_Send (*sendarray, 0, MPI_TAG_MESH); - - delete sendarray; - } - - else - { - Array * recvarray = new Array; - - for ( int sender = 1; sender < ntasks; sender++ ) - { - MyMPI_Recv ( *recvarray, sender, MPI_TAG_MESH); - - int distnel = (*recvarray)[0]; - int distnfa = (*recvarray)[1]; - int distned = (*recvarray)[2]; - int distnp = (*recvarray)[3]; - - int ii = 4; - - for ( int el = 1; el <= distnel; el++ ) - SetDistantEl ( sender, (*recvarray)[ii++], el ); - - for ( int fa = 1; fa <= distnfa; fa++ ) - { - int distfa = (*recvarray)[ii++]; - SetDistantFaceNum ( sender, (*recvarray)[ii++], distfa ); - } - for ( int ed = 1; ed <= distned; ed++ ) - { - int disted = (*recvarray)[ii++]; - SetDistantEdgeNum ( sender, (*recvarray)[ii++], disted ); - } - for ( int vnum = 1; vnum <= distnp; vnum++ ) - SetDistantPNum ( sender, (*recvarray)[ii++], vnum ); - } - - delete recvarray; - } -#ifdef SCALASCA -#pragma pomp inst end(updatecoarsegrid) -#endif - } void ParallelMeshTopology :: UpdateTopology () { - // loop over parallel faces and edges, find new local face/edge number, - const MeshTopology & topology = mesh.GetTopology(); int nfa = topology.GetNFaces(); int ned = topology.GetNEdges(); + /* isghostedge.SetSize(ned); isghostface.SetSize(nfa); isghostedge.Clear(); @@ -1542,8 +1174,9 @@ namespace netgen if ( IsGhostVert(v1) || IsGhostVert(v2) ) SetGhostEdge ( ed ); } + */ - + /* Array pnums; for ( int fa = 1; fa <= nfa; fa++) { @@ -1555,174 +1188,11 @@ namespace netgen break; } } + */ } - void ParallelMeshTopology :: UpdateExchangeElements() - { - (*testout) << "UPDATE EXCHANGE ELEMENTS " << endl; - const MeshTopology & topology = mesh.GetTopology(); - isexchangeedge->SetSize ( (ntasks+1) * topology.GetNEdges() ); - isexchangeface->SetSize ( (ntasks+1) * topology.GetNFaces() ); - - isexchangeedge->Clear(); - isexchangeface->Clear(); - - for ( int eli = 1; eli <= mesh.GetNE(); eli++) - { - if ( ! IsExchangeElement ( eli ) ) continue; - const Element & el = mesh.VolumeElement(eli); - Array faces, edges; - int np = el.NP(); - - topology.GetElementEdges ( eli, edges ); - topology.GetElementFaces ( eli, faces ); - for ( int i = 0; i < edges.Size(); i++) - { - SetExchangeEdge ( edges[i] ); - } - for ( int i = 0; i < faces.Size(); i++) - { - SetExchangeFace ( faces[i] ); - } - for ( int i = 0; i < np; i++) - { - SetExchangeVert ( el[i] ); - } - } - - if ( id == 0 ) return; - - - - Array ** elementonproc, ** recvelonproc; - elementonproc = new Array*[ntasks]; - recvelonproc = new Array*[ntasks]; - - for ( int i = 1; i < ntasks; i++ ) - { - elementonproc[i] = new Array(0); - recvelonproc[i] = new Array(0); - } - - - for ( int eli = 1; eli <= mesh.GetNE(); eli++ ) - { - if ( !IsExchangeElement(eli) ) continue; - for ( int i = 1; i < ntasks; i++ ) - if ( GetDistantElNum(i, eli) != -1 && i != id ) - { - elementonproc[i] -> Append(eli); - elementonproc[i] -> Append(GetDistantElNum(i, eli)); - } - - } - - for ( int sender = 1; sender < ntasks; sender ++ ) - { - if ( id == sender ) - for ( int dest = 1; dest < ntasks; dest ++ ) - if ( dest != id) - { - MyMPI_Send ( *(elementonproc[dest]), dest, MPI_TAG_MESH); - elementonproc[dest] -> SetSize(0); - } - - - if ( id != sender ) - { - MyMPI_Recv (*( recvelonproc[sender]), sender, MPI_TAG_MESH); - } - } - - - int ii = 0; - for ( int sender = 1; sender < ntasks; sender++ ) - { - if ( sender == id ) continue; - - ii = 0; - while ( recvelonproc[sender]->Size() > ii ) - { - int distelnum = (*recvelonproc[sender])[ii++]; - int locelnum = (*recvelonproc[sender])[ii++]; - SetDistantEl ( sender, locelnum, distelnum); - } - recvelonproc[sender]->SetSize(0); - } - - - BitArray procs(ntasks); - procs.Clear(); - for ( int eli = 1; eli <= mesh.GetNE(); eli++) - { - if ( IsGhostEl(eli) ) continue; - if ( !IsExchangeElement(eli) ) continue; - - procs.Clear(); - int sumprocs = 0; - for ( int i = 1; i < ntasks; i++ ) - if ( GetDistantElNum(i, eli) != -1 && i != id ) - { - procs.Set(i); - sumprocs++; - } - - for ( int dest = 1; dest < ntasks; dest++) - { - if ( !procs.Test(dest) ) continue; - elementonproc[dest]->Append(GetDistantElNum(dest, eli)); - elementonproc[dest]->Append(sumprocs); - for ( int i = 1; i < ntasks; i++ ) - if ( procs.Test(i) ) - { - elementonproc[dest]->Append(i); - elementonproc[dest]->Append(GetDistantElNum(i, eli)); - } - } - } - - for ( int sender = 1; sender < ntasks; sender ++ ) - { - if ( id == sender ) - for ( int dest = 1; dest < ntasks; dest ++ ) - if ( dest != id) - { - MyMPI_Send ( *(elementonproc[dest]), dest, MPI_TAG_MESH); - delete elementonproc[dest]; - } - - - if ( id != sender ) - { - MyMPI_Recv (*( recvelonproc[sender]), sender, MPI_TAG_MESH); - } - } - - for ( int sender = 1; sender < ntasks; sender++ ) - { - if ( sender == id ) continue; - ii = 0; - while ( recvelonproc[sender]->Size() > ii ) - { - int locelnum = (*recvelonproc[sender])[ii++]; - int nprocs = (*recvelonproc[sender])[ii++]; - for ( int iproc = 0; iproc < nprocs; iproc++) - { - int proc = (*recvelonproc[sender])[ii++]; - int distelnum = (*recvelonproc[sender])[ii++]; - if ( id == proc ) continue; - SetExchangeElement (locelnum, proc); - SetDistantEl( proc, locelnum, distelnum ); - } - } - delete recvelonproc[sender]; - } - - delete [] elementonproc; - delete [] recvelonproc; - } @@ -1738,40 +1208,16 @@ namespace netgen if (loc2distvert.EntrySize(i) == 0) loc2distvert.Add (i, -1); // will be the global nr - BitArray * isexchangevert2 = new BitArray( (ntasks+1) * anv ); - isexchangevert2->Clear(); - if ( isexchangevert ) - { - for ( int i = 0; i < min2( isexchangevert->Size(), isexchangevert2->Size() ); i++ ) - if ( isexchangevert->Test(i) ) isexchangevert2->Set(i); - delete isexchangevert; - } - - isexchangevert = isexchangevert2; nv = anv; - } void ParallelMeshTopology :: SetNE ( const int ane ) { loc2distel.ChangeSize (ane); for (int i = 0; i < ane; i++) - { - if (loc2distel[i].Size() == 0) - loc2distel.Add (i, -1); // will be the global nr - } - BitArray * isexchangeel2 = new BitArray ( (ntasks+1) * ane ); - isexchangeel2->Clear(); - if ( isexchangeel ) - { - for ( int i = 0; i < min2(isexchangeel->Size(), isexchangeel2->Size() ) ; i++ ) - if ( isexchangeel->Test(i) ) isexchangeel2->Set(i); - delete isexchangeel; - } - + if (loc2distel[i].Size() == 0) + loc2distel.Add (i, -1); // will be the global nr ne = ane; - isexchangeel = isexchangeel2; - } void ParallelMeshTopology :: SetNSE ( int anse ) diff --git a/libsrc/meshing/paralleltop.hpp b/libsrc/meshing/paralleltop.hpp index f8429d98..9edf9339 100644 --- a/libsrc/meshing/paralleltop.hpp +++ b/libsrc/meshing/paralleltop.hpp @@ -4,7 +4,7 @@ namespace netgen { - extern int ntasks; + // extern int ntasks; class ParallelMeshTopology { @@ -27,15 +27,10 @@ namespace netgen each row contains a list of pairs (procnr, dist_vnum) */ TABLE loc2distvert; - TABLE loc2distedge, loc2distface, loc2distel; - TABLE loc2distsegm, loc2distsurfel; - - BitArray * isexchangeface, * isexchangevert, * isexchangeedge, * isexchangeel; - - BitArray isghostedge, isghostface; + TABLE loc2distedge, loc2distface; + TABLE loc2distel, loc2distsegm, loc2distsurfel; bool coarseupdate; - int overlap; public: @@ -43,10 +38,17 @@ namespace netgen ~ParallelMeshTopology (); /// set number of local vertices, reset sizes of loc2dist_vert, isexchangevert... - void SetNV ( int anv ); - void SetNE ( int ane ); - void SetNSE ( int anse ); - void SetNSegm ( int anseg ); + void SetNV (int anv); + void SetNE (int ane); + void SetNSE (int anse); + void SetNSegm (int anseg); + + void SetNVGlob ( int anvglob ) { nvglob = anvglob; } + void SetNEGlob ( int aneglob ) { neglob = aneglob; } + + int GetNVGlob () { return nvglob; } + int GetNEGlob () { return neglob; } + void Reset (); @@ -65,10 +67,17 @@ namespace netgen int Glob2Loc_Segm ( int globnum ); int Glob2Loc_Vert ( int globnum ); - int GetNDistantPNums ( int locpnum ) const { return loc2distvert[locpnum].Size() / 2 + 1; } - int GetNDistantFaceNums ( int locfacenum ) const { return loc2distface[locfacenum-1].Size() / 2 + 1; } - int GetNDistantEdgeNums ( int locedgenum ) const { return loc2distedge[locedgenum-1].Size() / 2 + 1; } - int GetNDistantElNums ( int locelnum ) const { return loc2distel[locelnum-1].Size() / 2 + 1; } + int GetNDistantPNums ( int locpnum ) const + { return loc2distvert[locpnum].Size() / 2 + 1; } + + int GetNDistantFaceNums ( int locfacenum ) const + { return loc2distface[locfacenum-1].Size() / 2 + 1; } + + int GetNDistantEdgeNums ( int locedgenum ) const + { return loc2distedge[locedgenum-1].Size() / 2 + 1; } + + int GetNDistantElNums ( int locelnum ) const + { return loc2distel[locelnum-1].Size() / 2 + 1; } int GetDistantPNum ( int proc, int locpnum ) const; int GetDistantEdgeNum ( int proc, int locedgenum ) const; @@ -81,60 +90,23 @@ namespace netgen int GetDistantElNums ( int locelnum, int * distfacenums ) const; void Print() const; + /* + void SetExchangeFace ( int fnr ) {}; // { isexchangeface->Set( (fnr-1)*(ntasks+1) ); } + void SetExchangeVert ( int vnum ) {}; // { isexchangevert->Set( (vnum-1)*(ntasks+1) ); } + void SetExchangeElement ( int elnum ) {}; // { isexchangeel->Set((elnum-1)*(ntasks+1) ); } + void SetExchangeEdge ( int ednum ) {}; // { isexchangeedge->Set ((ednum-1)*(ntasks+1)); } + */ - void SetExchangeFace ( int fnr ) { isexchangeface->Set( (fnr-1)*(ntasks+1) ); } - void SetExchangeVert ( int vnum ) { isexchangevert->Set( (vnum-1)*(ntasks+1) ); } - void SetExchangeElement ( int elnum ) { isexchangeel->Set((elnum-1)*(ntasks+1) ); } - void SetExchangeEdge ( int ednum ) { isexchangeedge->Set ((ednum-1)*(ntasks+1)); } + bool IsExchangeVert ( PointIndex vnum ) const { return loc2distvert[vnum].Size() > 1; } + bool IsExchangeEdge ( int ednum ) const { return loc2distedge[ednum-1].Size() > 1; } + bool IsExchangeFace ( int fnum ) const { return loc2distface[fnum-1].Size() > 1; } + bool IsExchangeElement ( int elnum ) const { return false; } - // fuer diese Fkts kann man ja O(N) - bitarrays lassen - bool IsExchangeVert ( PointIndex vnum ) const - { - return loc2distvert[vnum].Size() > 1; - // return isexchangevert->Test((vnum-1)*(ntasks+1)); - } - - bool IsExchangeEdge ( int ednum ) const - { - /* - if ( isexchangeedge->Test( (ednum-1)*(ntasks+1) ) != - ( loc2distedge[ednum-1].Size() > 1) ) - { - cerr << "isexedge differs, id = " << id << ", ednum = " << ednum << endl; - } - */ - // return loc2distedge[ednum-1].Size() > 1; - int i = (ednum-1)*(ntasks+1); - return isexchangeedge->Test(i); - } - - bool IsExchangeFace ( int fnum ) const - { - /* - if ( isexchangeface->Test( (fnum-1)*(ntasks+1) ) != - ( loc2distface[fnum-1].Size() > 1) ) - { - cerr << "it differs, id = " << id << ", fnum = " << fnum << endl; - } - */ - // nur hier funktioniert's so nicht ???? (JS) - // return loc2distface[fnum-1].Size() > 1; - return isexchangeface->Test( (fnum-1)*(ntasks+1) ); - } - - bool IsExchangeElement ( int elnum ) const - { - // return loc2distel[elnum-1].Size() > 1; - return isexchangeel->Test((elnum-1)*(ntasks+1)); - } - - bool IsExchangeSEl ( int selnum ) const - { - return loc2distsurfel[selnum-1].Size() > 1; - } + bool IsExchangeSEl ( int selnum ) const { return loc2distsurfel[selnum-1].Size() > 1; } + /* void SetExchangeFace (int dest, int fnr ) { // isexchangeface->Set((fnr-1)*(ntasks+1) + (dest+1)); @@ -147,13 +119,14 @@ namespace netgen void SetExchangeElement (int dest, int vnum ) { - isexchangeel->Set( (vnum-1)*(ntasks+1) + (dest+1) ); + ; // isexchangeel->Set( (vnum-1)*(ntasks+1) + (dest+1) ); } void SetExchangeEdge (int dest, int ednum ) { // isexchangeedge->Set ( (ednum-1)*(ntasks+1) + (dest+1) ); } + */ bool IsExchangeVert (int dest, int vnum ) const @@ -162,7 +135,6 @@ namespace netgen for (int i = 1; i < exchange.Size(); i += 2) if (exchange[i] == dest) return true; return false; - // return isexchangevert->Test((vnum-1)*(ntasks+1) + (dest+1) ); } bool IsExchangeEdge (int dest, int ednum ) const @@ -171,8 +143,6 @@ namespace netgen for (int i = 1; i < exchange.Size(); i += 2) if (exchange[i] == dest) return true; return false; - - // return isexchangeedge->Test((ednum-1)*(ntasks+1) + (dest+1)); } bool IsExchangeFace (int dest, int fnum ) const @@ -181,41 +151,20 @@ namespace netgen for (int i = 1; i < exchange.Size(); i += 2) if (exchange[i] == dest) return true; return false; - - // return isexchangeface->Test((fnum-1)*(ntasks+1) + (dest+1) ); } - bool IsExchangeElement (int dest, int elnum ) const - { - // das geht auch nicht - /* - FlatArray exchange = loc2distel[elnum-1]; - for (int i = 1; i < exchange.Size(); i += 2) - if (exchange[i] == dest) return true; - return false; - */ - return isexchangeel->Test((elnum-1)*(ntasks+1) + (dest+1)); - } - - int Overlap() const - { return overlap; } - - int IncreaseOverlap () - { overlap++; return overlap; } + bool IsExchangeElement (int dest, int elnum ) const { return false; } void Update(); void UpdateCoarseGrid(); - void UpdateCoarseGridOverlap(); void UpdateRefinement (); void UpdateTopology (); void UpdateExchangeElements(); void UpdateCoarseGridGlobal(); - bool DoCoarseUpdate() const - { return !coarseupdate; } - + bool DoCoarseUpdate() const { return !coarseupdate; } void SetDistantFaceNum ( int dest, int locnum, int distnum ); void SetDistantPNum ( int dest, int locnum, int distnum ); @@ -225,40 +174,8 @@ namespace netgen void SetDistantSegm ( int dest, int locnum, int distnum ); bool IsGhostEl ( int elnum ) const { return mesh.VolumeElement(elnum).IsGhost(); } - bool IsGhostFace ( int fanum ) const { return isghostface.Test(fanum-1); } - bool IsGhostEdge ( int ednum ) const { return isghostedge.Test(ednum-1); } - bool IsGhostVert ( int pnum ) const { return mesh.Point(pnum).IsGhost(); } - // inline void SetGhostVert ( const int pnum ) - // { isghostvert.Set(pnum-1); } - void SetGhostEdge ( int ednum ) - { isghostedge.Set(ednum-1); } - - void ClearGhostEdge ( int ednum ) - { isghostedge.Clear(ednum-1); } - - void SetGhostFace ( int fanum ) - { isghostface.Set(fanum-1); } - - void ClearGhostFace ( int fanum ) - { isghostface.Clear(fanum-1); } - - bool IsElementInPartition ( int elnum, int dest ) const - { - return IsExchangeElement ( dest, elnum); - } - - void SetElementInPartition ( int elnum, int dest ) - { - SetExchangeElement ( dest, elnum ); - } - - void SetNVGlob ( int anvglob ) { nvglob = anvglob; } - void SetNEGlob ( int aneglob ) { neglob = aneglob; } - - int GetNVGlob () { return nvglob; } - int GetNEGlob () { return neglob; } }; diff --git a/libsrc/meshing/topology.cpp b/libsrc/meshing/topology.cpp index 8823b959..67b538f2 100644 --- a/libsrc/meshing/topology.cpp +++ b/libsrc/meshing/topology.cpp @@ -89,7 +89,8 @@ namespace netgen int nfa = 0; int ned = edge2vert.Size(); - PrintMessage (3, "Update mesh topology"); + if (id == 0) + PrintMessage (3, "Update mesh topology"); (*testout) << " UPDATE MESH TOPOLOGY " << endl; (*testout) << "ne = " << ne << endl; @@ -167,7 +168,8 @@ namespace netgen static int timer1 = NgProfiler::CreateTimer ("topology::buildedges"); NgProfiler::RegionTimer reg1 (timer1); - PrintMessage (5, "Update edges "); + if (id == 0) + PrintMessage (5, "Update edges "); edges.SetSize(ne); surfedges.SetSize(nse); @@ -499,7 +501,8 @@ namespace netgen static int timer2 = NgProfiler::CreateTimer ("topology::buildfaces"); NgProfiler::RegionTimer reg2 (timer2); - PrintMessage (5, "Update faces "); + if (id == 0) + PrintMessage (5, "Update faces "); @@ -1011,7 +1014,7 @@ namespace netgen // *testout << "faces = " << face2vert << endl; if (pass == 1) { - *testout << "sort from " << first_fa << " to " << nfa << endl; + // *testout << "sort from " << first_fa << " to " << nfa << endl; for (int i = first_fa; i < nfa; i++) for (int j = first_fa+1; j < nfa; j++) if (face2vert[j] < face2vert[j-1]) @@ -1026,7 +1029,7 @@ namespace netgen } - *testout << "face2vert = " << endl << face2vert << endl; + // *testout << "face2vert = " << endl << face2vert << endl; @@ -1125,6 +1128,8 @@ namespace netgen // if ( !paralleltop.IsExchangeFace (i+1) ) // paralleltop.SetRefinementFace (i+1); + + /* paralleltop.SetExchangeFace (i+1); for (int j = 0; j < 4; j++) @@ -1141,7 +1146,8 @@ namespace netgen int v1, v2; GetEdgeVertices(faceedges[j], v1, v2 ); } - + */ + /* (*testout) << "pos = "; for (int j = 0; j < 4; j++) diff --git a/libsrc/visualization/mvdraw.cpp b/libsrc/visualization/mvdraw.cpp index 475a9a7e..d348fc7b 100644 --- a/libsrc/visualization/mvdraw.cpp +++ b/libsrc/visualization/mvdraw.cpp @@ -765,25 +765,15 @@ namespace netgen cout << "extensionstring = " << glXQueryExtensionsString( dpy, 0 ) << endl; */ - - Array request(ntasks); - MPI_Status status; + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("init"); + for (int dest = 1; dest < ntasks; dest++) { - cout << "Initparallelgl, send to " << dest << endl; - MyMPI_Send ("redraw", dest, MPI_TAG_CMD); - MyMPI_Send ("init", dest, MPI_TAG_VIS); - MyMPI_Send (displname, dest, MPI_TAG_VIS); MyMPI_Send (int (drawable), dest, MPI_TAG_VIS); MyMPI_Send (int (xid), dest, MPI_TAG_VIS); - - int hi; - MPI_Irecv( &hi, 1, MPI_INT, dest, MPI_TAG_VIS, MPI_COMM_WORLD, &request[dest]); - // MyMPI_IRecv (hi, dest, request[dest]); } - for ( int dest = 1; dest < ntasks; dest++ ) - MPI_Wait(&request[dest], &status); } } } @@ -795,11 +785,16 @@ namespace netgen if (id == 0) { + /* for (int dest = 1; dest < ntasks; dest++) { MyMPI_Send ("redraw", dest, MPI_TAG_CMD); MyMPI_Send ("broadcast", dest, MPI_TAG_VIS); } + */ + + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("broadcast"); } MyMPI_Bcast (selface); diff --git a/libsrc/visualization/vsmesh.cpp b/libsrc/visualization/vsmesh.cpp index 4569a252..2dcecf3b 100644 --- a/libsrc/visualization/vsmesh.cpp +++ b/libsrc/visualization/vsmesh.cpp @@ -893,7 +893,7 @@ namespace netgen NgProfiler::RegionTimer reg (timer); #ifdef PARALLELGL - cout << "buildfillelist, id = " << id << ", nse = " << mesh -> GetNSE() << endl; + // cout << "buildfillelist, id = " << id << ", nse = " << mesh -> GetNSE() << endl; if (id == 0 && ntasks > 1) { @@ -901,11 +901,16 @@ namespace netgen par_filledlists.SetSize (ntasks); + /* for ( int dest = 1; dest < ntasks; dest++ ) { MyMPI_Send ("redraw", dest, MPI_TAG_CMD); - MyMPI_Send ("filledlist", dest, MPI_TAG_VIS); + // MyMPI_Send ("filledlist", dest, MPI_TAG_VIS); } + */ + + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("filledlist"); for ( int dest = 1; dest < ntasks; dest++ ) { MyMPI_Recv (par_filledlists[dest], dest, MPI_TAG_VIS); @@ -1344,11 +1349,16 @@ namespace netgen par_linelists.SetSize (ntasks); + /* for ( int dest = 1; dest < ntasks; dest++ ) { - MyMPI_Send ("redraw", dest, MPI_TAG_CMD); - MyMPI_Send ("linelist", dest, MPI_TAG_VIS); + MyMPI_Send ("redraw", dest, MPI_TAG_CMD); + MyMPI_Send ("linelist", dest, MPI_TAG_VIS); } + */ + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("linelist"); + for ( int dest = 1; dest < ntasks; dest++ ) MyMPI_Recv (par_linelists[dest], dest, MPI_TAG_VIS); diff --git a/libsrc/visualization/vssolution.cpp b/libsrc/visualization/vssolution.cpp index 6a8db2c5..106c541f 100644 --- a/libsrc/visualization/vssolution.cpp +++ b/libsrc/visualization/vssolution.cpp @@ -994,11 +994,16 @@ namespace netgen par_surfellists.SetSize (ntasks); + /* for ( int dest = 1; dest < ntasks; dest++ ) { - MyMPI_Send ("redraw", dest, MPI_TAG_CMD); - MyMPI_Send ("solsurfellist", dest, MPI_TAG_VIS); + MyMPI_Send ("redraw", dest, MPI_TAG_CMD); + MyMPI_Send ("solsurfellist", dest, MPI_TAG_VIS); } + */ + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("solsurfellist"); + for ( int dest = 1; dest < ntasks; dest++ ) MyMPI_Recv (par_surfellists[dest], dest, MPI_TAG_VIS); @@ -3706,11 +3711,16 @@ namespace netgen Array parlists (ntasks); + /* for ( int dest = 1; dest < ntasks; dest++ ) { MyMPI_Send ("redraw", dest, MPI_TAG_CMD); MyMPI_Send ("clipplanetrigs", dest, MPI_TAG_VIS); } + */ + MyMPI_SendCmd ("redraw"); + MyMPI_SendCmd ("clipplanetrigs"); + for ( int dest = 1; dest < ntasks; dest++ ) MyMPI_Recv (parlists[dest], dest, MPI_TAG_VIS); @@ -4086,6 +4096,47 @@ namespace netgen void VisualSceneSolution :: Broadcast () { + MPI_Datatype type; + int blocklen[] = + { + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 4 + }; + MPI_Aint displ[] = { (char*)&usetexture - (char*)this, + (char*)&clipsolution - (char*)this, + (char*)&scalfunction - (char*)this, + (char*)&scalcomp - (char*)this, + + (char*)&vecfunction - (char*)this, + (char*)&gridsize - (char*)this, + (char*)&autoscale - (char*)this, + (char*)&logscale - (char*)this, + + (char*)&minval - (char*)this, + (char*)&maxval - (char*)this, + (char*)&numisolines - (char*)this, + (char*)&subdivisions - (char*)this, + + (char*)&clipplane[0] - (char*)this }; + + + MPI_Datatype types[] = { + MPI_INT, MPI_INT, MPI_INT, MPI_INT, + MPI_INT, MPI_INT, MPI_INT, MPI_INT, + MPI_DOUBLE, MPI_DOUBLE, MPI_INT, MPI_INT, + MPI_DOUBLE + }; + + MPI_Type_create_struct (13, blocklen, displ, types, &type); + MPI_Type_commit ( &type ); + + MPI_Bcast (this, 1, type, 0, MPI_COMM_WORLD); + + MPI_Type_free (&type); + + /* MyMPI_Bcast (usetexture); MyMPI_Bcast (clipsolution); MyMPI_Bcast (scalfunction); @@ -4104,6 +4155,7 @@ namespace netgen MyMPI_Bcast (clipplane[1]); MyMPI_Bcast (clipplane[2]); MyMPI_Bcast (clipplane[3]); + */ } #endif diff --git a/ng/ng.tcl b/ng/ng.tcl index 848c98fd..85444604 100644 --- a/ng/ng.tcl +++ b/ng/ng.tcl @@ -55,23 +55,24 @@ if { $shellmode == "defined" } { if { $batchmode != "defined" } { - catch { - wm withdraw . + catch { + wm withdraw . - wm title . $progname - wm geometry . =800x600 - wm minsize . 400 300 - } + wm title . $progname + wm geometry . =800x600 + wm minsize . 400 300 + } } source ${ngdir}/variables.tcl source ${ngdir}/parameters.tcl + if { $batchmode != "defined" } { - catch { - source ${ngdir}/menustat.tcl - } + catch { + source ${ngdir}/menustat.tcl + } } catch { @@ -83,7 +84,6 @@ catch { } - if { [catch { load libgeom2dvis[info sharedlibextension] Ng_Geom2d } result ] } { puts "cannot load 2d meshing module" puts "error: $result" diff --git a/ng/ngappinit.cpp b/ng/ngappinit.cpp index ad05dade..7baca21f 100644 --- a/ng/ngappinit.cpp +++ b/ng/ngappinit.cpp @@ -49,7 +49,7 @@ using netgen::RegisterUserFormats; extern "C" int Ng_ServerSocketManagerInit (int port); extern "C" int Ng_ServerSocketManagerRun (void); - + bool nodisplay = false; bool shellmode = false; @@ -63,12 +63,17 @@ int main(int argc, char ** argv) { #ifdef PARALLEL + // MPI_Init(&argc, &argv); + int required = MPI_THREAD_MULTIPLE; int provided; MPI_Init_thread(&argc, &argv, required, &provided); - cout << "requ = " << required << ", provided = " << provided << endl; + MPI_Comm_size(MPI_COMM_WORLD, &netgen::ntasks); MPI_Comm_rank(MPI_COMM_WORLD, &netgen::id); + + if (netgen::id == 0 && provided == MPI_THREAD_MULTIPLE) + cout << "multithreaded mpi is supported" << endl; #endif diff --git a/ng/ngpkg.cpp b/ng/ngpkg.cpp index 9aff4efd..e8231f0d 100644 --- a/ng/ngpkg.cpp +++ b/ng/ngpkg.cpp @@ -2997,8 +2997,7 @@ void PlayAnimFile(const char* name, int speed, int maxcnt) #endif #ifdef PARALLEL - for ( int dest = 1; dest < ntasks; dest++) - MyMPI_Send ( "end", dest, MPI_TAG_CMD ); + MyMPI_SendCmd ("end"); #endif mesh.Reset (NULL); diff --git a/ng/parallelfunc.cpp b/ng/parallelfunc.cpp index 364af101..03f24d78 100644 --- a/ng/parallelfunc.cpp +++ b/ng/parallelfunc.cpp @@ -102,8 +102,8 @@ void ParallelRun() bool test = true; - testout = new ostream(0); - // new ofstream (string("testout_proc") + id ); + // testout = new ostream(0); + testout = new ofstream (string("testout_proc") + id ); while ( test ) { @@ -111,7 +111,8 @@ void ParallelRun() #pragma pomp inst begin (message) #endif - MyMPI_Recv ( message, 0, MPI_TAG_CMD ); + // MyMPI_Recv ( message, 0, MPI_TAG_CMD ); + message = MyMPI_RecvCmd(); #ifdef SCALASCA #pragma pomp inst end (message) @@ -224,8 +225,9 @@ void ParallelRun() // did not manage to get glXImportContextEXT working on Laptop (JS) string redraw_cmd; - MyMPI_Recv (redraw_cmd, 0, MPI_TAG_VIS); - + // MyMPI_Recv (redraw_cmd, 0, MPI_TAG_VIS); + redraw_cmd = MyMPI_RecvCmd(); + // PrintMessage (1, "Redraw - ", redraw_cmd); static string displname; @@ -351,8 +353,6 @@ void ParallelRun() #endif // PrintMessage (1, "redraw - init complete"); - int hi = id; - MyMPI_Send (hi, 0, MPI_TAG_VIS); } if (redraw_cmd == "broadcast") diff --git a/ng/parallelinterface.cpp b/ng/parallelinterface.cpp index 1abcebd7..4409e194 100644 --- a/ng/parallelinterface.cpp +++ b/ng/parallelinterface.cpp @@ -138,13 +138,14 @@ namespace netgen mesh -> GetParallelTopology().Print (); } + +/* bool NgPar_IsElementInPartition ( const int elnum, const int dest ) { return mesh -> GetParallelTopology().IsElementInPartition ( elnum+1, dest ); } - bool NgPar_IsGhostFace ( const int facenum ) { return mesh -> GetParallelTopology().IsGhostFace ( facenum+1); @@ -154,7 +155,7 @@ namespace netgen { return mesh -> GetParallelTopology().IsGhostEdge ( edgenum+1); } - +*/ #endif