From 0d39c64fbe520b3039896e67684adc11b71cd7ac Mon Sep 17 00:00:00 2001 From: Joachim Schoeberl Date: Sat, 16 Jun 2012 12:04:04 +0000 Subject: [PATCH] mpi - topology --- libsrc/meshing/curvedelems.cpp | 12 +- libsrc/meshing/paralleltop.cpp | 1002 +++++++++++++++----------------- libsrc/meshing/paralleltop.hpp | 29 +- 3 files changed, 493 insertions(+), 550 deletions(-) diff --git a/libsrc/meshing/curvedelems.cpp b/libsrc/meshing/curvedelems.cpp index 7957054f..313ca3bf 100644 --- a/libsrc/meshing/curvedelems.cpp +++ b/libsrc/meshing/curvedelems.cpp @@ -545,7 +545,8 @@ namespace netgen edgecoeffsindex[0] = 0; for (int i = 0; i < top.GetNEdges(); i++) { - int glob = partop.GetDistantEdgeNum (0, i+1); + // int glob = partop.GetDistantEdgeNum (0, i+1); + int glob = partop.GetGlobalEdgeNum (i+1); edgeorder[i] = master_edgeorder[glob-1]; int ncoefs = master_edgecoeffsindex[glob]-master_edgecoeffsindex[glob-1]; edgecoeffsindex[i+1] = edgecoeffsindex[i] + ncoefs; @@ -554,7 +555,8 @@ namespace netgen for (int i = 0; i < top.GetNEdges(); i++) { - int glob = partop.GetDistantEdgeNum (0, i+1); + // int glob = partop.GetDistantEdgeNum (0, i+1); + int glob = partop.GetGlobalEdgeNum (i+1); int ncoefs = master_edgecoeffsindex[glob]-master_edgecoeffsindex[glob-1]; for (int j = 0; j < ncoefs; j++) edgecoeffs[edgecoeffsindex[i]+j] = master_edgecoeffs[master_edgecoeffsindex[glob-1]+j]; @@ -567,7 +569,8 @@ namespace netgen facecoeffsindex[0] = 0; for (int i = 0; i < top.GetNFaces(); i++) { - int glob = partop.GetDistantFaceNum (0, i+1); + // int glob = partop.GetDistantFaceNum (0, i+1); + int glob = partop.GetGlobalFaceNum (i+1); faceorder[i] = master_faceorder[glob-1]; int ncoefs = master_facecoeffsindex[glob]-master_facecoeffsindex[glob-1]; facecoeffsindex[i+1] = facecoeffsindex[i] + ncoefs; @@ -576,7 +579,8 @@ namespace netgen for (int i = 0; i < top.GetNFaces(); i++) { - int glob = partop.GetDistantFaceNum (0, i+1); + // int glob = partop.GetDistantFaceNum (0, i+1); + int glob = partop.GetGlobalFaceNum (i+1); int ncoefs = master_facecoeffsindex[glob]-master_facecoeffsindex[glob-1]; for (int j = 0; j < ncoefs; j++) facecoeffs[facecoeffsindex[i]+j] = master_facecoeffs[master_facecoeffsindex[glob-1]+j]; diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index f613b651..de64e85c 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -8,37 +8,6 @@ namespace netgen { - static MPI_Group MPI_HIGHORDER_WORLD; - static MPI_Comm MPI_HIGHORDER_COMM; - - template - void MyMPI_ExchangeTable (TABLE & send_verts, - TABLE & recv_verts, int tag, - MPI_Comm comm = MPI_COMM_WORLD) - { - int ntasks, rank; - MPI_Comm_size(comm, &ntasks); - MPI_Comm_rank(comm, &rank); - - Array requests; - for (int dest = 0; dest < ntasks; dest++) - if (dest != rank) - requests.Append (MyMPI_ISend (send_verts[dest], dest, tag, comm)); - - for (int i = 0; i < ntasks-1; i++) - { - MPI_Status status; - MPI_Probe (MPI_ANY_SOURCE, tag, comm, &status); - int size, src = status.MPI_SOURCE; - MPI_Get_count (&status, MPI_INT, &size); - recv_verts.SetEntrySize (src, size, sizeof(int)); - requests.Append (MyMPI_IRecv (recv_verts[src], src, tag, comm)); - } - MPI_Waitall (requests.Size(), &requests[0], MPI_STATUS_IGNORE); - } - - - void ParallelMeshTopology :: Reset () { @@ -102,7 +71,7 @@ namespace netgen void ParallelMeshTopology :: Print() const { - + /* (*testout) << endl << "TOPOLOGY FOR PARALLEL MESHES" << endl << endl; for ( int i = 1; i <= nv; i++ ) @@ -145,6 +114,8 @@ namespace netgen (*testout) << " p" << dest << ": " << GetDistantFaceNum ( dest, i ) << endl; } } + */ + /* @@ -167,72 +138,20 @@ namespace netgen - - - int ParallelMeshTopology :: GetDistantPNum ( int proc, int locpnum ) const - { - if ( proc == 0 ) - return loc2distvert[locpnum][0]; - - for (int i = 1; i < loc2distvert[locpnum].Size(); i += 2) - if ( loc2distvert[locpnum][i] == proc ) - return loc2distvert[locpnum][i+1]; - - return -1; - } - - int ParallelMeshTopology :: GetDistantFaceNum ( int proc, int locfacenum ) const - { - if ( proc == 0 ) - return loc2distface[locfacenum-1][0]; - - for ( int i = 1; i < loc2distface[locfacenum-1].Size(); i+=2 ) - if ( loc2distface[locfacenum-1][i] == proc ) - return loc2distface[locfacenum-1][i+1]; - - return -1; - } - - int ParallelMeshTopology :: GetDistantEdgeNum ( int proc, int locedgenum ) const - { - if ( proc == 0 ) - return loc2distedge[locedgenum-1][0]; - - for ( int i = 1; i < loc2distedge[locedgenum-1].Size(); i+=2 ) - if ( loc2distedge[locedgenum-1][i] == proc ) - return loc2distedge[locedgenum-1][i+1]; - - return -1; - } - - int ParallelMeshTopology :: GetDistantElNum ( int proc, int locelnum ) const - { - if ( proc == 0 ) - return loc2distel[locelnum-1][0]; - - for ( int i = 1; i < loc2distel[locelnum-1].Size(); i+=2 ) - if ( loc2distel[locelnum-1][i] == proc ) - return loc2distel[locelnum-1][i+1]; - - return -1; - } - - - // gibt anzahl an distant pnums zurueck // * pnums entspricht Array - int ParallelMeshTopology :: GetDistantPNums ( int locpnum, int * distpnums ) const + void ParallelMeshTopology :: GetDistantPNums ( int locpnum, int * distpnums ) const { distpnums[0] = 0; distpnums[1] = loc2distvert[locpnum][0]; for ( int i = 1; i < loc2distvert[locpnum].Size(); i++ ) distpnums[i+1] = loc2distvert[locpnum][i]; - int size = loc2distvert[locpnum].Size() / 2 + 1; - return size; + // int size = loc2distvert[locpnum].Size() / 2 + 1; + // return size; } - int ParallelMeshTopology :: GetDistantFaceNums ( int locfacenum, int * distfacenums ) const + void ParallelMeshTopology :: GetDistantFaceNums ( int locfacenum, int * distfacenums ) const { distfacenums[0] = 0; distfacenums[1] = loc2distface[locfacenum-1][0]; @@ -240,11 +159,11 @@ namespace netgen for ( int i = 1; i < loc2distface[locfacenum-1].Size(); i++ ) distfacenums[i+1] = loc2distface[locfacenum-1][i]; - int size = loc2distface[locfacenum-1].Size() / 2 + 1; - return size; + // int size = loc2distface[locfacenum-1].Size() / 2 + 1; + // return size; } - int ParallelMeshTopology :: GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const + void ParallelMeshTopology :: GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const { distedgenums[0] = 0; distedgenums[1] = loc2distedge[locedgenum-1][0]; @@ -252,11 +171,11 @@ namespace netgen for ( int i = 1; i < loc2distedge[locedgenum-1].Size(); i++ ) distedgenums[i+1] = loc2distedge[locedgenum-1][i]; - int size = loc2distedge[locedgenum-1].Size() / 2 + 1; - return size; + // int size = loc2distedge[locedgenum-1].Size() / 2 + 1; + // return size; } - int ParallelMeshTopology :: GetDistantElNums ( int locelnum, int * distelnums ) const + void ParallelMeshTopology :: GetDistantElNums ( int locelnum, int * distelnums ) const { distelnums[0] = 0; distelnums[1] = loc2distel[locelnum-1][0]; @@ -264,8 +183,8 @@ namespace netgen for ( int i = 1; i < loc2distel[locelnum-1].Size(); i++ ) distelnums[i+1] = loc2distel[locelnum-1][i]; - int size = loc2distel[locelnum-1].Size() / 2 + 1; - return size; + // int size = loc2distel[locelnum-1].Size() / 2 + 1; + // return size; } @@ -399,453 +318,13 @@ namespace netgen } - void ParallelMeshTopology :: Update () - { - ne = mesh.GetNE(); - nv = mesh.GetNV(); - nseg = mesh.GetNSeg(); - nsurfel = mesh.GetNSE(); - - ned = mesh.GetTopology().GetNEdges(); - nfa = mesh.GetTopology().GetNFaces(); - } - - - void ParallelMeshTopology :: UpdateRefinement () - { - ; - } - - - - - void ParallelMeshTopology :: UpdateCoarseGridGlobal () - { - if (id == 0) - PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS - - - MPI_Group MPI_GROUP_WORLD; - int process_ranks[] = { 0 }; - MPI_Comm_group (MPI_COMM_WORLD, &MPI_GROUP_WORLD); - MPI_Group_excl (MPI_GROUP_WORLD, 1, process_ranks, &MPI_HIGHORDER_WORLD); - MPI_Comm_create (MPI_COMM_WORLD, MPI_HIGHORDER_WORLD, &MPI_HIGHORDER_COMM); - - - int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal"); - NgProfiler::RegionTimer reg(timer); - - - *testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl; - - const MeshTopology & topology = mesh.GetTopology(); - - Array sendarray, recvarray; - - nfa = topology . GetNFaces(); - ned = topology . GetNEdges(); - np = mesh . GetNP(); - nv = mesh . GetNV(); - ne = mesh . GetNE(); - nseg = mesh.GetNSeg(); - nsurfel = mesh.GetNSE(); - - // low order processor - save mesh partition - if ( id == 0 ) - { - for ( int eli = 1; eli <= ne; eli++ ) - loc2distel[eli-1][0] = eli; - - for ( int i = 1; i <= mesh .GetNV(); i++) - loc2distvert[i][0] = i; - - for ( int i = 0; i < mesh . GetNSeg(); i++) - loc2distsegm[i][0] = i+1; - - for ( int i = 0; i < mesh . GetNSE(); i++) - loc2distsurfel[i][0] = i+1; - - for ( int i = 0; i < topology .GetNEdges(); i++) - loc2distedge[i][0] = i+1; - - for ( int i = 0; i < topology .GetNFaces(); i++) - loc2distface[i][0] = i+1; - } - - - - if ( id == 0 ) - { - sendarray.Append (nfa); - sendarray.Append (ned); - - Array edges, pnums, faces, elpnums; - sendarray.Append (ne); - for ( int el = 1; el <= ne; el++ ) - { - topology.GetElementFaces (el, faces); - topology.GetElementEdges ( el, edges ); - const Element & volel = mesh.VolumeElement (el); - - int globeli = GetLoc2Glob_VolEl(el); - // cout << "el = " << el << ", globeli = " << globeli << endl; - - sendarray. Append ( globeli ); - sendarray. Append ( faces.Size() ); - sendarray. Append ( edges.Size() ); - sendarray. Append ( volel.GetNP() ); - - for ( int i = 0; i < faces.Size(); i++ ) - sendarray. Append(faces[i] ); - for ( int i = 0; i < edges.Size(); i++ ) - sendarray. Append(edges[i] ); - - for ( int i = 0; i < volel.GetNP(); i++ ) - if (id == 0) - sendarray. Append(volel[i] ); - else - sendarray. Append(GetLoc2Glob_Vert (volel[i])); - } - - - sendarray.Append (nsurfel); - for (int el = 1; el <= nsurfel; el++) - { - topology.GetSurfaceElementEdges ( el, edges ); - const Element2d & surfel = mesh.SurfaceElement (el); - - sendarray. Append ( el ); - sendarray. Append ( edges.Size() ); - sendarray. Append ( surfel.GetNP() ); - - for ( int i = 0; i < edges.Size(); i++ ) - sendarray. Append(edges[i] ); - - for ( int i = 0; i < surfel.GetNP(); i++ ) - sendarray. Append(surfel[i] ); - } - - } - - - if (id == 0) - MyMPI_Bcast ( sendarray ); - else - MyMPI_Bcast ( recvarray ); - - - if (id != 0) - { - Array glob2loc_el; - - glob2loc_el.SetSize (neglob); - glob2loc_el = -1; - for ( int locel = 1; locel <= mesh.GetNE(); locel++) - glob2loc_el[GetLoc2Glob_VolEl(locel)] = locel; - - int ii = 0; - nfaglob = recvarray[ii++]; - nedglob = recvarray[ii++]; - - Array faces, edges; - Array pnums, globalpnums; - - int recv_ne = recvarray[ii++]; - for (int hi = 0; hi < recv_ne; hi++) - { - int globvolel = recvarray[ii++]; - int distnfa = recvarray[ii++]; - int distned = recvarray[ii++]; - int distnp = recvarray[ii++]; - - int volel = glob2loc_el[globvolel]; - if (volel != -1) - { - topology.GetElementFaces( volel, faces); - topology.GetElementEdges ( volel, edges); - const Element & volelement = mesh.VolumeElement (volel); - - for ( int i = 0; i < faces.Size(); i++) - SetDistantFaceNum ( 0, faces[i], recvarray[ii++]); - - for ( int i = 0; i < edges.Size(); i++) - SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]); - - for ( int i = 0; i < volelement.GetNP(); i++) - SetDistantPNum ( 0, volelement[i], recvarray[ii++]); - } - else - ii += distnfa + distned + distnp; - } - - - Array glob2loc_sel; - - int recv_nse = recvarray[ii++]; - nseglob = recv_nse; - - glob2loc_sel.SetSize (nseglob); - glob2loc_sel = -1; - for ( int locel = 1; locel <= mesh.GetNSE(); locel++) - glob2loc_sel[GetLoc2Glob_SurfEl(locel)] = locel; - - - for (int hi = 0; hi < recv_nse; hi++) - { - int globvolel = recvarray[ii++]; - int distned = recvarray[ii++]; - int distnp = recvarray[ii++]; - - int surfel = glob2loc_sel[globvolel]; - if (surfel != -1) - { - topology.GetSurfaceElementEdges ( surfel, edges); - const Element2d & element = mesh.SurfaceElement (surfel); - - for ( int i = 0; i < edges.Size(); i++) - SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]); - - for ( int i = 0; i < element.GetNP(); i++) - SetDistantPNum ( 0, element[i], recvarray[ii++]); - } - else - ii += distned + distnp; - } - - - } - - if (id != 0) - { - *testout << "l2d - vert = " << loc2distvert << endl; - *testout << "l2d - edge = " << loc2distedge << endl; - *testout << "l2d - el = " << loc2distel << endl; - *testout << "l2d - sel = " << loc2distsurfel << endl; - } - - coarseupdate = 1; - } - - - - - - - void ParallelMeshTopology :: UpdateCoarseGrid () - { - static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid"); - NgProfiler::RegionTimer reg(timer); - - - (*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl; - if (id == 0) - PrintMessage (1, "UPDATE COARSE GRID PARALLEL TOPOLOGY "); - - - // find exchange edges - first send exchangeedges locnum, v1, v2 - // receive distant distnum, v1, v2 - // find matching - const MeshTopology & topology = mesh.GetTopology(); - - UpdateCoarseGridGlobal(); - - MPI_Barrier (MPI_COMM_WORLD); - - if ( id == 0 ) - { - return; - } - - - Array sendarray, recvarray; - - nfa = topology . GetNFaces(); - ned = topology . GetNEdges(); - np = mesh . GetNP(); - nv = mesh . GetNV(); - ne = mesh . GetNE(); - nseg = mesh.GetNSeg(); - nsurfel = mesh.GetNSE(); - - - static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); - static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges"); - static int timerf = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex faces"); - - - Array glob2loc; - Array cnt_send(ntasks-1); - - NgProfiler::StartTimer (timere); - - // exchange edges - int maxedge = 0; - for (int edge = 1; edge <= ned; edge++) - maxedge = max (maxedge, GetDistantEdgeNum (0, edge)); - - glob2loc.SetSize (maxedge); - glob2loc = -1; - - for (int edge = 1; edge <= ned; edge++) - glob2loc[GetDistantEdgeNum(0, edge)] = edge; - - cnt_send = 0; - int v1, v2; - for (int edge = 1; edge <= ned; edge++) - { - topology.GetEdgeVertices (edge, v1, v2); - for (int dest = 1; dest < ntasks; dest++) - if (IsExchangeVert (dest, v1) && - IsExchangeVert (dest, v2)) - { - cnt_send[dest-1]+=2; - } - } - - TABLE send_edges(cnt_send); - for (int edge = 1; edge <= ned; edge++) - { - topology.GetEdgeVertices (edge, v1, v2); - for (int dest = 1; dest < ntasks; dest++) - { - if (IsExchangeVert (dest, v1) && - IsExchangeVert (dest, v2)) - { - send_edges.Add (dest-1, GetDistantEdgeNum(0, edge)); - send_edges.Add (dest-1, edge); - } - } - } - - - // *testout << "send exchange edges: " << send_edges << endl; - - TABLE recv_edges(ntasks-1); - MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_HIGHORDER_COMM); - - // *testout << "recv exchange edges: " << recv_edges << endl; - - for (int sender = 1; sender < ntasks; sender ++) - if (id != sender) - { - FlatArray recvarray = recv_edges[sender-1]; - - for (int ii = 0; ii < recvarray.Size(); ) - { - int globe = recvarray[ii++]; - int diste = recvarray[ii++]; - - if (globe <= maxedge) - { - int loce = glob2loc[globe]; - if (loce != -1) - SetDistantEdgeNum (sender, loce, diste); - } - } - } - - - - NgProfiler::StopTimer (timere); - - - MPI_Barrier (MPI_HIGHORDER_COMM); - - if (mesh.GetDimension() == 3) - { - - NgProfiler::StartTimer (timerf); - - glob2loc.SetSize (nfaglob); - glob2loc = -1; - - for (int loc = 1; loc <= nfa; loc++) - glob2loc[GetDistantFaceNum(0, loc)] = loc; - - cnt_send = 0; - Array verts; - for (int face = 1; face <= nfa; face++) - { - topology.GetFaceVertices (face, verts); - for (int dest = 1; dest < ntasks; dest++) - if (IsExchangeVert (dest, verts[0]) && - IsExchangeVert (dest, verts[1]) && - IsExchangeVert (dest, verts[2])) - { - cnt_send[dest-1]+=2; - } - } - - TABLE send_faces(cnt_send); - for (int face = 1; face <= nfa; face++) - { - topology.GetFaceVertices (face, verts); - for (int dest = 1; dest < ntasks; dest++) - { - if (IsExchangeVert (dest, verts[0]) && - IsExchangeVert (dest, verts[1]) && - IsExchangeVert (dest, verts[2])) - { - send_faces.Add (dest-1, GetDistantFaceNum(0, face)); - send_faces.Add (dest-1, face); - } - } - } - TABLE recv_faces(ntasks-1); - MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_HIGHORDER_COMM); - - // *testout << "send exchange faces: " << send_faces << endl; - // *testout << "recv exchange faces: " << recv_faces << endl; - - for (int sender = 1; sender < ntasks; sender ++) - if (id != sender) - { - FlatArray recvarray = recv_faces[sender-1]; - - for (int ii = 0; ii < recvarray.Size(); ) - { - int globf = recvarray[ii++]; - int distf = recvarray[ii++]; - int locf = glob2loc[globf]; - - // *testout << "set distant face, sender = " << sender << ", locf = " << locf << "; distf = " << distf << endl; - if (locf != -1) - SetDistantFaceNum (sender, locf, distf); - } - } - - - NgProfiler::StopTimer (timerf); - } - - - // set which elements are where for the master processor - - coarseupdate = 1; - } - - - - - - void ParallelMeshTopology :: UpdateTopology () - { - ; - } - - - - - - - - void ParallelMeshTopology :: SetNV ( const int anv ) + void ParallelMeshTopology :: SetNV (int anv) { *testout << "called setnv" << endl << "old size: " << loc2distvert.Size() << endl @@ -889,6 +368,455 @@ namespace netgen } + + + + + void ParallelMeshTopology :: Update () + { + ne = mesh.GetNE(); + nv = mesh.GetNV(); + nseg = mesh.GetNSeg(); + nsurfel = mesh.GetNSE(); + + ned = mesh.GetTopology().GetNEdges(); + nfa = mesh.GetTopology().GetNFaces(); + } + + + + + + + + void ParallelMeshTopology :: UpdateCoarseGridGlobal () + { + if (id == 0) + PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); // JS + + + + + int timer = NgProfiler::CreateTimer ("UpdateCoarseGridGlobal"); + NgProfiler::RegionTimer reg(timer); + + + *testout << "ParallelMeshTopology :: UpdateCoarseGridGlobal" << endl; + + const MeshTopology & topology = mesh.GetTopology(); + + + nfa = topology . GetNFaces(); + ned = topology . GetNEdges(); + np = mesh . GetNP(); + nv = mesh . GetNV(); + ne = mesh . GetNE(); + nseg = mesh.GetNSeg(); + nsurfel = mesh.GetNSE(); + + // low order processor - save mesh partition + if ( id == 0 ) + { + for ( int eli = 1; eli <= ne; eli++ ) + loc2distel[eli-1][0] = eli; + + for ( int i = 1; i <= mesh .GetNV(); i++) + loc2distvert[i][0] = i; + + for ( int i = 0; i < mesh . GetNSeg(); i++) + loc2distsegm[i][0] = i+1; + + for ( int i = 0; i < mesh . GetNSE(); i++) + loc2distsurfel[i][0] = i+1; + + for ( int i = 0; i < topology .GetNEdges(); i++) + loc2distedge[i][0] = i+1; + + for ( int i = 0; i < topology .GetNFaces(); i++) + loc2distface[i][0] = i+1; + } + + + // Array sendarray, recvarray; + + if ( id == 0 ) + { + Array*> sendarrays(ntasks); + for (int dest = 1; dest < ntasks; dest++) + { + sendarrays[dest] = new Array; + sendarrays[dest]->Append (nfa); + sendarrays[dest]->Append (ned); + } + + Array edges, pnums, faces, elpnums; + for (int el = 1; el <= ne; el++) + { + topology.GetElementFaces (el, faces); + topology.GetElementEdges (el, edges); + const Element & volel = mesh.VolumeElement (el); + + Array & sendarray = *sendarrays[volel.GetPartition()]; + sendarray. Append ( el ); + + sendarray. Append ( faces.Size() ); + sendarray. Append ( edges.Size() ); + sendarray. Append ( volel.GetNP() ); + + for ( int i = 0; i < faces.Size(); i++ ) + sendarray. Append(faces[i] ); + for ( int i = 0; i < edges.Size(); i++ ) + sendarray. Append(edges[i] ); + for ( int i = 0; i < volel.GetNP(); i++ ) + sendarray. Append(volel[i] ); + } + + for (int dest = 1; dest < ntasks; dest++) + sendarrays[dest]->Append (-1); + + for (int el = 1; el <= nsurfel; el++) + { + topology.GetSurfaceElementEdges (el, edges); + const Element2d & surfel = mesh.SurfaceElement (el); + Array & sendarray = *sendarrays[surfel.GetPartition()]; + + sendarray.Append (el); + sendarray.Append (edges.Size()); + sendarray.Append (surfel.GetNP()); + + for ( int i = 0; i < edges.Size(); i++ ) + sendarray.Append (edges[i]); + + for ( int i = 0; i < surfel.GetNP(); i++ ) + sendarray.Append (surfel[i]); + } + for (int dest = 1; dest < ntasks; dest++) + sendarrays[dest]->Append (-1); + + Array sendrequests; + for (int dest = 1; dest < ntasks; dest++) + sendrequests.Append (MyMPI_ISend (*sendarrays[dest], dest, MPI_TAG_MESH+10)); + MPI_Waitall (sendrequests.Size(), &sendrequests[0], MPI_STATUS_IGNORE); + + for (int dest = 1; dest < ntasks; dest++) + delete sendarrays[dest]; + } + + else + + { + Array recvarray; + // MyMPI_Bcast ( recvarray ); + MyMPI_Recv (recvarray, 0, MPI_TAG_MESH+10); + + Array glob2loc_el; + + glob2loc_el.SetSize (neglob); + glob2loc_el = -1; + for ( int locel = 1; locel <= mesh.GetNE(); locel++) + glob2loc_el[GetLoc2Glob_VolEl(locel)] = locel; + + int ii = 0; + nfaglob = recvarray[ii++]; + nedglob = recvarray[ii++]; + + Array faces, edges; + Array pnums, globalpnums; + + // int recv_ne = recvarray[ii++]; + // for (int hi = 0; hi < recv_ne; hi++) + while (1) + { + int globvolel = recvarray[ii++]; + if (globvolel == -1) break; + + int distnfa = recvarray[ii++]; + int distned = recvarray[ii++]; + int distnp = recvarray[ii++]; + + int volel = glob2loc_el[globvolel]; + if (volel != -1) + { + topology.GetElementFaces( volel, faces); + topology.GetElementEdges ( volel, edges); + const Element & volelement = mesh.VolumeElement (volel); + + for ( int i = 0; i < faces.Size(); i++) + SetDistantFaceNum ( 0, faces[i], recvarray[ii++]); + + for ( int i = 0; i < edges.Size(); i++) + SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]); + + for ( int i = 0; i < volelement.GetNP(); i++) + SetDistantPNum ( 0, volelement[i], recvarray[ii++]); + } + else + ii += distnfa + distned + distnp; + } + + + Array glob2loc_sel; + + // int recv_nse = recvarray[ii++]; + // nseglob = recv_nse; + + glob2loc_sel.SetSize (nseglob); + glob2loc_sel = -1; + for ( int locel = 1; locel <= mesh.GetNSE(); locel++) + glob2loc_sel[GetLoc2Glob_SurfEl(locel)] = locel; + + + // for (int hi = 0; hi < recv_nse; hi++) + while (1) + { + int globvolel = recvarray[ii++]; + if (globvolel == -1) break; + + int distned = recvarray[ii++]; + int distnp = recvarray[ii++]; + + int surfel = glob2loc_sel[globvolel]; + if (surfel != -1) + { + topology.GetSurfaceElementEdges ( surfel, edges); + const Element2d & element = mesh.SurfaceElement (surfel); + + for ( int i = 0; i < edges.Size(); i++) + SetDistantEdgeNum ( 0, edges[i], recvarray[ii++]); + + for ( int i = 0; i < element.GetNP(); i++) + SetDistantPNum ( 0, element[i], recvarray[ii++]); + } + else + ii += distned + distnp; + } + + + } + + + if (id != 0) + { + *testout << "l2d - vert = " << loc2distvert << endl; + *testout << "l2d - edge = " << loc2distedge << endl; + *testout << "l2d - el = " << loc2distel << endl; + *testout << "l2d - sel = " << loc2distsurfel << endl; + } + + coarseupdate = 1; + } + + + + + + + void ParallelMeshTopology :: UpdateCoarseGrid () + { + static int timer = NgProfiler::CreateTimer ("UpdateCoarseGrid"); + NgProfiler::RegionTimer reg(timer); + + + (*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl; + if (id == 0) + PrintMessage (1, "UPDATE COARSE GRID PARALLEL TOPOLOGY "); + + + // find exchange edges - first send exchangeedges locnum, v1, v2 + // receive distant distnum, v1, v2 + // find matching + const MeshTopology & topology = mesh.GetTopology(); + + UpdateCoarseGridGlobal(); + + MPI_Barrier (MPI_COMM_WORLD); + + + MPI_Group MPI_GROUP_WORLD; + MPI_Group MPI_LocalGroup; + MPI_Comm MPI_LocalComm; + + int process_ranks[] = { 0 }; + MPI_Comm_group (MPI_COMM_WORLD, &MPI_GROUP_WORLD); + MPI_Group_excl (MPI_GROUP_WORLD, 1, process_ranks, &MPI_LocalGroup); + MPI_Comm_create (MPI_COMM_WORLD, MPI_LocalGroup, &MPI_LocalComm); + + if (id == 0) return; + + + + Array sendarray, recvarray; + + nfa = topology . GetNFaces(); + ned = topology . GetNEdges(); + np = mesh . GetNP(); + nv = mesh . GetNV(); + ne = mesh . GetNE(); + nseg = mesh.GetNSeg(); + nsurfel = mesh.GetNSE(); + + + static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); + static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges"); + static int timerf = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex faces"); + + + Array glob2loc; + Array cnt_send(ntasks-1); + + NgProfiler::StartTimer (timere); + + // exchange edges + int maxedge = 0; + for (int edge = 1; edge <= ned; edge++) + maxedge = max (maxedge, GetGlobalEdgeNum (edge)); + + glob2loc.SetSize (maxedge); + glob2loc = -1; + + for (int edge = 1; edge <= ned; edge++) + glob2loc[GetGlobalEdgeNum(edge)] = edge; + + cnt_send = 0; + int v1, v2; + for (int edge = 1; edge <= ned; edge++) + { + topology.GetEdgeVertices (edge, v1, v2); + for (int dest = 1; dest < ntasks; dest++) + if (IsExchangeVert (dest, v1) && + IsExchangeVert (dest, v2)) + { + cnt_send[dest-1]+=2; + } + } + + TABLE send_edges(cnt_send); + for (int edge = 1; edge <= ned; edge++) + { + topology.GetEdgeVertices (edge, v1, v2); + for (int dest = 1; dest < ntasks; dest++) + { + if (IsExchangeVert (dest, v1) && + IsExchangeVert (dest, v2)) + { + send_edges.Add (dest-1, GetGlobalEdgeNum(edge)); + send_edges.Add (dest-1, edge); + } + } + } + + + // *testout << "send exchange edges: " << send_edges << endl; + + TABLE recv_edges(ntasks-1); + MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_LocalComm); + + // *testout << "recv exchange edges: " << recv_edges << endl; + + for (int sender = 1; sender < ntasks; sender ++) + if (id != sender) + { + FlatArray recvarray = recv_edges[sender-1]; + + for (int ii = 0; ii < recvarray.Size(); ) + { + int globe = recvarray[ii++]; + int diste = recvarray[ii++]; + + if (globe <= maxedge) + { + int loce = glob2loc[globe]; + if (loce != -1) + SetDistantEdgeNum (sender, loce, diste); + } + } + } + + + + NgProfiler::StopTimer (timere); + + + MPI_Barrier (MPI_LocalComm); + + if (mesh.GetDimension() == 3) + { + + NgProfiler::StartTimer (timerf); + + glob2loc.SetSize (nfaglob); + glob2loc = -1; + + for (int loc = 1; loc <= nfa; loc++) + glob2loc[GetGlobalFaceNum(loc)] = loc; + + cnt_send = 0; + Array verts; + for (int face = 1; face <= nfa; face++) + { + topology.GetFaceVertices (face, verts); + for (int dest = 1; dest < ntasks; dest++) + if (IsExchangeVert (dest, verts[0]) && + IsExchangeVert (dest, verts[1]) && + IsExchangeVert (dest, verts[2])) + { + cnt_send[dest-1]+=2; + } + } + + TABLE send_faces(cnt_send); + for (int face = 1; face <= nfa; face++) + { + topology.GetFaceVertices (face, verts); + for (int dest = 1; dest < ntasks; dest++) + { + if (IsExchangeVert (dest, verts[0]) && + IsExchangeVert (dest, verts[1]) && + IsExchangeVert (dest, verts[2])) + { + send_faces.Add (dest-1, GetGlobalFaceNum(face)); + send_faces.Add (dest-1, face); + } + } + } + TABLE recv_faces(ntasks-1); + MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_LocalComm); + + // *testout << "send exchange faces: " << send_faces << endl; + // *testout << "recv exchange faces: " << recv_faces << endl; + + for (int sender = 1; sender < ntasks; sender ++) + if (id != sender) + { + FlatArray recvarray = recv_faces[sender-1]; + + for (int ii = 0; ii < recvarray.Size(); ) + { + int globf = recvarray[ii++]; + int distf = recvarray[ii++]; + int locf = glob2loc[globf]; + + // *testout << "set distant face, sender = " << sender << ", locf = " << locf << "; distf = " << distf << endl; + if (locf != -1) + SetDistantFaceNum (sender, locf, distf); + } + } + + + NgProfiler::StopTimer (timerf); + } + + + // set which elements are where for the master processor + + coarseupdate = 1; + } + + + + + + } diff --git a/libsrc/meshing/paralleltop.hpp b/libsrc/meshing/paralleltop.hpp index 52c03c52..86f24b35 100644 --- a/libsrc/meshing/paralleltop.hpp +++ b/libsrc/meshing/paralleltop.hpp @@ -78,15 +78,29 @@ namespace netgen int GetNDistantElNums ( int locelnum ) const { return loc2distel[locelnum-1].Size() / 2 + 1; } + int GetGlobalPNum ( int locpnum ) const + { return loc2distvert[locpnum][0]; } + + int GetGlobalEdgeNum ( int locedgenum ) const + { return loc2distedge[locedgenum-1][0]; } + + int GetGlobalFaceNum ( int locfacenum ) const + { return loc2distface[locfacenum-1][0]; } + + int GetGlobalElNum ( int locelnum ) const + { return loc2distel[locelnum-1][0]; } + + /* int GetDistantPNum ( int proc, int locpnum ) const; int GetDistantEdgeNum ( int proc, int locedgenum ) const; int GetDistantFaceNum ( int proc, int locedgenum ) const; int GetDistantElNum ( int proc, int locelnum ) const; + */ - int GetDistantPNums ( int locpnum, int * distpnums ) const; - int GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const; - int GetDistantFaceNums ( int locedgenum, int * distfacenums ) const; - int GetDistantElNums ( int locelnum, int * distfacenums ) const; + void GetDistantPNums ( int locpnum, int * distpnums ) const; + void GetDistantEdgeNums ( int locedgenum, int * distedgenums ) const; + void GetDistantFaceNums ( int locedgenum, int * distfacenums ) const; + void GetDistantElNums ( int locelnum, int * distfacenums ) const; void Print() const; @@ -97,8 +111,7 @@ namespace netgen bool IsExchangeElement ( int elnum ) const { return false; } - bool IsExchangeSEl ( int selnum ) const { return loc2distsurfel[selnum-1].Size() > 1; } - + // bool IsExchangeSEl ( int selnum ) const { return loc2distsurfel[selnum-1].Size() > 1; } bool IsExchangeVert (int dest, int vnum ) const { @@ -124,13 +137,11 @@ namespace netgen return false; } - bool IsExchangeElement (int dest, int elnum ) const { return false; } + // bool IsExchangeElement (int dest, int elnum ) const { return false; } void Update(); void UpdateCoarseGrid(); - void UpdateRefinement (); - void UpdateTopology (); void UpdateExchangeElements(); void UpdateCoarseGridGlobal();