diff --git a/libsrc/meshing/meshclass.cpp b/libsrc/meshing/meshclass.cpp index a89a6ff3..d49e37a4 100644 --- a/libsrc/meshing/meshclass.cpp +++ b/libsrc/meshing/meshclass.cpp @@ -44,6 +44,7 @@ namespace netgen cd2names.SetSize(0); #ifdef PARALLEL + this->comm = MPI_COMM_WORLD; paralleltop = new ParallelMeshTopology (*this); #endif } diff --git a/libsrc/meshing/meshclass.hpp b/libsrc/meshing/meshclass.hpp index 1befedce..7b6ac3cd 100644 --- a/libsrc/meshing/meshclass.hpp +++ b/libsrc/meshing/meshclass.hpp @@ -32,6 +32,11 @@ namespace netgen /// point coordinates T_POINTS points; +#ifdef PARALLEL + // The communicator for this mesh. (more or less dummy for now!) + MPI_Comm comm; +#endif + /// line-segments at edges Array segments; /// surface elements, 2d-inner elements diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 50f64c1c..f574f1f6 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -35,53 +35,25 @@ namespace netgen void Mesh :: SendRecvMesh () { + int id, np; + MPI_Comm_rank(this->comm, &id); + MPI_Comm_size(this->comm, &np); + + if (np == 1) { + throw NgException("SendRecvMesh called, but only one rank in communicator!!"); + } + if (id == 0) PrintMessage (1, "Send/Receive mesh"); - { - // distribute global information - int nelglob, nseglob, nvglob; - if (id == 0) - { - paralleltop -> SetNV (GetNV()); - paralleltop -> SetNE (GetNE()); - paralleltop -> SetNSegm (GetNSeg()); - paralleltop -> SetNSE (GetNSE()); - - nelglob = GetNE(); - nseglob = GetNSE(); - nvglob = GetNV(); - } - - MyMPI_Bcast (dimension); - MyMPI_Bcast (nelglob); - MyMPI_Bcast (nseglob); - MyMPI_Bcast (nvglob); - } - - - - { - // distribute number of local elements - if (id == 0) - { - Array num_els_on_proc(ntasks); - num_els_on_proc = 0; - for (ElementIndex ei = 0; ei < GetNE(); ei++) - num_els_on_proc[(*this)[ei].GetPartition()]++; - - MPI_Scatter (&num_els_on_proc[0], 1, MPI_INT, - MPI_IN_PLACE, -1, MPI_INT, 0, MPI_COMM_WORLD); - } - else - { - int nelloc; - MPI_Scatter (NULL, 0, MPI_INT, - &nelloc, 1, MPI_INT, 0, MPI_COMM_WORLD); - - paralleltop -> SetNE (nelloc); - } - } + // Why is this here?? + if (id == 0) + { + paralleltop -> SetNV (GetNV()); + paralleltop -> SetNE (GetNE()); + paralleltop -> SetNSegm (GetNSeg()); + paralleltop -> SetNSE (GetNSE()); + } if (id == 0) SendMesh (); @@ -101,18 +73,29 @@ namespace netgen { Array sendrequests; - PrintMessage ( 3, "Sending vertices"); + int dim = GetDimension(); + MyMPI_Bcast(dim); + + const_cast(GetTopology()).Update(); + + PrintMessage ( 3, "Sending nr of elements"); + + MPI_Comm comm = this->comm; Array num_els_on_proc(ntasks); num_els_on_proc = 0; for (ElementIndex ei = 0; ei < GetNE(); ei++) num_els_on_proc[(*this)[ei].GetPartition()]++; + MPI_Scatter (&num_els_on_proc[0], 1, MPI_INT, + MPI_IN_PLACE, -1, MPI_INT, 0, comm); + TABLE els_of_proc (num_els_on_proc); for (ElementIndex ei = 0; ei < GetNE(); ei++) els_of_proc.Add ( (*this)[ei].GetPartition(), ei); + PrintMessage ( 3, "Building vertex/proc mapping"); Array num_sels_on_proc(ntasks); num_sels_on_proc = 0; @@ -133,132 +116,160 @@ namespace netgen segs_of_proc.Add ( (*this)[ei].GetPartition(), ei); + /** + ----- STRATEGY FOR PERIODIC MESHES ----- + Whenever two vertices are identified by periodicity, any proc + that gets one of the vertices actually gets both of them. + This has to be transitive, that is, if + a <-> b and b <-> c, + then any proc that has vertex a also has vertices b and c! + Surfaceelements and Segments that are identified by + periodicity are treated the same way. + + We need to duplicate these so we have containers to + hold the edges/facets. Afaik, a mesh cannot have nodes + that are not part of some sort of element. + + **/ + + /** First, we build tables for vertex identification. **/ + Array per_pairs; + Array pp2; + auto & idents = GetIdentifications(); + bool has_periodic = false; + for (int idnr = 1; idnr < idents.GetMaxNr()+1; idnr++) + { + if(idents.GetType(idnr)!=Identifications::PERIODIC) continue; + has_periodic = true; + idents.GetPairs(idnr, pp2); + per_pairs.Append(pp2); + } + Array npvs(GetNV()); + npvs = 0; + for (int k = 0; k < per_pairs.Size(); k++) { + npvs[per_pairs[k].I1()]++; + npvs[per_pairs[k].I2()]++; + } + + /** for each vertex, gives us all identified vertices **/ + TABLE per_verts(npvs); + for (int k = 0; k < per_pairs.Size(); k++) { + per_verts.Add(per_pairs[k].I1(), per_pairs[k].I2()); + per_verts.Add(per_pairs[k].I2(), per_pairs[k].I1()); + } + for (int k = PointIndex::BASE; k < GetNV()+PointIndex::BASE; k++) { + BubbleSort(per_verts[k]); + } + + /** The same table as per_verts, but TRANSITIVE!! **/ + auto iterate_per_verts_trans = [&](auto f){ + Array allvs; + for (int k = PointIndex::BASE; k < GetNV()+PointIndex::BASE; k++) + { + allvs.SetSize(0); + allvs.Append(per_verts[k]); + bool changed = true; + while(changed) { + changed = false; + for (int j = 0; j per_verts_trans(npvs); + iterate_per_verts_trans([&](auto k, auto & allvs) { + for (int j = 0; j vert_flag (GetNV()); Array num_procs_on_vert (GetNV()); Array num_verts_on_proc (ntasks); - num_verts_on_proc = 0; num_procs_on_vert = 0; - vert_flag = -1; - - - for (int dest = 1; dest < ntasks; dest++) - { - FlatArray els = els_of_proc[dest]; - for (int hi = 0; hi < els.Size(); hi++) - { - const Element & el = (*this) [ els[hi] ]; - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - - num_verts_on_proc[dest]++; - num_procs_on_vert[epi]++; - - paralleltop -> SetDistantPNum (dest, epi); - } - } - } - - - FlatArray sels = sels_of_proc[dest]; - for (int hi = 0; hi < sels.Size(); hi++) - { - const Element2d & el = (*this) [ sels[hi] ]; - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - - num_verts_on_proc[dest]++; - num_procs_on_vert[epi]++; - - paralleltop -> SetDistantPNum (dest, epi); - } - } - } - - FlatArray segs = segs_of_proc[dest]; - for (int hi = 0; hi < segs.Size(); hi++) - { - const Segment & el = (*this) [segs[hi]]; - for (int i = 0; i < 2; i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - - num_verts_on_proc[dest]++; - num_procs_on_vert[epi]++; - - paralleltop -> SetDistantPNum (dest, epi); - } - } - } - } - + auto iterate_vertices = [&](auto f) { + vert_flag = -1; + for (int dest = 1; dest < ntasks; dest++) + { + FlatArray els = els_of_proc[dest]; + for (int hi = 0; hi < els.Size(); hi++) + { + const Element & el = (*this) [ els[hi] ]; + for (int i = 0; i < el.GetNP(); i++) + f(el[i], dest); + } + FlatArray sels = sels_of_proc[dest]; + for (int hi = 0; hi < sels.Size(); hi++) + { + const Element2d & el = (*this) [ sels[hi] ]; + for (int i = 0; i < el.GetNP(); i++) + f(el[i], dest); + } + FlatArray segs = segs_of_proc[dest]; + for (int hi = 0; hi < segs.Size(); hi++) + { + const Segment & el = (*this) [segs[hi]]; + for (int i = 0; i < 2; i++) + f(el[i], dest); + } + } + }; + /** count vertices per proc and procs per vertex **/ + iterate_vertices([&](auto vertex, auto dest){ + auto countit = [&] (auto vertex, auto dest) { + if (vert_flag[vertex] < dest) + { + vert_flag[vertex] = dest; + num_verts_on_proc[dest]++; + num_procs_on_vert[vertex]++; + GetParallelTopology().SetDistantPNum (dest, vertex); + } + }; + countit(vertex, dest); + auto pers = per_verts_trans[vertex]; + for(int j = 0; j < pers.Size(); j++) + countit(pers[j], dest); + }); TABLE verts_of_proc (num_verts_on_proc); TABLE procs_of_vert (num_procs_on_vert); TABLE loc_num_of_vert (num_procs_on_vert); - - vert_flag = -1; - for (int dest = 1; dest < ntasks; dest++) - { - FlatArray els = els_of_proc[dest]; - for (int hi = 0; hi < els.Size(); hi++) - { - const Element & el = (*this) [ els[hi] ]; - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - procs_of_vert.Add (epi, dest); - } - } - } - - FlatArray sels = sels_of_proc[dest]; - for (int hi = 0; hi < sels.Size(); hi++) - { - const Element2d & el = (*this) [ sels[hi] ]; - for (int i = 0; i < el.GetNP(); i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - procs_of_vert.Add (epi, dest); - } - } - } - - FlatArray segs = segs_of_proc[dest]; - for (int hi = 0; hi < segs.Size(); hi++) - { - const Segment & el = (*this) [ segs[hi] ]; - for (int i = 0; i < 2; i++) - { - PointIndex epi = el[i]; - if (vert_flag[epi] < dest) - { - vert_flag[epi] = dest; - procs_of_vert.Add (epi, dest); - } - } - } - - } - + /** Write vertex/proc mappingfs to tables **/ + iterate_vertices([&](auto vertex, auto dest) { + auto addit = [&] (auto vertex, auto dest) { + if (vert_flag[vertex] < dest) + { + vert_flag[vertex] = dest; + procs_of_vert.Add (vertex, dest); + } + }; + addit(vertex, dest); + auto pers = per_verts_trans[vertex]; + for(int j = 0; j < pers.Size(); j++) + addit(pers[j], dest); + }); + /** + local vertex numbers on distant procs + (I think this was only used for debugging??) + **/ for (int vert = 1; vert <= GetNP(); vert++ ) { FlatArray procs = procs_of_vert[vert]; @@ -269,6 +280,7 @@ namespace netgen loc_num_of_vert.Add (vert, verts_of_proc[dest].Size()); } } + PrintMessage ( 3, "Sending Vertices - vertices"); for (int dest = 1; dest < ntasks; dest++) { @@ -296,6 +308,70 @@ namespace netgen Array num_distpnums(ntasks); num_distpnums = 0; + + /** + Next, we send the identifications themselfs. + + Info about periodic identifications sent to each proc is an array of + integers. + - maxidentnr + - type for each identification + - nr of pairs for each identification (each pair is local!) + - pairs for each periodic ident (global numbers) + **/ + PrintMessage ( 3, "Sending Vertices - identifications"); + int maxidentnr = idents.GetMaxNr(); + Array ppd_sizes(ntasks); + ppd_sizes = 1 + 2*maxidentnr; + for (int idnr = 1; idnr < idents.GetMaxNr()+1; idnr++) + { + if(idents.GetType(idnr)!=Identifications::PERIODIC) continue; + idents.GetPairs(idnr, pp2); + for(int j = 0; j pp_data(ppd_sizes); + for(int dest = 0; dest < ntasks; dest++) + pp_data.Add(dest, maxidentnr); + for (int dest = 0; dest < ntasks; dest++) + { + for (int idnr = 1; idnr < idents.GetMaxNr()+1; idnr++) + pp_data.Add(dest, idents.GetType(idnr)); + for (int idnr = 1; idnr < idents.GetMaxNr()+1; idnr++) + pp_data.Add(dest, 0); + } + for (int idnr = 1; idnr < idents.GetMaxNr()+1; idnr++) + { + if(idents.GetType(idnr)!=Identifications::PERIODIC) continue; + idents.GetPairs(idnr, pp2); + for(int j = 0; j req_per; + for(int dest = 1; dest < ntasks; dest++) + req_per.Append(MyMPI_ISend(pp_data[dest], dest, MPI_TAG_MESH+1)); + MPI_Waitall(req_per.Size(), &req_per[0], MPI_STATUS_IGNORE); + + PrintMessage ( 3, "Sending Vertices - distprocs"); + for (int vert = 1; vert <= GetNP(); vert++) { FlatArray procs = procs_of_vert[vert]; @@ -368,161 +444,240 @@ namespace netgen for (int dest = 1; dest < ntasks; dest++) sendrequests.Append (MyMPI_ISend (fddata, dest, MPI_TAG_MESH+3)); + /** Surface Elements **/ + PrintMessage ( 3, "Sending Surface elements" ); - - Array nlocsel(ntasks), bufsize(ntasks); + // build sel-identification + size_t nse = GetNSE(); + Array ided_sel(nse); + ided_sel = -1; + bool has_ided_sels = false; + if(GetNE() && has_periodic) //we can only have identified surf-els if we have vol-els (right?) + { + Array os1, os2; + for(SurfaceElementIndex sei = 0; sei < GetNSE(); sei++) + { + if(ided_sel[sei]!=-1) continue; + const Element2d & sel = (*this)[sei]; + FlatArray points = sel.PNums(); + auto ided1 = per_verts[points[0]]; + os1.SetSize(0); + for (int j = 0; j < ided1.Size(); j++) + os1.Append(GetTopology().GetVertexSurfaceElements(ided1[j])); + for (int j = 1; j < points.Size(); j++) + { + os2.SetSize(0); + auto p2 = points[j]; + auto ided2 = per_verts[p2]; + for (int l = 0; l < ided2.Size(); l++) + os2.Append(GetTopology().GetVertexSurfaceElements(ided2[l])); + for (int m = 0; m1) { + throw NgException("SurfaceElement identified with more than one other??"); + } + const Element2d & sel2 = (*this)[sei]; + FlatArray points2 = sel2.PNums(); + has_ided_sels = true; + ided_sel[sei] = os1[0]; + ided_sel[os1[0]] = sei; + } + } + // build sel data to send + auto iterate_sels = [&](auto f) { + for (SurfaceElementIndex sei = 0; sei < GetNSE(); sei++ ) + { + const Element2d & sel = (*this)[sei]; + int dest = (*this)[sei].GetPartition(); + f(sei, sel, dest); + if(ided_sel[sei]!=-1) + { + int dest2 = (*this)[ided_sel[sei]].GetPartition(); + f(sei, sel, dest2); + } + } + }; + Array nlocsel(ntasks), bufsize(ntasks); nlocsel = 0; bufsize = 1; - - for (int sei = 1; sei <= GetNSE(); sei++ ) - { - const Element2d & sel = SurfaceElement (sei); - int dest = sel.GetPartition(); - nlocsel[dest] ++; + iterate_sels([&](SurfaceElementIndex sei, const Element2d & sel, int dest){ + nlocsel[dest]++; bufsize[dest] += 4 + 2*sel.GetNP(); - } - + }); TABLE selbuf(bufsize); - for (int dest = 1; dest < ntasks; dest++ ) selbuf.Add (dest, nlocsel[dest]); - - for (int sei = 1; sei <= GetNSE(); sei ++ ) - { - const Element2d & sel = SurfaceElement (sei); - int dest = sel.GetPartition(); - + iterate_sels([&](SurfaceElementIndex sei, const auto & sel, int dest) { selbuf.Add (dest, sei); selbuf.Add (dest, sel.GetIndex()); // selbuf.Add (dest, 0); selbuf.Add (dest, sel.GetNP()); - for ( int ii = 1; ii <= sel.GetNP(); ii++) { selbuf.Add (dest, sel.PNum(ii)); selbuf.Add (dest, sel.GeomInfoPi(ii).trignum); } - } - + }); + // distribute sel data for (int dest = 1; dest < ntasks; dest++) sendrequests.Append (MyMPI_ISend(selbuf[dest], dest, MPI_TAG_MESH+4)); + - + /** Segments **/ PrintMessage ( 3, "Sending Edge Segments"); - - - Array nloc(ntasks); - nloc = 0; - bufsize = 1; - - for (int i = 1; i <= GetNSeg(); i++ ) + auto iterate_segs1 = [&](auto f) { + Array osegs1, osegs2, osegs_both; + Array type1, type2; + for(SegmentIndex segi = 0; segi < GetNSeg(); segi++) + { + const Segment & seg = (*this)[segi]; + int segnp = seg.GetNP(); + PointIndex pi1 = seg[0]; + auto ided1 = per_verts[pi1]; + PointIndex pi2 = seg[1]; + auto ided2 = per_verts[pi2]; + if (!(ided1.Size() && ided2.Size())) continue; + osegs1.SetSize(0); + type1.SetSize(0); + for (int l = 0; l per_seg_size(GetNSeg()); + per_seg_size = 0; + iterate_segs1([&](SegmentIndex segi1, SegmentIndex segi2) + { per_seg_size[segi1]++; }); + TABLE per_seg(per_seg_size); + iterate_segs1([&](SegmentIndex segi1, SegmentIndex segi2) + { per_seg.Add(segi1, segi2); }); + // make per_seg transitive + auto iterate_per_seg_trans = [&](auto f){ + Array allsegs; + for (SegmentIndex segi = 0; segi < GetNSeg(); segi++) + { + allsegs.SetSize(0); + allsegs.Append(per_seg[segi]); + bool changed = true; + while (changed) + { + changed = false; + for (int j = 0; j & segs){ + for (int j = 0; j < segs.Size(); j++) + per_seg_size[segi] = segs.Size(); + }); + TABLE per_seg_trans(per_seg_size); + iterate_per_seg_trans([&](SegmentIndex segi, Array & segs){ + for (int j = 0; j < segs.Size(); j++) + per_seg_trans.Add(segi, segs[j]); + }); + // build segment data + Array dests; + auto iterate_segs2 = [&](auto f) { - const Segment & seg = LineSegment (i); - int dest = seg.GetPartition(); - nloc[dest] ++; - bufsize[dest] += 14; - } - + for (SegmentIndex segi = 0; segi nloc_seg(ntasks); + // bufsize = 1; //was originally this - why?? + bufsize = 0; + nloc_seg = 0; + iterate_segs2([&](auto segi, const auto & seg, int dest) + { + nloc_seg[dest]++; + bufsize[dest] += 14; + }); TABLE segm_buf(bufsize); - - /* - for (int dest = 1; dest < ntasks; dest++ ) - segm_buf.Add (dest, nloc[dest]); - */ - for (int i = 1; i <= GetNSeg(); i++ ) - { - const Segment & seg = LineSegment (i); - int dest = seg.GetPartition(); - - segm_buf.Add (dest, i); - segm_buf.Add (dest, seg.si); - segm_buf.Add (dest, seg.pnums[0]); - segm_buf.Add (dest, seg.pnums[1]); - segm_buf.Add (dest, seg.geominfo[0].trignum); - segm_buf.Add (dest, seg.geominfo[1].trignum); - segm_buf.Add (dest, seg.surfnr1); - segm_buf.Add (dest, seg.surfnr2); - segm_buf.Add (dest, seg.edgenr); - segm_buf.Add (dest, seg.epgeominfo[0].dist); - segm_buf.Add (dest, seg.epgeominfo[1].edgenr); - segm_buf.Add (dest, seg.epgeominfo[1].dist); - segm_buf.Add (dest, seg.singedge_right); - segm_buf.Add (dest, seg.singedge_left); - } - + iterate_segs2([&](auto segi, const auto & seg, int dest) + { + segm_buf.Add (dest, segi); + segm_buf.Add (dest, seg.si); + segm_buf.Add (dest, seg.pnums[0]); + segm_buf.Add (dest, seg.pnums[1]); + segm_buf.Add (dest, seg.geominfo[0].trignum); + segm_buf.Add (dest, seg.geominfo[1].trignum); + segm_buf.Add (dest, seg.surfnr1); + segm_buf.Add (dest, seg.surfnr2); + segm_buf.Add (dest, seg.edgenr); + segm_buf.Add (dest, seg.epgeominfo[0].dist); + segm_buf.Add (dest, seg.epgeominfo[1].edgenr); + segm_buf.Add (dest, seg.epgeominfo[1].dist); + segm_buf.Add (dest, seg.singedge_right); + segm_buf.Add (dest, seg.singedge_left); + }); + // distrubute segment data for (int dest = 1; dest < ntasks; dest++) sendrequests.Append (MyMPI_ISend(segm_buf[dest], dest, MPI_TAG_MESH+5)); - /* - Array nlocseg(ntasks), segi(ntasks); - for ( int i = 0; i < ntasks; i++) - { - nlocseg[i] = 0; - bufsize[i] = 0; - segi[i] = 0; - } - - for (int segi = 1; segi <= GetNSeg(); segi ++ ) - { - Array volels; - const MeshTopology & topol = GetTopology(); - topol . GetSegmentSurfaceElements ( segi, volels ); - for (int j = 0; j < volels.Size(); j++) - { - int ei = volels[j]; - if ( ei > 0 && ei <= GetNSE() ) - { - const Element2d & el = SurfaceElement (ei); - int dest = el.GetPartition(); - nlocseg[dest] ++; - bufsize[dest] += 14; - } - } - } - - TABLE segmbuf(bufsize); - - for ( int ls=1; ls <= GetNSeg(); ls++) - { - Array volels; - GetTopology().GetSegmentSurfaceElements ( ls, volels ); - const Segment & seg = LineSegment (ls); - - for (int j = 0; j < volels.Size(); j++) - { - int ei = volels[j]; - if ( ei > 0 && ei <= GetNSE() ) - { - const Element2d & el = SurfaceElement (ei); - int dest = el.GetPartition(); - - if ( dest > 0 ) - { - segmbuf.Add (dest, ls); - segmbuf.Add (dest, seg.si); - segmbuf.Add (dest, seg.pnums[0]); - segmbuf.Add (dest, seg.pnums[1]); - segmbuf.Add (dest, seg.geominfo[0].trignum); - segmbuf.Add (dest, seg.geominfo[1].trignum); - segmbuf.Add (dest, seg.surfnr1); - segmbuf.Add (dest, seg.surfnr2); - segmbuf.Add (dest, seg.edgenr); - segmbuf.Add (dest, seg.epgeominfo[0].dist); - segmbuf.Add (dest, seg.epgeominfo[1].edgenr); - segmbuf.Add (dest, seg.epgeominfo[1].dist); - segmbuf.Add (dest, seg.singedge_right); - segmbuf.Add (dest, seg.singedge_left); - segi[dest] += 14; - } - // paralleltop -> SetDistantSegm ( dest, ls, int ( segi[dest] / 14 ) ); - } - } - } - - for ( int dest = 1; dest < ntasks; dest++) - sendrequests.Append (MyMPI_ISend(segmbuf[dest], dest, MPI_TAG_MESH+5)); - */ - PrintMessage ( 3, "now wait ..."); MPI_Waitall (sendrequests.Size(), &sendrequests[0], MPI_STATUS_IGNORE); @@ -595,7 +750,16 @@ namespace netgen int timer_sels = NgProfiler::CreateTimer ("Receive surface elements"); NgProfiler::RegionTimer reg(timer); - + int dim; + MyMPI_Bcast(dim); + SetDimension(dim); + + // Receive number of local elements + int nelloc; + MPI_Scatter (NULL, 0, MPI_INT, + &nelloc, 1, MPI_INT, 0, MPI_COMM_WORLD); + paralleltop -> SetNE (nelloc); + // string st; // receive vertices @@ -604,6 +768,7 @@ namespace netgen Array verts; MyMPI_Recv (verts, 0, MPI_TAG_MESH+1); + int numvert = verts.Size(); paralleltop -> SetNV (numvert); @@ -623,6 +788,31 @@ namespace netgen MPI_Datatype mptype = MeshPoint::MyGetMPIType(); MPI_Status status; MPI_Recv( &points[1], numvert, mptype, 0, MPI_TAG_MESH+1, MPI_COMM_WORLD, &status); + + Array pp_data; + MyMPI_Recv(pp_data, 0, MPI_TAG_MESH+1); + + + int maxidentnr = pp_data[0]; + auto & idents = GetIdentifications(); + for (int idnr = 1; idnr < maxidentnr+1; idnr++) + { + + idents.SetType(idnr, (Identifications::ID_TYPE)pp_data[idnr]); + } + + int offset = 2*maxidentnr+1; + for(int idnr = 1; idnr < maxidentnr+1; idnr++) + { + int npairs = pp_data[maxidentnr+idnr]; + FlatArray pairdata(2*npairs, &pp_data[offset]); + offset += 2*npairs; + for (int k = 0; k dist_pnums; MyMPI_Recv (dist_pnums, 0, MPI_TAG_MESH+1); @@ -694,7 +884,6 @@ namespace netgen tri.PNum(j) = glob2loc_vert_ht.Get (selbuf[ii++]); tri.GeomInfoPi(j).trignum = selbuf[ii++]; } - paralleltop->SetLoc2Glob_SurfEl ( sel+1, globsel ); AddSurfaceElement (tri); sel ++; @@ -903,51 +1092,87 @@ namespace netgen LineSegment(i+1).SetPartition(epart[i+GetNE()+GetNSE()] + 1); + // surface elements attached to volume elements + Array boundarypoints (GetNP()); + boundarypoints = false; + + if(GetDimension() == 3) + for (SurfaceElementIndex sei = 0; sei < GetNSE(); sei++) + { + const Element2d & el = (*this)[sei]; + for (int j = 0; j < el.GetNP(); j++) + boundarypoints[el[j]] = true; + } + else + for (SegmentIndex segi = 0; segi < GetNSeg(); segi++) + { + const Segment & seg = (*this)[segi]; + for (int j = 0; j < 2; j++) + boundarypoints[seg[j]] = true; + } + + + // Build Pnt2Element table, boundary points only + Array cnt(GetNP()); + cnt = 0; + + auto loop_els_2d = [&](auto f) { + for (SurfaceElementIndex sei = 0; sei < GetNSE(); sei++) + { + const Element2d & el = (*this)[sei]; + cout << "surf-el " << sei << " verts: " << endl; + for (int j = 0; j < el.GetNP(); j++) { + cout << el[j] << " "; + f(el[j], sei); + } + cout << endl; + } + }; + auto loop_els_3d = [&](auto f) { + for (ElementIndex ei = 0; ei < GetNE(); ei++) + { + const Element & el = (*this)[ei]; + for (int j = 0; j < el.GetNP(); j++) + f(el[j], ei); + } + }; + auto loop_els = [&](auto f) + { + if (GetDimension() == 3 ) + loop_els_3d(f); + else + loop_els_2d(f); + }; + + + loop_els([&](auto vertex, int index) + { + if(boundarypoints[vertex]) + cnt[vertex]++; + }); + cout << "count: " << endl << cnt << endl; + TABLE pnt2el(cnt); + loop_els([&](auto vertex, int index) + { + if(boundarypoints[vertex]) + pnt2el.Add(vertex, index); + }); + if (GetDimension() == 3) { - - // surface elements attached to volume elements - Array boundarypoints (GetNP()); - boundarypoints = false; - - for (SurfaceElementIndex sei = 0; sei < GetNSE(); sei++) - { - const Element2d & el = (*this)[sei]; - for (int j = 0; j < el.GetNP(); j++) - boundarypoints[el[j]] = true; - } - // Build Pnt2Element table, boundary points only - Array cnt(GetNP()); - cnt = 0; - for (ElementIndex ei = 0; ei < GetNE(); ei++) - { - const Element & el = (*this)[ei]; - for (int j = 0; j < el.GetNP(); j++) - if (boundarypoints[el[j]]) - cnt[el[j]]++; - } - TABLE pnt2el(cnt); - cnt = 0; - for (ElementIndex ei = 0; ei < GetNE(); ei++) - { - const Element & el = (*this)[ei]; - for (int j = 0; j < el.GetNP(); j++) - if (boundarypoints[el[j]]) - pnt2el.Add (el[j], ei); - } - for (SurfaceElementIndex sei = 0; sei < GetNSE(); sei++) { Element2d & sel = (*this)[sei]; PointIndex pi1 = sel[0]; - FlatArray els = pnt2el[pi1]; + // FlatArray els = pnt2el[pi1]; + FlatArray els = pnt2el[pi1]; sel.SetPartition (-1); for (int j = 0; j < els.Size(); j++) { - const Element & el = (*this)[els[j]]; + const Element & el = (*this)[ElementIndex(els[j])]; bool hasall = true; @@ -976,13 +1201,13 @@ namespace netgen { Segment & sel = (*this)[si]; PointIndex pi1 = sel[0]; - FlatArray els = pnt2el[pi1]; + FlatArray els = pnt2el[pi1]; sel.SetPartition (-1); for (int j = 0; j < els.Size(); j++) { - const Element & el = (*this)[els[j]]; + const Element & el = (*this)[ElementIndex(els[j])]; bool haspi[9] = { false }; // max surfnp @@ -1004,7 +1229,36 @@ namespace netgen if (sel.GetPartition() == -1) cerr << "no volume element found" << endl; } - + } + else + { + for (SegmentIndex segi = 0; segi < GetNSeg(); segi++) + { + Segment & seg = (*this)[segi]; + seg.SetPartition(-1); + PointIndex pi1 = seg[0]; + + FlatArray sels = pnt2el[pi1]; + for (int j = 0; j < sels.Size(); j++) + { + SurfaceElementIndex sei = sels[j]; + Element2d & se = (*this)[sei]; + bool found = false; + for (int l = 0; l < se.GetNP(); l++ && !found) + found |= (se[l]==seg[1]); + if(found) { + seg.SetPartition(se.GetPartition()); + break; + } + } + + if (seg.GetPartition() == -1) { + cout << endl << "segi: " << segi << endl; + cout << "points: " << seg[0] << " " << seg[1] << endl; + cout << "surfels: " << endl << sels << endl; + throw NgException("no surface element found"); + } + } } }