From 383ced496b8930925ee941fea0d042829708d5f3 Mon Sep 17 00:00:00 2001 From: lkogler Date: Mon, 6 Mar 2017 14:32:20 +0100 Subject: [PATCH 1/7] Changed MyMPI_RecvCmd and MyMPI_SendCmd. Disabled USE_BUFFERS for now. --- libsrc/general/mpi_interface.cpp | 34 ++++++++++++----------------- libsrc/visualization/vssolution.hpp | 2 +- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/libsrc/general/mpi_interface.cpp b/libsrc/general/mpi_interface.cpp index 36df5542..8ea6a3c5 100644 --- a/libsrc/general/mpi_interface.cpp +++ b/libsrc/general/mpi_interface.cpp @@ -16,35 +16,29 @@ namespace netgen void MyMPI_SendCmd (const char * cmd) { - char buf[10000]; - strcpy (buf, cmd); - // MPI_Bcast (&buf, 100, MPI_CHAR, 0, MPI_COMM_WORLD); + int ntasks; + MPI_Comm_size(MPI_COMM_WORLD, &ntasks); + if(ntasks==1) + return; + for (int dest = 1; dest < ntasks; dest++) - MPI_Send( &buf, 10000, MPI_CHAR, dest, MPI_TAG_CMD, MPI_COMM_WORLD); + MPI_Send( cmd, (strlen(cmd)+1), MPI_CHAR, dest, MPI_TAG_CMD, MPI_COMM_WORLD); } string MyMPI_RecvCmd () { - char buf[10000]; - // MPI_Bcast (&buf, 100, MPI_CHAR, 0, MPI_COMM_WORLD); - - // VT_OFF(); MPI_Status status; int flag; - do - { - MPI_Iprobe (0, MPI_TAG_CMD, MPI_COMM_WORLD, &flag, &status); - if (!flag) - { - VT_TRACER ("sleep"); - usleep (1000); - } - } - while (!flag); - // VT_ON(); + int size_of_msg = -1; - MPI_Recv( &buf, 10000, MPI_CHAR, 0, MPI_TAG_CMD, MPI_COMM_WORLD, &status); + MPI_Probe(0, MPI_TAG_CMD, MPI_COMM_WORLD, &status); + MPI_Get_count(&status, MPI_CHAR, &size_of_msg); + + //char* buf = (char*)malloc(size_of_msg*sizeof(char)); + char buf[100000]; //1MB should be enough... + + MPI_Recv( &buf, size_of_msg, MPI_CHAR, 0, MPI_TAG_CMD, MPI_COMM_WORLD, &status); return string(buf); } diff --git a/libsrc/visualization/vssolution.hpp b/libsrc/visualization/vssolution.hpp index 80cf1c5b..d8193d56 100644 --- a/libsrc/visualization/vssolution.hpp +++ b/libsrc/visualization/vssolution.hpp @@ -43,7 +43,7 @@ class DLL_HEADER VisualSceneSolution : public VisualScene Point<3> p; }; -#define USE_BUFFERS + // #define USE_BUFFERS #ifdef USE_BUFFERS bool has_surfel_vbo = false; GLuint surfel_vbo[4]; // From 2cc28a960e763afc07de1cc9e96964c25216d523 Mon Sep 17 00:00:00 2001 From: Lukas Kogler Date: Tue, 6 Jun 2017 09:55:40 +0200 Subject: [PATCH 2/7] Removec some cout-output --- libsrc/meshing/parallelmesh.cpp | 2 +- libsrc/meshing/paralleltop.cpp | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index a9a84207..799f8bb3 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -707,7 +707,7 @@ namespace netgen NgProfiler::RegionTimer regloc(timerloc); stringstream str; str << "p" << id << ": got " << GetNE() << " elements and " - << GetNSE() << " surface elements"; + << GetNSE() << " surface elements"; cout << str.str() << endl; // PrintMessage (2, "Got ", GetNE(), " elements and ", GetNSE(), " surface elements"); // PrintMessage (2, "Got ", GetNSE(), " surface elements"); diff --git a/libsrc/meshing/paralleltop.cpp b/libsrc/meshing/paralleltop.cpp index 0998f09b..80c70121 100644 --- a/libsrc/meshing/paralleltop.cpp +++ b/libsrc/meshing/paralleltop.cpp @@ -112,7 +112,7 @@ namespace netgen void ParallelMeshTopology :: UpdateCoarseGridGlobal () { - cout << "updatecoarsegridglobal called" << endl; + // cout << "updatecoarsegridglobal called" << endl; if (id == 0) PrintMessage ( 3, "UPDATE GLOBAL COARSEGRID STARTS" ); @@ -203,7 +203,7 @@ namespace netgen void ParallelMeshTopology :: UpdateCoarseGrid () { - cout << "UpdateCoarseGrid" << endl; + // cout << "UpdateCoarseGrid" << endl; // if (is_updated) return; Reset(); @@ -241,7 +241,7 @@ namespace netgen // update new vertices after mesh-refinement if (mesh.mlbetweennodes.Size() > 0) { - cout << "UpdateCoarseGrid - vertices" << endl; + // cout << "UpdateCoarseGrid - vertices" << endl; int newnv = mesh.mlbetweennodes.Size(); loc2distvert.ChangeSize(mesh.mlbetweennodes.Size()); /* @@ -376,7 +376,7 @@ namespace netgen } Array sendarray, recvarray; - cout << "UpdateCoarseGrid - edges" << endl; + // cout << "UpdateCoarseGrid - edges" << endl; // static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices"); static int timere = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex edges"); @@ -442,10 +442,10 @@ namespace netgen } } - cout << "UpdateCoarseGrid - edges mpi-exchange" << endl; + // cout << "UpdateCoarseGrid - edges mpi-exchange" << endl; TABLE recv_edges(ntasks-1); MyMPI_ExchangeTable (send_edges, recv_edges, MPI_TAG_MESH+9, MPI_LocalComm); - cout << "UpdateCoarseGrid - edges mpi-exchange done" << endl; + // cout << "UpdateCoarseGrid - edges mpi-exchange done" << endl; /* for (int dest = 1; dest < ntasks; dest++) @@ -493,7 +493,7 @@ namespace netgen // MPI_Barrier (MPI_LocalComm); - cout << "UpdateCoarseGrid - faces" << endl; + // cout << "UpdateCoarseGrid - faces" << endl; if (mesh.GetDimension() == 3) { NgProfiler::StartTimer (timerf); @@ -558,10 +558,10 @@ namespace netgen } } - cout << "UpdateCoarseGrid - faces mpi-exchange" << endl; + // cout << "UpdateCoarseGrid - faces mpi-exchange" << endl; TABLE recv_faces(ntasks-1); MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+9, MPI_LocalComm); - cout << "UpdateCoarseGrid - faces mpi-exchange done" << endl; + // cout << "UpdateCoarseGrid - faces mpi-exchange done" << endl; /* for (int dest = 1; dest < ntasks; dest++) @@ -682,7 +682,7 @@ namespace netgen NgProfiler::StopTimer (timerf); } - cout << "UpdateCoarseGrid - done" << endl; + // cout << "UpdateCoarseGrid - done" << endl; is_updated = true; } From 7e21f0cd9c20e9ead36f2fbaae4c2d8fb14eb53b Mon Sep 17 00:00:00 2001 From: Lukas Kogler Date: Thu, 8 Jun 2017 17:50:37 +0200 Subject: [PATCH 3/7] BC-and Material-names are now working with MPI. --- libsrc/meshing/parallelmesh.cpp | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 799f8bb3..2f565a2f 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -527,6 +527,55 @@ namespace netgen MPI_Waitall (sendrequests.Size(), &sendrequests[0], MPI_STATUS_IGNORE); + PrintMessage ( 3, "Sending domain+bc - names"); + + sendrequests.SetSize(6*(ntasks-1)); + /** Send bc-names **/ + int nbcs = bcnames.Size(); + Array bcname_sizes(nbcs); + int tot_bcsize = 0; + for(int k=0;ksize(); + tot_bcsize += bcname_sizes[k]; + } + char compiled_bcnames[tot_bcsize]; + tot_bcsize = 0; + for(int k=0;k(1, &nbcs), k, MPI_TAG_MESH+6); + sendrequests[6*(k-1)+1] = MyMPI_ISend(bcname_sizes, k, MPI_TAG_MESH+6); + (void) MPI_Isend(compiled_bcnames, tot_bcsize, MPI_CHAR, k, MPI_TAG_MESH+6, MPI_COMM_WORLD, &sendrequests[6*(k-1)+2]); + } + + /** Send mat-names **/ + int nmats = materials.Size(); + Array mat_sizes(nmats); + int tot_matsize = 0; + for(int k=0;ksize(); + tot_matsize += mat_sizes[k]; + } + char compiled_mats[tot_matsize]; + tot_matsize = 0; + for(int k=0;k(1, &nmats), k, MPI_TAG_MESH+6); + sendrequests[6*(k-1)+4] = MyMPI_ISend(mat_sizes, k, MPI_TAG_MESH+6); + (void) MPI_Isend(compiled_mats, tot_matsize, MPI_CHAR, k, MPI_TAG_MESH+6, MPI_COMM_WORLD, &sendrequests[6*(k-1)+5]); + } + + /* now wait ... **/ + PrintMessage( 3, "now wait for domain+bc - names"); + + MPI_Waitall (sendrequests.Size(), &sendrequests[0], MPI_STATUS_IGNORE); + + PrintMessage( 3, "send mesh complete"); + MPI_Barrier(MPI_COMM_WORLD); } @@ -698,7 +747,43 @@ namespace netgen } } + + /** Recv bc-names **/ + int nbcs; + MyMPI_Recv(nbcs, 0, MPI_TAG_MESH+6); + Array bcs(nbcs); + MyMPI_Recv(bcs, 0, MPI_TAG_MESH+6); + int size_bc = 0; + for(int k=0;k matsz(nmats); + MyMPI_Recv(matsz, 0, MPI_TAG_MESH+6); + int size_mats = 0; + for(int k=0;k Date: Fri, 9 Jun 2017 20:21:55 +0200 Subject: [PATCH 4/7] Fixed curvedelems+mesh loaded from file via python. Fixed MPI+curvedelems --- libsrc/meshing/curvedelems.cpp | 4 +-- libsrc/meshing/parallelmesh.cpp | 2 +- libsrc/meshing/python_mesh.cpp | 50 +++++++++++++++++++++++++-------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/libsrc/meshing/curvedelems.cpp b/libsrc/meshing/curvedelems.cpp index 4b2238a7..16655e78 100644 --- a/libsrc/meshing/curvedelems.cpp +++ b/libsrc/meshing/curvedelems.cpp @@ -577,8 +577,8 @@ namespace netgen PrintMessage (1, "Curve elements, order = ", aorder); if (rational) PrintMessage (1, "curved elements with rational splines"); - if (working) - const_cast (mesh).UpdateTopology(); + // if (working) + const_cast (mesh).UpdateTopology(); const MeshTopology & top = mesh.GetTopology(); rational = arational; diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 2f565a2f..2a6c4e84 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -783,7 +783,7 @@ namespace netgen SetMaterial(k+1, string(&compiled_mats[cnt], matsz[k])); cnt += matsz[k]; } - + MPI_Barrier(MPI_COMM_WORLD); int timerloc = NgProfiler::CreateTimer ("Update local mesh"); diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index 90037fd5..b75562e7 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -423,26 +423,48 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) .def("Load", FunctionPointer ([](Mesh & self, const string & filename) { - istream * infile; + istream * infile; + +#ifdef PARALLEL + MPI_Comm_rank(MPI_COMM_WORLD, &id); + MPI_Comm_size(MPI_COMM_WORLD, &ntasks); + + char* buf = nullptr; + int strs = 0; + if(id==0) { +#endif if (filename.find(".vol.gz") != string::npos) infile = new igzstream (filename.c_str()); else infile = new ifstream (filename.c_str()); // ifstream input(filename); #ifdef PARALLEL - // int id; - MPI_Comm_rank(MPI_COMM_WORLD, &id); - MPI_Comm_size(MPI_COMM_WORLD, &ntasks); - - if (id == 0) - { - self.Load(*infile); + //still inside id==0-bracket... + self.Load(*infile); self.Distribute(); + + /** Copy the rest of the file into a string (for geometry) **/ + stringstream geom_part; + geom_part << infile->rdbuf(); + string geom_part_string = geom_part.str(); + strs = geom_part_string.size(); + buf = new char[strs]; + memcpy(buf, geom_part_string.c_str(), strs*sizeof(char)); } - else - { - self.SendRecvMesh(); - } + else { + self.SendRecvMesh(); + } + + /** Scatter the geometry-string **/ + MPI_Bcast(&strs, 1, MPI_INT, 0, MPI_COMM_WORLD); + if(id!=0) + buf = new char[strs]; + MPI_Bcast(buf, strs, MPI_CHAR, 0, MPI_COMM_WORLD); + if(id==0) + delete infile; + infile = new istringstream(string((const char*)buf, (size_t)strs)); + delete buf; + #else self.Load(*infile); #endif @@ -455,6 +477,10 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) break; } } + if (!ng_geometry) + ng_geometry = make_shared(); + self.SetGeometry(ng_geometry); + delete infile; })) // static_cast(&Mesh::Load)) .def("Save", static_cast(&Mesh::Save)) From 1e9e376d2e7eed044c8ec89ec8a0fde388d35e82 Mon Sep 17 00:00:00 2001 From: lkogler Date: Fri, 9 Jun 2017 20:56:19 +0200 Subject: [PATCH 5/7] prettied it all up a bit --- CMakeLists.txt | 4 ++-- cmake/SuperBuild.cmake | 2 +- libsrc/meshing/parallelmesh.cpp | 2 +- ng/CMakeLists.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 340a9fbd..1008999d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,7 @@ if(WIN32) endif(WIN32) set(NG_INSTALL_DIR EXPORT netgen-targets RUNTIME DESTINATION ${NG_INSTALL_DIR_BIN} COMPONENT netgen LIBRARY DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen_devel ARCHIVE DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen_devel) -install(EXPORT netgen-targets DESTINATION ${NG_INSTALL_DIR_CMAKE} ) +install(EXPORT netgen-targets DESTINATION ${NG_INSTALL_DIR_CMAKE} COMPONENT netgen_devel) set(CMAKE_MACOSX_RPATH TRUE) set(CMAKE_INSTALL_RPATH "${NG_RPATH_TOKEN};${NG_RPATH_TOKEN}/${NETGEN_RPATH}") @@ -440,7 +440,7 @@ get_directory_property(NETGEN_COMPILE_DEFINITIONS COMPILE_DEFINITIONS) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/NetgenConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/NetgenConfig.cmake @ONLY ESCAPE_QUOTES) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/NetgenConfig.cmake DESTINATION ${NG_INSTALL_DIR_CMAKE}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/NetgenConfig.cmake DESTINATION ${NG_INSTALL_DIR_CMAKE} COMPONENT netgen_devel) ####################################################################### # Configure message diff --git a/cmake/SuperBuild.cmake b/cmake/SuperBuild.cmake index 91ede9e7..efd8ad15 100644 --- a/cmake/SuperBuild.cmake +++ b/cmake/SuperBuild.cmake @@ -182,7 +182,7 @@ ExternalProject_Add (netgen ) -install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} --build . --target install --config ${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/netgen)") +install(CODE "execute_process(COMMAND \"${CMAKE_COMMAND}\" --build . --target install --config ${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/netgen)") add_custom_target(test_netgen ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR}/netgen diff --git a/libsrc/meshing/parallelmesh.cpp b/libsrc/meshing/parallelmesh.cpp index 2a6c4e84..50f64c1c 100644 --- a/libsrc/meshing/parallelmesh.cpp +++ b/libsrc/meshing/parallelmesh.cpp @@ -792,7 +792,7 @@ namespace netgen NgProfiler::RegionTimer regloc(timerloc); stringstream str; str << "p" << id << ": got " << GetNE() << " elements and " - << GetNSE() << " surface elements"; + << GetNSE() << " surface elements"; cout << str.str() << endl; // PrintMessage (2, "Got ", GetNE(), " elements and ", GetNSE(), " surface elements"); // PrintMessage (2, "Got ", GetNSE(), " surface elements"); diff --git a/ng/CMakeLists.txt b/ng/CMakeLists.txt index 7c4a1f81..23991339 100644 --- a/ng/CMakeLists.txt +++ b/ng/CMakeLists.txt @@ -65,7 +65,7 @@ if(USE_PYTHON) set_target_properties( ngpy PROPERTIES OUTPUT_NAME "libngpy") endif() set_target_properties(ngpy PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}") - install(TARGETS ngpy DESTINATION ${NG_INSTALL_DIR_PYTHON}/${NG_INSTALL_SUFFIX}) + install(TARGETS ngpy DESTINATION ${NG_INSTALL_DIR_PYTHON}/${NG_INSTALL_SUFFIX} COMPONENT netgen) endif(USE_PYTHON) if(USE_GUI) From 4c0c5aac8cda3a36ebd0c5c394a0d2eb2c191980 Mon Sep 17 00:00:00 2001 From: Lukas Kogler Date: Tue, 20 Jun 2017 10:27:26 +0200 Subject: [PATCH 6/7] correctly use delete[] instead of delete in python_mesh.cpp --- libsrc/meshing/python_mesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index b75562e7..dff080f7 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -463,7 +463,7 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) if(id==0) delete infile; infile = new istringstream(string((const char*)buf, (size_t)strs)); - delete buf; + delete[] buf; #else self.Load(*infile); From 80355b43cb06d587440bf95d7425412d8d9d99de Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 22 Jun 2017 15:01:11 +0200 Subject: [PATCH 7/7] Set path to TCL/TK libs explicitly on Windows --- cmake/external_projects/tcltk.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/external_projects/tcltk.cmake b/cmake/external_projects/tcltk.cmake index 6be407b3..7a939539 100644 --- a/cmake/external_projects/tcltk.cmake +++ b/cmake/external_projects/tcltk.cmake @@ -65,6 +65,11 @@ elseif(WIN32) LOG_DOWNLOAD 1 ) + set (TK_INCLUDE_PATH ${CMAKE_INSTALL_PREFIX}/include) + set (TCL_INCLUDE_PATH ${CMAKE_INSTALL_PREFIX}/include) + set (TCL_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tcl86.lib) + set (TK_LIBRARY ${CMAKE_INSTALL_PREFIX}/lib/tk86.lib) + list(APPEND NETGEN_DEPENDENCIES project_win_extlibs) else(WIN32) find_package(TCL 8.5 REQUIRED)