From c18b6cbbe1c952b1f19a88187e4fc2b869ad5199 Mon Sep 17 00:00:00 2001 From: Lukas Date: Mon, 26 Aug 2019 13:13:53 +0200 Subject: [PATCH 1/5] sub-communicators as methods of NgMPI_Comm, using ngcore-arrays --- libsrc/core/mpi_wrapper.hpp | 14 ++++++++++++-- libsrc/general/mpi_interface.hpp | 15 --------------- libsrc/meshing/python_mesh.cpp | 9 ++++----- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index b42c551e..07b15867 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -188,8 +188,15 @@ namespace ngcore MPI_Bcast (&s[0], len, MPI_CHAR, root, comm); } - - }; + NgMPI_Comm SubCommunicator (FlatArray procs) const + { + MPI_Comm subcomm; + MPI_Group gcomm, gsubcomm; + MPI_Comm_group(comm, &gcomm); + MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm); + MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm); + return NgMPI_Comm(subcomm, true); + } #else // PARALLEL @@ -239,6 +246,9 @@ namespace ngcore template void Bcast (T & s, int root = 0) const { ; } + + NgMPI_Comm SubCommunicator (FlatArray procs) const + { return *this; } }; #endif // PARALLEL diff --git a/libsrc/general/mpi_interface.hpp b/libsrc/general/mpi_interface.hpp index 08a15662..b7de6d64 100644 --- a/libsrc/general/mpi_interface.hpp +++ b/libsrc/general/mpi_interface.hpp @@ -94,21 +94,6 @@ namespace netgen template inline MPI_Datatype MyGetMPIType ( ) { return 0; } #endif -#ifdef PARALLEL - inline MPI_Comm MyMPI_SubCommunicator(MPI_Comm comm, NgArray & procs) - { - MPI_Comm subcomm; - MPI_Group gcomm, gsubcomm; - MPI_Comm_group(comm, &gcomm); - MPI_Group_incl(gcomm, procs.Size(), &(procs[0]), &gsubcomm); - MPI_Comm_create_group(comm, gsubcomm, 6969, &subcomm); - return subcomm; - } -#else - inline MPI_Comm MyMPI_SubCommunicator(MPI_Comm comm, NgArray & procs) - { return comm; } -#endif - #ifdef PARALLEL enum { MPI_TAG_CMD = 110 }; enum { MPI_TAG_MESH = 210 }; diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index 3e55ff75..dc28bab1 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -88,13 +88,12 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m) .def("Min", [](NgMPI_Comm & c, size_t x) { return MyMPI_AllReduceNG(x, MPI_MIN, c); }) .def("Max", [](NgMPI_Comm & c, size_t x) { return MyMPI_AllReduceNG(x, MPI_MAX, c); }) .def("SubComm", [](NgMPI_Comm & c, std::vector proc_list) { - NgArray procs(proc_list.size()); + Array procs(proc_list.size()); for (int i = 0; i < procs.Size(); i++) - procs[i] = proc_list[i]; + { procs[i] = proc_list[i]; } if (!procs.Contains(c.Rank())) - throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); - MPI_Comm subcomm = MyMPI_SubCommunicator(c, procs); - return NgMPI_Comm(subcomm, true); + { throw Exception("rank "+ToString(c.Rank())+" not in subcomm"); } + return c.SubCommunicator(procs); }, py::arg("procs")); ; From 608381d7742797dfc2a0ac77b86079db4eb7a563 Mon Sep 17 00:00:00 2001 From: Lukas Date: Tue, 27 Aug 2019 11:16:55 +0200 Subject: [PATCH 2/5] include array header --- libsrc/core/mpi_wrapper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index 07b15867..ae3e2c4c 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -6,6 +6,7 @@ #include #endif +#include "array.hpp" #include "exception.hpp" namespace ngcore From a37c84e6018bb5f04aeb10489588280a15ef8d8f Mon Sep 17 00:00:00 2001 From: Lukas Date: Tue, 27 Aug 2019 14:10:55 +0200 Subject: [PATCH 3/5] fix typo --- libsrc/core/mpi_wrapper.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index ae3e2c4c..2b376959 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -199,7 +199,8 @@ namespace ngcore return NgMPI_Comm(subcomm, true); } - + }; // class NgMPI_Comm + #else // PARALLEL class MPI_Comm { int nr; From 476a1e39b640f95787694322b0a0636c2361add8 Mon Sep 17 00:00:00 2001 From: Lukas Date: Tue, 27 Aug 2019 16:52:54 +0200 Subject: [PATCH 4/5] remove some whitespace --- libsrc/core/mpi_wrapper.hpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index 2b376959..4075bb44 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -255,15 +255,7 @@ namespace ngcore #endif // PARALLEL - - - - - - - - -} +} // namespace ngcore #endif // NGCORE_MPIWRAPPER_HPP From 36a7c507b8e583a7e287519e7fd39b7599fc33c7 Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 28 Aug 2019 14:59:32 +0200 Subject: [PATCH 5/5] check in forgotten lines --- libsrc/core/mpi_wrapper.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libsrc/core/mpi_wrapper.hpp b/libsrc/core/mpi_wrapper.hpp index bf9de365..14e8ccd8 100644 --- a/libsrc/core/mpi_wrapper.hpp +++ b/libsrc/core/mpi_wrapper.hpp @@ -219,6 +219,16 @@ namespace ngcore MPI_Bcast (&s[0], len, MPI_CHAR, root, comm); } + NgMPI_Comm SubCommunicator (FlatArray procs) const + { + MPI_Comm subcomm; + MPI_Group gcomm, gsubcomm; + MPI_Comm_group(comm, &gcomm); + MPI_Group_incl(gcomm, procs.Size(), procs.Data(), &gsubcomm); + MPI_Comm_create_group(comm, gsubcomm, 4242, &subcomm); + return NgMPI_Comm(subcomm, true); + } + }; // class NgMPI_Comm NETGEN_INLINE void MyMPI_WaitAll (FlatArray requests)