ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-25 21:40:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

Fix mpi4py converters, some cleanup

Browse Source
This commit is contained in:
Matthias Hochsteger 2024-05-06 12:31:30 +02:00
parent 63d3df9cd9
commit f02bd53573
9 changed files with 138 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -338,6 +338,7 @@ endif (USE_PYTHON)
add_library(netgen_mpi INTERFACE)
add_library(netgen_metis INTERFACE)
if (USE_MPI)
set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
find_package(MPI REQUIRED)
target_include_directories(netgen_mpi INTERFACE ${MPI_CXX_INCLUDE_PATH})
target_link_libraries(netgen_mpi INTERFACE ${MPI_mpi_LIBRARY} ${MPI_CXX_LIBRARIES} )
@ -351,8 +352,8 @@ if (USE_MPI)
if(USE_MPI4PY AND USE_PYTHON)
execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE)
find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED)
target_include_directories(netgen_metis INTERFACE ${MPI4PY_INCLUDE_DIR})
target_compile_definitions(netgen_metis INTERFACE NG_MPI4PY )
target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR})
target_compile_definitions(netgen_python INTERFACE NG_MPI4PY )
message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}")
endif(USE_MPI4PY AND USE_PYTHON)
endif (USE_MPI)

36
libsrc/core/CMakeLists.txt
View File

@ -104,7 +104,7 @@ add_dependencies(ngcore ng_generate_version_file)
if(USE_PYTHON)
pybind11_add_module(pyngcore MODULE python_ngcore_export.cpp)
target_link_libraries(pyngcore PUBLIC ngcore netgen_python)
target_link_libraries(pyngcore PUBLIC ngcore PRIVATE netgen_python)
set_target_properties(pyngcore PROPERTIES INSTALL_RPATH "${NG_RPATH_TOKEN}/../${NETGEN_PYTHON_RPATH}")
if(EMSCRIPTEN)
target_compile_definitions(pyngcore PRIVATE NGCORE_EXPORTS)
@ -112,22 +112,34 @@ if(USE_PYTHON)
install(TARGETS pyngcore DESTINATION ${NG_INSTALL_DIR_PYTHON}/pyngcore COMPONENT netgen)
endif(USE_PYTHON)
function (build_mpi_variant)
set(target ng_${ARGV0})
set(include_dir ${ARGV1})
message("Building MPI variant: ${ARGV0} ${ARGV1}")
add_library(${target} SHARED ng_mpi.cpp)
target_link_libraries(${target} PUBLIC ngcore PRIVATE netgen_python)
target_compile_definitions(${target} PUBLIC PARALLEL)
target_include_directories(${target} PRIVATE ${include_dir})
install(TARGETS ${target} DESTINATION ${NG_INSTALL_DIR_LIB} COMPONENT netgen)
endfunction()
if(USE_MPI)
target_compile_definitions(ngcore PUBLIC PARALLEL)
# assume openmpi here
add_library(ng_openmpi SHARED ng_mpi.cpp)
target_link_libraries(ng_openmpi PUBLIC ngcore)
target_include_directories(ng_openmpi PRIVATE ${MPI_C_HEADER_DIR})
install(TARGETS ng_openmpi DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
if(MPI_C_LIBRARY_VERSION_STRING MATCHES "Open MPI.*")
set(OPENMPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
endif()
# set(MPICH_INCLUDE_DIR /opt/mpich/include)
if(MPI_C_LIBRARY_VERSION_STRING MATCHES "MPICH.*")
set(MPICH_INCLUDE_DIR ${MPI_C_INCLUDE_PATH})
endif()
if(OPENMPI_INCLUDE_DIR)
build_mpi_variant(openmpi ${OPENMPI_INCLUDE_DIR})
endif()
if(MPICH_INCLUDE_DIR)
add_library(ng_mpich SHARED ng_mpi.cpp)
target_include_directories(ng_mpich PRIVATE ${MPICH_INCLUDE_DIR})
target_link_libraries(ng_mpich PUBLIC ngcore)
install(TARGETS ng_mpich DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
endif(MPICH_INCLUDE_DIR)
build_mpi_variant(mpich ${MPICH_INCLUDE_DIR})
endif()
endif(USE_MPI)

34
libsrc/core/ng_mpi.cpp
View File

@ -5,6 +5,15 @@
#include <type_traits>
#include "ngcore_api.hpp"
#include "pybind11/pytypes.h"
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#include <mpi4py.h>
#include "python_ngcore.hpp"
namespace py = pybind11;
#endif
namespace ngcore {
@ -103,6 +112,31 @@ using namespace ngcore;
NGCORE_API_EXPORT extern "C" void ng_init_mpi();
static bool imported_mpi4py = false;
void ng_init_mpi() {
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool {
if (!imported_mpi4py) {
import_mpi4py();
imported_mpi4py = true;
}
PyObject* py_src = src.ptr();
auto type = Py_TYPE(py_src);
if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
dst = mpi2ng(*PyMPIComm_Get(py_src));
return !PyErr_Occurred();
}
return false;
};
NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle {
if (!imported_mpi4py) {
import_mpi4py();
imported_mpi4py = true;
}
return py::handle(PyMPIComm_New(ng2mpi(src)));
};
#endif
#include "ng_mpi_generated_init.hpp"
}

11
libsrc/core/ng_mpi.hpp
View File

@ -7,6 +7,12 @@
#include "ngcore_api.hpp"
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#include <pybind11/pybind11.h>
namespace py = pybind11;
#endif
namespace ngcore {
NGCORE_API void InitMPI(
@ -69,6 +75,11 @@ struct NG_MPI_Aint {
NG_MPI_Aint() = default;
};
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
#endif
#include "ng_mpi_generated_declarations.hpp"
} // namespace ngcore

54
libsrc/core/ng_mpi_wrapper.cpp
View File

@ -1,11 +1,12 @@
#include <filesystem>
#include <iostream>
#include <stdexcept>
#include <filesystem>
#include "ng_mpi.hpp"
#include "ngstream.hpp"
#include "utils.hpp"
using std::cerr;
using std::cout;
using std::endl;
@ -13,7 +14,8 @@ namespace ngcore {
static std::unique_ptr<SharedLibrary> mpi_lib, ng_mpi_lib;
void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_dir) {
void InitMPI(std::filesystem::path mpi_lib_path,
std::filesystem::path ng_libs_dir) {
if (ng_mpi_lib) return;
cout << IM(3) << "InitMPI" << endl;
@ -25,58 +27,49 @@ void InitMPI(std::filesystem::path mpi_lib_path, std::filesystem::path ng_libs_d
init_handle mpi_init;
mpi_initialized_handle mpi_initialized;
get_version_handle get_version;
mpi_lib = std::make_unique<SharedLibrary>(mpi_lib_path, std::nullopt, true);
try {
mpi_init = GetSymbol<init_handle>("MPI_Init");
cout << IM(3) << "MPI already loaded " << mpi_init << endl;
mpi_initialized = GetSymbol<mpi_initialized_handle>("MPI_Initialized");
get_version = GetSymbol<get_version_handle>("MPI_Get_library_version");
} catch (std::runtime_error &e) {
cout << IM(3) << "MPI not loaded" << endl;
mpi_lib = std::make_unique<SharedLibrary>(mpi_lib_path,
std::nullopt, true);
mpi_init = mpi_lib->GetSymbol<init_handle>("MPI_Init");
mpi_initialized =
mpi_lib->GetSymbol<mpi_initialized_handle>("MPI_Initialized");
get_version =
mpi_lib->GetSymbol<get_version_handle>("MPI_Get_library_version");
cerr << "Could not load MPI symbols: " << e.what() << endl;
throw e;
}
int flag = 0;
mpi_initialized(&flag);
if (!flag) {
cout << IM(3) << "Calling MPI_Init" << endl;
mpi_init(nullptr, nullptr);
typedef const char *pchar;
int argc = 1;
pchar args[] = {"netgen", nullptr};
pchar *argv = &args[0];
cout << IM(5) << "Calling MPI_Init" << endl;
mpi_init(&argc, (char ***)argv);
}
char version_string[65536];
int resultlen = 0;
get_version(version_string, &resultlen);
mpi_library_version = version_string;
cout << IM(3) << "MPI version: " << version_string << endl;
cout << IM(7) << "MPI version: " << version_string << endl;
std::string libname = "";
if (mpi_library_version.substr(0, 8) == "Open MPI") {
cout << IM(3) << "Have Open MPI" << endl;
cout << IM(5) << "Have Open MPI" << endl;
libname = std::string("libng_openmpi") + NETGEN_SHARED_LIBRARY_SUFFIX;
} else if (mpi_library_version.substr(0, 5) == "MPICH") {
cout << IM(3) << "Have MPICH" << endl;
libname = std::string("libng_mpich.so") + NETGEN_SHARED_LIBRARY_SUFFIX;
cout << IM(5) << "Have MPICH" << endl;
libname = std::string("libng_mpich") + NETGEN_SHARED_LIBRARY_SUFFIX;
} else
cout << IM(3) << "Unknown MPI" << endl;
cerr << "Unknown MPI version, skipping init: " << version_string<< endl;
if (libname.size()) {
cout << IM(3) << "loading " << libname << endl;
cout << IM(3) << "NG_MPI_INT before " << NG_MPI_INT.value << endl;
ng_mpi_lib = std::make_unique<SharedLibrary>(libname);
auto ng_init = ng_mpi_lib->GetSymbol<ng_init_handle>("ng_init_mpi");
cout << IM(3) << "have ng_init " << ng_init << endl;
ng_init();
cout << IM(3) << "NG_MPI_INT after " << NG_MPI_INT.value << endl;
int size, rank;
NG_MPI_Comm_size(NG_MPI_COMM_WORLD, &size);
NG_MPI_Comm_rank(NG_MPI_COMM_WORLD, &rank);
cout << IM(3) << "Hello from " << rank << " of " << size << endl;
}
}
@ -86,6 +79,13 @@ static std::runtime_error no_mpi() {
std::string mpi_library_version = "";
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
[](py::handle, NG_MPI_Comm &) -> bool { throw no_mpi(); };
decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
[](NG_MPI_Comm) -> py::handle { throw no_mpi(); };
#endif
#include "ng_mpi_generated_dummy_init.hpp"
} // namespace ngcore

23
libsrc/core/python_ngcore.hpp
View File

@ -14,6 +14,8 @@
#include "flags.hpp"
#include "ngcore_api.hpp"
#include "profiler.hpp"
#include "ng_mpi.hpp"
namespace py = pybind11;
namespace ngcore
@ -39,6 +41,27 @@ namespace ngcore
namespace pybind11 {
namespace detail {
#ifdef NG_MPI4PY
template <> struct type_caster<ngcore::NG_MPI_Comm> {
public:
PYBIND11_TYPE_CASTER(ngcore::NG_MPI_Comm, _("mpi4py_comm"));
// Python -> C++
bool load(handle src, bool) {
return ngcore::NG_MPI_CommFromMPI4Py(src, value);
}
// C++ -> Python
static handle cast(ngcore::NG_MPI_Comm src,
return_value_policy /* policy */,
handle /* parent */)
{
// Create an mpi4py handle
return ngcore::NG_MPI_CommToMPI4Py(src);
}
};
#endif // NG_MPI4PY
template <typename Type, typename Value> struct ngcore_list_caster {
using value_conv = make_caster<Value>;

10
libsrc/core/python_ngcore_export.cpp
View File

@ -7,6 +7,11 @@ using namespace ngcore;
using namespace std;
using namespace pybind11::literals;
namespace pybind11 { namespace detail {
}} // namespace pybind11::detail
PYBIND11_MODULE(pyngcore, m) // NOLINT
{
@ -31,7 +36,10 @@ PYBIND11_MODULE(pyngcore, m) // NOLINT
ExportArray<uint64_t>(m);
ExportTable<int>(m);
py::class_<NG_MPI_Comm> (m, "_NG_MPI_Comm")
;
py::class_<BitArray, shared_ptr<BitArray>> (m, "BitArray")
.def(py::init([] (size_t n) { return make_shared<BitArray>(n); }),py::arg("n"))
.def(py::init([] (const BitArray& a) { return make_shared<BitArray>(a); } ), py::arg("ba"))

2
libsrc/core/utils.cpp
View File

@ -117,7 +117,7 @@ namespace ngcore
const std::chrono::time_point<TClock> wall_time_start = TClock::now();
int printmessage_importance = 0;
int printmessage_importance = getenv("NG_MESSAGE_LEVEL") ? atoi(getenv("NG_MESSAGE_LEVEL")) : 0;
bool NGSOStream :: glob_active = true;
NGCORE_API int GetCompiledSIMDSize()

65
libsrc/meshing/python_mesh.cpp
View File

@ -1,3 +1,4 @@
#include "pybind11/pytypes.h"
#ifdef NG_PYTHON
#include <regex>
@ -23,52 +24,6 @@ public:
~ClearSolutionClass() { Ng_ClearSolutionData(); }
};
#undef NG_MPI4PY
#ifdef NG_MPI4PY
#include <mpi4py.h>
struct mpi4py_comm {
mpi4py_comm() = default;
mpi4py_comm(MPI_Comm value) : value(value) {}
operator MPI_Comm () { return value; }
MPI_Comm value;
};
namespace pybind11 { namespace detail {
template <> struct type_caster<mpi4py_comm> {
public:
PYBIND11_TYPE_CASTER(mpi4py_comm, _("mpi4py_comm"));
// Python -> C++
bool load(handle src, bool) {
PyObject *py_src = src.ptr();
// Check that we have been passed an mpi4py communicator
if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) {
// Convert to regular MPI communicator
value.value = *PyMPIComm_Get(py_src);
} else {
return false;
}
return !PyErr_Occurred();
}
// C++ -> Python
static handle cast(mpi4py_comm src,
return_value_policy /* policy */,
handle /* parent */)
{
// Create an mpi4py handle
return PyMPIComm_New(src.value);
}
};
}} // namespace pybind11::detail
#endif // NG_MPI4PY
using namespace netgen;
@ -101,9 +56,6 @@ static Transformation<3> global_trafo(Vec<3> (0,0,0));
DLL_HEADER void ExportNetgenMeshing(py::module &m)
{
#ifdef NG_MPI4PY
import_mpi4py();
#endif // NG_MPI4PY
py::register_exception<NgException>(m, "NgException");
m.attr("_netgen_executable_started") = py::cast(netgen::netgen_executable_started);
string script;
@ -133,13 +85,10 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m)
py::class_<NgMPI_Comm> (m, "MPI_Comm")
.def(py::init([] () { return NgMPI_Comm(NG_MPI_COMM_WORLD); }))
#ifdef NG_MPI4PY
.def(py::init([] (mpi4py_comm comm)
{
return NgMPI_Comm(comm);
}))
.def_property_readonly ("mpi4py", [] (NgMPI_Comm comm) { return mpi4py_comm(comm); })
#endif // NG_MPI4PY
.def(py::init([] (NG_MPI_Comm comm) { return NgMPI_Comm(comm); }))
.def_property_readonly ("mpi4py", [](NgMPI_Comm & self) {
return static_cast<NG_MPI_Comm>(self);
})
.def_property_readonly ("rank", &NgMPI_Comm::Rank)
.def_property_readonly ("size", &NgMPI_Comm::Size)
.def("Barrier", &NgMPI_Comm::Barrier)
@ -169,9 +118,7 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m)
;
#ifdef NG_MPI4PY
py::implicitly_convertible<mpi4py_comm, NgMPI_Comm>();
#endif // NG_MPI4PY
py::implicitly_convertible<NG_MPI_Comm, NgMPI_Comm>();
py::class_<NGDummyArgument>(m, "NGDummyArgument")