ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-24 21:10:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dalcinl-mpi4py-limited-api'

Browse Source
This commit is contained in:
Matthias Hochsteger 2024-05-29 09:36:59 +02:00
commit e475224359
9 changed files with 291 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -16,8 +16,7 @@ option( USE_GUI "build with GUI" ON )
option( USE_PYTHON "build with python interface" ON )
cmake_dependent_option( PREFER_SYSTEM_PYBIND11 "Use system wide PyBind11" OFF "USE_PYTHON" OFF)
option( USE_MPI "enable mpi parallelization" OFF )
option( USE_MPI4PY "enable mpi4py interface" ON )
option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" ON )
option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" OFF )
option( USE_OCC "build with OpenCascade geometry kernel interface" ON)
option( USE_STLGEOM "build with STL geometry support" ON)
option( USE_CSG "build with CSG kernel" ON)
@ -323,6 +322,7 @@ if (USE_PYTHON)
add_subdirectory(external_dependencies/pybind11)
endif()
target_compile_definitions(netgen_python INTERFACE NG_PYTHON NETGEN_PYTHON)
target_include_directories(netgen_python INTERFACE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
target_include_directories(nglib PRIVATE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS})
if(Python3_LIBRARIES AND (WIN32 OR NOT BUILD_FOR_CONDA))
@ -345,14 +345,6 @@ if (USE_MPI)
target_include_directories(netgen_metis INTERFACE ${METIS_INCLUDE_DIR})
target_link_libraries(netgen_metis INTERFACE ${METIS_LIBRARY} )
target_compile_definitions(netgen_metis INTERFACE METIS )
if(USE_MPI4PY AND USE_PYTHON)
execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE)
find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED)
target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR})
target_compile_definitions(netgen_python INTERFACE NG_MPI4PY )
message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}")
endif(USE_MPI4PY AND USE_PYTHON)
endif (USE_MPI)
#######################################################################

3
libsrc/core/CMakeLists.txt
View File

@ -93,7 +93,7 @@ install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp
xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp
simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp
register_archive.hpp autodiff.hpp autodiffdiff.hpp
ng_mpi.hpp ng_mpi_generated_declarations.hpp ng_mpi_native.hpp
ng_mpi.hpp ng_mpi_generated_declarations.hpp mpi4py_pycapi.h
DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
if(ENABLE_CPP_CORE_GUIDELINES_CHECK)
@ -166,6 +166,7 @@ if(USE_MPI)
endif()
else()
target_link_libraries(ngcore PUBLIC ${MPI_C_LIBRARIES})
target_include_directories(ngcore PUBLIC ${MPI_C_INCLUDE_PATH})
endif(USE_MPI_WRAPPER)
endif(USE_MPI)

245
libsrc/core/mpi4py_pycapi.h Normal file
View File

@ -0,0 +1,245 @@
/* Author: Lisandro Dalcin */
/* Contact: dalcinl@gmail.com */
#ifndef MPI4PY_PYCAPI_H
#define MPI4PY_PYCAPI_H
#include <mpi.h>
#include <Python.h>
#define _mpi4py_declare_pycapi(Type, star) \
static PyTypeObject *_mpi4py_PyMPI##Type = NULL; \
static PyObject *(*_mpi4py_PyMPI##Type##_New)(MPI_##Type star) = NULL; \
static MPI_##Type *(*_mpi4py_PyMPI##Type##_Get)(PyObject *) = NULL;
#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
_mpi4py_declare_pycapi(Datatype,)
#define PyMPIDatatype_Type (*_mpi4py_PyMPIDatatype)
#define PyMPIDatatype_New _mpi4py_PyMPIDatatype_New
#define PyMPIDatatype_Get _mpi4py_PyMPIDatatype_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
_mpi4py_declare_pycapi(Status,*)
#define PyMPIStatus_Type (*_mpi4py_PyMPIStatus)
#define PyMPIStatus_New _mpi4py_PyMPIStatus_New
#define PyMPIStatus_Get _mpi4py_PyMPIStatus_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
_mpi4py_declare_pycapi(Request,)
#define PyMPIRequest_Type (*_mpi4py_PyMPIRequest)
#define PyMPIRequest_New _mpi4py_PyMPIRequest_New
#define PyMPIRequest_Get _mpi4py_PyMPIRequest_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
_mpi4py_declare_pycapi(Message,)
#define PyMPIMessage_Type (*_mpi4py_PyMPIMessage)
#define PyMPIMessage_New _mpi4py_PyMPIMessage_New
#define PyMPIMessage_Get _mpi4py_PyMPIMessage_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_OP
_mpi4py_declare_pycapi(Op,)
#define PyMPIOp_Type (*_mpi4py_PyMPIOp)
#define PyMPIOp_New _mpi4py_PyMPIOp_New
#define PyMPIOp_Get _mpi4py_PyMPIOp_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
_mpi4py_declare_pycapi(Group,)
#define PyMPIGroup_Type (*_mpi4py_PyMPIGroup)
#define PyMPIGroup_New _mpi4py_PyMPIGroup_New
#define PyMPIGroup_Get _mpi4py_PyMPIGroup_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_INFO
_mpi4py_declare_pycapi(Info,)
#define PyMPIInfo_Type (*_mpi4py_PyMPIInfo)
#define PyMPIInfo_New _mpi4py_PyMPIInfo_New
#define PyMPIInfo_Get _mpi4py_PyMPIInfo_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
_mpi4py_declare_pycapi(Errhandler,)
#define PyMPIErrhandler_Type (*_mpi4py_PyMPIErrhandler)
#define PyMPIErrhandler_New _mpi4py_PyMPIErrhandler_New
#define PyMPIErrhandler_Get _mpi4py_PyMPIErrhandler_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
_mpi4py_declare_pycapi(Session,)
#define PyMPISession_Type (*_mpi4py_PyMPISession)
#define PyMPISession_New _mpi4py_PyMPISession_New
#define PyMPISession_Get _mpi4py_PyMPISession_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_COMM
_mpi4py_declare_pycapi(Comm,)
#define PyMPIComm_Type (*_mpi4py_PyMPIComm)
#define PyMPIComm_New _mpi4py_PyMPIComm_New
#define PyMPIComm_Get _mpi4py_PyMPIComm_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_WIN
_mpi4py_declare_pycapi(Win,)
#define PyMPIWin_Type (*_mpi4py_PyMPIWin)
#define PyMPIWin_New _mpi4py_PyMPIWin_New
#define PyMPIWin_Get _mpi4py_PyMPIWin_Get
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_FILE
_mpi4py_declare_pycapi(File,)
#define PyMPIFile_Type (*_mpi4py_PyMPIFile)
#define PyMPIFile_New _mpi4py_PyMPIFile_New
#define PyMPIFile_Get _mpi4py_PyMPIFile_Get
#endif
#undef _mpi4py_define_pycapi
static int _mpi4py_ImportType(PyObject *module,
const char *type_name,
PyTypeObject **type)
{
PyObject *attr = NULL;
attr = PyObject_GetAttrString(module, type_name);
if (!attr)
goto fn_fail;
if (!PyType_Check(attr)) {
PyErr_Format(PyExc_TypeError,
"%.200s.%.200s is not a type object",
PyModule_GetName(module), type_name);
goto fn_fail;
}
*type = (PyTypeObject *)attr;
return 0;
fn_fail:
Py_DecRef(attr);
return -1;
}
static int _mpi4py_ImportFunc(PyObject *module,
const char *func_name,
const char *signature,
void (**func)(void))
{
PyObject *pyxcapi = NULL;
PyObject *capsule = NULL;
union { void *obj; void (*fcn)(void); } ptr;
pyxcapi = PyObject_GetAttrString(module, (char *)"__pyx_capi__");
if (!pyxcapi)
goto fn_fail;
capsule = PyDict_GetItemString(pyxcapi, func_name);
if (!capsule) {
PyErr_Format(PyExc_ImportError,
"%.200s does not export expected C function %.200s",
PyModule_GetName(module), func_name);
goto fn_fail;
}
if (!PyCapsule_CheckExact(capsule)) {
PyErr_Format(PyExc_TypeError,
"%.200s.%.200s is not a capsule",
PyModule_GetName(module), func_name);
}
if (!signature) {
signature = PyCapsule_GetName(capsule);
}
if (!PyCapsule_IsValid(capsule, signature)) {
PyErr_Format(PyExc_TypeError,
"C function %.200s.%.200s has wrong signature "
"(expected %.500s, got %.500s)",
PyModule_GetName(module), func_name,
signature, PyCapsule_GetName(capsule));
goto fn_fail;
}
ptr.obj = PyCapsule_GetPointer(capsule, signature);
if (!ptr.obj)
goto fn_fail;
*func = ptr.fcn;
Py_DecRef(pyxcapi);
return 0;
fn_fail:
Py_DecRef(pyxcapi);
return -1;
}
static int import_mpi4py_MPI(void)
{
PyObject *module = PyImport_ImportModule("mpi4py.MPI");
if (!module)
goto fn_fail;
#define _mpi4py_import_pycapi(Type) do { \
if (_mpi4py_ImportType(module, #Type, &_mpi4py_PyMPI##Type) < 0) \
goto fn_fail; \
if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_New", NULL, \
(void (**)(void))&_mpi4py_PyMPI##Type##_New) < 0) \
goto fn_fail; \
if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_Get", NULL, \
(void (**)(void))&_mpi4py_PyMPI##Type##_Get) < 0) \
goto fn_fail; \
} while (0)
#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE
_mpi4py_import_pycapi(Datatype);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_STATUS
_mpi4py_import_pycapi(Status);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST
_mpi4py_import_pycapi(Request);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE
_mpi4py_import_pycapi(Message);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_OP
_mpi4py_import_pycapi(Op);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_GROUP
_mpi4py_import_pycapi(Group);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_INFO
_mpi4py_import_pycapi(Info);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER
_mpi4py_import_pycapi(Errhandler);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_SESSION
_mpi4py_import_pycapi(Session);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_COMM
_mpi4py_import_pycapi(Comm);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_WIN
_mpi4py_import_pycapi(Win);
#endif
#ifndef MPI4PY_LIMITED_API_SKIP_FILE
_mpi4py_import_pycapi(File);
#endif
#undef _mpi4py_import_pycapi
Py_DecRef(module);
return 0;
fn_fail:
Py_DecRef(module);
return -1;
}
#define __PYX_HAVE_API__mpi4py__MPI
#define import_mpi4py__MPI import_mpi4py_MPI
#endif /* MPI4PY_PYCAPI_H */

23
libsrc/core/ng_mpi.cpp
View File

@ -1,22 +1,23 @@
#define OMPI_SKIP_MPICXX
#include <mpi.h>
#include "ng_mpi.hpp"
#include <mpi.h>
#include <type_traits>
#include "array.hpp"
#include "ngcore_api.hpp"
#include "pybind11/pytypes.h"
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#include <mpi4py.h>
#ifdef NG_PYTHON
#include "python_ngcore.hpp"
namespace py = pybind11;
#endif
#define MPI4PY_LIMITED_API 1
#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
#define MPI4PY_LIMITED_API_SKIP_SESSION 1
#include "mpi4py_pycapi.h" // mpi4py < 4.0.0
#ifdef MSMPI_VER
int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2,
MPI_Comm* arg3) {
@ -156,10 +157,10 @@ NGCORE_API_EXPORT void ng_init_mpi();
static bool imported_mpi4py = false;
void ng_init_mpi() {
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#ifdef NG_PYTHON
NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool {
if (!imported_mpi4py) {
import_mpi4py();
import_mpi4py__MPI();
imported_mpi4py = true;
}
PyObject* py_src = src.ptr();
@ -172,12 +173,12 @@ void ng_init_mpi() {
};
NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle {
if (!imported_mpi4py) {
import_mpi4py();
import_mpi4py__MPI();
imported_mpi4py = true;
}
return py::handle(PyMPIComm_New(ng2mpi(src)));
};
#endif
#endif // NG_PYTHON
#include "ng_mpi_generated_init.hpp"
}

21
libsrc/core/ng_mpi.hpp
View File

@ -9,17 +9,9 @@
#include "ngcore_api.hpp"
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#include <pybind11/pybind11.h>
namespace py = pybind11;
#endif
#ifndef NG_MPI_WRAPPER
#define OMPI_SKIP_MPICXX
#include <mpi.h>
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#include <mpi4py.h>
#endif
#endif // NG_MPI_WRAPPER
namespace ngcore {
@ -83,23 +75,18 @@ struct NG_MPI_Aint {
NG_MPI_Aint() = default;
};
#else
#else // NG_MPI_WRAPPER
using NG_MPI_Comm = MPI_Comm;
using NG_MPI_Status = MPI_Status;
using NG_MPI_Comm = MPI_Comm;
using NG_MPI_Datatype = MPI_Datatype;
using NG_MPI_Request = MPI_Request;
using NG_MPI_Op = MPI_Op;
using NG_MPI_Group = MPI_Group;
using NG_MPI_Aint = MPI_Aint;
#endif
#endif // NG_MPI_WRAPPER
#include "ng_mpi_generated_declarations.hpp"
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
#endif
} // namespace ngcore
#endif // PARALLEL

21
libsrc/core/ng_mpi_native.hpp
View File

@ -1,21 +0,0 @@
#ifndef NG_MPI_NATIVE_HPP
#define NG_MPI_NATIVE_HPP
#include <mpi.h>
#include "mpi_wrapper.hpp"
#include "ng_mpi.hpp"
namespace ngcore {
MPI_Comm NG_MPI_Native(NG_MPI_Comm comm) {
return reinterpret_cast<MPI_Comm>(comm.value);
}
MPI_Comm NG_MPI_Native(NgMPI_Comm comm) {
return reinterpret_cast<MPI_Comm>(static_cast<NG_MPI_Comm>(comm).value);
}
} // namespace ngcore
#endif // NG_MPI_NATIVE_HPP

25
libsrc/core/ng_mpi_wrapper.cpp
View File

@ -13,6 +13,13 @@ using std::cerr;
using std::cout;
using std::endl;
#ifndef NG_MPI_WRAPPER
#define MPI4PY_LIMITED_API 1
#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1
#define MPI4PY_LIMITED_API_SKIP_SESSION 1
#include "mpi4py_pycapi.h" // mpi4py < 4.0.0
#endif // NG_MPI_WRAPPER
namespace ngcore {
#ifdef NG_MPI_WRAPPER
@ -28,9 +35,7 @@ struct MPIFinalizer {
}
} mpi_finalizer;
bool MPI_Loaded() {
return ng_mpi_lib != nullptr;
}
bool MPI_Loaded() { return ng_mpi_lib != nullptr; }
void InitMPI(std::optional<std::filesystem::path> mpi_lib_path) {
if (ng_mpi_lib) return;
@ -128,7 +133,7 @@ static std::runtime_error no_mpi() {
return std::runtime_error("MPI not enabled");
}
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#ifdef NG_PYTHON
decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
[](py::handle py_obj, NG_MPI_Comm &ng_comm) -> bool {
// If this gets called, it means that we want to convert an mpi4py
@ -152,17 +157,17 @@ decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
};
decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
[](NG_MPI_Comm) -> py::handle { throw no_mpi(); };
#endif
#endif // NG_PYTHON
#include "ng_mpi_generated_dummy_init.hpp"
#else // NG_MPI_WRAPPER
static bool imported_mpi4py = false;
#if defined(NG_PYTHON) && defined(NG_MPI4PY)
#ifdef NG_PYTHON
decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
[](py::handle src, NG_MPI_Comm &dst) -> bool {
if (!imported_mpi4py) {
import_mpi4py();
import_mpi4py__MPI();
imported_mpi4py = true;
}
PyObject *py_src = src.ptr();
@ -177,19 +182,19 @@ decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py =
decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py =
[](NG_MPI_Comm src) -> py::handle {
if (!imported_mpi4py) {
import_mpi4py();
import_mpi4py__MPI();
imported_mpi4py = true;
}
return py::handle(PyMPIComm_New(src));
};
#endif
#endif // NG_PYTHON
bool MPI_Loaded() { return true; }
void InitMPI(std::optional<std::filesystem::path>) {}
#endif // NG_MPI_WRAPPER
} // namespace ngcore
#endif // PARALLEL

14
libsrc/core/python_ngcore.hpp
View File

@ -13,13 +13,17 @@
#include "archive.hpp"
#include "flags.hpp"
#include "ngcore_api.hpp"
#include "profiler.hpp"
#include "ng_mpi.hpp"
namespace py = pybind11;
namespace ngcore
{
#ifdef PARALLEL
NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &);
NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm);
#endif // PARALLEL
namespace detail
{
template<typename T>
@ -34,15 +38,15 @@ namespace ngcore
};
} // namespace detail
#ifdef PARALLEL
struct mpi4py_comm {
mpi4py_comm() = default;
#ifdef PARALLEL
mpi4py_comm(NG_MPI_Comm value) : value(value) {}
operator NG_MPI_Comm () { return value; }
NG_MPI_Comm value;
#endif // PARALLEL
};
#endif // PARALLEL
} // namespace ngcore
@ -51,7 +55,7 @@ namespace ngcore
namespace pybind11 {
namespace detail {
#ifdef NG_MPI4PY
#ifdef PARALLEL
template <> struct type_caster<ngcore::mpi4py_comm> {
public:
PYBIND11_TYPE_CASTER(ngcore::mpi4py_comm, _("mpi4py_comm"));
@ -70,7 +74,7 @@ template <> struct type_caster<ngcore::mpi4py_comm> {
return ngcore::NG_MPI_CommToMPI4Py(src.value);
}
};
#endif // NG_MPI4PY
#endif // PARALLEL
template <typename Type, typename Value> struct ngcore_list_caster {
using value_conv = make_caster<Value>;

2
libsrc/core/python_ngcore_export.cpp
View File

@ -371,5 +371,7 @@ threads : int
;
#ifdef PARALLEL
py::implicitly_convertible<mpi4py_comm, NgMPI_Comm>();
#endif // PARALLEL
}