diff --git a/CMakeLists.txt b/CMakeLists.txt index 71ac2d86..31faba7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,7 @@ option( USE_GUI "build with GUI" ON ) option( USE_PYTHON "build with python interface" ON ) cmake_dependent_option( PREFER_SYSTEM_PYBIND11 "Use system wide PyBind11" OFF "USE_PYTHON" OFF) option( USE_MPI "enable mpi parallelization" OFF ) -option( USE_MPI4PY "enable mpi4py interface" ON ) -option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" ON ) +option( USE_MPI_WRAPPER "enable mpi wrapper (run-time dispatch of MPI library calls)" OFF ) option( USE_OCC "build with OpenCascade geometry kernel interface" ON) option( USE_STLGEOM "build with STL geometry support" ON) option( USE_CSG "build with CSG kernel" ON) @@ -323,6 +322,7 @@ if (USE_PYTHON) add_subdirectory(external_dependencies/pybind11) endif() + target_compile_definitions(netgen_python INTERFACE NG_PYTHON NETGEN_PYTHON) target_include_directories(netgen_python INTERFACE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS}) target_include_directories(nglib PRIVATE ${pybind11_INCLUDE_DIR} ${Python3_INCLUDE_DIRS}) if(Python3_LIBRARIES AND (WIN32 OR NOT BUILD_FOR_CONDA)) @@ -345,14 +345,6 @@ if (USE_MPI) target_include_directories(netgen_metis INTERFACE ${METIS_INCLUDE_DIR}) target_link_libraries(netgen_metis INTERFACE ${METIS_LIBRARY} ) target_compile_definitions(netgen_metis INTERFACE METIS ) - - if(USE_MPI4PY AND USE_PYTHON) - execute_process(COMMAND ${Python3_EXECUTABLE} -c "import mpi4py;print(mpi4py.get_include())" OUTPUT_VARIABLE mpi4py_path OUTPUT_STRIP_TRAILING_WHITESPACE) - find_path(MPI4PY_INCLUDE_DIR mpi4py.h HINTS ${mpi4py_path}/mpi4py NO_DEFAULT_PATH REQUIRED) - target_include_directories(netgen_python INTERFACE ${MPI4PY_INCLUDE_DIR}) - target_compile_definitions(netgen_python INTERFACE NG_MPI4PY ) - message(STATUS "Found mpi4py: ${MPI4PY_INCLUDE_DIR}") - endif(USE_MPI4PY AND USE_PYTHON) endif (USE_MPI) ####################################################################### diff --git a/libsrc/core/CMakeLists.txt b/libsrc/core/CMakeLists.txt index eca94b8c..6b60875d 100644 --- a/libsrc/core/CMakeLists.txt +++ b/libsrc/core/CMakeLists.txt @@ -93,7 +93,7 @@ install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp xbool.hpp signal.hpp bitarray.hpp table.hpp hashtable.hpp ranges.hpp ngstream.hpp simd.hpp simd_avx.hpp simd_avx512.hpp simd_generic.hpp simd_sse.hpp simd_arm64.hpp register_archive.hpp autodiff.hpp autodiffdiff.hpp - ng_mpi.hpp ng_mpi_generated_declarations.hpp ng_mpi_native.hpp + ng_mpi.hpp ng_mpi_generated_declarations.hpp mpi4py_pycapi.h DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel) if(ENABLE_CPP_CORE_GUIDELINES_CHECK) @@ -166,6 +166,7 @@ if(USE_MPI) endif() else() target_link_libraries(ngcore PUBLIC ${MPI_C_LIBRARIES}) + target_include_directories(ngcore PUBLIC ${MPI_C_INCLUDE_PATH}) endif(USE_MPI_WRAPPER) endif(USE_MPI) diff --git a/libsrc/core/mpi4py_pycapi.h b/libsrc/core/mpi4py_pycapi.h new file mode 100644 index 00000000..87d0c69e --- /dev/null +++ b/libsrc/core/mpi4py_pycapi.h @@ -0,0 +1,245 @@ +/* Author: Lisandro Dalcin */ +/* Contact: dalcinl@gmail.com */ + +#ifndef MPI4PY_PYCAPI_H +#define MPI4PY_PYCAPI_H + +#include +#include + +#define _mpi4py_declare_pycapi(Type, star) \ +static PyTypeObject *_mpi4py_PyMPI##Type = NULL; \ +static PyObject *(*_mpi4py_PyMPI##Type##_New)(MPI_##Type star) = NULL; \ +static MPI_##Type *(*_mpi4py_PyMPI##Type##_Get)(PyObject *) = NULL; + +#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE +_mpi4py_declare_pycapi(Datatype,) +#define PyMPIDatatype_Type (*_mpi4py_PyMPIDatatype) +#define PyMPIDatatype_New _mpi4py_PyMPIDatatype_New +#define PyMPIDatatype_Get _mpi4py_PyMPIDatatype_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_STATUS +_mpi4py_declare_pycapi(Status,*) +#define PyMPIStatus_Type (*_mpi4py_PyMPIStatus) +#define PyMPIStatus_New _mpi4py_PyMPIStatus_New +#define PyMPIStatus_Get _mpi4py_PyMPIStatus_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST +_mpi4py_declare_pycapi(Request,) +#define PyMPIRequest_Type (*_mpi4py_PyMPIRequest) +#define PyMPIRequest_New _mpi4py_PyMPIRequest_New +#define PyMPIRequest_Get _mpi4py_PyMPIRequest_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE +_mpi4py_declare_pycapi(Message,) +#define PyMPIMessage_Type (*_mpi4py_PyMPIMessage) +#define PyMPIMessage_New _mpi4py_PyMPIMessage_New +#define PyMPIMessage_Get _mpi4py_PyMPIMessage_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_OP +_mpi4py_declare_pycapi(Op,) +#define PyMPIOp_Type (*_mpi4py_PyMPIOp) +#define PyMPIOp_New _mpi4py_PyMPIOp_New +#define PyMPIOp_Get _mpi4py_PyMPIOp_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_GROUP +_mpi4py_declare_pycapi(Group,) +#define PyMPIGroup_Type (*_mpi4py_PyMPIGroup) +#define PyMPIGroup_New _mpi4py_PyMPIGroup_New +#define PyMPIGroup_Get _mpi4py_PyMPIGroup_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_INFO +_mpi4py_declare_pycapi(Info,) +#define PyMPIInfo_Type (*_mpi4py_PyMPIInfo) +#define PyMPIInfo_New _mpi4py_PyMPIInfo_New +#define PyMPIInfo_Get _mpi4py_PyMPIInfo_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER +_mpi4py_declare_pycapi(Errhandler,) +#define PyMPIErrhandler_Type (*_mpi4py_PyMPIErrhandler) +#define PyMPIErrhandler_New _mpi4py_PyMPIErrhandler_New +#define PyMPIErrhandler_Get _mpi4py_PyMPIErrhandler_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_SESSION +_mpi4py_declare_pycapi(Session,) +#define PyMPISession_Type (*_mpi4py_PyMPISession) +#define PyMPISession_New _mpi4py_PyMPISession_New +#define PyMPISession_Get _mpi4py_PyMPISession_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_COMM +_mpi4py_declare_pycapi(Comm,) +#define PyMPIComm_Type (*_mpi4py_PyMPIComm) +#define PyMPIComm_New _mpi4py_PyMPIComm_New +#define PyMPIComm_Get _mpi4py_PyMPIComm_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_WIN +_mpi4py_declare_pycapi(Win,) +#define PyMPIWin_Type (*_mpi4py_PyMPIWin) +#define PyMPIWin_New _mpi4py_PyMPIWin_New +#define PyMPIWin_Get _mpi4py_PyMPIWin_Get +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_FILE +_mpi4py_declare_pycapi(File,) +#define PyMPIFile_Type (*_mpi4py_PyMPIFile) +#define PyMPIFile_New _mpi4py_PyMPIFile_New +#define PyMPIFile_Get _mpi4py_PyMPIFile_Get +#endif + +#undef _mpi4py_define_pycapi + +static int _mpi4py_ImportType(PyObject *module, + const char *type_name, + PyTypeObject **type) +{ + PyObject *attr = NULL; + attr = PyObject_GetAttrString(module, type_name); + if (!attr) + goto fn_fail; + if (!PyType_Check(attr)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + PyModule_GetName(module), type_name); + goto fn_fail; + } + *type = (PyTypeObject *)attr; + return 0; + fn_fail: + Py_DecRef(attr); + return -1; +} + +static int _mpi4py_ImportFunc(PyObject *module, + const char *func_name, + const char *signature, + void (**func)(void)) +{ + PyObject *pyxcapi = NULL; + PyObject *capsule = NULL; + union { void *obj; void (*fcn)(void); } ptr; + pyxcapi = PyObject_GetAttrString(module, (char *)"__pyx_capi__"); + if (!pyxcapi) + goto fn_fail; + + capsule = PyDict_GetItemString(pyxcapi, func_name); + if (!capsule) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C function %.200s", + PyModule_GetName(module), func_name); + goto fn_fail; + } + if (!PyCapsule_CheckExact(capsule)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a capsule", + PyModule_GetName(module), func_name); + } + if (!signature) { + signature = PyCapsule_GetName(capsule); + } + if (!PyCapsule_IsValid(capsule, signature)) { + PyErr_Format(PyExc_TypeError, + "C function %.200s.%.200s has wrong signature " + "(expected %.500s, got %.500s)", + PyModule_GetName(module), func_name, + signature, PyCapsule_GetName(capsule)); + goto fn_fail; + } + ptr.obj = PyCapsule_GetPointer(capsule, signature); + if (!ptr.obj) + goto fn_fail; + *func = ptr.fcn; + Py_DecRef(pyxcapi); + return 0; + fn_fail: + Py_DecRef(pyxcapi); + return -1; +} + +static int import_mpi4py_MPI(void) +{ + PyObject *module = PyImport_ImportModule("mpi4py.MPI"); + if (!module) + goto fn_fail; + +#define _mpi4py_import_pycapi(Type) do { \ + if (_mpi4py_ImportType(module, #Type, &_mpi4py_PyMPI##Type) < 0) \ + goto fn_fail; \ + if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_New", NULL, \ + (void (**)(void))&_mpi4py_PyMPI##Type##_New) < 0) \ + goto fn_fail; \ + if (_mpi4py_ImportFunc(module, "PyMPI" #Type "_Get", NULL, \ + (void (**)(void))&_mpi4py_PyMPI##Type##_Get) < 0) \ + goto fn_fail; \ + } while (0) + +#ifndef MPI4PY_LIMITED_API_SKIP_DATATYPE + _mpi4py_import_pycapi(Datatype); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_STATUS + _mpi4py_import_pycapi(Status); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_REQUEST + _mpi4py_import_pycapi(Request); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_MESSAGE + _mpi4py_import_pycapi(Message); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_OP + _mpi4py_import_pycapi(Op); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_GROUP + _mpi4py_import_pycapi(Group); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_INFO + _mpi4py_import_pycapi(Info); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_ERRHANDLER + _mpi4py_import_pycapi(Errhandler); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_SESSION + _mpi4py_import_pycapi(Session); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_COMM + _mpi4py_import_pycapi(Comm); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_WIN + _mpi4py_import_pycapi(Win); +#endif + +#ifndef MPI4PY_LIMITED_API_SKIP_FILE + _mpi4py_import_pycapi(File); +#endif + +#undef _mpi4py_import_pycapi + + Py_DecRef(module); + return 0; + fn_fail: + Py_DecRef(module); + return -1; +} + +#define __PYX_HAVE_API__mpi4py__MPI +#define import_mpi4py__MPI import_mpi4py_MPI + +#endif /* MPI4PY_PYCAPI_H */ diff --git a/libsrc/core/ng_mpi.cpp b/libsrc/core/ng_mpi.cpp index 9dce98c8..bac75e47 100644 --- a/libsrc/core/ng_mpi.cpp +++ b/libsrc/core/ng_mpi.cpp @@ -1,22 +1,23 @@ #define OMPI_SKIP_MPICXX +#include #include "ng_mpi.hpp" -#include - #include +#include "array.hpp" #include "ngcore_api.hpp" #include "pybind11/pytypes.h" -#if defined(NG_PYTHON) && defined(NG_MPI4PY) -#include - +#ifdef NG_PYTHON #include "python_ngcore.hpp" - -namespace py = pybind11; #endif +#define MPI4PY_LIMITED_API 1 +#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1 +#define MPI4PY_LIMITED_API_SKIP_SESSION 1 +#include "mpi4py_pycapi.h" // mpi4py < 4.0.0 + #ifdef MSMPI_VER int MPI_Comm_create_group(MPI_Comm arg0, MPI_Group arg1, int arg2, MPI_Comm* arg3) { @@ -156,10 +157,10 @@ NGCORE_API_EXPORT void ng_init_mpi(); static bool imported_mpi4py = false; void ng_init_mpi() { -#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#ifdef NG_PYTHON NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm& dst) -> bool { if (!imported_mpi4py) { - import_mpi4py(); + import_mpi4py__MPI(); imported_mpi4py = true; } PyObject* py_src = src.ptr(); @@ -172,12 +173,12 @@ void ng_init_mpi() { }; NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle { if (!imported_mpi4py) { - import_mpi4py(); + import_mpi4py__MPI(); imported_mpi4py = true; } return py::handle(PyMPIComm_New(ng2mpi(src))); }; -#endif +#endif // NG_PYTHON #include "ng_mpi_generated_init.hpp" } diff --git a/libsrc/core/ng_mpi.hpp b/libsrc/core/ng_mpi.hpp index f29bb664..36151c09 100644 --- a/libsrc/core/ng_mpi.hpp +++ b/libsrc/core/ng_mpi.hpp @@ -9,17 +9,9 @@ #include "ngcore_api.hpp" -#if defined(NG_PYTHON) && defined(NG_MPI4PY) -#include - -namespace py = pybind11; -#endif - #ifndef NG_MPI_WRAPPER +#define OMPI_SKIP_MPICXX #include -#if defined(NG_PYTHON) && defined(NG_MPI4PY) -#include -#endif #endif // NG_MPI_WRAPPER namespace ngcore { @@ -83,23 +75,18 @@ struct NG_MPI_Aint { NG_MPI_Aint() = default; }; -#else +#else // NG_MPI_WRAPPER +using NG_MPI_Comm = MPI_Comm; using NG_MPI_Status = MPI_Status; -using NG_MPI_Comm = MPI_Comm; using NG_MPI_Datatype = MPI_Datatype; using NG_MPI_Request = MPI_Request; using NG_MPI_Op = MPI_Op; using NG_MPI_Group = MPI_Group; using NG_MPI_Aint = MPI_Aint; -#endif +#endif // NG_MPI_WRAPPER #include "ng_mpi_generated_declarations.hpp" -#if defined(NG_PYTHON) && defined(NG_MPI4PY) -NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &); -NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm); -#endif - } // namespace ngcore #endif // PARALLEL diff --git a/libsrc/core/ng_mpi_native.hpp b/libsrc/core/ng_mpi_native.hpp deleted file mode 100644 index 6c8f40ce..00000000 --- a/libsrc/core/ng_mpi_native.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef NG_MPI_NATIVE_HPP -#define NG_MPI_NATIVE_HPP - -#include - -#include "mpi_wrapper.hpp" -#include "ng_mpi.hpp" - -namespace ngcore { - -MPI_Comm NG_MPI_Native(NG_MPI_Comm comm) { - return reinterpret_cast(comm.value); -} - -MPI_Comm NG_MPI_Native(NgMPI_Comm comm) { - return reinterpret_cast(static_cast(comm).value); -} - -} // namespace ngcore - -#endif // NG_MPI_NATIVE_HPP diff --git a/libsrc/core/ng_mpi_wrapper.cpp b/libsrc/core/ng_mpi_wrapper.cpp index ac9d3894..835b0c31 100644 --- a/libsrc/core/ng_mpi_wrapper.cpp +++ b/libsrc/core/ng_mpi_wrapper.cpp @@ -13,6 +13,13 @@ using std::cerr; using std::cout; using std::endl; +#ifndef NG_MPI_WRAPPER +#define MPI4PY_LIMITED_API 1 +#define MPI4PY_LIMITED_API_SKIP_MESSAGE 1 +#define MPI4PY_LIMITED_API_SKIP_SESSION 1 +#include "mpi4py_pycapi.h" // mpi4py < 4.0.0 +#endif // NG_MPI_WRAPPER + namespace ngcore { #ifdef NG_MPI_WRAPPER @@ -28,9 +35,7 @@ struct MPIFinalizer { } } mpi_finalizer; -bool MPI_Loaded() { - return ng_mpi_lib != nullptr; -} +bool MPI_Loaded() { return ng_mpi_lib != nullptr; } void InitMPI(std::optional mpi_lib_path) { if (ng_mpi_lib) return; @@ -128,7 +133,7 @@ static std::runtime_error no_mpi() { return std::runtime_error("MPI not enabled"); } -#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#ifdef NG_PYTHON decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = [](py::handle py_obj, NG_MPI_Comm &ng_comm) -> bool { // If this gets called, it means that we want to convert an mpi4py @@ -152,17 +157,17 @@ decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = }; decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py = [](NG_MPI_Comm) -> py::handle { throw no_mpi(); }; -#endif +#endif // NG_PYTHON #include "ng_mpi_generated_dummy_init.hpp" #else // NG_MPI_WRAPPER static bool imported_mpi4py = false; -#if defined(NG_PYTHON) && defined(NG_MPI4PY) +#ifdef NG_PYTHON decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = [](py::handle src, NG_MPI_Comm &dst) -> bool { if (!imported_mpi4py) { - import_mpi4py(); + import_mpi4py__MPI(); imported_mpi4py = true; } PyObject *py_src = src.ptr(); @@ -177,19 +182,19 @@ decltype(NG_MPI_CommFromMPI4Py) NG_MPI_CommFromMPI4Py = decltype(NG_MPI_CommToMPI4Py) NG_MPI_CommToMPI4Py = [](NG_MPI_Comm src) -> py::handle { if (!imported_mpi4py) { - import_mpi4py(); + import_mpi4py__MPI(); imported_mpi4py = true; } return py::handle(PyMPIComm_New(src)); }; -#endif +#endif // NG_PYTHON +bool MPI_Loaded() { return true; } void InitMPI(std::optional) {} #endif // NG_MPI_WRAPPER - } // namespace ngcore #endif // PARALLEL diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index cd3d14fb..c8ad53ac 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -13,13 +13,17 @@ #include "archive.hpp" #include "flags.hpp" #include "ngcore_api.hpp" -#include "profiler.hpp" #include "ng_mpi.hpp" namespace py = pybind11; namespace ngcore { +#ifdef PARALLEL + NGCORE_API extern bool (*NG_MPI_CommFromMPI4Py)(py::handle, NG_MPI_Comm &); + NGCORE_API extern py::handle (*NG_MPI_CommToMPI4Py)(NG_MPI_Comm); +#endif // PARALLEL + namespace detail { template @@ -34,15 +38,15 @@ namespace ngcore }; } // namespace detail +#ifdef PARALLEL struct mpi4py_comm { mpi4py_comm() = default; -#ifdef PARALLEL mpi4py_comm(NG_MPI_Comm value) : value(value) {} operator NG_MPI_Comm () { return value; } NG_MPI_Comm value; -#endif // PARALLEL }; +#endif // PARALLEL } // namespace ngcore @@ -51,7 +55,7 @@ namespace ngcore namespace pybind11 { namespace detail { -#ifdef NG_MPI4PY +#ifdef PARALLEL template <> struct type_caster { public: PYBIND11_TYPE_CASTER(ngcore::mpi4py_comm, _("mpi4py_comm")); @@ -70,7 +74,7 @@ template <> struct type_caster { return ngcore::NG_MPI_CommToMPI4Py(src.value); } }; -#endif // NG_MPI4PY +#endif // PARALLEL template struct ngcore_list_caster { using value_conv = make_caster; diff --git a/libsrc/core/python_ngcore_export.cpp b/libsrc/core/python_ngcore_export.cpp index 8c9e1a0e..fdcc4bb2 100644 --- a/libsrc/core/python_ngcore_export.cpp +++ b/libsrc/core/python_ngcore_export.cpp @@ -371,5 +371,7 @@ threads : int ; +#ifdef PARALLEL py::implicitly_convertible(); +#endif // PARALLEL }