From d89b328979a9a6200eb3a980d70d4b359442bd04 Mon Sep 17 00:00:00 2001 From: Matthias Hochsteger Date: Thu, 11 Jul 2019 16:23:22 +0200 Subject: [PATCH] Get rid of pybind11 include in archive.hpp Forward-declare pybind11::object and move implementation of Archive::Shallow() to new header python_ngcore.hpp All files using the Shallow/Python archive functionality must include core/python_ngcore.hpp. Missing includes result in link errors, due to missing instantiations of Archive::Shallow(); --- libsrc/core/CMakeLists.txt | 2 +- libsrc/core/archive.hpp | 123 ++--------------------------- libsrc/core/python_ngcore.hpp | 132 ++++++++++++++++++++++++++++++++ libsrc/csg/python_csg.cpp | 1 + libsrc/geom2d/python_geom2d.cpp | 1 + libsrc/meshing/meshclass.cpp | 1 + libsrc/meshing/python_mesh.cpp | 1 + libsrc/occ/python_occ.cpp | 1 + libsrc/stlgeom/python_stl.cpp | 1 + 9 files changed, 145 insertions(+), 118 deletions(-) create mode 100644 libsrc/core/python_ngcore.hpp diff --git a/libsrc/core/CMakeLists.txt b/libsrc/core/CMakeLists.txt index de4b33c0..15e00ce0 100644 --- a/libsrc/core/CMakeLists.txt +++ b/libsrc/core/CMakeLists.txt @@ -43,7 +43,7 @@ target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE ${CMAKE_THREAD_LIBS_INIT} install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp - array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp + array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp python_ngcore.hpp DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel) if(ENABLE_CPP_CORE_GUIDELINES_CHECK) diff --git a/libsrc/core/archive.hpp b/libsrc/core/archive.hpp index c708c596..73c8f75b 100644 --- a/libsrc/core/archive.hpp +++ b/libsrc/core/archive.hpp @@ -22,7 +22,10 @@ #include "version.hpp" // for VersionInfo #ifdef NETGEN_PYTHON -#include +namespace pybind11 +{ + class object; +} #endif // NETGEN_PYTHON namespace ngcore @@ -122,27 +125,12 @@ namespace ngcore // Python should be archived using this Shallow function. If Shallow is called from C++ code // it archives the object normally. template - Archive& Shallow(T& val) - { - static_assert(detail::is_any_pointer, "ShallowArchive must be given pointer type!"); -#ifdef NETGEN_PYTHON - if(shallow_to_python) - { - if(is_output) - ShallowOutPython(pybind11::cast(val)); - else - val = pybind11::cast(ShallowInPython()); - } - else -#endif // NETGEN_PYTHON - *this & val; - return *this; - } + Archive& Shallow(T& val); // implemented in python_ngcore.hpp #ifdef NETGEN_PYTHON virtual void ShallowOutPython(const pybind11::object& /*unused*/) { throw UnreachableCodeException{}; } - virtual pybind11::object ShallowInPython() + virtual void ShallowInPython(pybind11::object &) { throw UnreachableCodeException{}; } #endif // NETGEN_PYTHON @@ -909,105 +897,6 @@ namespace ngcore } }; -#ifdef NETGEN_PYTHON - - template - class PyArchive : public ARCHIVE - { - private: - pybind11::list lst; - size_t index = 0; - std::map version_needed; - protected: - using ARCHIVE::stream; - using ARCHIVE::version_map; - using ARCHIVE::logger; - using ARCHIVE::GetLibraryVersions; - public: - PyArchive(const pybind11::object& alst = pybind11::none()) : - ARCHIVE(std::make_shared()), - lst(alst.is_none() ? pybind11::list() : pybind11::cast(alst)) - { - ARCHIVE::shallow_to_python = true; - if(Input()) - { - stream = std::make_shared - (pybind11::cast(lst[pybind11::len(lst)-1])); - *this & version_needed; - logger->debug("versions needed for unpickling = {}", version_needed); - for(auto& libversion : version_needed) - if(libversion.second > GetLibraryVersion(libversion.first)) - throw Exception("Error in unpickling data:\nLibrary " + libversion.first + - " must be at least " + libversion.second.to_string()); - stream = std::make_shared - (pybind11::cast(lst[pybind11::len(lst)-2])); - *this & version_map; - stream = std::make_shared - (pybind11::cast(lst[pybind11::len(lst)-3])); - } - } - - void NeedsVersion(const std::string& library, const std::string& version) override - { - if(Output()) - { - logger->debug("Need version {} of library {}.", version, library); - version_needed[library] = version_needed[library] > version ? version_needed[library] : version; - } - } - - using ARCHIVE::Output; - using ARCHIVE::Input; - using ARCHIVE::FlushBuffer; - using ARCHIVE::operator&; - using ARCHIVE::operator<<; - using ARCHIVE::GetVersion; - void ShallowOutPython(const pybind11::object& val) override { lst.append(val); } - pybind11::object ShallowInPython() override { return lst[index++]; } - - pybind11::list WriteOut() - { - FlushBuffer(); - lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); - stream = std::make_shared(); - *this & GetLibraryVersions(); - FlushBuffer(); - lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); - stream = std::make_shared(); - logger->debug("Writeout version needed = {}", version_needed); - *this & version_needed; - FlushBuffer(); - lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); - return lst; - } - }; - - template - auto NGSPickle() - { - return pybind11::pickle([](T* self) - { - PyArchive ar; - ar & self; - auto output = pybind11::make_tuple(ar.WriteOut()); - GetLogger("Archive")->trace("Pickling output for object of type {} = {}", - Demangle(typeid(T).name()), - std::string(pybind11::str(output))); - return output; - }, - [](const pybind11::tuple & state) - { - T* val = nullptr; - GetLogger("Archive")->trace("State for unpickling of object of type {} = {}", - Demangle(typeid(T).name()), - std::string(pybind11::str(state[0]))); - PyArchive ar(state[0]); - ar & val; - return val; - }); - } - -#endif // NETGEN_PYTHON } // namespace ngcore #endif // NETGEN_CORE_ARCHIVE_HPP diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp new file mode 100644 index 00000000..07f348c7 --- /dev/null +++ b/libsrc/core/python_ngcore.hpp @@ -0,0 +1,132 @@ +#ifndef NETGEN_CORE_PYTHON_NGCORE_HPP +#define NETGEN_CORE_PYTHON_NGCORE_HPP + +#include + +#include "archive.hpp" + + +namespace ngcore +{ + template + Archive& Archive :: Shallow(T& val) + { + static_assert(detail::is_any_pointer, "ShallowArchive must be given pointer type!"); +#ifdef NETGEN_PYTHON + if(shallow_to_python) + { + if(is_output) + ShallowOutPython(pybind11::cast(val)); + else + { + pybind11::object obj; + ShallowInPython(obj); + val = pybind11::cast(obj); + } + } + else +#endif // NETGEN_PYTHON + *this & val; + return *this; + } + + template + class PyArchive : public ARCHIVE + { + private: + pybind11::list lst; + size_t index = 0; + std::map version_needed; + protected: + using ARCHIVE::stream; + using ARCHIVE::version_map; + using ARCHIVE::logger; + using ARCHIVE::GetLibraryVersions; + public: + PyArchive(const pybind11::object& alst = pybind11::none()) : + ARCHIVE(std::make_shared()), + lst(alst.is_none() ? pybind11::list() : pybind11::cast(alst)) + { + ARCHIVE::shallow_to_python = true; + if(Input()) + { + stream = std::make_shared + (pybind11::cast(lst[pybind11::len(lst)-1])); + *this & version_needed; + logger->debug("versions needed for unpickling = {}", version_needed); + for(auto& libversion : version_needed) + if(libversion.second > GetLibraryVersion(libversion.first)) + throw Exception("Error in unpickling data:\nLibrary " + libversion.first + + " must be at least " + libversion.second.to_string()); + stream = std::make_shared + (pybind11::cast(lst[pybind11::len(lst)-2])); + *this & version_map; + stream = std::make_shared + (pybind11::cast(lst[pybind11::len(lst)-3])); + } + } + + void NeedsVersion(const std::string& library, const std::string& version) override + { + if(Output()) + { + logger->debug("Need version {} of library {}.", version, library); + version_needed[library] = version_needed[library] > version ? version_needed[library] : version; + } + } + + using ARCHIVE::Output; + using ARCHIVE::Input; + using ARCHIVE::FlushBuffer; + using ARCHIVE::operator&; + using ARCHIVE::operator<<; + using ARCHIVE::GetVersion; + void ShallowOutPython(const pybind11::object& val) override { lst.append(val); } + void ShallowInPython(pybind11::object& val) override { val = lst[index++]; } + + pybind11::list WriteOut() + { + FlushBuffer(); + lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); + stream = std::make_shared(); + *this & GetLibraryVersions(); + FlushBuffer(); + lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); + stream = std::make_shared(); + logger->debug("Writeout version needed = {}", version_needed); + *this & version_needed; + FlushBuffer(); + lst.append(pybind11::bytes(std::static_pointer_cast(stream)->str())); + return lst; + } + }; + + template + auto NGSPickle() + { + return pybind11::pickle([](T* self) + { + PyArchive ar; + ar & self; + auto output = pybind11::make_tuple(ar.WriteOut()); + GetLogger("Archive")->trace("Pickling output for object of type {} = {}", + Demangle(typeid(T).name()), + std::string(pybind11::str(output))); + return output; + }, + [](const pybind11::tuple & state) + { + T* val = nullptr; + GetLogger("Archive")->trace("State for unpickling of object of type {} = {}", + Demangle(typeid(T).name()), + std::string(pybind11::str(state[0]))); + PyArchive ar(state[0]); + ar & val; + return val; + }); + } + + +} // namespace ngcore + +#endif // NETGEN_CORE_PYTHON_NGCORE_HPP diff --git a/libsrc/csg/python_csg.cpp b/libsrc/csg/python_csg.cpp index 47f85d58..d404a32d 100644 --- a/libsrc/csg/python_csg.cpp +++ b/libsrc/csg/python_csg.cpp @@ -1,6 +1,7 @@ #ifdef NG_PYTHON #include <../general/ngpython.hpp> +#include #include diff --git a/libsrc/geom2d/python_geom2d.cpp b/libsrc/geom2d/python_geom2d.cpp index e42450ae..d04b8cd7 100644 --- a/libsrc/geom2d/python_geom2d.cpp +++ b/libsrc/geom2d/python_geom2d.cpp @@ -1,6 +1,7 @@ #ifdef NG_PYTHON #include <../general/ngpython.hpp> +#include #include #include diff --git a/libsrc/meshing/meshclass.cpp b/libsrc/meshing/meshclass.cpp index d6d15ea5..695ea807 100644 --- a/libsrc/meshing/meshclass.cpp +++ b/libsrc/meshing/meshclass.cpp @@ -1,6 +1,7 @@ #include #include #include "meshing.hpp" +#include namespace netgen { diff --git a/libsrc/meshing/python_mesh.cpp b/libsrc/meshing/python_mesh.cpp index e027bb0d..448775d8 100644 --- a/libsrc/meshing/python_mesh.cpp +++ b/libsrc/meshing/python_mesh.cpp @@ -1,6 +1,7 @@ #ifdef NG_PYTHON #include <../general/ngpython.hpp> +#include #include #include "meshing.hpp" diff --git a/libsrc/occ/python_occ.cpp b/libsrc/occ/python_occ.cpp index eec271a1..18514c8e 100644 --- a/libsrc/occ/python_occ.cpp +++ b/libsrc/occ/python_occ.cpp @@ -2,6 +2,7 @@ #ifdef OCCGEOMETRY #include <../general/ngpython.hpp> +#include #include #include diff --git a/libsrc/stlgeom/python_stl.cpp b/libsrc/stlgeom/python_stl.cpp index 4968078e..0d2bb32f 100644 --- a/libsrc/stlgeom/python_stl.cpp +++ b/libsrc/stlgeom/python_stl.cpp @@ -2,6 +2,7 @@ #ifdef NG_PYTHON #include <../general/ngpython.hpp> +#include #include #ifdef WIN32