Get rid of pybind11 include in archive.hpp

Forward-declare pybind11::object and move implementation of
Archive::Shallow() to new header python_ngcore.hpp

All files using the Shallow/Python archive functionality must include
core/python_ngcore.hpp. Missing includes result in link errors, due to
missing instantiations of Archive::Shallow<T>();

libsrc/core/CMakeLists.txt

@@ -43,7 +43,7 @@ target_link_libraries(ngcore PUBLIC netgen_mpi PRIVATE ${CMAKE_THREAD_LIBS_INIT}
 
 install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp logging.hpp
   exception.hpp symboltable.hpp paje_trace.hpp utils.hpp profiler.hpp mpi_wrapper.hpp
-  array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp
+  array.hpp taskmanager.hpp concurrentqueue.h localheap.hpp python_ngcore.hpp
   DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)
 
 if(ENABLE_CPP_CORE_GUIDELINES_CHECK)

libsrc/core/archive.hpp

@@ -22,7 +22,10 @@
 #include "version.hpp"          // for VersionInfo
 
 #ifdef NETGEN_PYTHON
-#include <pybind11/pybind11.h>
+namespace pybind11
+{
+  class object;
+}
 #endif // NETGEN_PYTHON
 
 namespace ngcore
@@ -122,27 +125,12 @@ namespace ngcore
     // Python should be archived using this Shallow function. If Shallow is called from C++ code
     // it archives the object normally.
     template<typename T>
-    Archive& Shallow(T& val)
+    Archive& Shallow(T& val); // implemented in python_ngcore.hpp
-    {
-      static_assert(detail::is_any_pointer<T>, "ShallowArchive must be given pointer type!");
-#ifdef NETGEN_PYTHON
-      if(shallow_to_python)
-        {
-          if(is_output)
-            ShallowOutPython(pybind11::cast(val));
-          else
-            val = pybind11::cast<T>(ShallowInPython());
-        }
-      else
-#endif // NETGEN_PYTHON
-        *this & val;
-      return *this;
-    }
 
 #ifdef NETGEN_PYTHON
     virtual void ShallowOutPython(const pybind11::object& /*unused*/)
     { throw UnreachableCodeException{}; }
-    virtual pybind11::object ShallowInPython()
+    virtual void ShallowInPython(pybind11::object &)
     { throw UnreachableCodeException{}; }
 #endif // NETGEN_PYTHON
 
@@ -909,105 +897,6 @@ namespace ngcore
     }
   };
 
-#ifdef NETGEN_PYTHON
-
-  template<typename ARCHIVE>
-  class PyArchive : public ARCHIVE
-  {
-  private:
-    pybind11::list lst;
-    size_t index = 0;
-    std::map<std::string, VersionInfo> version_needed;
-  protected:
-    using ARCHIVE::stream;
-    using ARCHIVE::version_map;
-    using ARCHIVE::logger;
-    using ARCHIVE::GetLibraryVersions;
-  public:
-    PyArchive(const pybind11::object& alst = pybind11::none()) :
-      ARCHIVE(std::make_shared<std::stringstream>()),
-      lst(alst.is_none() ? pybind11::list() : pybind11::cast<pybind11::list>(alst))
-    {
-      ARCHIVE::shallow_to_python = true;
-      if(Input())
-        {
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-1]));
-          *this & version_needed;
-          logger->debug("versions needed for unpickling = {}", version_needed);
-          for(auto& libversion : version_needed)
-            if(libversion.second > GetLibraryVersion(libversion.first))
-              throw Exception("Error in unpickling data:\nLibrary " + libversion.first +
-                              " must be at least " + libversion.second.to_string());
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-2]));
-          *this & version_map;
-          stream = std::make_shared<std::stringstream>
-            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-3]));
-        }
-    }
-
-    void NeedsVersion(const std::string& library, const std::string& version) override
-    {
-      if(Output())
-        {
-          logger->debug("Need version {} of library {}.", version, library);
-          version_needed[library] = version_needed[library] > version ? version_needed[library] : version;
-        }
-    }
-
-    using ARCHIVE::Output;
-    using ARCHIVE::Input;
-    using ARCHIVE::FlushBuffer;
-    using ARCHIVE::operator&;
-    using ARCHIVE::operator<<;
-    using ARCHIVE::GetVersion;
-    void ShallowOutPython(const pybind11::object& val) override { lst.append(val); }
-    pybind11::object ShallowInPython() override { return lst[index++]; }
-
-    pybind11::list WriteOut()
-    {
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      stream = std::make_shared<std::stringstream>();
-      *this & GetLibraryVersions();
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      stream = std::make_shared<std::stringstream>();
-      logger->debug("Writeout version needed = {}", version_needed);
-      *this & version_needed;
-      FlushBuffer();
-      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
-      return lst;
-    }
-  };
-
-  template<typename T, typename T_ARCHIVE_OUT=BinaryOutArchive, typename T_ARCHIVE_IN=BinaryInArchive>
-  auto NGSPickle()
-  {
-    return pybind11::pickle([](T* self)
-                      {
-                        PyArchive<T_ARCHIVE_OUT> ar;
-                        ar & self;
-                        auto output = pybind11::make_tuple(ar.WriteOut());
-                        GetLogger("Archive")->trace("Pickling output for object of type {} = {}",
-                                                    Demangle(typeid(T).name()),
-                                                    std::string(pybind11::str(output)));
-                        return output;
-                      },
-                      [](const pybind11::tuple & state)
-                      {
-                        T* val = nullptr;
-                        GetLogger("Archive")->trace("State for unpickling of object of type {} = {}",
-                                                    Demangle(typeid(T).name()),
-                                                    std::string(pybind11::str(state[0])));
-                        PyArchive<T_ARCHIVE_IN> ar(state[0]);
-                        ar & val;
-                        return val;
-                      });
-  }
-
-#endif // NETGEN_PYTHON
 } // namespace ngcore
 
 #endif // NETGEN_CORE_ARCHIVE_HPP

libsrc/core/python_ngcore.hpp

@@ -0,0 +1,132 @@
+#ifndef NETGEN_CORE_PYTHON_NGCORE_HPP
+#define NETGEN_CORE_PYTHON_NGCORE_HPP
+
+#include <pybind11/pybind11.h>
+
+#include "archive.hpp"
+
+
+namespace ngcore
+{
+    template<typename T>
+    Archive& Archive :: Shallow(T& val)
+    {
+      static_assert(detail::is_any_pointer<T>, "ShallowArchive must be given pointer type!");
+#ifdef NETGEN_PYTHON
+      if(shallow_to_python)
+        {
+          if(is_output)
+            ShallowOutPython(pybind11::cast(val));
+          else
+          {
+            pybind11::object obj;
+            ShallowInPython(obj);
+            val = pybind11::cast<T>(obj);
+          }
+        }
+      else
+#endif // NETGEN_PYTHON
+        *this & val;
+      return *this;
+    }
+
+  template<typename ARCHIVE>
+  class PyArchive : public ARCHIVE
+  {
+  private:
+    pybind11::list lst;
+    size_t index = 0;
+    std::map<std::string, VersionInfo> version_needed;
+  protected:
+    using ARCHIVE::stream;
+    using ARCHIVE::version_map;
+    using ARCHIVE::logger;
+    using ARCHIVE::GetLibraryVersions;
+  public:
+    PyArchive(const pybind11::object& alst = pybind11::none()) :
+      ARCHIVE(std::make_shared<std::stringstream>()),
+      lst(alst.is_none() ? pybind11::list() : pybind11::cast<pybind11::list>(alst))
+    {
+      ARCHIVE::shallow_to_python = true;
+      if(Input())
+        {
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-1]));
+          *this & version_needed;
+          logger->debug("versions needed for unpickling = {}", version_needed);
+          for(auto& libversion : version_needed)
+            if(libversion.second > GetLibraryVersion(libversion.first))
+              throw Exception("Error in unpickling data:\nLibrary " + libversion.first +
+                              " must be at least " + libversion.second.to_string());
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-2]));
+          *this & version_map;
+          stream = std::make_shared<std::stringstream>
+            (pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-3]));
+        }
+    }
+
+    void NeedsVersion(const std::string& library, const std::string& version) override
+    {
+      if(Output())
+        {
+          logger->debug("Need version {} of library {}.", version, library);
+          version_needed[library] = version_needed[library] > version ? version_needed[library] : version;
+        }
+    }
+
+    using ARCHIVE::Output;
+    using ARCHIVE::Input;
+    using ARCHIVE::FlushBuffer;
+    using ARCHIVE::operator&;
+    using ARCHIVE::operator<<;
+    using ARCHIVE::GetVersion;
+    void ShallowOutPython(const pybind11::object& val) override { lst.append(val); }
+    void ShallowInPython(pybind11::object& val) override { val = lst[index++]; }
+
+    pybind11::list WriteOut()
+    {
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      stream = std::make_shared<std::stringstream>();
+      *this & GetLibraryVersions();
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      stream = std::make_shared<std::stringstream>();
+      logger->debug("Writeout version needed = {}", version_needed);
+      *this & version_needed;
+      FlushBuffer();
+      lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
+      return lst;
+    }
+  };
+
+  template<typename T, typename T_ARCHIVE_OUT=BinaryOutArchive, typename T_ARCHIVE_IN=BinaryInArchive>
+  auto NGSPickle()
+  {
+    return pybind11::pickle([](T* self)
+                      {
+                        PyArchive<T_ARCHIVE_OUT> ar;
+                        ar & self;
+                        auto output = pybind11::make_tuple(ar.WriteOut());
+                        GetLogger("Archive")->trace("Pickling output for object of type {} = {}",
+                                                    Demangle(typeid(T).name()),
+                                                    std::string(pybind11::str(output)));
+                        return output;
+                      },
+                      [](const pybind11::tuple & state)
+                      {
+                        T* val = nullptr;
+                        GetLogger("Archive")->trace("State for unpickling of object of type {} = {}",
+                                                    Demangle(typeid(T).name()),
+                                                    std::string(pybind11::str(state[0])));
+                        PyArchive<T_ARCHIVE_IN> ar(state[0]);
+                        ar & val;
+                        return val;
+                      });
+  }
+
+
+} // namespace ngcore
+
+#endif // NETGEN_CORE_PYTHON_NGCORE_HPP

libsrc/csg/python_csg.cpp

@@ -1,6 +1,7 @@
 #ifdef NG_PYTHON
 
 #include <../general/ngpython.hpp>
+#include <core/python_ngcore.hpp>
 #include <csg.hpp>
 
 

libsrc/geom2d/python_geom2d.cpp

@@ -1,6 +1,7 @@
 #ifdef NG_PYTHON
 
 #include <../general/ngpython.hpp>
+#include <core/python_ngcore.hpp>
 
 #include <meshing.hpp>
 #include <geometry2d.hpp>

libsrc/meshing/meshclass.cpp

@@ -1,6 +1,7 @@
 #include <mystdlib.h>
 #include <atomic>
 #include "meshing.hpp"
+#include <core/python_ngcore.hpp>
 
 namespace netgen
 {

libsrc/meshing/python_mesh.cpp

@@ -1,6 +1,7 @@
 #ifdef NG_PYTHON
 
 #include <../general/ngpython.hpp>
+#include <core/python_ngcore.hpp>
 
 #include <mystdlib.h>
 #include "meshing.hpp"

libsrc/occ/python_occ.cpp

@@ -2,6 +2,7 @@
 #ifdef OCCGEOMETRY
 
 #include <../general/ngpython.hpp>
+#include <core/python_ngcore.hpp>
 
 #include <meshing.hpp>
 #include <occgeom.hpp>

libsrc/stlgeom/python_stl.cpp

@@ -2,6 +2,7 @@
 #ifdef NG_PYTHON
 
 #include <../general/ngpython.hpp>
+#include <core/python_ngcore.hpp>
 #include <stlgeom.hpp>
 
 #ifdef WIN32