ForkMaster/netgen
1
0
Fork 0
mirror of https://github.com/NGSolve/netgen.git synced 2024-12-24 21:10:33 +05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'archive_python_pickle' into 'master'

Browse Source 
archive now support python exported objects

See merge request jschoeberl/netgen!112
This commit is contained in:
Joachim Schöberl 2018-12-30 14:34:03 +00:00
commit cdf50f2cd6
11 changed files with 189 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libsrc/core/CMakeLists.txt
View File

@ -4,6 +4,10 @@ target_compile_definitions(ngcore PRIVATE -DNGCORE_EXPORTS)
install(TARGETS ngcore DESTINATION ${NG_INSTALL_DIR} COMPONENT netgen)
if(USE_PYTHON)
target_include_directories(ngcore PUBLIC ${PYTHON_INCLUDE_DIRS})
endif(USE_PYTHON)
install(FILES ngcore.hpp archive.hpp type_traits.hpp version.hpp ngcore_api.hpp
DESTINATION ${NG_INSTALL_DIR_INCLUDE}/core COMPONENT netgen_devel)

218
libsrc/core/archive.hpp
View File

@ -18,6 +18,10 @@
#include "type_traits.hpp" // for all_of_tmpl
#include "version.hpp" // for VersionInfo
#ifdef NG_PYTHON
#include <pybind11/pybind11.h>
#endif // NG_PYTHON
namespace ngcore
{
// Libraries using this archive can store their version here to implement backwards compatibility
@ -98,7 +102,8 @@ namespace ngcore
// vectors for storing the unarchived (shared) pointers
std::vector<std::shared_ptr<void>> nr2shared_ptr;
std::vector<void*> nr2ptr;
protected:
bool shallow_to_python = false;
public:
Archive() = delete;
Archive(const Archive&) = delete;
@ -108,6 +113,31 @@ namespace ngcore
virtual ~Archive() { ; }
template<typename T>
Archive& Shallow(T& val)
{
static_assert(detail::is_any_pointer<T>, "ShallowArchive must be given pointer type!");
#ifdef NG_PYTHON
if(shallow_to_python)
{
if(is_output)
ShallowOutPython(pybind11::cast(val));
else
val = pybind11::cast<T>(ShallowInPython());
}
else
#endif // NG_PYTHON
*this & val;
return *this;
}
#ifdef NG_PYTHON
virtual void ShallowOutPython(pybind11::object /*unused*/) // NOLINT (copy by val is ok for this virt func)
{ throw std::runtime_error("Should not get in ShallowToPython base class implementation!"); }
virtual pybind11::object ShallowInPython()
{ throw std::runtime_error("Should not get in ShallowFromPython base class implementation!"); }
#endif // NG_PYTHON
Archive& operator=(const Archive&) = delete;
Archive& operator=(Archive&&) = delete;
@ -526,16 +556,15 @@ namespace ngcore
static constexpr size_t BUFFERSIZE = 1024;
char buffer[BUFFERSIZE] = {};
size_t ptr = 0;
std::shared_ptr<std::ostream> fout;
protected:
std::shared_ptr<std::ostream> stream;
public:
BinaryOutArchive() = delete;
BinaryOutArchive(const BinaryOutArchive&) = delete;
BinaryOutArchive(BinaryOutArchive&&) = delete;
BinaryOutArchive(std::shared_ptr<std::ostream>&& afout)
: Archive(true), fout(std::move(afout))
{
(*this) & GetLibraryVersions();
}
BinaryOutArchive(std::shared_ptr<std::ostream>&& astream)
: Archive(true), stream(std::move(astream))
{ }
BinaryOutArchive(const std::string& filename)
: BinaryOutArchive(std::make_shared<std::ofstream>(filename)) {}
~BinaryOutArchive () override { FlushBuffer(); }
@ -543,9 +572,6 @@ namespace ngcore
BinaryOutArchive& operator=(const BinaryOutArchive&) = delete;
BinaryOutArchive& operator=(BinaryOutArchive&&) = delete;
const VersionInfo& GetVersion(const std::string& library) override
{ return GetLibraryVersions()[library]; }
using Archive::operator&;
Archive & operator & (double & d) override
{ return Write(d); }
@ -567,7 +593,7 @@ namespace ngcore
(*this) & len;
FlushBuffer();
if(len)
fout->write (&str[0], len);
stream->write (&str[0], len);
return *this;
}
Archive & operator & (char *& str) override
@ -576,14 +602,14 @@ namespace ngcore
(*this) & len;
FlushBuffer();
if(len > 0)
fout->write (&str[0], len); // NOLINT
stream->write (&str[0], len); // NOLINT
return *this;
}
void FlushBuffer() override
{
if (ptr > 0)
{
fout->write(&buffer[0], ptr);
stream->write(&buffer[0], ptr);
ptr = 0;
}
}
@ -594,7 +620,7 @@ namespace ngcore
{
if (unlikely(ptr > BUFFERSIZE-sizeof(T)))
{
fout->write(&buffer[0], ptr);
stream->write(&buffer[0], ptr);
*reinterpret_cast<T*>(&buffer[0]) = x; // NOLINT
ptr = sizeof(T);
return *this;
@ -608,20 +634,15 @@ namespace ngcore
// BinaryInArchive ======================================================================
class NGCORE_API BinaryInArchive : public Archive
{
std::map<std::string, VersionInfo> vinfo{};
std::shared_ptr<std::istream> fin;
protected:
std::shared_ptr<std::istream> stream;
public:
BinaryInArchive (std::shared_ptr<std::istream>&& afin)
: Archive(false), fin(std::move(afin))
{
(*this) & vinfo;
}
BinaryInArchive (std::shared_ptr<std::istream>&& astream)
: Archive(false), stream(std::move(astream))
{ }
BinaryInArchive (const std::string& filename)
: BinaryInArchive(std::make_shared<std::ifstream>(filename)) { ; }
const VersionInfo& GetVersion(const std::string& library) override
{ return vinfo[library]; }
using Archive::operator&;
Archive & operator & (double & d) override
{ Read(d); return *this; }
@ -643,7 +664,7 @@ namespace ngcore
(*this) & len;
str.resize(len);
if(len)
fin->read(&str[0], len); // NOLINT
stream->read(&str[0], len); // NOLINT
return *this;
}
Archive & operator & (char *& str) override
@ -655,64 +676,60 @@ namespace ngcore
else
{
str = new char[len+1]; // NOLINT
fin->read(&str[0], len); // NOLINT
stream->read(&str[0], len); // NOLINT
str[len] = '\0'; // NOLINT
}
return *this;
}
Archive & Do (double * d, size_t n) override
{ fin->read(reinterpret_cast<char*>(d), n*sizeof(double)); return *this; } // NOLINT
{ stream->read(reinterpret_cast<char*>(d), n*sizeof(double)); return *this; } // NOLINT
Archive & Do (int * i, size_t n) override
{ fin->read(reinterpret_cast<char*>(i), n*sizeof(int)); return *this; } // NOLINT
{ stream->read(reinterpret_cast<char*>(i), n*sizeof(int)); return *this; } // NOLINT
Archive & Do (size_t * i, size_t n) override
{ fin->read(reinterpret_cast<char*>(i), n*sizeof(size_t)); return *this; } // NOLINT
{ stream->read(reinterpret_cast<char*>(i), n*sizeof(size_t)); return *this; } // NOLINT
private:
template<typename T>
inline void Read(T& val)
{ fin->read(reinterpret_cast<char*>(&val), sizeof(T)); } // NOLINT
{ stream->read(reinterpret_cast<char*>(&val), sizeof(T)); } // NOLINT
};
// TextOutArchive ======================================================================
class NGCORE_API TextOutArchive : public Archive
{
std::shared_ptr<std::ostream> fout;
protected:
std::shared_ptr<std::ostream> stream;
public:
TextOutArchive (std::shared_ptr<std::ostream>&& afout)
: Archive(true), fout(std::move(afout))
{
(*this) & GetLibraryVersions();
}
TextOutArchive (std::shared_ptr<std::ostream>&& astream)
: Archive(true), stream(std::move(astream))
{ }
TextOutArchive (const std::string& filename) :
TextOutArchive(std::make_shared<std::ofstream>(filename)) { }
const VersionInfo& GetVersion(const std::string& library) override
{ return GetLibraryVersions()[library]; }
using Archive::operator&;
Archive & operator & (double & d) override
{ *fout << d << '\n'; return *this; }
{ *stream << d << '\n'; return *this; }
Archive & operator & (int & i) override
{ *fout << i << '\n'; return *this; }
{ *stream << i << '\n'; return *this; }
Archive & operator & (short & i) override
{ *fout << i << '\n'; return *this; }
{ *stream << i << '\n'; return *this; }
Archive & operator & (long & i) override
{ *fout << i << '\n'; return *this; }
{ *stream << i << '\n'; return *this; }
Archive & operator & (size_t & i) override
{ *fout << i << '\n'; return *this; }
{ *stream << i << '\n'; return *this; }
Archive & operator & (unsigned char & i) override
{ *fout << int(i) << '\n'; return *this; }
{ *stream << int(i) << '\n'; return *this; }
Archive & operator & (bool & b) override
{ *fout << (b ? 't' : 'f') << '\n'; return *this; }
{ *stream << (b ? 't' : 'f') << '\n'; return *this; }
Archive & operator & (std::string & str) override
{
int len = str.length();
*fout << len << '\n';
*stream << len << '\n';
if(len)
{
fout->write(&str[0], len); // NOLINT
*fout << '\n';
stream->write(&str[0], len); // NOLINT
*stream << '\n';
}
return *this;
}
@ -722,8 +739,8 @@ namespace ngcore
*this & len;
if(len > 0)
{
fout->write (&str[0], len); // NOLINT
*fout << '\n';
stream->write (&str[0], len); // NOLINT
*stream << '\n';
}
return *this;
}
@ -732,44 +749,39 @@ namespace ngcore
// TextInArchive ======================================================================
class NGCORE_API TextInArchive : public Archive
{
std::map<std::string, VersionInfo> vinfo{};
std::shared_ptr<std::istream> fin;
protected:
std::shared_ptr<std::istream> stream;
public:
TextInArchive (std::shared_ptr<std::istream>&& afin) :
Archive(false), fin(std::move(afin))
{
(*this) & vinfo;
}
TextInArchive (std::shared_ptr<std::istream>&& astream) :
Archive(false), stream(std::move(astream))
{ }
TextInArchive (const std::string& filename)
: TextInArchive(std::make_shared<std::ifstream>(filename)) {}
const VersionInfo& GetVersion(const std::string& library) override
{ return vinfo[library]; }
using Archive::operator&;
Archive & operator & (double & d) override
{ *fin >> d; return *this; }
{ *stream >> d; return *this; }
Archive & operator & (int & i) override
{ *fin >> i; return *this; }
{ *stream >> i; return *this; }
Archive & operator & (short & i) override
{ *fin >> i; return *this; }
{ *stream >> i; return *this; }
Archive & operator & (long & i) override
{ *fin >> i; return *this; }
{ *stream >> i; return *this; }
Archive & operator & (size_t & i) override
{ *fin >> i; return *this; }
{ *stream >> i; return *this; }
Archive & operator & (unsigned char & i) override
{ int _i; *fin >> _i; i = _i; return *this; }
{ int _i; *stream >> _i; i = _i; return *this; }
Archive & operator & (bool & b) override
{ char c; *fin >> c; b = (c=='t'); return *this; }
{ char c; *stream >> c; b = (c=='t'); return *this; }
Archive & operator & (std::string & str) override
{
int len;
*fin >> len;
*stream >> len;
char ch;
fin->get(ch); // '\n'
stream->get(ch); // '\n'
str.resize(len);
if(len)
fin->get(&str[0], len+1, '\0');
stream->get(&str[0], len+1, '\0');
return *this;
}
Archive & operator & (char *& str) override
@ -785,13 +797,73 @@ namespace ngcore
str = new char[len+1]; // NOLINT
if(len)
{
fin->get(ch); // \n
fin->get(&str[0], len+1, '\0'); // NOLINT
stream->get(ch); // \n
stream->get(&str[0], len+1, '\0'); // NOLINT
}
str[len] = '\0'; // NOLINT
return *this;
}
};
#ifdef NG_PYTHON
template<typename ARCHIVE>
class PyArchive : public ARCHIVE
{
private:
pybind11::list lst;
size_t index = 0;
using ARCHIVE::stream;
public:
PyArchive(const pybind11::object& alst = pybind11::none()) :
ARCHIVE(std::make_shared<std::stringstream>()),
lst(alst.is_none() ? pybind11::list() : pybind11::cast<pybind11::list>(alst))
{
ARCHIVE::shallow_to_python = true;
if(Input())
stream = std::make_shared<std::stringstream>
(pybind11::cast<pybind11::bytes>(lst[pybind11::len(lst)-1]));
}
using ARCHIVE::Output;
using ARCHIVE::Input;
using ARCHIVE::FlushBuffer;
using ARCHIVE::operator&;
using ARCHIVE::operator<<;
using ARCHIVE::GetVersion;
void ShallowOutPython(pybind11::object val) override { lst.append(val); }
pybind11::object ShallowInPython() override { return lst[index++]; }
pybind11::list WriteOut()
{
FlushBuffer();
lst.append(pybind11::bytes(std::static_pointer_cast<std::stringstream>(stream)->str()));
return lst;
}
};
template<typename T, typename T_ARCHIVE_OUT=BinaryOutArchive, typename T_ARCHIVE_IN=BinaryInArchive>
auto NGSPickle(bool printoutput=false)
{
return pybind11::pickle([printoutput](T* self)
{
PyArchive<T_ARCHIVE_OUT> ar;
ar & self;
auto output = pybind11::make_tuple(ar.WriteOut());
if(printoutput)
pybind11::print("pickle output of", Demangle(typeid(T).name()),"=", output);
return output;
},
[](pybind11::tuple state)
{
T* val = nullptr;
PyArchive<T_ARCHIVE_IN> ar(state[0]);
ar & val;
return val;
});
}
#endif // NG_PYTHON
} // namespace ngcore
#endif // NETGEN_CORE_ARCHIVE_HPP

16
libsrc/core/type_traits.hpp
View File

@ -1,6 +1,7 @@
#ifndef NETGEN_CORE_TYPE_TRAITS_HPP
#define NETGEN_CORE_TYPE_TRAITS_HPP
#include <memory>
#include <type_traits>
namespace ngcore
@ -11,6 +12,21 @@ namespace ngcore
template<bool ... vals>
constexpr bool all_of_tmpl = std::is_same<_BoolArray<vals...>, _BoolArray<(vals || true)...>>::value; // NOLINT
template<typename T>
struct is_any_pointer_impl : std::false_type {};
template<typename T>
struct is_any_pointer_impl<T*> : std::true_type {};
template<typename T>
struct is_any_pointer_impl<std::shared_ptr<T>> : std::true_type {};
template<typename T>
struct is_any_pointer_impl<std::unique_ptr<T>> : std::true_type {};
template<typename T>
constexpr bool is_any_pointer = is_any_pointer_impl<T>::value;
} // namespace detail
} // namespace ngcore

18
libsrc/csg/python_csg.cpp
View File

@ -372,23 +372,7 @@ However, when r = 0, the top part becomes a point(tip) and meshing fails!
geo->FindIdenticSurfaces(1e-8 * geo->MaxSize());
return geo;
}), py::arg("filename"))
.def(py::pickle(
[](CSGeometry& self)
{
auto ss = make_shared<stringstream>();
BinaryOutArchive archive(ss);
archive & self;
archive.FlushBuffer();
return py::make_tuple(py::bytes(ss->str()));
},
[](py::tuple state)
{
auto geo = make_shared<CSGeometry>();
auto ss = make_shared<stringstream> (py::cast<py::bytes>(state[0]));
BinaryInArchive archive(ss);
archive & (*geo);
return geo;
}))
.def(NGSPickle<CSGeometry>())
.def("Save", FunctionPointer([] (CSGeometry & self, string filename)
{
cout << "save geometry to file " << filename << endl;

21
libsrc/geom2d/python_geom2d.cpp
View File

@ -27,25 +27,8 @@ DLL_HEADER void ExportGeom2d(py::module &m)
ng_geometry = geo;
return geo;
}))
.def(py::pickle(
[](SplineGeometry2d& self)
{
auto ss = make_shared<stringstream>();
BinaryOutArchive archive(ss);
archive & self;
archive.FlushBuffer();
return py::make_tuple(py::bytes(ss->str()));
},
[](py::tuple state)
{
auto geo = make_shared<SplineGeometry2d>();
auto ss = make_shared<stringstream> (py::cast<py::bytes>(state[0]));
BinaryInArchive archive(ss);
archive & (*geo);
return geo;
}))
.def("Load",&SplineGeometry2d::Load)
.def(NGSPickle<SplineGeometry2d>())
.def("Load",&SplineGeometry2d::Load)
.def("AppendPoint", FunctionPointer
([](SplineGeometry2d &self, double px, double py, double maxh, double hpref, string name)
{

2
libsrc/meshing/meshclass.cpp
View File

@ -1316,7 +1316,7 @@ namespace netgen
archive & *ident;
archive & geometry;
archive.Shallow(geometry);
archive & *curvedelems;
if (archive.Input())

1
libsrc/meshing/python_mesh.cpp
View File

@ -493,6 +493,7 @@ DLL_HEADER void ExportNetgenMeshing(py::module &m)
} ),
py::arg("dim")=3
)
.def(NGSPickle<Mesh>())
/*
.def("__init__",

18
libsrc/occ/python_occ.cpp
View File

@ -18,23 +18,7 @@ DLL_HEADER void ExportNgOCC(py::module &m)
{
py::class_<OCCGeometry, shared_ptr<OCCGeometry>, NetgenGeometry> (m, "OCCGeometry", R"raw_string(Use LoadOCCGeometry to load the geometry from a *.step file.)raw_string")
.def(py::init<>())
.def(py::pickle(
[](OCCGeometry& self)
{
auto ss = make_shared<stringstream>();
BinaryOutArchive archive(ss);
archive & self;
archive.FlushBuffer();
return py::make_tuple(py::bytes(ss->str()));
},
[](py::tuple state)
{
auto geo = make_shared<OCCGeometry>();
auto ss = make_shared<stringstream> (py::cast<py::bytes>(state[0]));
BinaryInArchive archive(ss);
archive & (*geo);
return geo;
}))
.def(NGSPickle<OCCGeometry>())
.def("Heal",[](OCCGeometry & self, double tolerance, bool fixsmalledges, bool fixspotstripfaces, bool sewfaces, bool makesolids, bool splitpartitions)
{
self.tolerance = tolerance;

18
libsrc/stlgeom/python_stl.cpp
View File

@ -20,23 +20,7 @@ DLL_HEADER void ExportSTL(py::module & m)
{
py::class_<STLGeometry,shared_ptr<STLGeometry>, NetgenGeometry> (m,"STLGeometry")
.def(py::init<>())
.def(py::pickle(
[](STLGeometry& self)
{
auto ss = make_shared<stringstream>();
BinaryOutArchive archive(ss);
archive & self;
archive.FlushBuffer();
return py::make_tuple(py::bytes(ss->str()));
},
[](py::tuple state)
{
auto geo = make_shared<STLGeometry>();
auto ss = make_shared<stringstream> (py::cast<py::bytes>(state[0]));
BinaryInArchive archive(ss);
archive & (*geo);
return geo;
}))
.def(NGSPickle<STLGeometry>())
.def("_visualizationData", [](shared_ptr<STLGeometry> stl_geo)
{
std::vector<float> vertices;

11
tests/catch/archive.cpp
View File

@ -236,13 +236,6 @@ void testMultipleInheritance(Archive& in, Archive& out)
}
}
void testLibraryVersion(Archive& in, Archive& out)
{
SetLibraryVersion("netgen","v6.2.1812");
CHECK(in.GetVersion("netgen") == "v6.2.1811");
CHECK(out.GetVersion("netgen") == "v6.2.1812");
}
void testArchive(Archive& in, Archive& out)
{
SECTION("Empty String")
@ -292,10 +285,6 @@ void testArchive(Archive& in, Archive& out)
{
testNullPtr(in, out);
}
SECTION("Library Version")
{
testLibraryVersion(in,out);
}
}
TEST_CASE("BinaryArchive")

18
tests/pytest/test_pickling.py
View File

@ -85,5 +85,21 @@ def test_pickle_geom2d():
for val1, val2 in zip(vd1.values(), vd2.values()):
assert numpy.array_equal(val1, val2)
def test_pickle_mesh():
import netgen.csg as csg
geo = csg.CSGeometry()
brick = csg.OrthoBrick(csg.Pnt(-3,-3,-3), csg.Pnt(3,3,3))
mesh = geo.GenerateMesh(maxh=0.2)
assert geo == mesh.GetGeometry()
dump = pickle.dumps([geo,mesh])
geo2, mesh2 = pickle.loads(dump)
assert geo2 == mesh2.GetGeometry()
mesh.Save("msh1.vol.gz")
mesh2.Save("msh2.vol.gz")
import filecmp, os
assert filecmp.cmp("msh1.vol.gz", "msh2.vol.gz")
os.remove("msh1.vol.gz")
os.remove("msh2.vol.gz")
if __name__ == "__main__":
test_pickle_csg()
test_pickle_mesh()