From 5288af641c4592644ad092cf7d23b9440355b334 Mon Sep 17 00:00:00 2001 From: Christopher Lackner Date: Tue, 10 Sep 2019 23:01:05 +0200 Subject: [PATCH 1/3] array numpy buffer protocol --- libsrc/core/python_ngcore.cpp | 2 +- libsrc/core/python_ngcore.hpp | 43 +++++++++++++++++++++++++++- libsrc/core/python_ngcore_export.cpp | 7 ++++- tests/pytest/test_array.py | 17 +++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/pytest/test_array.py diff --git a/libsrc/core/python_ngcore.cpp b/libsrc/core/python_ngcore.cpp index 5e64d1b2..f7addf3c 100644 --- a/libsrc/core/python_ngcore.cpp +++ b/libsrc/core/python_ngcore.cpp @@ -7,7 +7,7 @@ using std::string; namespace ngcore { - + bool ngcore_have_numpy = false; void SetFlag(Flags &flags, string s, py::object value) { if (py::isinstance(value)) diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index f52c104e..49136e53 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -4,6 +4,7 @@ #include "ngcore_api.hpp" // for operator new #include #include +#include #include "array.hpp" #include "archive.hpp" @@ -13,6 +14,7 @@ namespace py = pybind11; namespace ngcore { + NGCORE_API extern bool ngcore_have_numpy; template Array makeCArray(const py::object& obj) @@ -29,6 +31,20 @@ namespace ngcore return arr; } + namespace detail + { + template + struct HasPyFormat + { + private: + template + static auto check(T2*) -> std::enable_if_t>().format()), std::string>, std::true_type>; + static auto check(...) -> std::false_type; + public: + static constexpr bool value = decltype(check((T*) nullptr))::value; + }; + } // namespace detail + template ::index_type> void ExportArray (py::module &m) { @@ -36,7 +52,8 @@ namespace ngcore using TArray = Array; std::string suffix = std::string(typeid(T).name()) + "_" + typeid(TIND).name(); std::string fname = std::string("FlatArray_") + suffix; - py::class_(m, fname.c_str()) + auto flatarray_class = py::class_(m, fname.c_str(), + py::buffer_protocol()) .def ("__len__", [] ( TFlat &self ) { return self.Size(); } ) .def ("__getitem__", [](TFlat & self, TIND i) -> T& @@ -77,6 +94,30 @@ namespace ngcore ; + if constexpr (detail::HasPyFormat::value) + { + if(ngcore_have_numpy && !py::detail::npy_format_descriptor::dtype().is_none()) + { + flatarray_class + .def_buffer([](TFlat& self) + { + return py::buffer_info( + self.Addr(0), + sizeof(T), + py::format_descriptor::format(), + 1, + { self.Size() }, + { sizeof(T) * (self.Addr(1) - self.Addr(0)) }); + }) + .def("NumPy", [](py::object self) + { + return py::module::import("numpy") + .attr("frombuffer")(self, py::detail::npy_format_descriptor::dtype()); + }) + ; + } + } + std::string aname = std::string("Array_") + suffix; py::class_(m, aname.c_str()) .def(py::init([] (size_t n) { return new TArray(n); }),py::arg("n"), "Makes array of given length") diff --git a/libsrc/core/python_ngcore_export.cpp b/libsrc/core/python_ngcore_export.cpp index 162f8163..d6cc9d6e 100644 --- a/libsrc/core/python_ngcore_export.cpp +++ b/libsrc/core/python_ngcore_export.cpp @@ -2,13 +2,18 @@ #include "bitarray.hpp" #include "taskmanager.hpp" - using namespace ngcore; using namespace std; using namespace pybind11::literals; PYBIND11_MODULE(pyngcore, m) // NOLINT { + try + { + auto numpy = py::module::import("numpy"); + ngcore_have_numpy = !numpy.is_none(); + } + catch(...) {} ExportArray(m); ExportArray(m); ExportArray(m); diff --git a/tests/pytest/test_array.py b/tests/pytest/test_array.py new file mode 100644 index 00000000..ccd186c1 --- /dev/null +++ b/tests/pytest/test_array.py @@ -0,0 +1,17 @@ +from pyngcore import * +from numpy import sort, array + +def test_array_numpy(): + a = Array_i_m(5) + a[:] = 0 + a[3:] = 2 + assert(sum(a) == 4) + a[1] = 5 + b = sort(a) + assert(all(b == array([0,0,2,2,5]))) + assert(all(a == array([0,5,0,2,2]))) + a.NumPy().sort() + assert(all(a == array([0,0,2,2,5]))) + +if __name__ == "__main__": + test_array_numpy() From 59087f5c2c93e9ba57f7382ec5bd1d1cdf974a17 Mon Sep 17 00:00:00 2001 From: Christopher Lackner Date: Wed, 11 Sep 2019 08:27:04 +0200 Subject: [PATCH 2/3] make array name platform independent --- libsrc/core/python_ngcore.hpp | 4 +++- tests/pytest/test_array.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index 49136e53..b3e428b3 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -50,7 +50,9 @@ namespace ngcore { using TFlat = FlatArray; using TArray = Array; - std::string suffix = std::string(typeid(T).name()) + "_" + typeid(TIND).name(); + std::string suffix = std::string(Demangle(typeid(T).name())) + "_" + Demangle(typeid(TIND).name()); + std::replace(suffix.begin(), suffix.end(), ':', '_'); + std::replace(suffix.begin(), suffix.end(), ' ', '_'); std::string fname = std::string("FlatArray_") + suffix; auto flatarray_class = py::class_(m, fname.c_str(), py::buffer_protocol()) diff --git a/tests/pytest/test_array.py b/tests/pytest/test_array.py index ccd186c1..54c48755 100644 --- a/tests/pytest/test_array.py +++ b/tests/pytest/test_array.py @@ -2,7 +2,7 @@ from pyngcore import * from numpy import sort, array def test_array_numpy(): - a = Array_i_m(5) + a = Array_int_unsigned_long(5) a[:] = 0 a[3:] = 2 assert(sum(a) == 4) From 485d9f230bb230f5445c89f4fe3c2e1c97c79b11 Mon Sep 17 00:00:00 2001 From: Christopher Lackner Date: Wed, 11 Sep 2019 13:04:50 +0200 Subject: [PATCH 3/3] platform independent and readable names for array export suffix --- libsrc/core/python_ngcore.hpp | 69 ++++++++++++++++++++++++++++++----- tests/pytest/test_array.py | 2 +- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/libsrc/core/python_ngcore.hpp b/libsrc/core/python_ngcore.hpp index b3e428b3..21f610e3 100644 --- a/libsrc/core/python_ngcore.hpp +++ b/libsrc/core/python_ngcore.hpp @@ -16,6 +16,56 @@ namespace ngcore { NGCORE_API extern bool ngcore_have_numpy; + // Python class name type traits + template + struct PyNameTraits { + static const std::string & GetName() + { + static const std::string name = + py::cast(py::cast(T()).attr("__class__").attr("__name__")); + return name; + } + }; + + template + std::string GetPyName(const char *prefix = 0) { + std::string s; + if(prefix) s = std::string(prefix); + s+= PyNameTraits::GetName(); + return s; + } + + template<> + struct PyNameTraits { + static std::string GetName() { return "I"; } + }; + + template<> + struct PyNameTraits { + static std::string GetName() { return "U"; } + }; + + template<> + struct PyNameTraits { + static std::string GetName() { return "F"; } + }; + + template<> + struct PyNameTraits { + static std::string GetName() { return "D"; } + }; + + template<> + struct PyNameTraits { + static std::string GetName() { return "S"; } + }; + + template + struct PyNameTraits> { + static std::string GetName() + { return std::string("sp_")+GetPyName(); } + }; + template Array makeCArray(const py::object& obj) { @@ -44,15 +94,14 @@ namespace ngcore static constexpr bool value = decltype(check((T*) nullptr))::value; }; } // namespace detail - + template ::index_type> void ExportArray (py::module &m) { using TFlat = FlatArray; using TArray = Array; - std::string suffix = std::string(Demangle(typeid(T).name())) + "_" + Demangle(typeid(TIND).name()); - std::replace(suffix.begin(), suffix.end(), ':', '_'); - std::replace(suffix.begin(), suffix.end(), ' ', '_'); + std::string suffix = GetPyName() + "_" + + GetPyName(); std::string fname = std::string("FlatArray_") + suffix; auto flatarray_class = py::class_(m, fname.c_str(), py::buffer_protocol()) @@ -73,7 +122,7 @@ namespace ngcore if (i < base || i >= self.Size()+base) throw py::index_error(); self[i] = val; - return self[i]; + return self[i]; }, py::return_value_policy::reference) @@ -85,11 +134,11 @@ namespace ngcore throw py::error_already_set(); static constexpr int base = IndexBASE(); if (start < base || start+(slicelength-1)*step >= self.Size()+base) - throw py::index_error(); - for (size_t i = 0; i < slicelength; i++, start+=step) - self[start] = val; + throw py::index_error(); + for (size_t i = 0; i < slicelength; i++, start+=step) + self[start] = val; }) - + .def("__iter__", [] ( TFlat & self) { return py::make_iterator (self.begin(),self.end()); }, py::keep_alive<0,1>()) // keep array alive while iterator is used @@ -143,7 +192,7 @@ namespace ngcore py::dict NGCORE_API CreateDictFromFlags(const Flags& flags); // *************** Archiving functionality ************** - + template Archive& Archive :: Shallow(T& val) { diff --git a/tests/pytest/test_array.py b/tests/pytest/test_array.py index 54c48755..db73452e 100644 --- a/tests/pytest/test_array.py +++ b/tests/pytest/test_array.py @@ -2,7 +2,7 @@ from pyngcore import * from numpy import sort, array def test_array_numpy(): - a = Array_int_unsigned_long(5) + a = Array_I_S(5) a[:] = 0 a[3:] = 2 assert(sum(a) == 4)