Merge pull request #267 from tdegeus/qad

Adding possibility to 'cast' or copy to `xt::xarray` etc

Author: JohanMabille

.azure-pipelines/unix-build.yml

@@ -45,6 +45,17 @@ steps:
     displayName: Example - readme 1
     workingDirectory: $(Build.SourcesDirectory)/docs/source/examples/readme_example_1
 
+  - script: |
+      source activate xtensor-python
+      cmake -Bbuild -DPython_EXECUTABLE=`which python`
+      cd build
+      cmake --build .
+      cp ../example.py .
+      python example.py
+      cd ..
+    displayName: Example - Copy 'cast'
+    workingDirectory: $(Build.SourcesDirectory)/docs/source/examples/copy_cast
+
   - script: |
       source activate xtensor-python
       cmake -Bbuild -DPython_EXECUTABLE=`which python`

docs/source/examples.rst

@@ -143,5 +143,54 @@ Then we can test the module:
     Since we did not install the module,
     we should compile and run the example from the same folder.
     To install, please consult
-    `this *pybind11* / *CMake* example <https://github.com/pybind/cmake_example>`_.
+    `this pybind11 / CMake example <https://github.com/pybind/cmake_example>`_.
     **Tip**: take care to modify that example with the correct *CMake* case ``Python_EXECUTABLE``.
+
+Fall-back cast
+==============
+
+The previous example showed you how to design your module to be flexible in accepting data.
+From C++ we used ``xt::xarray<double>``,
+whereas for the Python API we used ``xt::pyarray<double>`` to operate directly on the memory
+of a NumPy array from Python (without copying the data).
+
+Sometimes, you might not have the flexibility to design your module's methods
+with template parameters.
+This might occur when you want to ``override`` functions
+(though it is recommended to use CRTP to still use templates).
+In this case we can still bind the module in Python using *xtensor-python*,
+however, we have to copy the data from a (NumPy) array.
+This means that although the following signatures are quite different when used from C++,
+as follows:
+
+1.  *Constant reference*: read from the data, without copying it.
+
+    .. code-block:: cpp
+
+         void foo(const xt::xarray<double>& a);
+
+2.  *Reference*: read from and/or write to the data, without copying it.
+
+    .. code-block:: cpp
+
+         void foo(xt::xarray<double>& a);
+
+3.   *Copy*: copy the data.
+
+     .. code-block:: cpp
+
+         void foo(xt::xarray<double> a);
+
+The Python will all cases result in a copy to a temporary variable
+(though the last signature will lead to a copy to a temporary variable, and another copy to ``a``).
+On the one hand, this is more costly than when using ``xt::pyarray`` and ``xt::pyxtensor``,
+on the other hand, it means that all changes you make to a reference, are made to the temporary
+copy, and are thus lost.
+
+Still, it might be a convenient way to create Python bindings, using a minimal effort.
+Consider this example:
+
+:download:`main.cpp <examples/copy_cast/main.cpp>`
+
+.. literalinclude:: examples/copy_cast/main.cpp
+   :language: cpp

docs/source/examples/copy_cast/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.1..3.19)
+
+project(mymodule)
+
+find_package(pybind11 CONFIG REQUIRED)
+find_package(xtensor REQUIRED)
+find_package(xtensor-python REQUIRED)
+find_package(Python REQUIRED COMPONENTS NumPy)
+
+pybind11_add_module(mymodule main.cpp)
+target_link_libraries(mymodule PUBLIC pybind11::module xtensor-python Python::NumPy)
+
+target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0)

docs/source/examples/copy_cast/example.py

@@ -0,0 +1,6 @@
+import mymodule
+import numpy as np
+
+c = np.array([[1, 2, 3], [4, 5, 6]])
+assert np.isclose(np.sum(np.sin(c)), mymodule.sum_of_sines(c))
+assert np.isclose(np.sum(np.cos(c)), mymodule.sum_of_cosines(c))

docs/source/examples/copy_cast/main.cpp

@@ -0,0 +1,27 @@
+#include <numeric>
+#include <xtensor.hpp>
+#include <pybind11/pybind11.h>
+#define FORCE_IMPORT_ARRAY
+#include <xtensor-python/pyarray.hpp>
+
+template <class T>
+double sum_of_sines(T& m)
+{
+    auto sines = xt::sin(m);  // sines does not actually hold values.
+    return std::accumulate(sines.begin(), sines.end(), 0.0);
+}
+
+// In the Python API this a reference to a temporary variable
+double sum_of_cosines(const xt::xarray<double>& m)
+{
+    auto cosines = xt::cos(m);  // cosines does not actually hold values.
+    return std::accumulate(cosines.begin(), cosines.end(), 0.0);
+}
+
+PYBIND11_MODULE(mymodule, m)
+{
+    xt::import_numpy();
+    m.doc() = "Test module for xtensor python bindings";
+    m.def("sum_of_sines", sum_of_sines<xt::pyarray<double>>, "Sum the sines of the input values");
+    m.def("sum_of_cosines", sum_of_cosines, "Sum the cosines of the input values");
+}

include/xtensor-python/pynative_casters.hpp

@@ -12,7 +12,6 @@
 
 #include "xtensor_type_caster_base.hpp"
 
-
 namespace pybind11
 {
     namespace detail

include/xtensor-python/xtensor_type_caster_base.hpp

@@ -23,6 +23,97 @@ namespace pybind11
 {
     namespace detail
     {
+        template <typename T, xt::layout_type L>
+        struct pybind_array_getter_impl
+        {
+            static auto run(handle src)
+            {
+                return array_t<T, array::c_style | array::forcecast>::ensure(src);
+            }
+        };
+
+        template <typename T>
+        struct pybind_array_getter_impl<T, xt::layout_type::column_major>
+        {
+            static auto run(handle src)
+            {
+                return array_t<T, array::f_style | array::forcecast>::ensure(src);
+            }
+        };
+
+        template <class T>
+        struct pybind_array_getter
+        {
+        };
+
+        template <class T, xt::layout_type L>
+        struct pybind_array_getter<xt::xarray<T, L>>
+        {
+            static auto run(handle src)
+            {
+                return pybind_array_getter_impl<T, L>::run(src);
+            }
+        };
+
+        template <class T, std::size_t N, xt::layout_type L>
+        struct pybind_array_getter<xt::xtensor<T, N, L>>
+        {
+            static auto run(handle src)
+            {
+                return pybind_array_getter_impl<T, L>::run(src);
+            }
+        };
+
+        template <class CT, class S, xt::layout_type L, class FST>
+        struct pybind_array_getter<xt::xstrided_view<CT, S, L, FST>>
+        {
+            static auto run(handle /*src*/)
+            {
+                return false;
+            }
+        };
+
+        template <class EC, xt::layout_type L, class SC, class Tag>
+        struct pybind_array_getter<xt::xarray_adaptor<EC, L, SC, Tag>>
+        {
+            static auto run(handle src)
+            {
+                auto buf = pybind_array_getter_impl<EC, L>::run(src);
+                return buf;
+            }
+        };
+
+        template <class EC, std::size_t N, xt::layout_type L, class Tag>
+        struct pybind_array_getter<xt::xtensor_adaptor<EC, N, L, Tag>>
+        {
+            static auto run(handle /*src*/)
+            {
+                return false;
+            }
+        };
+
+
+        template <class T>
+        struct pybind_array_dim_checker
+        {
+            template <class B>
+            static bool run(const B& buf)
+            {
+                return true;
+            }
+        };
+
+        template <class T, std::size_t N, xt::layout_type L>
+        struct pybind_array_dim_checker<xt::xtensor<T, N, L>>
+        {
+            template <class B>
+            static bool run(const B& buf)
+            {
+                return buf.ndim() == N;
+            }
+        };
+
+
         // Casts a strided expression type to numpy array.If given a base,
         // the numpy array references the src data, otherwise it'll make a copy.
         // The writeable attributes lets you specify writeable flag for the array.
@@ -74,10 +165,6 @@ namespace pybind11
         template <class Type>
         struct xtensor_type_caster_base
         {
-            bool load(handle /*src*/, bool)
-            {
-                return false;
-            }
 
         private:
 
@@ -106,6 +193,36 @@ namespace pybind11
 
         public:
 
+            PYBIND11_TYPE_CASTER(Type, _("numpy.ndarray[") + npy_format_descriptor<typename Type::value_type>::name + _("]"));
+
+            bool load(handle src, bool convert)
+            {
+                using T = typename Type::value_type;
+
+                if (!convert && !array_t<T>::check_(src))
+                {
+                    return false;
+                }
+
+                auto buf = pybind_array_getter<Type>::run(src);
+
+                if (!buf)
+                {
+                    return false;
+                }
+                if (!pybind_array_dim_checker<Type>::run(buf))
+                {
+                    return false;
+                }
+
+                std::vector<size_t> shape(buf.ndim());
+                std::copy(buf.shape(), buf.shape() + buf.ndim(), shape.begin());
+                value = Type::from_shape(shape);
+                std::copy(buf.data(), buf.data() + buf.size(), value.data());
+
+                return true;
+            }
+
             // Normal returned non-reference, non-const value:
             static handle cast(Type&& src, return_value_policy /* policy */, handle parent)
             {
@@ -151,18 +268,6 @@ namespace pybind11
             {
                 return cast_impl(src, policy, parent);
             }
-
-#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3
-            static PYBIND11_DESCR name()
-            {
-                return _("xt::xtensor");
-            }
-#else
-            static constexpr auto name = _("xt::xtensor");
-#endif
-
-            template <typename T>
-            using cast_op_type = cast_op_type<T>;
         };
     }
 }

test_python/main.cpp

@@ -33,6 +33,33 @@ xt::pyarray<double> example2(xt::pyarray<double>& m)
     return m + 2;
 }
 
+xt::xarray<int> example3_xarray(const xt::xarray<int>& m)
+{
+    return xt::transpose(m) + 2;
+}
+
+xt::xarray<int, xt::layout_type::column_major> example3_xarray_colmajor(
+    const xt::xarray<int, xt::layout_type::column_major>& m)
+{
+    return xt::transpose(m) + 2;
+}
+
+xt::xtensor<int, 3> example3_xtensor3(const xt::xtensor<int, 3>& m)
+{
+    return xt::transpose(m) + 2;
+}
+
+xt::xtensor<int, 2> example3_xtensor2(const xt::xtensor<int, 2>& m)
+{
+    return xt::transpose(m) + 2;
+}
+
+xt::xtensor<int, 2, xt::layout_type::column_major> example3_xtensor2_colmajor(
+    const xt::xtensor<int, 2, xt::layout_type::column_major>& m)
+{
+    return xt::transpose(m) + 2;
+}
+
 // Readme Examples
 
 double readme_example1(xt::pyarray<double>& m)
@@ -249,6 +276,11 @@ PYBIND11_MODULE(xtensor_python_test, m)
 
     m.def("example1", example1);
     m.def("example2", example2);
+    m.def("example3_xarray", example3_xarray);
+    m.def("example3_xarray_colmajor", example3_xarray_colmajor);
+    m.def("example3_xtensor3", example3_xtensor3);
+    m.def("example3_xtensor2", example3_xtensor2);
+    m.def("example3_xtensor2_colmajor", example3_xtensor2_colmajor);
 
     m.def("complex_overload", no_complex_overload);
     m.def("complex_overload", complex_overload);

test_python/test_pyarray.py

@@ -36,6 +36,22 @@ def test_example2(self):
         y = xt.example2(x)
         np.testing.assert_allclose(y, res, 1e-12)
 
+    def test_example3(self):
+        x = np.arange(2 * 3).reshape(2, 3)
+        xc = np.asfortranarray(x)
+        y = np.arange(2 * 3 * 4).reshape(2, 3, 4)
+        v = y[1:, 1:, 0]
+        z = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5)
+        np.testing.assert_array_equal(xt.example3_xarray(x), x.T + 2)
+        np.testing.assert_array_equal(xt.example3_xarray_colmajor(xc), xc.T + 2)
+        np.testing.assert_array_equal(xt.example3_xtensor3(y), y.T + 2)
+        np.testing.assert_array_equal(xt.example3_xtensor2(x), x.T + 2)
+        np.testing.assert_array_equal(xt.example3_xtensor2(y[1:, 1:, 0]), v.T + 2)
+        np.testing.assert_array_equal(xt.example3_xtensor2_colmajor(xc), xc.T + 2)
+
+        with self.assertRaises(TypeError):
+            xt.example3_xtensor3(x)
+
     def test_vectorize(self):
         x1 = np.array([[0, 1], [2, 3]])
         x2 = np.array([0, 1])