Eigen3 support?

[rancheng]

Eigen3 support?

[rancheng]

never mind, I've wrote a tiny function for this:

void cnpy2eigen(string data_fname, double* data_ptr, Eigen::MatrixXd& mat_out){
    cnpy::NpyArray npy_data = cnpy::npy_load(data_fname);
    // double* ptr = npy_data.data<double>();
    int data_row = npy_data.shape[0];
    int data_col = npy_data.shape[1];
    data_ptr = static_cast<double *>(malloc(data_row * data_col * sizeof(double)));
    memcpy(ptr, npy_data.data<double>(), data_row * data_col * sizeof(double));
    cv::Mat dmat = cv::Mat(cv::Size(data_col, data_row), CV_64F, data_ptr); // CV_64F is equivalent double
    new (&mat_out) Eigen::Map<Eigen::Matrix<double,Eigen::Dynamic,Eigen::Dynamic>>(reinterpret_cast<double *>(dmat.data), data_col, data_row);
}

reference from Pierluigi's answer in StackOverFlow.

Note that you need to manually delete *data_ptr after mat_out.release()

[dariusarnold]

I wrote a function to load N-dimensional numpy arrays using Eigen::Tensor, since the Eigen::Matrix is 2D only. Maybe someone knows a way to use the type and rank specified in the npy file instead of requiring them as template parameters.

namespace impl {
    /**
     * Create Eigen::Tensor with sizes of dimensions specified in a vector.
     * @tparam Scalar Scalar type of tensor elements.
     * @tparam Dimensions Rank of tensor (number of dimensions).
     * @tparam Layout Layout of returned tensor, either column major or row major.
     * @tparam I Index sequence for vector.
     * @param data Pointer to raw array data of tensor.
     * @param shape Array containing shape of tensor (size of dimensions).
     * @return Tensor with data given from pointer in given shape.
     */
    template <typename Scalar, int Dimensions, Eigen::StorageOptions Layout = Eigen::ColMajor,
              size_t... I>
    Eigen::Tensor<Scalar, Dimensions, Layout> make_tensor(Scalar* data, std::vector<size_t> shape,
                                                          std::index_sequence<I...>) {
        return Eigen::TensorMap<Eigen::Tensor<Scalar, Dimensions, Layout>>(data, shape[I]...);
    }
} // namespace impl


/**
 * Load npy file and return it as tensor.
 * While the datatype and the dimensions are specified in the npy file, they are required to declare
 * the return type of this function, so they must be given by the user.
 * @tparam Scalar Scalar element type of npy file.
 * @tparam Dimensions Rank of tensor/number of dimensions of the numpy array in the file.
 * @param path Path to .npy file.
 * @return
 */
template <typename Scalar, int Dimensions>
Eigen::Tensor<Scalar, Dimensions> load_npy(const std::filesystem::path& path) {

    std::ifstream file{path, std::ios::binary};
    if (not file.is_open()) {
        throw std::runtime_error(("Failed to open file " + path.string()));
    }
    const int initial_buffer_size = 250;
    std::vector<unsigned char> buffer(initial_buffer_size);
    // size of file content in bytes
    const int SIZE_MAGIC_STRING = 6;
    const int SIZE_VERSION_NUMBER = 2;
    const int SIZE_HEADER_LEN = 2;
    int offset = 0;
    // ifstream::read expects pointer to signed char, while the file content is unsigned
    auto buffer_as_signed = [&]() { return reinterpret_cast<char*>(buffer.data()); };
    file.read(buffer_as_signed(), SIZE_MAGIC_STRING);
    if (not std::strcmp(buffer_as_signed(), "\x93NUMPY")) {
        throw std::runtime_error(path.string() +
                                 " is not a valid npy file. "
                                 "Read invalid magic string " +
                                 std::string(buffer.begin(), buffer.end() + SIZE_MAGIC_STRING) +
                                 ". Expected \x93NUMPY");
    }
    offset += SIZE_MAGIC_STRING;
    file.get(buffer_as_signed() + offset, SIZE_VERSION_NUMBER);
    if (not std::strcmp(buffer_as_signed() + offset, "\x00\x01")) {
        throw std::runtime_error(
            "Invalid version number " +
            std::string(buffer.begin() + offset, buffer.end() + offset + SIZE_VERSION_NUMBER));
    }
    offset += SIZE_VERSION_NUMBER;
    file.read(buffer_as_signed() + offset, SIZE_HEADER_LEN);
    // extract header length as little endian 2 byte unsigned int
    std::uint16_t header_length = *reinterpret_cast<uint16_t*>(buffer.data());
    offset += SIZE_HEADER_LEN;
    if (header_length > initial_buffer_size - offset) {
        buffer.resize(offset + header_length);
    }
    file.read(buffer_as_signed() + offset, header_length);

    size_t word_size;
    std::vector<size_t> shape;
    bool col_wise_order;
    // delegate header parsing
    cnpy::parse_npy_header(reinterpret_cast<unsigned char*>(buffer.data()), word_size, shape,
                           col_wise_order);
    cnpy::NpyArray P = cnpy::npy_load(path.string());
    if (Dimensions != P.shape.size()) {
        throw std::runtime_error(impl::Formatter()
                                 << "Got " << Dimensions
                                 << " dimensions as template argument, but file contains "
                                 << P.shape.size() << ".");
    }
    auto is = std::make_index_sequence<Dimensions>();
    if (not col_wise_order) {
        // load the tensor in row wise order and shuffle it
        auto tensor =
            impl::make_tensor<Scalar, Dimensions, Eigen::RowMajor>(P.data<Scalar>(), P.shape, is);
        auto shuffle = std::vector<int>(Dimensions);
        std::iota(shuffle.begin(), shuffle.end(), 0);
        std::reverse(shuffle.begin(), shuffle.end());
        return tensor.swap_layout().shuffle(shuffle).eval();
    }
    return impl::make_tensor<Scalar, Dimensions>(P.data<Scalar>(), P.shape, is);
}

impl::Formatter is defined elsewhere but it just returns everything streamed into it as a std::string.

EDIT: New code version should work with column major and row major npy files.

[rin-23]

Just FYI in the post above, you need to be careful about the storage order.
you should be converting any col major matrices to row major order before saving them to disk to be read in python with numpy.load, otherwise your data will be out of order( transpose wont help either)

smth like

using Eigen;
using RowMajorMat = Matrix<double, Dynamic, Dynamic, RowMajor>;
using ColMajorMat = Matrix<double, Dynamic, Dynamic, ColMajor>;

ColMajorMat Matrix;
RowMajorMat Vout = Map(Matrix.data(), Matrix.rows(), Matrix.cols());

see issue #64

[kabukunz]

Faster, Fitter, Happier

    // create mats
    MatrixXd V(32768, 3);
    MatrixXi F(32768, 3);
    
    V.setRandom();
    F.setRandom();

    // SAVE

    // map to const mats in memory
    Map<const MatrixXd> VOut(&V(0), V.rows(), V.cols());
    Map<const MatrixXi> FOut(&F(0), F.rows(), F.cols());

    // save to np-arrays files
    cnpy::npy_save("vertices.npy", VOut.data(), {(size_t)V.rows(), (size_t)V.cols()}, "w");
    cnpy::npy_save("faces.npy", FOut.data(), {(size_t)F.rows(), (size_t)F.cols()}, "w");

    // LOAD

    // reset mats
    V = {};
    F = {};

    // load from np-arrays files
    cnpy::NpyArray vertices = cnpy::npy_load("vertices.npy");
    cnpy::NpyArray faces = cnpy::npy_load("faces.npy");

    // map to const mats in memory
    Map<const MatrixXd> Vin(vertices.data<double>(), vertices.shape[0], vertices.shape[1]);
    Map<const MatrixXi> Fin(faces.data<int>(), faces.shape[0], faces.shape[1]);

    // assign
    V = Vin;
    F = Fin;

Note the Map that spares recreating matrices
Not sure while all those const in the sdk are for. My use of Eigen mats is dynamic

[hibiki-kato]

I use this header(not smart, but simple.)
You can use this like below:

#include "Eigen_npy_converter.hpp"

// load
Eigen::VectorXd Vec = np::load<double>("path/to/the/.npy");
Eigen::MatrixXcd Mat = np::load<std::complex<double>>("path/to/the/.npy")

// save
np::save("fileName.npy", Vec)
np::save("fileName.npy", Mat)

Eigen_npy_converter.hpp :

#pragma once
#include <complex>
#include <eigen3/Eigen/Dense>
#include <string>

#include "cnpy/cnpy.h"

namespace np {
    template <typename Num>
    Eigen::Matrix<Num, Eigen::Dynamic, Eigen::Dynamic> load(const char *fname) {
        cnpy::NpyArray arr = cnpy::npy_load(fname);
        // shape
        std::vector<size_t> shape = arr.shape;
        // vector case
        if (shape.size() == 1) {
            Eigen::Matrix<Num, Eigen::Dynamic, 1> vec(shape[0]);
            std::copy(arr.data<Num>(), arr.data<Num>() + shape[0], vec.data());
            return vec;
        } else {
            Num *loaded_data = arr.data<Num>();
            Eigen::Matrix<Num, Eigen::Dynamic, Eigen::Dynamic> mat(shape[0], shape[1]);
            for (int i = 0; i < shape[0]; i++) {
                for (int j = 0; j < shape[1]; j++) {
                    mat(i, j) = loaded_data[i * shape[1] + j];
                }
            }
            return mat;
        }
    }

    template <typename Derived>
    void save(std::string fname, const Eigen::MatrixBase<Derived> &Mat) {
        int rows = Mat.rows();
        int cols = Mat.cols();
        // Vector case
        if (cols == 1) {
            // copy to std::vector
            std::vector<typename Derived::Scalar> data(rows);
            for (int i = 0; i < rows; i++) {
                data[i] = Mat(i);
            }
            // save
            cnpy::npy_save(fname, &data[0], {(size_t)rows}, "w");
            return;
        } else {
            // Matrix case
            // transpose
            Eigen::Matrix<typename Derived::Scalar, Eigen::Dynamic, Eigen::Dynamic> transposed = Mat.transpose();
            // Eigen::Map
            Eigen::Map<const Eigen::Matrix<typename Derived::Scalar, Eigen::Dynamic, Eigen::Dynamic>> MOut(transposed.data(), transposed.rows(), transposed.cols());
            // save to file
            cnpy::npy_save(fname, MOut.data(), {(size_t)transposed.cols(), (size_t)transposed.rows()}, "w");
        }
    }
} // namespace np

This didn't support npz, tensor.
Good Luck.