From 6772ad8a18714cb695bcf9ded9d4a060348dd0dc Mon Sep 17 00:00:00 2001 From: Konstantinos Papadakis Date: Mon, 23 Sep 2024 11:54:24 +0300 Subject: [PATCH 1/3] Allow for memory reuse by reading into already existing buffers --- include/npy.hpp | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/include/npy.hpp b/include/npy.hpp index 3898e87..bf4b7f2 100644 --- a/include/npy.hpp +++ b/include/npy.hpp @@ -503,6 +503,36 @@ inline npy_data read_npy(std::istream &in) { return data; } +template +inline void read_npy_into(std::istream &in,npy_data& data) { + std::string header_s = read_header(in); + + // parse header + header_t header = parse_header(header_s); + + // check if the typestring matches the given one + const dtype_t dtype = dtype_map.at(std::type_index(typeid(Scalar))); + + if (header.dtype.tie() != dtype.tie()) { + throw std::runtime_error("formatting error: typestrings not matching"); + } + + // compute the data size based on the shape + auto size = static_cast(comp_size(header.shape)); + + data.shape = header.shape; + data.fortran_order = header.fortran_order; + + //This will now do nothing + if (data.data.size()!=size){ + data.data.resize(size); + } + + // read the data + in.read(reinterpret_cast(data.data.data()), sizeof(Scalar) * size); + return; +} + template inline npy_data read_npy(const std::string &filename) { std::ifstream stream(filename, std::ifstream::binary); @@ -513,6 +543,16 @@ inline npy_data read_npy(const std::string &filename) { return read_npy(stream); } +template +inline void read_npy_into(const std::string &filename,npy_data& data) { + std::ifstream stream(filename, std::ifstream::binary); + if (!stream) { + throw std::runtime_error("io error: failed to open a file."); + } + read_npy_into(stream,data); + return; +} + template inline void write_npy(std::ostream &out, const npy_data &data) { // static_assert(has_typestring::value, "scalar type not From 18af465792e96e68d08b2806830b4f179b3a8be7 Mon Sep 17 00:00:00 2001 From: Konstantinos Papadakis Date: Tue, 24 Sep 2024 11:26:20 +0300 Subject: [PATCH 2/3] Clang format --- include/npy.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/npy.hpp b/include/npy.hpp index bf4b7f2..37a032b 100644 --- a/include/npy.hpp +++ b/include/npy.hpp @@ -504,7 +504,7 @@ inline npy_data read_npy(std::istream &in) { } template -inline void read_npy_into(std::istream &in,npy_data& data) { +inline void read_npy_into(std::istream &in, npy_data &data) { std::string header_s = read_header(in); // parse header @@ -523,8 +523,8 @@ inline void read_npy_into(std::istream &in,npy_data& data) { data.shape = header.shape; data.fortran_order = header.fortran_order; - //This will now do nothing - if (data.data.size()!=size){ + // This will now do nothing + if (data.data.size() != size) { data.data.resize(size); } @@ -544,12 +544,12 @@ inline npy_data read_npy(const std::string &filename) { } template -inline void read_npy_into(const std::string &filename,npy_data& data) { +inline void read_npy_into(const std::string &filename, npy_data &data) { std::ifstream stream(filename, std::ifstream::binary); if (!stream) { throw std::runtime_error("io error: failed to open a file."); } - read_npy_into(stream,data); + read_npy_into(stream, data); return; } From cedcb23359fb1f94451963c1c4d94d677a875003 Mon Sep 17 00:00:00 2001 From: Konstantinos Papadakis Date: Mon, 3 Mar 2025 17:41:12 +0200 Subject: [PATCH 3/3] Feature: add partial reads --- include/npy.hpp | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/include/npy.hpp b/include/npy.hpp index 37a032b..2c215fa 100644 --- a/include/npy.hpp +++ b/include/npy.hpp @@ -503,6 +503,41 @@ inline npy_data read_npy(std::istream &in) { return data; } +template +inline npy_data read_npy_partial(std::istream &in,shape_t shape_requested,std::size_t byte_offset) { + std::string header_s = read_header(in); + + // parse header + const header_t header = parse_header(header_s); + + // check if the typestring matches the given one + const dtype_t dtype = dtype_map.at(std::type_index(typeid(Scalar))); + + if (header.dtype.tie() != dtype.tie()) { + throw std::runtime_error("formatting error: typestrings not matching"); + } + + // compute the data size based on the shape the user requested + const auto requested_size=static_cast(comp_size(shape_requested)); + + npy_data data; + + data.shape = shape_requested; + data.fortran_order = header.fortran_order; + + data.data.resize(requested_size); + + // read the data + if (byte_offset>0){ + in.seekg(byte_offset,std::ios::beg); + } + in.read(reinterpret_cast(data.data.data()), sizeof(Scalar) * requested_size); + if (!in){ + std::cerr<<"ERROR: partial read failed!"< inline void read_npy_into(std::istream &in, npy_data &data) { std::string header_s = read_header(in); @@ -533,6 +568,13 @@ inline void read_npy_into(std::istream &in, npy_data &data) { return; } + +inline shape_t read_npy_shape(std::istream& in) { + std::string header_s = read_header(in); + header_t header = parse_header(header_s); + return header.shape; +} + template inline npy_data read_npy(const std::string &filename) { std::ifstream stream(filename, std::ifstream::binary); @@ -543,6 +585,25 @@ inline npy_data read_npy(const std::string &filename) { return read_npy(stream); } +template +inline npy_data read_npy_partial(const std::string &filename,shape_t shape,std::size_t byte_offset) { + std::ifstream stream(filename, std::ifstream::binary); + if (!stream) { + throw std::runtime_error("io error: failed to open a file."); + } + + return read_npy_partial(stream, shape, byte_offset); +} + +inline shape_t read_npy_shape(const std::string &filename) { + std::ifstream stream(filename, std::ifstream::binary); + if (!stream) { + throw std::runtime_error("io error: failed to open a file."); + } + + return read_npy_shape(stream); +} + template inline void read_npy_into(const std::string &filename, npy_data &data) { std::ifstream stream(filename, std::ifstream::binary);