19#ifndef ALEXANDRIA_NDARRAY_IMPL_NPYCOMMON_H
20#define ALEXANDRIA_NDARRAY_IMPL_NPYCOMMON_H
23#include <boost/endian/arithmetic.hpp>
24#include <boost/filesystem/operations.hpp>
25#include <boost/iostreams/device/mapped_file.hpp>
31using boost::endian::little_uint16_t;
32using boost::endian::little_uint32_t;
37constexpr const char NPY_MAGIC[] = {
'\x93',
'N',
'U',
'M',
'P',
'Y'};
42#if BYTE_ORDER == LITTLE_ENDIAN
44#elif BYTE_ORDER == BIG_ENDIAN
47#error "PDP_ENDIAN not supported"
58 static constexpr const char* str =
"b";
63 static constexpr const char* str =
"i2";
68 static constexpr const char* str =
"i4";
73 static constexpr const char* str =
"i8";
78 static constexpr const char* str =
"B";
83 static constexpr const char* str =
"u2";
88 static constexpr const char* str =
"u4";
93 static constexpr const char* str =
"u8";
98 static constexpr const char* str =
"f4";
103 static constexpr const char* str =
"f8";
171 for (
auto s : shape) {
172 shape_stream << s <<
',';
175 return shape_stream.
str();
184 for (
auto& attr : attrs) {
185 dtype <<
"('" << attr <<
"', '" <<
ENDIAN_MARKER << type <<
"'), ";
197 if (!attrs.
empty()) {
207 <<
", 'fortran_order': False, 'shape': " <<
npyShape(shape) <<
"}";
208 auto header_str = header.
str();
209 little_uint32_t header_len = header_str.size();
212 size_t total_length =
sizeof(
NPY_MAGIC) +
sizeof(
NPY_VERSION) +
sizeof(header_len) + header_len + 1;
213 if (total_length % 64 > 0) {
214 size_t padding = 64 - total_length % 64;
218 header_str = header.
str();
219 header_len = header_str.size();
226 out.
write(
reinterpret_cast<char*
>(&header_len),
sizeof(header_len));
229 out.
write(header_str.data(), header_str.size());
241 MappedContainer(
const boost::filesystem::path& path,
size_t data_offset,
size_t n_elements,
249 ,
m_data(reinterpret_cast<T*>(const_cast<char*>(
m_mapped.const_data()) + data_offset)) {}
267 auto header_str = header.
str();
268 auto header_size = header_str.size();
272 "The new header length must match the allocated space.";
276 size_t new_size = header_size +
sizeof(T) *
m_n_elements;
278 throw Elements::Exception() <<
"resize request bigger than maximum allocated size: " << new_size <<
" > "
281 boost::filesystem::resize_file(
m_path, new_size);
MappedContainer(const boost::filesystem::path &path, size_t data_offset, size_t n_elements, const std::vector< std::string > &attr_names, boost::iostreams::mapped_file &&input, size_t max_size)
boost::filesystem::path m_path
std::vector< std::string > m_attr_names
void resize(const std::vector< size_t > &shape)
boost::iostreams::mapped_file m_mapped
std::string typeDescription(const std::string &type, const std::vector< std::string > &attrs)
void writeNpyHeader(std::ostream &out, std::vector< size_t > shape, const std::vector< std::string > &attrs)
void parseSingleValue(const std::string &descr, bool &big_endian, std::string &dtype)
std::string npyShape(std::vector< size_t > shape)
void readNpyHeader(std::istream &input, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
void parseFieldValues(const std::string &descr, bool &big_endian, std::vector< std::string > &attrs, std::string &dtype)
constexpr const uint8_t NPY_VERSION[]
constexpr const char * ENDIAN_MARKER
void parseNpyDict(const std::string &header, bool &fortran_order, bool &big_endian, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
constexpr const char NPY_MAGIC[]