34 #include <visp3/core/vpConfig.h>
35 #include <visp3/core/vpIoTools.h>
37 #if defined(VISP_HAVE_MINIZ) && defined(VISP_HAVE_WORKING_REGEX)
38 #define USE_ZLIB_API 0
43 #include "basisu_miniz.h"
45 using namespace buminiz;
51 #define UNUSED(x) ((void)(x))
60 return (((
reinterpret_cast<char *
>(&x))[0]) ?
'<' :
'>');
65 if (t ==
typeid(
float)) {
return 'f'; }
66 if (t ==
typeid(
double)) {
return 'f'; }
67 if (t ==
typeid(
long double)) {
return 'f'; }
69 if (t ==
typeid(
int)) {
return 'i'; }
70 if (t ==
typeid(
char)) {
return 'i'; }
71 if (t ==
typeid(
short)) {
return 'i'; }
72 if (t ==
typeid(
long)) {
return 'i'; }
73 if (t ==
typeid(
long long)) {
return 'i'; }
75 if (t ==
typeid(
unsigned char)) {
return 'u'; }
76 if (t ==
typeid(
unsigned short)) {
return 'u'; }
77 if (t ==
typeid(
unsigned long)) {
return 'u'; }
78 if (t ==
typeid(
unsigned long long)) {
return 'u'; }
79 if (t ==
typeid(
unsigned int)) {
return 'u'; }
81 if (t ==
typeid(
bool)) {
return 'b'; }
83 if (t ==
typeid(std::complex<float>)) {
return 'c'; }
84 if (t ==
typeid(std::complex<double>)) {
return 'c'; }
85 if (t ==
typeid(std::complex<long double>)) {
return 'c'; }
92 uint16_t header_len = *
reinterpret_cast<uint16_t *
>(buffer+8);
93 std::string header(
reinterpret_cast<char *
>(buffer+9), header_len);
96 size_t loc1 = header.find(
"fortran_order")+16;
97 fortran_order = (header.substr(loc1, 4) ==
"True" ? true :
false);
100 loc1 = header.find(
"(");
101 size_t loc2 = header.find(
")");
103 std::regex num_regex(
"[0-9][0-9]*");
107 std::string str_shape = header.substr(loc1+1, loc2-loc1-1);
108 while (std::regex_search(str_shape, sm, num_regex)) {
109 shape.push_back(std::stoi(sm[0].str()));
110 str_shape = sm.suffix().str();
116 loc1 = header.find(
"descr")+9;
117 bool littleEndian = (((header[loc1] ==
'<') || (header[loc1] ==
'|')) ?
true :
false);
118 UNUSED(littleEndian); assert(littleEndian);
120 std::string str_ws = header.substr(loc1+2);
121 loc2 = str_ws.find(
"'");
122 word_size = atoi(str_ws.substr(0, loc2).c_str());
128 size_t res = fread(buffer,
sizeof(
char), 11, fp);
130 throw std::runtime_error(
"parse_npy_header: failed fread");
132 std::string header = fgets(buffer, 256, fp);
133 assert(header[header.size()-1] ==
'\n');
138 loc1 = header.find(
"fortran_order");
139 if (loc1 == std::string::npos) {
140 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: 'fortran_order'");
143 fortran_order = (header.substr(loc1, 4) ==
"True" ? true :
false);
146 loc1 = header.find(
"(");
147 loc2 = header.find(
")");
148 if ((loc1 == std::string::npos) || (loc2 == std::string::npos)) {
149 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: '(' or ')'");
152 std::regex num_regex(
"[0-9][0-9]*");
156 std::string str_shape = header.substr(loc1+1, loc2-loc1-1);
157 while (std::regex_search(str_shape, sm, num_regex)) {
158 shape.push_back(std::stoi(sm[0].str()));
159 str_shape = sm.suffix().str();
165 loc1 = header.find(
"descr");
166 if (loc1 == std::string::npos) {
167 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: 'descr'");
170 bool littleEndian = ((header[loc1] ==
'<') || (header[loc1] ==
'|') ? true :
false);
171 UNUSED(littleEndian); assert(littleEndian);
176 std::string str_ws = header.substr(loc1+2);
177 loc2 = str_ws.find(
"'");
178 word_size = atoi(str_ws.substr(0, loc2).c_str());
183 std::vector<char> footer(22);
184 fseek(fp, -22, SEEK_END);
185 size_t res = fread(&footer[0],
sizeof(
char), 22, fp);
187 throw std::runtime_error(
"parse_zip_footer: failed fread");
190 uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
191 disk_no = *(uint16_t *)&footer[4];
192 disk_start = *(uint16_t *)&footer[6];
193 nrecs_on_disk = *(uint16_t *)&footer[8];
194 nrecs = *(uint16_t *)&footer[10];
195 global_header_size = *(uint32_t *)&footer[12];
196 global_header_offset = *(uint32_t *)&footer[16];
197 comment_len = *(uint16_t *)&footer[20];
199 UNUSED(disk_no); assert(disk_no == 0);
200 UNUSED(disk_start); assert(disk_start == 0);
201 UNUSED(nrecs_on_disk); assert(nrecs_on_disk == nrecs);
202 UNUSED(comment_len); assert(comment_len == 0);
207 std::vector<size_t> shape;
213 size_t nread = fread(arr.data<
char>(), 1, arr.num_bytes(), fp);
214 if (nread != arr.num_bytes()) {
215 throw std::runtime_error(
"load_the_npy_file: failed fread");
222 std::vector<unsigned char> buffer_compr(compr_bytes);
223 std::vector<unsigned char> buffer_uncompr(uncompr_bytes);
224 size_t nread = fread(&buffer_compr[0], 1, compr_bytes, fp);
225 if (nread != compr_bytes) {
226 throw std::runtime_error(
"load_the_npy_file: failed fread");
231 d_stream.zalloc = Z_NULL;
232 d_stream.zfree = Z_NULL;
233 d_stream.opaque = Z_NULL;
234 d_stream.avail_in = 0;
235 d_stream.next_in = Z_NULL;
236 int err = inflateInit2(&d_stream, -MAX_WBITS);
237 UNUSED(err); assert(err == 0);
239 d_stream.avail_in = compr_bytes;
240 d_stream.next_in = &buffer_compr[0];
241 d_stream.avail_out = uncompr_bytes;
242 d_stream.next_out = &buffer_uncompr[0];
244 err = inflate(&d_stream, Z_FINISH);
245 UNUSED(err); assert(err == 0);
246 err = inflateEnd(&d_stream);
247 UNUSED(err); assert(err == 0);
249 std::vector<size_t> shape;
256 size_t offset = uncompr_bytes - array.num_bytes();
257 memcpy(array.data<
unsigned char>(), &buffer_uncompr[0]+offset, array.num_bytes());
272 FILE *fp = fopen(fname.c_str(),
"rb");
275 throw std::runtime_error(
"npz_load: Error! Unable to open file "+fname+
"!");
280 const unsigned int index_2 = 2;
281 const unsigned int index_3 = 3;
282 const unsigned int index_26 = 26;
283 const unsigned int index_28 = 28;
284 const unsigned int val_8 = 8;
285 const unsigned int val_18 = 18;
286 const unsigned int val_22 = 22;
287 const unsigned int val_30 = 30;
289 std::vector<char> local_header(val_30);
290 size_t headerres = fread(&local_header[0],
sizeof(
char), val_30, fp);
291 if (headerres != 30) {
292 throw std::runtime_error(
"npz_load: failed fread");
296 if ((local_header[index_2] != 0x03) || (local_header[index_3] != 0x04)) {
301 uint16_t name_len = *(uint16_t *)&local_header[index_26];
302 std::string varname(name_len,
' ');
303 size_t vname_res = fread(&varname[0],
sizeof(
char), name_len, fp);
304 if (vname_res != name_len) {
305 throw std::runtime_error(
"npz_load: failed fread");
309 varname.erase(varname.end()-4, varname.end());
312 uint16_t extra_field_len = *(uint16_t *)&local_header[index_28];
313 if (extra_field_len > 0) {
314 std::vector<char> buff(extra_field_len);
315 size_t efield_res = fread(&buff[0],
sizeof(
char), extra_field_len, fp);
316 if (efield_res != extra_field_len) {
317 throw std::runtime_error(
"npz_load: failed fread");
321 uint16_t compr_method = *
reinterpret_cast<uint16_t *
>(&local_header[0] + val_8);
322 uint32_t compr_bytes = *
reinterpret_cast<uint32_t *
>(&local_header[0] + val_18);
323 uint32_t uncompr_bytes = *
reinterpret_cast<uint32_t *
>(&local_header[0] + val_22);
325 if (compr_method == 0) {
326 arrays[varname] = load_the_npy_file(fp);
329 arrays[varname] = load_the_npz_array(fp, compr_bytes, uncompr_bytes);
349 FILE *fp = fopen(fname.c_str(),
"rb");
352 throw std::runtime_error(
"npz_load: Unable to open file "+fname);
356 const unsigned int index_2 = 2;
357 const unsigned int index_3 = 3;
358 const unsigned int index_26 = 26;
359 const unsigned int index_28 = 28;
360 const unsigned int val_8 = 8;
361 const unsigned int val_18 = 18;
362 const unsigned int val_22 = 22;
363 const unsigned int val_30 = 30;
365 std::vector<char> local_header(val_30);
366 size_t header_res = fread(&local_header[0],
sizeof(
char), val_30, fp);
367 if (header_res != 30) {
368 throw std::runtime_error(
"npz_load: failed fread");
372 if ((local_header[index_2] != 0x03) || (local_header[index_3] != 0x04)) {
377 uint16_t name_len = *(uint16_t *)&local_header[index_26];
378 std::string vname(name_len,
' ');
379 size_t vname_res = fread(&vname[0],
sizeof(
char), name_len, fp);
380 if (vname_res != name_len) {
381 throw std::runtime_error(
"npz_load: failed fread");
383 vname.erase(vname.end()-4, vname.end());
386 uint16_t extra_field_len = *(uint16_t *)&local_header[index_28];
387 fseek(fp, extra_field_len, SEEK_CUR);
389 uint16_t compr_method = *
reinterpret_cast<uint16_t *
>(&local_header[0] + val_8);
390 uint32_t compr_bytes = *
reinterpret_cast<uint32_t *
>(&local_header[0] + val_18);
391 uint32_t uncompr_bytes = *
reinterpret_cast<uint32_t *
>(&local_header[0] + val_22);
393 if (vname == varname) {
394 NpyArray array = (compr_method == 0) ? load_the_npy_file(fp) : load_the_npz_array(fp, compr_bytes, uncompr_bytes);
400 uint32_t size = *(uint32_t *)&local_header[22];
401 fseek(fp, size, SEEK_CUR);
409 throw std::runtime_error(
"npz_load: Variable name "+varname+
" not found in "+fname);
423 FILE *fp = fopen(fname.c_str(),
"rb");
426 throw std::runtime_error(
"npy_load: Unable to open file "+fname);
429 NpyArray arr = load_the_npy_file(fp);
VISP_EXPORT char map_type(const std::type_info &t)
VISP_EXPORT npz_t npz_load(std::string fname)
VISP_EXPORT void parse_zip_footer(FILE *fp, uint16_t &nrecs, size_t &global_header_size, size_t &global_header_offset)
VISP_EXPORT void parse_npy_header(FILE *fp, size_t &word_size, std::vector< size_t > &shape, bool &fortran_order)
std::map< std::string, NpyArray > npz_t
VISP_EXPORT NpyArray npy_load(std::string fname)
VISP_EXPORT char BigEndianTest()