Visual Servoing Platform  version 3.6.1 under development (2024-06-20)
vpIoTools.h
1 /*
2  * ViSP, open source Visual Servoing Platform software.
3  * Copyright (C) 2005 - 2023 by Inria. All rights reserved.
4  *
5  * This software is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * See the file LICENSE.txt at the root directory of this source
10  * distribution for additional information about the GNU GPL.
11  *
12  * For using ViSP with software that can not be combined with the GNU
13  * GPL, please contact Inria about acquiring a ViSP Professional
14  * Edition License.
15  *
16  * See https://visp.inria.fr for more information.
17  *
18  * This software was developed at:
19  * Inria Rennes - Bretagne Atlantique
20  * Campus Universitaire de Beaulieu
21  * 35042 Rennes Cedex
22  * France
23  *
24  * If you have questions regarding the use of this file, please contact
25  * Inria at visp@inria.fr
26  *
27  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
28  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
29  *
30  * Description:
31  * Directory management.
32  */
33 
39 #ifndef _vpIoTools_h_
40 #define _vpIoTools_h_
41 
42 #include <visp3/core/vpConfig.h>
43 
44 #include <iostream>
45 #include <sstream>
46 #include <stdint.h> //for uint32_t related types ; works also with >= VS2010 / _MSC_VER >= 1600
47 #include <stdlib.h>
48 #include <string>
49 #include <vector>
50 #include <numeric>
51 #include <visp3/core/vpColor.h>
52 
53 #include <memory>
54 #include <map>
55 #include <cassert>
56 #include <complex>
57 
58 #if VISP_CXX_STANDARD > VISP_CXX_STANDARD_98
59 namespace visp
60 {
61 // https://github.com/BinomialLLC/basis_universal/blob/ad9386a4a1cf2a248f7bbd45f543a7448db15267/encoder/basisu_miniz.h#L665
62 static inline unsigned long vp_mz_crc32(unsigned long crc, const unsigned char *ptr, size_t buf_len)
63 {
64  static const unsigned int s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
65  0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c };
66  unsigned int crcu32 = static_cast<unsigned int>(crc);
67  if (!ptr) return 0;
68  crcu32 = ~crcu32;
69  while (buf_len--) {
70  unsigned char b = *ptr++;
71  crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b & 0xF)];
72  crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b >> 4)];
73  }
74  return ~crcu32;
75 }
76 
77 #ifdef VISP_HAVE_MINIZ
78 namespace cnpy
79 {
80 // Copyright (C) 2011 Carl Rogers
81 // Released under MIT License
82 // license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php
83 struct NpyArray
84 {
85  NpyArray(const std::vector<size_t> &_shape, size_t _word_size, bool _fortran_order) :
86  shape(_shape), word_size(_word_size), fortran_order(_fortran_order)
87  {
88  num_vals = 1;
89  for (size_t i = 0; i < shape.size(); ++i) num_vals *= shape[i];
90  data_holder = std::shared_ptr<std::vector<char> >(
91  new std::vector<char>(num_vals * word_size));
92  }
93 
95 
96  template<typename T>
97  T *data()
98  {
99  return reinterpret_cast<T *>(&(*data_holder)[0]);
100  }
101 
102  template<typename T>
103  const T *data() const
104  {
105  return reinterpret_cast<T *>(&(*data_holder)[0]);
106  }
107 
108  template<typename T>
109  std::vector<T> as_vec() const
110  {
111  const T *p = data<T>();
112  return std::vector<T>(p, p+num_vals);
113  }
114 
115  size_t num_bytes() const
116  {
117  return data_holder->size();
118  }
119 
120  std::shared_ptr<std::vector<char> > data_holder;
121  std::vector<size_t> shape;
122  size_t word_size;
124  size_t num_vals;
125 };
126 
127 using npz_t = std::map<std::string, NpyArray>;
128 VISP_EXPORT npz_t npz_load(std::string fname);
129 
130 
131 VISP_EXPORT char BigEndianTest();
132 VISP_EXPORT char map_type(const std::type_info &t);
133 template<typename T> std::vector<char> create_npy_header(const std::vector<size_t> &shape);
134 VISP_EXPORT void parse_npy_header(FILE *fp, size_t &word_size, std::vector<size_t> &shape, bool &fortran_order);
135 VISP_EXPORT void parse_npy_header(unsigned char *buffer, size_t &word_size, std::vector<size_t> &shape, bool &fortran_order);
136 VISP_EXPORT void parse_zip_footer(FILE *fp, uint16_t &nrecs, size_t &global_header_size, size_t &global_header_offset);
137 VISP_EXPORT NpyArray npz_load(std::string fname, std::string varname);
138 VISP_EXPORT NpyArray npy_load(std::string fname);
139 
140 template<typename T> std::vector<char> &operator+=(std::vector<char> &lhs, const T rhs)
141 {
142  //write in little endian
143  for (size_t byte = 0; byte < sizeof(T); ++byte) {
144  char val = *((char *)&rhs+byte);
145  lhs.push_back(val);
146  }
147  return lhs;
148 }
149 
150 template<> inline std::vector<char> &operator+=(std::vector<char> &lhs, const std::string rhs)
151 {
152  lhs.insert(lhs.end(), rhs.begin(), rhs.end());
153  return lhs;
154 }
155 
156 template<> inline std::vector<char> &operator+=(std::vector<char> &lhs, const char *rhs)
157 {
158 //write in little endian
159  size_t len = strlen(rhs);
160  lhs.reserve(len);
161  for (size_t byte = 0; byte < len; ++byte) {
162  lhs.push_back(rhs[byte]);
163  }
164  return lhs;
165 }
166 
177 template<typename T> void npy_save(std::string fname, const T *data, const std::vector<size_t> shape, std::string mode = "w")
178 {
179  FILE *fp = NULL;
180  std::vector<size_t> true_data_shape; //if appending, the shape of existing + new data
181 
182  if (mode == "a") fp = fopen(fname.c_str(), "r+b");
183 
184  if (fp) {
185  //file exists. we need to append to it. read the header, modify the array size
186  size_t word_size;
187  bool fortran_order;
188  parse_npy_header(fp, word_size, true_data_shape, fortran_order);
189  assert(!fortran_order);
190 
191  if (word_size != sizeof(T)) {
192  std::cout<<"libnpy error: "<<fname<<" has word size "<<word_size<<" but npy_save appending data sized "<<sizeof(T)<<"\n";
193  assert(word_size == sizeof(T));
194  }
195  if (true_data_shape.size() != shape.size()) {
196  std::cout<<"libnpy error: npy_save attempting to append misdimensioned data to "<<fname<<"\n";
197  assert(true_data_shape.size() != shape.size());
198  }
199 
200  for (size_t i = 1; i < shape.size(); ++i) {
201  if (shape[i] != true_data_shape[i]) {
202  std::cout<<"libnpy error: npy_save attempting to append misshaped data to "<<fname<<"\n";
203  assert(shape[i] == true_data_shape[i]);
204  }
205  }
206  true_data_shape[0] += shape[0];
207  }
208  else {
209  fp = fopen(fname.c_str(), "wb");
210  true_data_shape = shape;
211  }
212 
213  std::vector<char> header = create_npy_header<T>(true_data_shape);
214  size_t nels = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<size_t>());
215 
216  fseek(fp, 0, SEEK_SET);
217  fwrite(&header[0], sizeof(char), header.size(), fp);
218  fseek(fp, 0, SEEK_END);
219  fwrite(data, sizeof(T), nels, fp);
220  fclose(fp);
221 }
222 
234 template<typename T> void npz_save(std::string zipname, std::string fname, const T *data, const std::vector<size_t> &shape, std::string mode = "w")
235 {
236  //first, append a .npy to the fname
237  fname += ".npy";
238 
239  //now, on with the show
240  FILE *fp = NULL;
241  uint16_t nrecs = 0;
242  size_t global_header_offset = 0;
243  std::vector<char> global_header;
244 
245  if (mode == "a") fp = fopen(zipname.c_str(), "r+b");
246 
247  if (fp) {
248  //zip file exists. we need to add a new npy file to it.
249  //first read the footer. this gives us the offset and size of the global header
250  //then read and store the global header.
251  //below, we will write the the new data at the start of the global header then append the global header and footer below it
252  size_t global_header_size;
253  parse_zip_footer(fp, nrecs, global_header_size, global_header_offset);
254  fseek(fp, static_cast<long>(global_header_offset), SEEK_SET);
255  global_header.resize(global_header_size);
256  size_t res = fread(&global_header[0], sizeof(char), global_header_size, fp);
257  if (res != global_header_size) {
258  throw std::runtime_error("npz_save: header read error while adding to existing zip");
259  }
260  fseek(fp, static_cast<long>(global_header_offset), SEEK_SET);
261  }
262  else {
263  fp = fopen(zipname.c_str(), "wb");
264  }
265 
266  std::vector<char> npy_header = create_npy_header<T>(shape);
267 
268  size_t nels = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<size_t>());
269  size_t nbytes = nels*sizeof(T) + npy_header.size();
270 
271  //get the CRC of the data to be added
272  uint32_t crc = vp_mz_crc32(0L, (uint8_t *)&npy_header[0], npy_header.size());
273  crc = vp_mz_crc32(crc, (uint8_t *)data, nels*sizeof(T));
274 
275  //build the local header
276  std::vector<char> local_header;
277  local_header += "PK"; //first part of sig
278  local_header += static_cast<uint16_t>(0x0403); //second part of sig
279  local_header += static_cast<uint16_t>(20); //min version to extract
280  local_header += static_cast<uint16_t>(0); //general purpose bit flag
281  local_header += static_cast<uint16_t>(0); //compression method
282  local_header += static_cast<uint16_t>(0); //file last mod time
283  local_header += static_cast<uint16_t>(0); //file last mod date
284  local_header += static_cast<uint32_t>(crc); //crc
285  local_header += static_cast<uint32_t>(nbytes); //compressed size
286  local_header += static_cast<uint32_t>(nbytes); //uncompressed size
287  local_header += static_cast<uint16_t>(fname.size()); //fname length
288  local_header += static_cast<uint16_t>(0); //extra field length
289  local_header += fname;
290 
291  //build global header
292  global_header += "PK"; //first part of sig
293  global_header += static_cast<uint16_t>(0x0201); //second part of sig
294  global_header += static_cast<uint16_t>(20); //version made by
295  global_header.insert(global_header.end(), local_header.begin()+4, local_header.begin()+30);
296  global_header += static_cast<uint16_t>(0); //file comment length
297  global_header += static_cast<uint16_t>(0); //disk number where file starts
298  global_header += static_cast<uint16_t>(0); //internal file attributes
299  global_header += static_cast<uint32_t>(0); //external file attributes
300  global_header += static_cast<uint32_t>(global_header_offset); //relative offset of local file header, since it begins where the global header used to begin
301  global_header += fname;
302 
303  //build footer
304  std::vector<char> footer;
305  footer += "PK"; //first part of sig
306  footer += static_cast<uint16_t>(0x0605); //second part of sig
307  footer += static_cast<uint16_t>(0); //number of this disk
308  footer += static_cast<uint16_t>(0); //disk where footer starts
309  footer += static_cast<uint16_t>(nrecs+1); //number of records on this disk
310  footer += static_cast<uint16_t>(nrecs+1); //total number of records
311  footer += static_cast<uint32_t>(global_header.size()); //nbytes of global headers
312  footer += static_cast<uint32_t>(global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array
313  footer += static_cast<uint16_t>(0); //zip file comment length
314 
315  //write everything
316  fwrite(&local_header[0], sizeof(char), local_header.size(), fp);
317  fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp);
318  fwrite(data, sizeof(T), nels, fp);
319  fwrite(&global_header[0], sizeof(char), global_header.size(), fp);
320  fwrite(&footer[0], sizeof(char), footer.size(), fp);
321  fclose(fp);
322 }
323 
333 template<typename T> void npy_save(std::string fname, const std::vector<T> data, std::string mode = "w")
334 {
335  std::vector<size_t> shape;
336  shape.push_back(data.size());
337  npy_save(fname, &data[0], shape, mode);
338 }
339 
350 template<typename T> void npz_save(std::string zipname, std::string fname, const std::vector<T> data, std::string mode = "w")
351 {
352  std::vector<size_t> shape;
353  shape.push_back(data.size());
354  npz_save(zipname, fname, &data[0], shape, mode);
355 }
356 
357 template<typename T> std::vector<char> create_npy_header(const std::vector<size_t> &shape)
358 {
359  std::vector<char> dict;
360  dict += "{'descr': '";
361  dict += BigEndianTest();
362  dict += map_type(typeid(T));
363  dict += std::to_string(sizeof(T));
364  dict += "', 'fortran_order': False, 'shape': (";
365  dict += std::to_string(shape[0]);
366  for (size_t i = 1; i < shape.size(); ++i) {
367  dict += ", ";
368  dict += std::to_string(shape[i]);
369  }
370  if (shape.size() == 1) dict += ",";
371  dict += "), }";
372  //pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 bytes. dict needs to end with \n
373  int remainder = 16 - (10 + dict.size()) % 16;
374  dict.insert(dict.end(), remainder, ' ');
375  dict.back() = '\n';
376 
377  std::vector<char> header;
378  header += static_cast<char>(0x93);
379  header += "NUMPY";
380  header += static_cast<char>(0x01); //major version of numpy format
381  header += static_cast<char>(0x00); //minor version of numpy format
382  header += static_cast<uint16_t>(dict.size());
383  header.insert(header.end(), dict.begin(), dict.end());
384 
385  return header;
386 }
387 
388 } // namespace cnpy
389 #endif
390 } // namespace VISP_NAMESPACE_NAME
391 #endif
392 
493 class VISP_EXPORT vpIoTools
494 {
495 
496 public:
497  static const std::string &getBuildInformation();
498  static std::string getTempPath();
499  static void getUserName(std::string &username);
500  static std::string getUserName();
501  static std::string getenv(const std::string &env);
502  static std::string getViSPImagesDataPath();
503  static void getVersion(const std::string &version, unsigned int &major, unsigned int &minor, unsigned int &patch);
504  static bool checkDirectory(const std::string &dirname);
505  static bool checkFifo(const std::string &filename);
506  static bool checkFilename(const std::string &filename);
507  static bool copy(const std::string &src, const std::string &dst);
508 
509  static void makeDirectory(const std::string &dirname);
510  static void makeFifo(const std::string &dirname);
511  static std::string makeTempDirectory(const std::string &dirname);
512  static std::string path(const std::string &pathname);
513 
514  static bool remove(const std::string &filename);
515  static bool rename(const std::string &oldfilename, const std::string &newfilename);
516 
521  static const char separator;
522 
523  static std::string toUpperCase(const std::string &input);
524  static std::string toLowerCase(const std::string &input);
525  static std::string getAbsolutePathname(const std::string &pathname);
526  static std::string getFileExtension(const std::string &pathname, bool checkFile = false);
527  static long getIndex(const std::string &filename, const std::string &format);
528  static std::string getName(const std::string &pathname);
529  static std::string getNameWE(const std::string &pathname);
530  static std::string getParent(const std::string &pathname);
531  static std::string createFilePath(const std::string &parent, const std::string &child);
532  static bool isAbsolutePathname(const std::string &pathname);
533  static bool isSamePathname(const std::string &pathname1, const std::string &pathname2);
534  static std::pair<std::string, std::string> splitDrive(const std::string &pathname);
535  static std::vector<std::string> splitChain(const std::string &chain, const std::string &sep);
536  static std::vector<std::string> getDirFiles(const std::string &dirname);
537 
542  // read configuration file
543  static bool loadConfigFile(const std::string &confFile);
544  static bool readConfigVar(const std::string &var, float &value);
545  static bool readConfigVar(const std::string &var, double &value);
546  static bool readConfigVar(const std::string &var, int &value);
547  static bool readConfigVar(const std::string &var, unsigned int &value);
548  static bool readConfigVar(const std::string &var, bool &value);
549  static bool readConfigVar(const std::string &var, std::string &value);
550  static bool readConfigVar(const std::string &var, vpColor &value);
551  static bool readConfigVar(const std::string &var, vpArray2D<double> &value, const unsigned int &nCols = 0,
552  const unsigned int &nRows = 0);
553 
554  // construct experiment filename & path
555  static void setBaseName(const std::string &s);
556  static void setBaseDir(const std::string &dir);
557  static void addNameElement(const std::string &strTrue, const bool &cond = true, const std::string &strFalse = "");
558  static void addNameElement(const std::string &strTrue, const double &val);
559  static std::string getBaseName();
560  static std::string getFullName();
561 
562  // write files
563  static void saveConfigFile(const bool &actuallySave = true);
564  static void createBaseNamePath(const bool &empty = false);
566 
567  static void readBinaryValueLE(std::ifstream &file, int16_t &short_value);
568  static void readBinaryValueLE(std::ifstream &file, uint16_t &ushort_value);
569  static void readBinaryValueLE(std::ifstream &file, int32_t &int_value);
570  static void readBinaryValueLE(std::ifstream &file, uint32_t &int_value);
571  static void readBinaryValueLE(std::ifstream &file, float &float_value);
572  static void readBinaryValueLE(std::ifstream &file, double &double_value);
573 
574  static void writeBinaryValueLE(std::ofstream &file, const int16_t short_value);
575  static void writeBinaryValueLE(std::ofstream &file, const uint16_t ushort_value);
576  static void writeBinaryValueLE(std::ofstream &file, const int32_t int_value);
577  static void writeBinaryValueLE(std::ofstream &file, const uint32_t int_value);
578  static void writeBinaryValueLE(std::ofstream &file, float float_value);
579  static void writeBinaryValueLE(std::ofstream &file, double double_value);
580 
581  static bool parseBoolean(std::string input);
582  static std::string trim(std::string s);
583 
584 protected:
585  static std::string baseName;
586  static std::string baseDir;
587  static std::string configFile;
588  static std::vector<std::string> configVars;
589  static std::vector<std::string> configValues;
590 
591 #ifndef DOXYGEN_SHOULD_SKIP_THIS
592  static int mkdir_p(const std::string &path, int mode);
593 #endif
594 };
596 #endif
Class to define RGB colors available for display functionalities.
Definition: vpColor.h:153
File and directories basic tools.
Definition: vpIoTools.h:494
static std::vector< std::string > configVars
Definition: vpIoTools.h:588
static std::string baseDir
Definition: vpIoTools.h:586
static std::string configFile
Definition: vpIoTools.h:587
static std::vector< std::string > configValues
Definition: vpIoTools.h:589
static std::string baseName
Definition: vpIoTools.h:585
static const char separator
Definition: vpIoTools.h:521
VISP_EXPORT char map_type(const std::type_info &t)
Definition: vpIoTools.cpp:135
std::vector< char > create_npy_header(const std::vector< size_t > &shape)
Definition: vpIoTools.h:357
void npz_save(std::string zipname, std::string fname, const T *data, const std::vector< size_t > &shape, std::string mode="w")
Definition: vpIoTools.h:234
VISP_EXPORT npz_t npz_load(std::string fname)
Definition: vpIoTools.cpp:342
VISP_EXPORT void parse_zip_footer(FILE *fp, uint16_t &nrecs, size_t &global_header_size, size_t &global_header_offset)
Definition: vpIoTools.cpp:253
void npy_save(std::string fname, const T *data, const std::vector< size_t > shape, std::string mode="w")
Definition: vpIoTools.h:177
VISP_EXPORT void parse_npy_header(FILE *fp, size_t &word_size, std::vector< size_t > &shape, bool &fortran_order)
Definition: vpIoTools.cpp:197
std::vector< char > & operator+=(std::vector< char > &lhs, const T rhs)
Definition: vpIoTools.h:140
std::map< std::string, NpyArray > npz_t
Definition: vpIoTools.h:127
VISP_EXPORT NpyArray npy_load(std::string fname)
Definition: vpIoTools.cpp:472
VISP_EXPORT char BigEndianTest()
Definition: vpIoTools.cpp:129
Definition: vpIoTools.h:60
static unsigned long vp_mz_crc32(unsigned long crc, const unsigned char *ptr, size_t buf_len)
Definition: vpIoTools.h:62
std::shared_ptr< std::vector< char > > data_holder
Definition: vpIoTools.h:120
size_t num_bytes() const
Definition: vpIoTools.h:115
std::vector< size_t > shape
Definition: vpIoTools.h:121
const T * data() const
Definition: vpIoTools.h:103
std::vector< T > as_vec() const
Definition: vpIoTools.h:109
NpyArray(const std::vector< size_t > &_shape, size_t _word_size, bool _fortran_order)
Definition: vpIoTools.h:85