OpenVDB  10.0.1
IO.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3 
4 /*!
5  \file IO.h
6 
7  \author Ken Museth
8 
9  \date May 1, 2020
10 
11  \brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
12  file compression, support for multiple grids per file as well as
13  multiple grid types.
14 
15  \note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
16 */
17 
18 #ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
19 #define NANOVDB_IO_H_HAS_BEEN_INCLUDED
20 
21 #include "../NanoVDB.h"
22 #include "GridHandle.h"
23 
24 #include <fstream> // for std::ifstream
25 #include <iostream> // for std::cerr/cout
26 #include <string> // for std::string
27 #include <sstream> // for std::stringstream
28 #include <cstring> // for std::strcmp
29 #include <memory> // for std::unique_ptr
30 #include <vector> // for std::vector
31 #ifdef NANOVDB_USE_ZIP
32 #include <zlib.h> // for ZIP compression
33 #endif
34 #ifdef NANOVDB_USE_BLOSC
35 #include <blosc.h> // for BLOSC compression
36 #endif
37 
38 // Due to a bug in older versions of gcc, including fstream might
39 // define "major" and "minor" which are used as member data below.
40 // See https://bugzilla.redhat.com/show_bug.cgi?id=130601
41 #if defined(major) || defined(minor)
42 #undef major
43 #undef minor
44 #endif
45 
46 namespace nanovdb {
47 
48 namespace io {
49 
50 /// We fix a specific size for counting bytes in files so that they
51 /// are saved the same regardless of machine precision. (Note there are
52 /// still little/bigendian issues, however)
53 using fileSize_t = uint64_t;
54 
55 /// @brief Optional compression codecs
56 ///
57 /// @note NONE is the default, ZIP is slow but compact and BLOSC offers a great balance.
58 ///
59 /// @warning NanoVDB optionally supports ZIP and BLOSC compression and will throw an exception
60 /// if it support is required but missing.
61 enum class Codec : uint16_t { NONE = 0,
62  ZIP = 1,
63  BLOSC = 2,
64  END = 3 };
65 
66 inline __hostdev__ const char* toStr(Codec codec)
67 {
68  static const char * LUT[] = { "NONE", "ZIP", "BLOSC" , "END" };
69  return LUT[static_cast<int>(codec)];
70 }
71 
72 /// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
73 ///
74 /// @warning These functions should never be called directly by client code
75 namespace Internal {
76 static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
77 
78 template<typename BufferT>
79 static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec);
80 
81 template<typename BufferT>
82 static void read(std::istream& is, GridHandle<BufferT>& handle, Codec codec);
83 }; // namespace Internal
84 
85 /// @brief Standard hash function to use on strings; std::hash may vary by
86 /// platform/implementation and is know to produce frequent collisions.
87 uint64_t stringHash(const char* cstr);
88 
89 /// @brief Return a uint64_t hash key of a std::string
90 inline uint64_t stringHash(const std::string& str)
91 {
92  return stringHash(str.c_str());
93 }
94 
95 /// @brief Return a uint64_t with its bytes reversed so we can check for endianness
96 inline uint64_t reverseEndianness(uint64_t val)
97 {
98  return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
99  (((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
100  (((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
101  (((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
102 }
103 
104 /// @brief Data encoded at the head of each segment of a file or stream.
105 ///
106 /// @note A file or stream is composed of one or more segments that each contain
107 // one or more grids.
108 // Magic number of NanoVDB files (uint64_t) |
109 // Version numbers of this file (uint32_t) | one header for each segment
110 // Number of grids in this segment (uint16_t) |
111 // Compression mode (uint16_t) |
112 struct Header
113 {// 16 bytes
114  uint64_t magic; // 8 bytes
115  Version version;// 4 bytes version numbers
116  uint16_t gridCount; // 2 bytes
117  Codec codec; // 2 bytes
119  : magic(NANOVDB_MAGIC_NUMBER) // Magic number: "NanoVDB" in hex
120  , version()// major, minor and patch version numbers
121  , gridCount(0)
122  , codec(c)
123  {
124  }
125 }; // Header ( 16 bytes = 2 words )
126 
127 /// @brief Data encoded for each of the grids associated with a segment.
128 // Grid size in memory (uint64_t) |
129 // Grid size on disk (uint64_t) |
130 // Grid name hash key (uint64_t) |
131 // Numer of active voxels (uint64_t) |
132 // Grid type (uint32_t) |
133 // Grid class (uint32_t) |
134 // Characters in grid name (uint32_t) |
135 // AABB in world space (2*3*double) | one per grid in file
136 // AABB in index space (2*3*int) |
137 // Size of a voxel in world units (3*double) |
138 // Byte size of the grid name (uint32_t) |
139 // Number of nodes per level (4*uint32_t) |
140 // Numer of active tiles per level (3*uint32_t) |
141 // Codec for file compression (uint16_t) |
142 // Padding due to 8B alignment (uint16_t) |
143 // Version number (uint32_t) |
144 struct MetaData
145 {// 176 bytes
146  uint64_t gridSize, fileSize, nameKey, voxelCount; // 4 * 8 = 32B.
149  BBox<Vec3d> worldBBox; // 2 * 3 * 8 = 48B.
150  CoordBBox indexBBox; // 2 * 3 * 4 = 24B.
151  Vec3R voxelSize; // 24B.
152  uint32_t nameSize; // 4B.
153  uint32_t nodeCount[4]; //4 x 4 = 16B
154  uint32_t tileCount[3];// 3 x 4 = 12B
155  Codec codec; // 2B
156  uint16_t padding;// 2B, due to 8B alignment from uint64_t
158 }; // MetaData
159 
160 struct GridMetaData : public MetaData
161 {
162  static_assert(sizeof(MetaData) == 176, "Unexpected sizeof(MetaData)");
163  std::string gridName;
164  void read(std::istream& is);
165  void write(std::ostream& os) const;
167  template<typename ValueT>
168  GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
169  uint64_t memUsage() const { return sizeof(MetaData) + nameSize; }
170 }; // GridMetaData
171 
172 struct Segment
173 {
174  // Check assumptions made during read and write of Header and MetaData
175  static_assert(sizeof(Header) == 16u, "Unexpected sizeof(Header)");
176  Header header;
177  std::vector<GridMetaData> meta;
179  : header(c)
180  , meta()
181  {
182  }
183  template<typename BufferT>
184  void add(const GridHandle<BufferT>& h);
185  bool read(std::istream& is);
186  void write(std::ostream& os) const;
187  uint64_t memUsage() const;
188 }; // Segment
189 
190 /// @brief Write a single grid to file (over-writing existing content of the file)
191 template<typename BufferT>
192 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE, int verbose = 0);
193 
194 /// @brief Write a single grid to stream (starting at the current position)
195 ///
196 /// @note This method can be used to append grid to an existing stream
197 template<typename BufferT>
198 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE);
199 
200 /// @brief Write multiple grids to file (over-writing existing content of the file)
201 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
202 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
203 
204 /// @brief Writes multiple grids to stream (starting at its current position)
205 ///
206 /// @note This method can be used to append multiple grids to an existing stream
207 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
208 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE);
209 
210 /// @brief Read the n'th grid from file (defaults to first grid)
211 ///
212 /// @throw If n exceeds the number of grids in the file
213 template<typename BufferT = HostBuffer>
214 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n = 0, int verbose = 0, const BufferT& buffer = BufferT());
215 
216 /// @brief Read the n'th grid from stream (defaults to first grid)
217 ///
218 /// @throw If n exceeds the number of grids in the stream
219 template<typename BufferT = HostBuffer>
220 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n = 0, const BufferT& buffer = BufferT());
221 
222 /// @brief Read the first grid with a specific name
223 ///
224 /// @warning If not grid exists with the specified name the resulting GridHandle is empty
225 template<typename BufferT = HostBuffer>
226 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
227 
228 /// @brief Read the first grid with a specific name
229 template<typename BufferT = HostBuffer>
230 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer = BufferT());
231 
232 /// @brief Read all the grids in the file
233 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
234 VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
235 
236 /// @brief Real all grids at the current position of the input stream
237 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
238 VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer = BufferT());
239 
240 /// @brief Return true if the file contains a grid with the specified name
241 bool hasGrid(const std::string& fileName, const std::string& gridName);
242 
243 /// @brief Return true if the stream contains a grid with the specified name
244 bool hasGrid(std::istream& is, const std::string& gridName);
245 
246 /// @brief Reads and returns a vector of meta data for all the grids found in the specified file
247 std::vector<GridMetaData> readGridMetaData(const std::string& fileName);
248 
249 /// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
250 std::vector<GridMetaData> readGridMetaData(std::istream& is);
251 
252 // --------------------------> Implementations for Internal <------------------------------------
253 
254 template<typename BufferT>
255 fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
256 {
257  const char* data = reinterpret_cast<const char*>(handle.data());
258  fileSize_t total = 0, residual = handle.size();
259 
260  switch (codec) {
261  case Codec::ZIP: {
262 #ifdef NANOVDB_USE_ZIP
263  uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
264  std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
265  const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
266  if (status != Z_OK)
267  std::runtime_error("Internal write error in ZIP");
268  if (size > residual)
269  std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
270  const fileSize_t outBytes = size;
271  os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
272  os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
273  total += sizeof(fileSize_t) + outBytes;
274 #else
275  throw std::runtime_error("ZIP compression codec was disabled during build");
276 #endif
277  break;
278  }
279  case Codec::BLOSC: {
280 #ifdef NANOVDB_USE_BLOSC
281  do {
282  fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
283  std::unique_ptr<char[]> tmp(new char[size]);
284  const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
285  if (count <= 0)
286  std::runtime_error("Internal write error in BLOSC");
287  const fileSize_t outBytes = count;
288  os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
289  os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
290  total += sizeof(fileSize_t) + outBytes;
291  data += chunk;
292  residual -= chunk;
293  } while (residual > 0);
294 #else
295  throw std::runtime_error("BLOSC compression codec was disabled during build");
296 #endif
297  break;
298  }
299  default:
300  os.write(data, residual);
301  total += residual;
302  }
303  if (!os) {
304  throw std::runtime_error("Failed to write Tree to file");
305  }
306  return total;
307 } // Internal::write
308 
309 template<typename BufferT>
310 void Internal::read(std::istream& is, GridHandle<BufferT>& handle, Codec codec)
311 {
312  char* data = reinterpret_cast<char*>(handle.buffer().data());
313  fileSize_t residual = handle.buffer().size();
314 
315  // read tree using optional compression
316  switch (codec) {
317  case Codec::ZIP: {
318 #ifdef NANOVDB_USE_ZIP
319  fileSize_t size;
320  is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
321  std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
322  is.read(reinterpret_cast<char*>(tmp.get()), size);
323  uLongf numBytes = residual;
324  int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
325  if (status != Z_OK)
326  std::runtime_error("Internal read error in ZIP");
327  if (fileSize_t(numBytes) != residual)
328  throw std::runtime_error("UNZIP failed on byte size");
329 #else
330  throw std::runtime_error("ZIP compression codec was disabled during build");
331 #endif
332  break;
333  }
334  case Codec::BLOSC: {
335 #ifdef NANOVDB_USE_BLOSC
336  do {
337  fileSize_t size;
338  is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
339  std::unique_ptr<char[]> tmp(new char[size]);
340  is.read(reinterpret_cast<char*>(tmp.get()), size);
341  const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
342  const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
343  if (count < 1)
344  std::runtime_error("Internal read error in BLOSC");
345  if (count != int(chunk))
346  throw std::runtime_error("BLOSC failed on byte size");
347  data += size_t(chunk);
348  residual -= chunk;
349  } while (residual > 0);
350 #else
351  throw std::runtime_error("BLOSC compression codec was disabled during build");
352 #endif
353  break;
354  }
355  default:
356  is.read(data, residual);
357  }
358  if (!is) {
359  throw std::runtime_error("Failed to read Tree from file");
360  }
361 } // Internal::read
362 
363 // --------------------------> Implementations for GridMetaData <------------------------------------
364 
365 template<typename ValueT>
366 inline GridMetaData::GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
367  : MetaData{size, // gridSize
368  0, // fileSize
369  0, // nameKey
370  grid.activeVoxelCount(), // voxelCount
371  grid.gridType(), // gridType
372  grid.gridClass(), // gridClass
373  grid.worldBBox(), // worldBBox
374  grid.tree().bbox(), // indexBBox
375  grid.voxelSize(), // voxelSize
376  0, // nameSize
377  {0, 0, 0, 1}, // nodeCount[4]
378  {0, 0, 0}, // tileCount[3]
379  c, // codec
380  0, // padding
381  Version()}// version
382  , gridName(grid.gridName())
383 {
384  nameKey = stringHash(gridName);
385  nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
386  const uint32_t* ptr = reinterpret_cast<const TreeData<3>*>(&grid.tree())->mNodeCount;
387  for (int i = 0; i < 3; ++i) {
388  MetaData::nodeCount[i] = *ptr++;
389  }
390  //MetaData::nodeCount[3] = 1;// one root node
391  for (int i = 0; i < 3; ++i) {
392  MetaData::tileCount[i] = *ptr++;
393  }
394 }
395 
396 inline void GridMetaData::write(std::ostream& os) const
397 {
398  os.write(reinterpret_cast<const char*>(this), sizeof(MetaData));
399  os.write(gridName.c_str(), nameSize);
400  if (!os) {
401  throw std::runtime_error("Failed writing GridMetaData");
402  }
403 }
404 
405 inline void GridMetaData::read(std::istream& is)
406 {
407  is.read(reinterpret_cast<char*>(this), sizeof(MetaData));
408  std::unique_ptr<char[]> tmp(new char[nameSize]);
409  is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
410  gridName.assign(tmp.get());
411  if (!is) {
412  throw std::runtime_error("Failed reading GridMetaData");
413  }
414 }
415 
416 // --------------------------> Implementations for Segment <------------------------------------
417 
418 inline uint64_t Segment::memUsage() const
419 {
420  uint64_t sum = sizeof(Header);
421  for (auto& m : meta) {
422  sum += m.memUsage();
423  }
424  return sum;
425 }
426 
427 template<typename BufferT>
428 inline void Segment::add(const GridHandle<BufferT>& h)
429 {
430  if (auto* grid = h.template grid<float>()) { // most common
431  meta.emplace_back(h.size(), header.codec, *grid);
432  } else if (auto* grid = h.template grid<Vec3f>()) {
433  meta.emplace_back(h.size(), header.codec, *grid);
434  } else if (auto* grid = h.template grid<double>()) {
435  meta.emplace_back(h.size(), header.codec, *grid);
436  } else if (auto* grid = h.template grid<int32_t>()) {
437  meta.emplace_back(h.size(), header.codec, *grid);
438  } else if (auto* grid = h.template grid<uint32_t>()) {
439  meta.emplace_back(h.size(), header.codec, *grid);
440  } else if (auto* grid = h.template grid<int64_t>()) {
441  meta.emplace_back(h.size(), header.codec, *grid);
442  } else if (auto* grid = h.template grid<int16_t>()) {
443  meta.emplace_back(h.size(), header.codec, *grid);
444  } else if (auto* grid = h.template grid<Vec3d>()) {
445  meta.emplace_back(h.size(), header.codec, *grid);
446  } else if (auto* grid = h.template grid<ValueMask>()) {
447  meta.emplace_back(h.size(), header.codec, *grid);
448  } else if (auto* grid = h.template grid<ValueIndex>()) {
449  meta.emplace_back(h.size(), header.codec, *grid);
450  } else if (auto* grid = h.template grid<bool>()) {
451  meta.emplace_back(h.size(), header.codec, *grid);
452  } else if (auto* grid = h.template grid<Rgba8>()) {
453  meta.emplace_back(h.size(), header.codec, *grid);
454  } else if (auto* grid = h.template grid<Fp4>()) {
455  meta.emplace_back(h.size(), header.codec, *grid);
456  } else if (auto* grid = h.template grid<Fp8>()) {
457  meta.emplace_back(h.size(), header.codec, *grid);
458  } else if (auto* grid = h.template grid<Fp16>()) {
459  meta.emplace_back(h.size(), header.codec, *grid);
460  } else if (auto* grid = h.template grid<FpN>()) {
461  meta.emplace_back(h.size(), header.codec, *grid);
462  } else if (auto* grid = h.template grid<Vec4f>()) {
463  meta.emplace_back(h.size(), header.codec, *grid);
464  } else if (auto* grid = h.template grid<Vec4d>()) {
465  meta.emplace_back(h.size(), header.codec, *grid);
466  } else {
467  throw std::runtime_error("nanovdb::io::Segment::add Cannot write grid of unknown type to file");
468  }
469  header.gridCount += 1;
470 }
471 
472 inline void Segment::write(std::ostream& os) const
473 {
474  if (header.gridCount == 0) {
475  throw std::runtime_error("Segment contains no grids");
476  } else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(Header))) {
477  throw std::runtime_error("Failed to write Header of Segment");
478  }
479  for (auto& m : meta) {
480  m.write(os);
481  }
482 }
483 
484 inline bool Segment::read(std::istream& is)
485 {
486  is.read(reinterpret_cast<char*>(&header), sizeof(Header));
487  if (is.eof()) {
488  return false;
489  }
490  if (!is || header.magic != NANOVDB_MAGIC_NUMBER) {
491  // first check for byte-swapped header magic.
492  if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER))
493  throw std::runtime_error("This nvdb file has reversed endianness");
494  throw std::runtime_error("Magic number error: This is not a valid nvdb file");
495  } else if ( header.version.getMajor() != NANOVDB_MAJOR_VERSION_NUMBER) {
496  std::stringstream ss;
497  if (header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
498  ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
499  << "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
500  } else {
501  ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
502  << "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
503  }
504  throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
505  }
506  meta.resize(header.gridCount);
507  for (auto& m : meta) {
508  m.read(is);
509  m.version = header.version;
510  }
511  return true;
512 }
513 
514 // --------------------------> Implementations for read/write <------------------------------------
515 
516 template<typename BufferT>
517 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
518 {
519  std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
520  if (!os.is_open()) {
521  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
522  }
523  writeGrid<BufferT>(os, handle, codec);
524  if (verbose) {
525  std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
526  }
527 }
528 
529 template<typename BufferT>
530 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
531 {
532  Segment s(codec);
533  s.add(handle);
534  const uint64_t headerSize = s.memUsage();
535  std::streamoff seek = headerSize;
536  os.seekp(seek, std::ios_base::cur); // skip forward from the current position
537  s.meta[0].fileSize = Internal::write(os, handle, codec);
538  seek += s.meta[0].fileSize;
539  os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
540  s.write(os); // write header
541  os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
542 }
543 
544 template<typename BufferT, template<typename...> class VecT>
545 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
546 {
547  std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
548  if (!os.is_open()) {
549  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
550  }
551  writeGrids<BufferT, VecT>(os, handles, codec);
552  if (verbose) {
553  std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
554  }
555 }
556 
557 template<typename BufferT, template<typename...> class VecT>
558 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec)
559 {
560  Segment s(codec);
561  for (auto& h : handles) {
562  s.add(h);
563  }
564  const uint64_t headerSize = s.memUsage();
565  std::streamoff seek = headerSize;
566  os.seekp(seek, std::ios_base::cur); // skip forward from the current position
567  for (size_t i = 0; i < handles.size(); ++i) {
568  s.meta[i].fileSize = Internal::write(os, handles[i], codec);
569  seek += s.meta[i].fileSize;
570  }
571  os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
572  s.write(os); // write header
573  os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
574 }
575 
576 /// @brief Read the n'th grid
577 template<typename BufferT>
578 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n, int verbose, const BufferT& buffer)
579 {
580  std::ifstream is(fileName, std::ios::in | std::ios::binary);
581  if (!is.is_open()) {
582  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
583  }
584  auto handle = readGrid<BufferT>(is, n, buffer);
585  if (verbose) {
586  std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
587  }
588  return handle; // is converted to r-value and return value is move constructed.
589 }
590 
591 template<typename BufferT>
592 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n, const BufferT& buffer)
593 {
594  Segment s;
595  uint64_t counter = 0;
596  while (s.read(is)) {
597  std::streamoff seek = 0;
598  for (auto& m : s.meta) {
599  if (counter == n) {
600  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
601  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
602  Internal::read(is, handle, s.header.codec);
603  return handle; // is converted to r-value and return value is move constructed.
604  } else {
605  seek += m.fileSize;
606  }
607  ++counter;
608  }
609  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
610  }
611  throw std::runtime_error("Grid index exceeds grid count in file");
612 }
613 
614 /// @brief Read the first grid with a specific name
615 template<typename BufferT>
616 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
617 {
618  std::ifstream is(fileName, std::ios::in | std::ios::binary);
619  if (!is.is_open()) {
620  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
621  }
622  auto handle = readGrid<BufferT>(is, gridName, buffer);
623  if (verbose) {
624  if (handle) {
625  std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
626  } else {
627  std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
628  }
629  }
630  return handle; // is converted to r-value and return value is move constructed.
631 }
632 
633 template<typename BufferT>
634 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer)
635 {
636  const auto key = stringHash(gridName);
637  Segment s;
638  while (s.read(is)) {
639  std::streamoff seek = 0;
640  for (auto& m : s.meta) {
641  if (m.nameKey == key && m.gridName == gridName) { // check for hash key collision
642  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
643  is.seekg(seek, std::ios_base::cur); // rewind
644  Internal::read(is, handle, s.header.codec);
645  return handle; // is converted to r-value and return value is move constructed.
646  } else {
647  seek += m.fileSize;
648  }
649  }
650  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
651  }
652  return GridHandle<BufferT>(); // empty handle
653 }
654 
655 /// @brief Read all the grids
656 template<typename BufferT, template<typename...> class VecT>
657 VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
658 {
659  std::ifstream is(fileName, std::ios::in | std::ios::binary);
660  if (!is.is_open()) {
661  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
662  }
663  auto handles = readGrids<BufferT, VecT>(is, buffer);
664  if (verbose) {
665  std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
666  }
667  return handles; // is converted to r-value and return value is move constructed.
668 }
669 
670 template<typename BufferT, template<typename...> class VecT>
671 VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer)
672 {
673  VecT<GridHandle<BufferT>> handles;
674  Segment seg;
675  while (seg.read(is)) {
676  for (auto& m : seg.meta) {
677  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
678  Internal::read(is, handle, seg.header.codec);
679  handles.push_back(std::move(handle)); // force move copy assignment
680  }
681  }
682  return handles; // is converted to r-value and return value is move constructed.
683 }
684 
685 inline std::vector<GridMetaData> readGridMetaData(const std::string& fileName)
686 {
687  std::ifstream is(fileName, std::ios::in | std::ios::binary);
688  if (!is.is_open()) {
689  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
690  }
691  return readGridMetaData(is); // is converted to r-value and return value is move constructed.
692 }
693 
694 inline std::vector<GridMetaData> readGridMetaData(std::istream& is)
695 {
696  std::vector<GridMetaData> meta;
697  Segment seg;
698  while (seg.read(is)) {
699  std::streamoff seek = 0;
700  for (auto& m : seg.meta) {
701  meta.push_back(m);
702  seek += m.fileSize;
703  }
704  is.seekg(seek, std::ios_base::cur);
705  }
706  return meta; // is converted to r-value and return value is move constructed.
707 }
708 
709 inline bool hasGrid(const std::string& fileName, const std::string& gridName)
710 {
711  std::ifstream is(fileName, std::ios::in | std::ios::binary);
712  if (!is.is_open()) {
713  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
714  }
715  return hasGrid(is, gridName);
716 }
717 
718 inline bool hasGrid(std::istream& is, const std::string& gridName)
719 {
720  const auto key = stringHash(gridName);
721  Segment s;
722  while (s.read(is)) {
723  std::streamoff seek = 0;
724  for (auto& m : s.meta) {
725  if (m.nameKey == key && m.gridName == gridName) {
726  return true; // check for hash key collision
727  }
728  seek += m.fileSize;
729  }
730  is.seekg(seek, std::ios_base::cur);
731  }
732  return false;
733 }
734 
735 inline uint64_t stringHash(const char* cstr)
736 {
737  uint64_t hash = 0;
738  if (!cstr) {
739  return hash;
740  }
741  for (auto* str = reinterpret_cast<const unsigned char*>(cstr); *str; ++str) {
742  uint64_t overflow = hash >> (64 - 8);
743  hash *= 67; // Next-ish prime after 26 + 26 + 10
744  hash += *str + overflow;
745  }
746  return hash;
747 }
748 
749 }
750 } // namespace nanovdb::io
751 
752 #endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridMetaData()
Definition: IO.h:166
const BBox< Vec3R > & worldBBox() const
Computes a AABB of active values in world space.
Definition: NanoVDB.h:2659
BBox< Vec3d > worldBBox
Definition: IO.h:149
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition: IO.h:96
std::vector< GridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition: IO.h:685
Highest level of the data structure. Contains a tree and a world->index transform (that currently onl...
Definition: NanoVDB.h:2555
Header(Codec c=Codec::NONE)
Definition: IO.h:118
GridClass
Classes (defined in OpenVDB) that are currently supported by NanoVDB.
Definition: NanoVDB.h:281
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition: IO.h:709
GridClass gridClass
Definition: IO.h:148
Defines two classes, a GridRegister the defines the value type (e.g. Double, Float etc) of a NanoVDB ...
Data encoded at the head of each segment of a file or stream.
Definition: IO.h:112
std::vector< GridMetaData > meta
Definition: IO.h:177
const TreeT & tree() const
Return a const reference to the tree.
Definition: NanoVDB.h:2598
uint32_t nameSize
Definition: IO.h:152
Segment(Codec c=Codec::NONE)
Definition: IO.h:178
Bit-compacted representation of all three version numbers.
Definition: NanoVDB.h:647
Codec codec
Definition: IO.h:117
Index64 memUsage(const TreeT &tree, bool threaded=true)
Return the total amount of memory in bytes occupied by this tree.
Definition: Count.h:493
uint64_t activeVoxelCount() const
Return the total number of active voxels in this tree.
Definition: NanoVDB.h:2668
uint64_t fileSize_t
Definition: IO.h:53
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition: IO.h:735
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
This class serves to manage a raw memory buffer of a NanoVDB Grid.
Definition: GridHandle.h:70
uint16_t padding
Definition: IO.h:156
Version version
Definition: IO.h:157
BufferT & buffer()
Return a reference to the buffer.
Definition: GridHandle.h:109
CoordBBox indexBBox
Definition: IO.h:150
uint16_t gridCount
Definition: IO.h:116
Definition: NanoVDB.h:208
uint32_t nodeCount[4]
Definition: IO.h:153
Header header
Definition: IO.h:175
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
uint64_t voxelCount
Definition: IO.h:146
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition: NanoVDB.h:123
GridType gridType
Definition: IO.h:147
uint8_t * data() override
Returns a non-const pointer to the data.
Definition: GridHandle.h:117
Data encoded for each of the grids associated with a segment.
Definition: IO.h:144
Definition: IO.h:160
Definition: NanoVDB.h:1655
void read(std::istream &is)
Definition: IO.h:405
#define NANOVDB_MAGIC_NUMBER
Definition: NanoVDB.h:121
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file)
Definition: IO.h:517
const GridType & gridType() const
Definition: NanoVDB.h:2672
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition: HostBuffer.h:114
Definition: IO.h:172
uint64_t memUsage() const
Definition: IO.h:418
__hostdev__ uint32_t hash(uint32_t x)
Definition: common.h:13
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file.
Definition: IO.h:657
GridType
List of types that are currently supported by NanoVDB.
Definition: NanoVDB.h:243
const Vec3R & voxelSize() const
Return a const reference to the size of a voxel in world units.
Definition: NanoVDB.h:2607
const GridClass & gridClass() const
Definition: NanoVDB.h:2673
std::string gridName
Definition: IO.h:162
uint64_t magic
Definition: IO.h:114
Codec
Optional compression codecs.
Definition: IO.h:61
uint64_t memUsage() const
Definition: IO.h:169
Definition: NanoVDB.h:2756
bool read(std::istream &is)
Definition: IO.h:484
uint64_t size() const override
Returns the size in bytes of the raw memory buffer managed by this GridHandle&#39;s allocator.
Definition: GridHandle.h:125
void write(std::ostream &os) const
Definition: IO.h:396
Vec3R voxelSize
Definition: IO.h:151
uint32_t tileCount[3]
Definition: IO.h:154
Codec codec
Definition: IO.h:155
#define __hostdev__
Definition: NanoVDB.h:192
GridHandle< BufferT > readGrid(const std::string &fileName, uint64_t n=0, int verbose=0, const BufferT &buffer=BufferT())
Read the n&#39;th grid from file (defaults to first grid)
Definition: IO.h:578
void add(const GridHandle< BufferT > &h)
Definition: IO.h:428
void write(std::ostream &os) const
Definition: IO.h:472
Version version
Definition: IO.h:115
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT >> &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file)
Definition: IO.h:545
uint64_t nameKey
Definition: IO.h:146
const char * toStr(Codec codec)
Definition: IO.h:66
static constexpr fileSize_t MAX_SIZE
Definition: IO.h:76