OpenVDB  9.1.1
IO.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3 
4 /*!
5  \file IO.h
6 
7  \author Ken Museth
8 
9  \date May 1, 2020
10 
11  \brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
12  file compression, support for multiple grids per file as well as
13  multiple grid types.
14 
15  \note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
16 */
17 
18 #ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
19 #define NANOVDB_IO_H_HAS_BEEN_INCLUDED
20 
21 #include "../NanoVDB.h"
22 #include "GridHandle.h"
23 
24 #include <fstream> // for std::ifstream
25 #include <iostream> // for std::cerr/cout
26 #include <string> // for std::string
27 #include <sstream> // for std::stringstream
28 #include <cstring> // for std::strcmp
29 #include <memory> // for std::unique_ptr
30 #include <vector> // for std::vector
31 #ifdef NANOVDB_USE_ZIP
32 #include <zlib.h> // for ZIP compression
33 #endif
34 #ifdef NANOVDB_USE_BLOSC
35 #include <blosc.h> // for BLOSC compression
36 #endif
37 
38 // Due to a bug in older versions of gcc, including fstream might
39 // define "major" and "minor" which are used as member data below.
40 // See https://bugzilla.redhat.com/show_bug.cgi?id=130601
41 #if defined(major) || defined(minor)
42 #undef major
43 #undef minor
44 #endif
45 
46 namespace nanovdb {
47 
48 namespace io {
49 
50 /// We fix a specific size for counting bytes in files so that they
51 /// are saved the same regardless of machine precision. (Note there are
52 /// still little/bigendian issues, however)
53 using fileSize_t = uint64_t;
54 
55 /// @brief Optional compression codecs
56 ///
57 /// @note NONE is the default, ZIP is slow but compact and BLOSC offers a great balance.
58 ///
59 /// @warning NanoVDB optionally supports ZIP and BLOSC compression and will throw an exception
60 /// if it support is required but missing.
61 enum class Codec : uint16_t { NONE = 0,
62  ZIP = 1,
63  BLOSC = 2,
64  END = 3 };
65 
66 inline __hostdev__ const char* toStr(Codec codec)
67 {
68  static const char * LUT[] = { "NONE", "ZIP", "BLOSC" , "END" };
69  return LUT[static_cast<int>(codec)];
70 }
71 
72 /// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
73 ///
74 /// @warning These functions should never be called directly by client code
75 namespace Internal {
76 static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
77 
78 template<typename BufferT>
79 static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec);
80 
81 template<typename BufferT>
82 static void read(std::istream& is, GridHandle<BufferT>& handle, Codec codec);
83 }; // namespace Internal
84 
85 /// @brief Standard hash function to use on strings; std::hash may vary by
86 /// platform/implementation and is know to produce frequent collisions.
87 uint64_t stringHash(const char* cstr);
88 
89 /// @brief Return a uint64_t hash key of a std::string
90 inline uint64_t stringHash(const std::string& str)
91 {
92  return stringHash(str.c_str());
93 }
94 
95 /// @brief Return a uint64_t with its bytes reversed so we can check for endianness
96 inline uint64_t reverseEndianness(uint64_t val)
97 {
98  return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
99  (((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
100  (((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
101  (((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
102 }
103 
104 /// @brief Data encoded at the head of each segment of a file or stream.
105 ///
106 /// @note A file or stream is composed of one or more segments that each contain
107 // one or more grids.
108 // Magic number of NanoVDB files (uint64_t) |
109 // Version numbers of this file (uint32_t) | one header for each segment
110 // Number of grids in this segment (uint16_t) |
111 // Compression mode (uint16_t) |
112 struct Header
113 {// 16 bytes
114  uint64_t magic; // 8 bytes
115  Version version;// 4 bytes version numbers
116  uint16_t gridCount; // 2 bytes
117  Codec codec; // 2 bytes
119  : magic(NANOVDB_MAGIC_NUMBER) // Magic number: "NanoVDB" in hex
120  , version()// major, minor and patch version numbers
121  , gridCount(0)
122  , codec(c)
123  {
124  }
125 }; // Header ( 16 bytes = 2 words )
126 
127 /// @brief Data encoded for each of the grids associated with a segment.
128 // Grid size in memory (uint64_t) |
129 // Grid size on disk (uint64_t) |
130 // Grid name hash key (uint64_t) |
131 // Numer of active voxels (uint64_t) |
132 // Grid type (uint32_t) |
133 // Grid class (uint32_t) |
134 // Characters in grid name (uint32_t) |
135 // AABB in world space (2*3*double) | one per grid in file
136 // AABB in index space (2*3*int) |
137 // Size of a voxel in world units (3*double) |
138 // Byte size of the grid name (uint32_t) |
139 // Number of nodes per level (4*uint32_t) |
140 // Numer of active tiles per level (3*uint32_t) |
141 // Codec for file compression (uint16_t) |
142 // Padding due to 8B alignment (uint16_t) |
143 // Version number (uint32_t) |
144 struct MetaData
145 {// 176 bytes
146  uint64_t gridSize, fileSize, nameKey, voxelCount; // 4 * 8 = 32B.
149  BBox<Vec3d> worldBBox; // 2 * 3 * 8 = 48B.
150  CoordBBox indexBBox; // 2 * 3 * 4 = 24B.
151  Vec3R voxelSize; // 24B.
152  uint32_t nameSize; // 4B.
153  uint32_t nodeCount[4]; //4 x 4 = 16B
154  uint32_t tileCount[3];// 3 x 4 = 12B
155  Codec codec; // 2B
156  uint16_t padding;// 2B, due to 8B alignment from uint64_t
158 }; // MetaData
159 
160 struct GridMetaData : public MetaData
161 {
162  static_assert(sizeof(MetaData) == 176, "Unexpected sizeof(MetaData)");
163  std::string gridName;
164  void read(std::istream& is);
165  void write(std::ostream& os) const;
167  template<typename ValueT>
168  GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
169  uint64_t memUsage() const { return sizeof(MetaData) + nameSize; }
170 }; // GridMetaData
171 
172 struct Segment
173 {
174  // Check assumptions made during read and write of Header and MetaData
175  static_assert(sizeof(Header) == 16u, "Unexpected sizeof(Header)");
176  Header header;
177  std::vector<GridMetaData> meta;
179  : header(c)
180  , meta()
181  {
182  }
183  template<typename BufferT>
184  void add(const GridHandle<BufferT>& h);
185  bool read(std::istream& is);
186  void write(std::ostream& os) const;
187  uint64_t memUsage() const;
188 }; // Segment
189 
190 /// @brief Write a single grid to file (over-writing existing content of the file)
191 template<typename BufferT>
192 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE, int verbose = 0);
193 
194 /// @brief Write a single grid to stream (starting at the current position)
195 ///
196 /// @note This method can be used to append grid to an existing stream
197 template<typename BufferT>
198 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE);
199 
200 /// @brief Write multiple grids to file (over-writing existing content of the file)
201 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
202 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
203 
204 /// @brief Writes multiple grids to stream (starting at its current position)
205 ///
206 /// @note This method can be used to append multiple grids to an existing stream
207 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
208 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE);
209 
210 /// @brief Read the n'th grid from file (defaults to first grid)
211 ///
212 /// @throw If n exceeds the number of grids in the file
213 template<typename BufferT = HostBuffer>
214 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n = 0, int verbose = 0, const BufferT& buffer = BufferT());
215 
216 /// @brief Read the n'th grid from stream (defaults to first grid)
217 ///
218 /// @throw If n exceeds the number of grids in the stream
219 template<typename BufferT = HostBuffer>
220 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n = 0, const BufferT& buffer = BufferT());
221 
222 /// @brief Read the first grid with a specific name
223 ///
224 /// @warning If not grid exists with the specified name the resulting GridHandle is empty
225 template<typename BufferT = HostBuffer>
226 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
227 
228 /// @brief Read the first grid with a specific name
229 template<typename BufferT = HostBuffer>
230 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer = BufferT());
231 
232 /// @brief Read all the grids in the file
233 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
234 VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
235 
236 /// @brief Real all grids at the current position of the input stream
237 template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
238 VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer = BufferT());
239 
240 /// @brief Return true if the file contains a grid with the specified name
241 bool hasGrid(const std::string& fileName, const std::string& gridName);
242 
243 /// @brief Return true if the stream contains a grid with the specified name
244 bool hasGrid(std::istream& is, const std::string& gridName);
245 
246 /// @brief Reads and returns a vector of meta data for all the grids found in the specified file
247 std::vector<GridMetaData> readGridMetaData(const std::string& fileName);
248 
249 /// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
250 std::vector<GridMetaData> readGridMetaData(std::istream& is);
251 
252 // --------------------------> Implementations for Internal <------------------------------------
253 
254 template<typename BufferT>
255 fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
256 {
257  const char* data = reinterpret_cast<const char*>(handle.data());
258  fileSize_t total = 0, residual = handle.size();
259 
260  switch (codec) {
261  case Codec::ZIP: {
262 #ifdef NANOVDB_USE_ZIP
263  uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
264  std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
265  const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
266  if (status != Z_OK)
267  std::runtime_error("Internal write error in ZIP");
268  if (size > residual)
269  std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
270  const fileSize_t outBytes = size;
271  os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
272  os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
273  total += sizeof(fileSize_t) + outBytes;
274 #else
275  throw std::runtime_error("ZIP compression codec was disabled during build");
276 #endif
277  break;
278  }
279  case Codec::BLOSC: {
280 #ifdef NANOVDB_USE_BLOSC
281  do {
282  fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
283  std::unique_ptr<char[]> tmp(new char[size]);
284  const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
285  if (count <= 0)
286  std::runtime_error("Internal write error in BLOSC");
287  const fileSize_t outBytes = count;
288  os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
289  os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
290  total += sizeof(fileSize_t) + outBytes;
291  data += chunk;
292  residual -= chunk;
293  } while (residual > 0);
294 #else
295  throw std::runtime_error("BLOSC compression codec was disabled during build");
296 #endif
297  break;
298  }
299  default:
300  os.write(data, residual);
301  total += residual;
302  }
303  if (!os) {
304  throw std::runtime_error("Failed to write Tree to file");
305  }
306  return total;
307 } // Internal::write
308 
309 template<typename BufferT>
310 void Internal::read(std::istream& is, GridHandle<BufferT>& handle, Codec codec)
311 {
312  char* data = reinterpret_cast<char*>(handle.buffer().data());
313  fileSize_t residual = handle.buffer().size();
314 
315  // read tree using optional compression
316  switch (codec) {
317  case Codec::ZIP: {
318 #ifdef NANOVDB_USE_ZIP
319  fileSize_t size;
320  is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
321  std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
322  is.read(reinterpret_cast<char*>(tmp.get()), size);
323  uLongf numBytes = residual;
324  int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
325  if (status != Z_OK)
326  std::runtime_error("Internal read error in ZIP");
327  if (fileSize_t(numBytes) != residual)
328  throw std::runtime_error("UNZIP failed on byte size");
329 #else
330  throw std::runtime_error("ZIP compression codec was disabled during build");
331 #endif
332  break;
333  }
334  case Codec::BLOSC: {
335 #ifdef NANOVDB_USE_BLOSC
336  do {
337  fileSize_t size;
338  is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
339  std::unique_ptr<char[]> tmp(new char[size]);
340  is.read(reinterpret_cast<char*>(tmp.get()), size);
341  const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
342  const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
343  if (count < 1)
344  std::runtime_error("Internal read error in BLOSC");
345  if (count != int(chunk))
346  throw std::runtime_error("BLOSC failed on byte size");
347  data += size_t(chunk);
348  residual -= chunk;
349  } while (residual > 0);
350 #else
351  throw std::runtime_error("BLOSC compression codec was disabled during build");
352 #endif
353  break;
354  }
355  default:
356  is.read(data, residual);
357  }
358  if (!is) {
359  throw std::runtime_error("Failed to read Tree from file");
360  }
361 } // Internal::read
362 
363 // --------------------------> Implementations for GridMetaData <------------------------------------
364 
365 template<typename ValueT>
366 inline GridMetaData::GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
367  : MetaData{size, // gridSize
368  0, // fileSize
369  0, // nameKey
370  grid.activeVoxelCount(), // voxelCount
371  grid.gridType(), // gridType
372  grid.gridClass(), // gridClass
373  grid.worldBBox(), // worldBBox
374  grid.tree().bbox(), // indexBBox
375  grid.voxelSize(), // voxelSize
376  0, // nameSize
377  {0, 0, 0, 1}, // nodeCount[4]
378  {0, 0, 0}, // tileCount[3]
379  c, // codec
380  0, // padding
381  Version()}// version
382  , gridName(grid.gridName())
383 {
384  nameKey = stringHash(gridName);
385  nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
386  const uint32_t* ptr = reinterpret_cast<const TreeData<3>*>(&grid.tree())->mNodeCount;
387  for (int i = 0; i < 3; ++i) {
388  MetaData::nodeCount[i] = *ptr++;
389  }
390  //MetaData::nodeCount[3] = 1;// one root node
391  for (int i = 0; i < 3; ++i) {
392  MetaData::tileCount[i] = *ptr++;
393  }
394 }
395 
396 inline void GridMetaData::write(std::ostream& os) const
397 {
398  os.write(reinterpret_cast<const char*>(this), sizeof(MetaData));
399  os.write(gridName.c_str(), nameSize);
400  if (!os) {
401  throw std::runtime_error("Failed writing GridMetaData");
402  }
403 }
404 
405 inline void GridMetaData::read(std::istream& is)
406 {
407  is.read(reinterpret_cast<char*>(this), sizeof(MetaData));
408  std::unique_ptr<char[]> tmp(new char[nameSize]);
409  is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
410  gridName.assign(tmp.get());
411  if (!is) {
412  throw std::runtime_error("Failed reading GridMetaData");
413  }
414 }
415 
416 // --------------------------> Implementations for Segment <------------------------------------
417 
418 inline uint64_t Segment::memUsage() const
419 {
420  uint64_t sum = sizeof(Header);
421  for (auto& m : meta) {
422  sum += m.memUsage();
423  }
424  return sum;
425 }
426 
427 template<typename BufferT>
428 inline void Segment::add(const GridHandle<BufferT>& h)
429 {
430  if (auto* grid = h.template grid<float>()) { // most common
431  meta.emplace_back(h.size(), header.codec, *grid);
432  } else if (auto* grid = h.template grid<Vec3f>()) {
433  meta.emplace_back(h.size(), header.codec, *grid);
434  } else if (auto* grid = h.template grid<double>()) {
435  meta.emplace_back(h.size(), header.codec, *grid);
436  } else if (auto* grid = h.template grid<int32_t>()) {
437  meta.emplace_back(h.size(), header.codec, *grid);
438  } else if (auto* grid = h.template grid<uint32_t>()) {
439  meta.emplace_back(h.size(), header.codec, *grid);
440  } else if (auto* grid = h.template grid<int64_t>()) {
441  meta.emplace_back(h.size(), header.codec, *grid);
442  } else if (auto* grid = h.template grid<int16_t>()) {
443  meta.emplace_back(h.size(), header.codec, *grid);
444  } else if (auto* grid = h.template grid<Vec3d>()) {
445  meta.emplace_back(h.size(), header.codec, *grid);
446  } else if (auto* grid = h.template grid<ValueMask>()) {
447  meta.emplace_back(h.size(), header.codec, *grid);
448  } else if (auto* grid = h.template grid<bool>()) {
449  meta.emplace_back(h.size(), header.codec, *grid);
450  } else if (auto* grid = h.template grid<Rgba8>()) {
451  meta.emplace_back(h.size(), header.codec, *grid);
452  } else if (auto* grid = h.template grid<Fp4>()) {
453  meta.emplace_back(h.size(), header.codec, *grid);
454  } else if (auto* grid = h.template grid<Fp8>()) {
455  meta.emplace_back(h.size(), header.codec, *grid);
456  } else if (auto* grid = h.template grid<Fp16>()) {
457  meta.emplace_back(h.size(), header.codec, *grid);
458  } else if (auto* grid = h.template grid<FpN>()) {
459  meta.emplace_back(h.size(), header.codec, *grid);
460  } else if (auto* grid = h.template grid<Vec4f>()) {
461  meta.emplace_back(h.size(), header.codec, *grid);
462  } else if (auto* grid = h.template grid<Vec4d>()) {
463  meta.emplace_back(h.size(), header.codec, *grid);
464  } else {
465  throw std::runtime_error("nanovdb::io::Segment::add Cannot write grid of unknown type to file");
466  }
467  header.gridCount += 1;
468 }
469 
470 inline void Segment::write(std::ostream& os) const
471 {
472  if (header.gridCount == 0) {
473  throw std::runtime_error("Segment contains no grids");
474  } else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(Header))) {
475  throw std::runtime_error("Failed to write Header of Segment");
476  }
477  for (auto& m : meta) {
478  m.write(os);
479  }
480 }
481 
482 inline bool Segment::read(std::istream& is)
483 {
484  is.read(reinterpret_cast<char*>(&header), sizeof(Header));
485  if (is.eof()) {
486  return false;
487  }
488  if (!is || header.magic != NANOVDB_MAGIC_NUMBER) {
489  // first check for byte-swapped header magic.
490  if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER))
491  throw std::runtime_error("This nvdb file has reversed endianness");
492  throw std::runtime_error("Magic number error: This is not a valid nvdb file");
493  } else if ( header.version.getMajor() != NANOVDB_MAJOR_VERSION_NUMBER) {
494  std::stringstream ss;
495  if (header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
496  ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
497  << "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
498  } else {
499  ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
500  << "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
501  }
502  throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
503  }
504  meta.resize(header.gridCount);
505  for (auto& m : meta) {
506  m.read(is);
507  m.version = header.version;
508  }
509  return true;
510 }
511 
512 // --------------------------> Implementations for read/write <------------------------------------
513 
514 template<typename BufferT>
515 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
516 {
517  std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
518  if (!os.is_open()) {
519  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
520  }
521  writeGrid<BufferT>(os, handle, codec);
522  if (verbose) {
523  std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
524  }
525 }
526 
527 template<typename BufferT>
528 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
529 {
530  Segment s(codec);
531  s.add(handle);
532  const uint64_t headerSize = s.memUsage();
533  std::streamoff seek = headerSize;
534  os.seekp(seek, std::ios_base::cur); // skip forward from the current position
535  s.meta[0].fileSize = Internal::write(os, handle, codec);
536  seek += s.meta[0].fileSize;
537  os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
538  s.write(os); // write header
539  os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
540 }
541 
542 template<typename BufferT, template<typename...> class VecT>
543 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
544 {
545  std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
546  if (!os.is_open()) {
547  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
548  }
549  writeGrids<BufferT, VecT>(os, handles, codec);
550  if (verbose) {
551  std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
552  }
553 }
554 
555 template<typename BufferT, template<typename...> class VecT>
556 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec)
557 {
558  Segment s(codec);
559  for (auto& h : handles) {
560  s.add(h);
561  }
562  const uint64_t headerSize = s.memUsage();
563  std::streamoff seek = headerSize;
564  os.seekp(seek, std::ios_base::cur); // skip forward from the current position
565  for (size_t i = 0; i < handles.size(); ++i) {
566  s.meta[i].fileSize = Internal::write(os, handles[i], codec);
567  seek += s.meta[i].fileSize;
568  }
569  os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
570  s.write(os); // write header
571  os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
572 }
573 
574 /// @brief Read the n'th grid
575 template<typename BufferT>
576 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n, int verbose, const BufferT& buffer)
577 {
578  std::ifstream is(fileName, std::ios::in | std::ios::binary);
579  if (!is.is_open()) {
580  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
581  }
582  auto handle = readGrid<BufferT>(is, n, buffer);
583  if (verbose) {
584  std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
585  }
586  return handle; // is converted to r-value and return value is move constructed.
587 }
588 
589 template<typename BufferT>
590 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n, const BufferT& buffer)
591 {
592  Segment s;
593  uint64_t counter = 0;
594  while (s.read(is)) {
595  std::streamoff seek = 0;
596  for (auto& m : s.meta) {
597  if (counter == n) {
598  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
599  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
600  Internal::read(is, handle, s.header.codec);
601  return handle; // is converted to r-value and return value is move constructed.
602  } else {
603  seek += m.fileSize;
604  }
605  ++counter;
606  }
607  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
608  }
609  throw std::runtime_error("Grid index exceeds grid count in file");
610 }
611 
612 /// @brief Read the first grid with a specific name
613 template<typename BufferT>
614 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
615 {
616  std::ifstream is(fileName, std::ios::in | std::ios::binary);
617  if (!is.is_open()) {
618  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
619  }
620  auto handle = readGrid<BufferT>(is, gridName, buffer);
621  if (verbose) {
622  if (handle) {
623  std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
624  } else {
625  std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
626  }
627  }
628  return handle; // is converted to r-value and return value is move constructed.
629 }
630 
631 template<typename BufferT>
632 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer)
633 {
634  const auto key = stringHash(gridName);
635  Segment s;
636  while (s.read(is)) {
637  std::streamoff seek = 0;
638  for (auto& m : s.meta) {
639  if (m.nameKey == key && m.gridName == gridName) { // check for hask key collision
640  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
641  is.seekg(seek, std::ios_base::cur); // rewind
642  Internal::read(is, handle, s.header.codec);
643  return handle; // is converted to r-value and return value is move constructed.
644  } else {
645  seek += m.fileSize;
646  }
647  }
648  is.seekg(seek, std::ios_base::cur); // skip forward from the current position
649  }
650  return GridHandle<BufferT>(); // empty handle
651 }
652 
653 /// @brief Read all the grids
654 template<typename BufferT, template<typename...> class VecT>
655 VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
656 {
657  std::ifstream is(fileName, std::ios::in | std::ios::binary);
658  if (!is.is_open()) {
659  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
660  }
661  auto handles = readGrids<BufferT, VecT>(is, buffer);
662  if (verbose) {
663  std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
664  }
665  return handles; // is converted to r-value and return value is move constructed.
666 }
667 
668 template<typename BufferT, template<typename...> class VecT>
669 VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer)
670 {
671  VecT<GridHandle<BufferT>> handles;
672  Segment seg;
673  while (seg.read(is)) {
674  for (auto& m : seg.meta) {
675  GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
676  Internal::read(is, handle, seg.header.codec);
677  handles.push_back(std::move(handle)); // force move copy assignment
678  }
679  }
680  return handles; // is converted to r-value and return value is move constructed.
681 }
682 
683 inline std::vector<GridMetaData> readGridMetaData(const std::string& fileName)
684 {
685  std::ifstream is(fileName, std::ios::in | std::ios::binary);
686  if (!is.is_open()) {
687  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
688  }
689  return readGridMetaData(is); // is converted to r-value and return value is move constructed.
690 }
691 
692 inline std::vector<GridMetaData> readGridMetaData(std::istream& is)
693 {
694  std::vector<GridMetaData> meta;
695  Segment seg;
696  while (seg.read(is)) {
697  std::streamoff seek = 0;
698  for (auto& m : seg.meta) {
699  meta.push_back(m);
700  seek += m.fileSize;
701  }
702  is.seekg(seek, std::ios_base::cur);
703  }
704  return meta; // is converted to r-value and return value is move constructed.
705 }
706 
707 inline bool hasGrid(const std::string& fileName, const std::string& gridName)
708 {
709  std::ifstream is(fileName, std::ios::in | std::ios::binary);
710  if (!is.is_open()) {
711  throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
712  }
713  return hasGrid(is, gridName);
714 }
715 
716 inline bool hasGrid(std::istream& is, const std::string& gridName)
717 {
718  const auto key = stringHash(gridName);
719  Segment s;
720  while (s.read(is)) {
721  std::streamoff seek = 0;
722  for (auto& m : s.meta) {
723  if (m.nameKey == key && m.gridName == gridName) {
724  return true; // check for hash key collision
725  }
726  seek += m.fileSize;
727  }
728  is.seekg(seek, std::ios_base::cur);
729  }
730  return false;
731 }
732 
733 inline uint64_t stringHash(const char* cstr)
734 {
735  uint64_t hash = 0;
736  if (!cstr) {
737  return hash;
738  }
739  for (auto* str = reinterpret_cast<const unsigned char*>(cstr); *str; ++str) {
740  uint64_t overflow = hash >> (64 - 8);
741  hash *= 67; // Next-ish prime after 26 + 26 + 10
742  hash += *str + overflow;
743  }
744  return hash;
745 }
746 
747 }
748 } // namespace nanovdb::io
749 
750 #endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridMetaData()
Definition: IO.h:166
const BBox< Vec3R > & worldBBox() const
Computes a AABB of active values in world space.
Definition: NanoVDB.h:2406
BBox< Vec3d > worldBBox
Definition: IO.h:149
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition: IO.h:96
std::vector< GridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition: IO.h:683
Highest level of the data structure. Contains a tree and a world->index transform (that currently onl...
Definition: NanoVDB.h:2308
Index64 memUsage(const TreeT &tree, bool threaded=true)
Return the total amount of memory in bytes occupied by this tree.
Definition: Count.h:493
Header(Codec c=Codec::NONE)
Definition: IO.h:118
GridClass
Classes (defined in OpenVDB) that are currently supported by NanoVDB.
Definition: NanoVDB.h:253
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition: IO.h:707
GridClass gridClass
Definition: IO.h:148
Defines two classes, a GridRegister the defines the value type (e.g. Double, Float etc) of a NanoVDB ...
Data encoded at the head of each segment of a file or stream.
Definition: IO.h:112
std::vector< GridMetaData > meta
Definition: IO.h:177
const TreeT & tree() const
Return a const reference to the tree.
Definition: NanoVDB.h:2345
uint32_t nameSize
Definition: IO.h:152
Segment(Codec c=Codec::NONE)
Definition: IO.h:178
Bit-compacted representation of all three version numbers.
Definition: NanoVDB.h:540
Codec codec
Definition: IO.h:117
uint64_t activeVoxelCount() const
Return the total number of active voxels in this tree.
Definition: NanoVDB.h:2415
uint64_t fileSize_t
Definition: IO.h:53
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition: IO.h:733
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
This class serves to manage a raw memory buffer of a NanoVDB Grid.
Definition: GridHandle.h:70
uint16_t padding
Definition: IO.h:156
Version version
Definition: IO.h:157
BufferT & buffer()
Return a reference to the buffer.
Definition: GridHandle.h:107
CoordBBox indexBBox
Definition: IO.h:150
uint16_t gridCount
Definition: IO.h:116
Definition: NanoVDB.h:184
uint32_t nodeCount[4]
Definition: IO.h:153
Header header
Definition: IO.h:175
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
uint64_t voxelCount
Definition: IO.h:146
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition: NanoVDB.h:104
GridType gridType
Definition: IO.h:147
uint8_t * data() override
Returns a non-const pointer to the data.
Definition: GridHandle.h:115
Data encoded for each of the grids associated with a segment.
Definition: IO.h:144
Definition: IO.h:160
Definition: NanoVDB.h:1524
void read(std::istream &is)
Definition: IO.h:405
#define NANOVDB_MAGIC_NUMBER
Definition: NanoVDB.h:102
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file)
Definition: IO.h:515
const GridType & gridType() const
Definition: NanoVDB.h:2419
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition: HostBuffer.h:109
Definition: IO.h:172
uint64_t memUsage() const
Definition: IO.h:418
__hostdev__ uint32_t hash(uint32_t x)
Definition: common.h:13
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file.
Definition: IO.h:655
GridType
List of types that are currently supported by NanoVDB.
Definition: NanoVDB.h:216
const Vec3R & voxelSize() const
Return a const reference to the size of a voxel in world units.
Definition: NanoVDB.h:2354
const GridClass & gridClass() const
Definition: NanoVDB.h:2420
std::string gridName
Definition: IO.h:162
uint64_t magic
Definition: IO.h:114
Codec
Optional compression codecs.
Definition: IO.h:61
uint64_t memUsage() const
Definition: IO.h:169
Definition: NanoVDB.h:2501
bool read(std::istream &is)
Definition: IO.h:482
uint64_t size() const override
Returns the size in bytes of the raw memory buffer managed by this GridHandle&#39;s allocator.
Definition: GridHandle.h:123
void write(std::ostream &os) const
Definition: IO.h:396
Vec3R voxelSize
Definition: IO.h:151
uint32_t tileCount[3]
Definition: IO.h:154
Codec codec
Definition: IO.h:155
#define __hostdev__
Definition: NanoVDB.h:168
GridHandle< BufferT > readGrid(const std::string &fileName, uint64_t n=0, int verbose=0, const BufferT &buffer=BufferT())
Read the n&#39;th grid from file (defaults to first grid)
Definition: IO.h:576
void add(const GridHandle< BufferT > &h)
Definition: IO.h:428
void write(std::ostream &os) const
Definition: IO.h:470
Version version
Definition: IO.h:115
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT >> &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file)
Definition: IO.h:543
uint64_t nameKey
Definition: IO.h:146
const char * toStr(Codec codec)
Definition: IO.h:66
static constexpr fileSize_t MAX_SIZE
Definition: IO.h:76