OpenVDB  7.0.0
StreamCompression.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3 
18 
19 #ifndef OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
20 #define OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
21 
22 #include <openvdb/io/io.h>
23 #include <tbb/spin_mutex.h>
24 #include <memory>
25 #include <string>
26 
27 
28 class TestStreamCompression;
29 
30 namespace openvdb {
32 namespace OPENVDB_VERSION_NAME {
33 namespace compression {
34 
35 
36 // This is the minimum number of bytes below which Blosc compression is not used to
37 // avoid unecessary computation, as Blosc offers minimal compression until this limit
38 static const int BLOSC_MINIMUM_BYTES = 48;
39 
40 // This is the minimum number of bytes below which the array is padded with zeros up
41 // to this number of bytes to allow Blosc to perform compression with small arrays
42 static const int BLOSC_PAD_BYTES = 128;
43 
44 
47 
51 OPENVDB_API size_t bloscUncompressedSize(const char* buffer);
52 
60 OPENVDB_API void bloscCompress(char* compressedBuffer, size_t& compressedBytes,
61  const size_t bufferBytes, const char* uncompressedBuffer, const size_t uncompressedBytes);
62 
71 OPENVDB_API std::unique_ptr<char[]> bloscCompress(const char* buffer,
72  const size_t uncompressedBytes, size_t& compressedBytes, const bool resize = true);
73 
78 OPENVDB_API size_t bloscCompressedSize(const char* buffer, const size_t uncompressedBytes);
79 
87 OPENVDB_API void bloscDecompress(char* uncompressedBuffer, const size_t expectedBytes,
88  const size_t bufferBytes, const char* compressedBuffer);
89 
97 OPENVDB_API std::unique_ptr<char[]> bloscDecompress(const char* buffer,
98  const size_t expectedBytes, const bool resize = true);
99 
100 
102 
103 
104 // 1MB = 1048576 Bytes
105 static const int PageSize = 1024 * 1024;
106 
107 
112 {
113 private:
114  struct Info
115  {
116  io::MappedFile::Ptr mappedFile;
118  std::streamoff filepos;
119  long compressedBytes;
120  long uncompressedBytes;
121  }; // Info
122 
123 public:
124  using Ptr = std::shared_ptr<Page>;
125 
126  Page() = default;
127 
129  void load() const;
130 
133  long uncompressedBytes() const;
134 
137  const char* buffer(const int index) const;
138 
140  void readHeader(std::istream&);
141 
144  void readBuffers(std::istream&, bool delayed);
145 
147  bool isOutOfCore() const;
148 
149 private:
151  void copy(const std::unique_ptr<char[]>& temp, int pageSize);
152 
154  void decompress(const std::unique_ptr<char[]>& temp);
155 
157  void doLoad() const;
158 
159  std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
160  std::unique_ptr<char[]> mData;
161  tbb::spin_mutex mMutex;
162 }; // class Page
163 
164 
168 {
169 public:
170 #if OPENVDB_ABI_VERSION_NUMBER >= 6
171  using Ptr = std::unique_ptr<PageHandle>;
172 #else
173  using Ptr = std::shared_ptr<PageHandle>;
174 #endif
175 
180  PageHandle(const Page::Ptr& page, const int index, const int size);
181 
183  Page& page();
184 
186  int size() const { return mSize; }
187 
190  std::unique_ptr<char[]> read();
191 
193  Ptr copy() { return Ptr(new PageHandle(mPage, mIndex, mSize)); }
194 
195 protected:
196  friend class ::TestStreamCompression;
197 
198 private:
199  Page::Ptr mPage;
200  int mIndex = -1;
201  int mSize = 0;
202 }; // class PageHandle
203 
204 
209 {
210 public:
211  using Ptr = std::shared_ptr<PagedInputStream>;
212 
213  PagedInputStream() = default;
214 
215  explicit PagedInputStream(std::istream& is);
216 
218  void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
219  bool sizeOnly() const { return mSizeOnly; }
220 
221  // @brief Set and get the input stream
222  std::istream& getInputStream() { assert(mIs); return *mIs; }
223  void setInputStream(std::istream& is) { mIs = &is; }
224 
226  PageHandle::Ptr createHandle(std::streamsize n);
227 
231  void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
232 
233 private:
234  int mByteIndex = 0;
235  int mUncompressedBytes = 0;
236  std::istream* mIs = nullptr;
237  Page::Ptr mPage;
238  bool mSizeOnly = false;
239 }; // class PagedInputStream
240 
241 
246 {
247 public:
248  using Ptr = std::shared_ptr<PagedOutputStream>;
249 
251 
252  explicit PagedOutputStream(std::ostream& os);
253 
255  void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
256  bool sizeOnly() const { return mSizeOnly; }
257 
259  std::ostream& getOutputStream() { assert(mOs); return *mOs; }
260  void setOutputStream(std::ostream& os) { mOs = &os; }
261 
263  PagedOutputStream& write(const char* str, std::streamsize n);
264 
266  void flush();
267 
268 private:
271  void compressAndWrite(const char* buffer, size_t size);
272 
274  void resize(size_t size);
275 
276  std::unique_ptr<char[]> mData = std::unique_ptr<char[]>(new char[PageSize]);
277  std::unique_ptr<char[]> mCompressedData = nullptr;
278  size_t mCapacity = PageSize;
279  int mBytes = 0;
280  std::ostream* mOs = nullptr;
281  bool mSizeOnly = false;
282 }; // class PagedOutputStream
283 
284 
285 } // namespace compression
286 } // namespace OPENVDB_VERSION_NAME
287 } // namespace openvdb
288 
289 #endif // OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:230
OPENVDB_API std::unique_ptr< char[]> bloscCompress(const char *buffer, const size_t uncompressedBytes, size_t &compressedBytes, const bool resize=true)
Compress and return the heap-allocated compressed buffer.
void setOutputStream(std::ostream &os)
Definition: StreamCompression.h:260
OPENVDB_API bool bloscCanCompress()
Returns true if compression is available.
Ptr copy()
Return a copy of this PageHandle.
Definition: StreamCompression.h:193
SharedPtr< MappedFile > Ptr
Definition: io.h:136
std::unique_ptr< PageHandle > Ptr
Definition: StreamCompression.h:171
A Paging wrapper to std::istream that is responsible for reading from a given input stream and creati...
Definition: StreamCompression.h:208
std::shared_ptr< T > SharedPtr
Definition: Types.h:91
OPENVDB_API std::unique_ptr< char[]> bloscDecompress(const char *buffer, const size_t expectedBytes, const bool resize=true)
Decompress and return the the heap-allocated uncompressed buffer.
OPENVDB_API size_t bloscCompressedSize(const char *buffer, const size_t uncompressedBytes)
Convenience wrapper to retrieve the compressed size of buffer when compressed.
static const int PageSize
Definition: StreamCompression.h:105
std::istream & getInputStream()
Definition: StreamCompression.h:222
void setInputStream(std::istream &is)
Definition: StreamCompression.h:223
OPENVDB_API size_t bloscUncompressedSize(const char *buffer)
Retrieves the uncompressed size of buffer when uncompressed.
Stores a variable-size, compressed, delayed-load Page of data that is loaded into memory when accesse...
Definition: StreamCompression.h:111
A Paging wrapper to std::ostream that is responsible for writing from a given output stream at interv...
Definition: StreamCompression.h:245
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h:102
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only reading size data.
Definition: StreamCompression.h:218
A PageHandle holds a unique ptr to a Page and a specific stream pointer to a point within the decompr...
Definition: StreamCompression.h:167
static const int BLOSC_MINIMUM_BYTES
Definition: StreamCompression.h:38
Definition: Exceptions.h:13
bool sizeOnly() const
Definition: StreamCompression.h:256
int size() const
Return the size of the buffer.
Definition: StreamCompression.h:186
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only writing size data.
Definition: StreamCompression.h:255
std::shared_ptr< PagedOutputStream > Ptr
Definition: StreamCompression.h:248
bool sizeOnly() const
Definition: StreamCompression.h:219
static const int BLOSC_PAD_BYTES
Definition: StreamCompression.h:42
std::shared_ptr< PagedInputStream > Ptr
Definition: StreamCompression.h:211
std::shared_ptr< Page > Ptr
Definition: StreamCompression.h:124
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:154
std::ostream & getOutputStream()
Set and get the output stream.
Definition: StreamCompression.h:259