OpenVDB  9.1.1
PagedArray.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3 ///
4 /// @file PagedArray.h
5 ///
6 /// @author Ken Museth
7 ///
8 /// @brief Concurrent, page-based, dynamically-sized linear data
9 /// structure with O(1) random access and STL-compliant
10 /// iterators. It is primarily intended for applications
11 /// that involve multi-threading push_back of (a possibly
12 /// unkown number of) elements into a dynamically growing
13 /// linear array, and fast random access to said elements.
14 
15 #ifndef OPENVDB_UTIL_PAGED_ARRAY_HAS_BEEN_INCLUDED
16 #define OPENVDB_UTIL_PAGED_ARRAY_HAS_BEEN_INCLUDED
17 
18 #include <openvdb/version.h>
19 #include <openvdb/Types.h>// SharedPtr
20 #include <deque>
21 #include <cassert>
22 #include <iostream>
23 #include <algorithm>// std::swap
24 #include <atomic>
25 #include <tbb/spin_mutex.h>
26 #include <tbb/parallel_for.h>
27 #include <tbb/parallel_sort.h>
28 
29 namespace openvdb {
31 namespace OPENVDB_VERSION_NAME {
32 namespace util {
33 
34 ////////////////////////////////////////
35 
36 
37 /// @brief Concurrent, page-based, dynamically-sized linear data structure
38 /// with O(1) random access and STL-compliant iterators. It is
39 /// primarily intended for applications that concurrently insert
40 /// (a possibly unkown number of) elements into a dynamically
41 /// growing linear array, and fast random access to said elements.
42 ///
43 /// @note Multiple threads can grow the page-table and push_back
44 /// new elements concurrently. A ValueBuffer provides accelerated
45 /// and threadsafe push_back at the cost of potentially re-ordering
46 /// elements (when multiple instances are used).
47 ///
48 /// @details This data structure employes contiguous pages of elements
49 /// (a std::deque) which avoids moving data when the
50 /// capacity is out-grown and new pages are allocated. The
51 /// size of the pages can be controlled with the Log2PageSize
52 /// template parameter (defaults to 1024 elements of type ValueT).
53 ///
54 /// There are three fundamentally different ways to insert elements to
55 /// this container - each with different advanteges and disadvanteges.
56 ///
57 /// The simplest way to insert elements is to use PagedArray::push_back_unsafe
58 /// which is @a not thread-safe:
59 /// @code
60 /// PagedArray<size_t> array;
61 /// for (size_t i=0; i<100000; ++i) array.push_back_unsafe(i);
62 /// @endcode
63 ///
64 /// The fastest way (by far) to insert elements is by means of a PagedArray::ValueBuffer:
65 /// @code
66 /// PagedArray<size_t> array;
67 /// auto buffer = array.getBuffer();
68 /// for (size_t i=0; i<100000; ++i) buffer.push_back(i);
69 /// buffer.flush();
70 /// @endcode
71 /// or
72 /// @code
73 /// PagedArray<size_t> array;
74 /// {
75 /// //local scope of a single thread
76 /// auto buffer = array.getBuffer();
77 /// for (size_t i=0; i<100000; ++i) buffer.push_back(i);
78 /// }
79 /// @endcode
80 /// or with TBB task-based multi-threading:
81 /// @code
82 /// PagedArray<size_t> array;
83 /// tbb::parallel_for(
84 /// tbb::blocked_range<size_t>(0, 10000, array.pageSize()),
85 /// [&array](const tbb::blocked_range<size_t>& range) {
86 /// auto buffer = array.getBuffer();
87 /// for (size_t i=range.begin(); i!=range.end(); ++i) buffer.push_back(i);
88 /// }
89 /// );
90 /// @endcode
91 /// or with TBB thread-local storage for even better performance (due
92 /// to fewer concurrent instantiations of partially full ValueBuffers)
93 /// @code
94 /// PagedArray<size_t> array;
95 /// auto exemplar = array.getBuffer();//dummy used for initialization
96 /// tbb::enumerable_thread_specific<PagedArray<size_t>::ValueBuffer>
97 /// pool(exemplar);//thread local storage pool of ValueBuffers
98 /// tbb::parallel_for(
99 /// tbb::blocked_range<size_t>(0, 10000, array.pageSize()),
100 /// [&pool](const tbb::blocked_range<size_t>& range) {
101 /// auto &buffer = pool.local();
102 /// for (size_t i=range.begin(); i!=range.end(); ++i) buffer.push_back(i);
103 /// }
104 /// );
105 /// for (auto i=pool.begin(); i!=pool.end(); ++i) i->flush();
106 /// @endcode
107 /// This technique generally outperforms PagedArray::push_back_unsafe,
108 /// std::vector::push_back, std::deque::push_back and even
109 /// tbb::concurrent_vector::push_back. Additionally it
110 /// is thread-safe as long as each thread has it's own instance of a
111 /// PagedArray::ValueBuffer. The only disadvantage is the ordering of
112 /// the elements is undefined if multiple instance of a
113 /// PagedArray::ValueBuffer are employed. This is typically the case
114 /// in the context of multi-threading, where the
115 /// ordering of inserts are undefined anyway. Note that a local scope
116 /// can be used to guarentee that the ValueBuffer has inserted all its
117 /// elements by the time the scope ends. Alternatively the ValueBuffer
118 /// can be explicitly flushed by calling ValueBuffer::flush.
119 ///
120 /// The third way to insert elements is to resize the container and use
121 /// random access, e.g.
122 /// @code
123 /// PagedArray<int> array;
124 /// array.resize(100000);
125 /// for (int i=0; i<100000; ++i) array[i] = i;
126 /// @endcode
127 /// or in terms of the random access iterator
128 /// @code
129 /// PagedArray<int> array;
130 /// array.resize(100000);
131 /// for (auto i=array.begin(); i!=array.end(); ++i) *i = i.pos();
132 /// @endcode
133 /// While this approach is both fast and thread-safe it suffers from the
134 /// major disadvantage that the problem size, i.e. number of elements, needs to
135 /// be known in advance. If that's the case you might as well consider
136 /// using std::vector or a raw c-style array! In other words the
137 /// PagedArray is most useful in the context of applications that
138 /// involve multi-threading of dynamically growing linear arrays that
139 /// require fast random access.
140 
141 template<typename ValueT, size_t Log2PageSize = 10UL>
143 {
144 private:
145  static_assert(Log2PageSize > 1UL, "Expected Log2PageSize > 1");
146  class Page;
147 
148  // must allow mutiple threads to call operator[] as long as only one thread calls push_back
149  using PageTableT = std::deque<Page*>;
150 
151 public:
152  using ValueType = ValueT;
154 
155  /// @brief Default constructor
156  PagedArray() : mCapacity{0} { mSize = 0; }
157 
158  /// @brief Destructor removed all allocated pages
159  ~PagedArray() { this->clear(); }
160 
161  // Disallow copy construction and assignment
162  PagedArray(const PagedArray&) = delete;
163  PagedArray& operator=(const PagedArray&) = delete;
164 
165  /// @brief Return a shared pointer to a new instance of this class
166  static Ptr create() { return Ptr(new PagedArray); }
167 
168  /// @brief Caches values into a local memory Page to improve
169  /// performance of push_back into a PagedArray.
170  ///
171  /// @note The ordering of inserted elements is undefined when
172  /// multiple ValueBuffers are used!
173  ///
174  /// @warning By design this ValueBuffer is not threadsafe so
175  /// make sure to create an instance per thread!
176  class ValueBuffer;
177 
178  /// @return a new instance of a ValueBuffer which supports thread-safe push_back!
179  ValueBuffer getBuffer() { return ValueBuffer(*this); }
180 
181  /// Const std-compliant iterator
182  class ConstIterator;
183 
184  /// Non-const std-compliant iterator
185  class Iterator;
186 
187  /// @brief This method is deprecated and will be removed shortly!
189  {
190  return this->push_back_unsafe(value);
191  }
192 
193  /// @param value value to be added to this PagedArray
194  ///
195  /// @note For best performance consider using the ValueBuffer!
196  ///
197  /// @warning Not thread-safe and mostly intended for debugging!
199  {
200  const size_t index = mSize.fetch_add(1);
201  if (index >= mCapacity) {
202  mPageTable.push_back( new Page() );
203  mCapacity += Page::Size;
204  }
205  (*mPageTable[index >> Log2PageSize])[index] = value;
206  return index;
207  }
208 
209  /// @brief Reduce the page table to fix the current size.
210  ///
211  /// @warning Not thread-safe!
212  void shrink_to_fit();
213 
214  /// @brief Return a reference to the value at the specified offset
215  ///
216  /// @param i linear offset of the value to be accessed.
217  ///
218  /// @note This random access has constant time complexity.
219  ///
220  /// @warning It is assumed that the i'th element is already allocated!
222  {
223  assert(i<mCapacity);
224  return (*mPageTable[i>>Log2PageSize])[i];
225  }
226 
227  /// @brief Return a const-reference to the value at the specified offset
228  ///
229  /// @param i linear offset of the value to be accessed.
230  ///
231  /// @note This random access has constant time complexity.
232  ///
233  /// @warning It is assumed that the i'th element is already allocated!
234  const ValueType& operator[](size_t i) const
235  {
236  assert(i<mCapacity);
237  return (*mPageTable[i>>Log2PageSize])[i];
238  }
239 
240  /// @brief Set all elements in the page table to the specified value
241  ///
242  /// @param v value to be filled in all the existing pages of this PagedArray.
243  ///
244  /// @note Multi-threaded
245  void fill(const ValueType& v)
246  {
247  auto op = [&](const tbb::blocked_range<size_t>& r){
248  for(size_t i=r.begin(); i!=r.end(); ++i) mPageTable[i]->fill(v);
249  };
250  tbb::parallel_for(tbb::blocked_range<size_t>(0, this->pageCount()), op);
251  }
252 
253  /// @brief Copy the first @a count values in this PageArray into
254  /// a raw c-style array, assuming it to be at least @a count
255  /// elements long.
256  ///
257  /// @param p pointer to an array that will used as the destination of the copy.
258  /// @param count number of elements to be copied.
259  ///
260  bool copy(ValueType *p, size_t count) const
261  {
262  size_t last_page = count >> Log2PageSize;
263  if (last_page >= this->pageCount()) return false;
264  auto op = [&](const tbb::blocked_range<size_t>& r){
265  for (size_t i=r.begin(); i!=r.end(); ++i) {
266  mPageTable[i]->copy(p+i*Page::Size, Page::Size);
267  }
268  };
269  if (size_t m = count & Page::Mask) {//count is not divisible by page size
270  tbb::parallel_for(tbb::blocked_range<size_t>(0, last_page, 32), op);
271  mPageTable[last_page]->copy(p+last_page*Page::Size, m);
272  } else {
273  tbb::parallel_for(tbb::blocked_range<size_t>(0, last_page+1, 32), op);
274  }
275  return true;
276  }
277  void copy(ValueType *p) const { this->copy(p, mSize); }
278 
279  /// @brief Resize this array to the specified size.
280  ///
281  /// @param size number of elements that this PageArray will contain.
282  ///
283  /// @details Will grow or shrink the page table to contain
284  /// the specified number of elements. It will affect the size(),
285  /// iteration will go over all those elements, push_back will
286  /// insert after them and operator[] can be used directly access
287  /// them.
288  ///
289  /// @note No reserve method is implemented due to efficiency concerns
290  /// (especially for the ValueBuffer) from having to deal with empty pages.
291  ///
292  /// @warning Not thread-safe!
293  void resize(size_t size)
294  {
295  mSize = size;
296  if (size > mCapacity) {
297  this->grow(size-1);
298  } else {
299  this->shrink_to_fit();
300  }
301  }
302 
303  /// @brief Resize this array to the specified size and initialize
304  /// all values to @a v.
305  ///
306  /// @param size number of elements that this PageArray will contain.
307  /// @param v value of all the @a size values.
308  ///
309  /// @details Will grow or shrink the page table to contain
310  /// the specified number of elements. It will affect the size(),
311  /// iteration will go over all those elements, push_back will
312  /// insert after them and operator[] can be used directly access them.
313  ///
314  /// @note No reserve method is implemented due to efficiency concerns
315  /// (especially for the ValueBuffer) from having to deal with empty pages.
316  ///
317  /// @warning Not thread-safe!
318  void resize(size_t size, const ValueType& v)
319  {
320  this->resize(size);
321  this->fill(v);
322  }
323 
324  /// @brief Return the number of elements in this array.
325  size_t size() const { return mSize; }
326 
327  /// @brief Return the maximum number of elements that this array
328  /// can contain without allocating more memory pages.
329  size_t capacity() const { return mCapacity; }
330 
331  /// @brief Return the number of additional elements that can be
332  /// added to this array without allocating more memory pages.
333  size_t freeCount() const { return mCapacity - mSize; }
334 
335  /// @brief Return the number of allocated memory pages.
336  size_t pageCount() const { return mPageTable.size(); }
337 
338  /// @brief Return the number of elements per memory page.
339  static size_t pageSize() { return Page::Size; }
340 
341  /// @brief Return log2 of the number of elements per memory page.
342  static size_t log2PageSize() { return Log2PageSize; }
343 
344  /// @brief Return the memory footprint of this array in bytes.
345  size_t memUsage() const
346  {
347  return sizeof(*this) + mPageTable.size() * Page::memUsage();
348  }
349 
350  /// @brief Return true if the container contains no elements.
351  bool isEmpty() const { return mSize == 0; }
352 
353  /// @brief Return true if the page table is partially full, i.e. the
354  /// last non-empty page contains less than pageSize() elements.
355  ///
356  /// @details When the page table is partially full calling merge()
357  /// or using a ValueBuffer will rearrange the ordering of
358  /// existing elements.
359  bool isPartiallyFull() const { return (mSize & Page::Mask) > 0; }
360 
361  /// @brief Removes all elements from the array and delete all pages.
362  ///
363  /// @warning Not thread-safe!
364  void clear()
365  {
366  for (size_t i=0, n=mPageTable.size(); i<n; ++i) delete mPageTable[i];
367  PageTableT().swap(mPageTable);
368  mSize = 0;
369  mCapacity = 0;
370  }
371 
372  /// @brief Return a non-const iterator pointing to the first element
373  Iterator begin() { return Iterator(*this, 0); }
374 
375  /// @brief Return a non-const iterator pointing to the
376  /// past-the-last element.
377  ///
378  /// @warning Iterator does not point to a valid element and should not
379  /// be dereferenced!
380  Iterator end() { return Iterator(*this, mSize); }
381 
382  //@{
383  /// @brief Return a const iterator pointing to the first element
384  ConstIterator cbegin() const { return ConstIterator(*this, 0); }
385  ConstIterator begin() const { return ConstIterator(*this, 0); }
386  //@}
387 
388  //@{
389  /// @brief Return a const iterator pointing to the
390  /// past-the-last element.
391  ///
392  /// @warning Iterator does not point to a valid element and should not
393  /// be dereferenced!
394  ConstIterator cend() const { return ConstIterator(*this, mSize); }
395  ConstIterator end() const { return ConstIterator(*this, mSize); }
396  //@}
397 
398  /// @brief Parallel sort of all the elements in ascending order.
399  void sort() { tbb::parallel_sort(this->begin(), this->end(), std::less<ValueT>() ); }
400 
401  /// @brief Parallel sort of all the elements in descending order.
402  void invSort() { tbb::parallel_sort(this->begin(), this->end(), std::greater<ValueT>()); }
403 
404  //@{
405  /// @brief Parallel sort of all the elements based on a custom
406  /// functor with the api:
407  /// @code bool operator()(const ValueT& a, const ValueT& b) @endcode
408  /// which returns true if a comes before b.
409  template <typename Functor>
410  void sort(Functor func) { tbb::parallel_sort(this->begin(), this->end(), func ); }
411  //@}
412 
413  /// @brief Transfer all the elements (and pages) from the other array to this array.
414  ///
415  /// @param other non-const reference to the PagedArray that will be merged into this PagedArray.
416  ///
417  /// @note The other PagedArray is empty on return.
418  ///
419  /// @warning The ordering of elements is undefined if this page table is partially full!
420  void merge(PagedArray& other);
421 
422  /// @brief Print information for debugging
423  void print(std::ostream& os = std::cout) const
424  {
425  os << "PagedArray:\n"
426  << "\tSize: " << this->size() << " elements\n"
427  << "\tPage table: " << this->pageCount() << " pages\n"
428  << "\tPage size: " << this->pageSize() << " elements\n"
429  << "\tCapacity: " << this->capacity() << " elements\n"
430  << "\tFootprint: " << this->memUsage() << " bytes\n";
431  }
432 
433 private:
434 
435  friend class ValueBuffer;
436 
437  void grow(size_t index)
438  {
439  tbb::spin_mutex::scoped_lock lock(mGrowthMutex);
440  while(index >= mCapacity) {
441  mPageTable.push_back( new Page() );
442  mCapacity += Page::Size;
443  }
444  }
445 
446  void add_full(Page*& page, size_t size);
447 
448  void add_partially_full(Page*& page, size_t size);
449 
450  void add(Page*& page, size_t size) {
451  tbb::spin_mutex::scoped_lock lock(mGrowthMutex);
452  if (size == Page::Size) {//page is full
453  this->add_full(page, size);
454  } else if (size>0) {//page is only partially full
455  this->add_partially_full(page, size);
456  }
457  }
458  PageTableT mPageTable;//holds points to allocated pages
459  std::atomic<size_t> mSize;// current number of elements in array
460  size_t mCapacity;//capacity of array given the current page count
461  tbb::spin_mutex mGrowthMutex;//Mutex-lock required to grow pages
462 }; // Public class PagedArray
463 
464 ////////////////////////////////////////////////////////////////////////////////
465 
466 template <typename ValueT, size_t Log2PageSize>
468 {
469  if (mPageTable.size() > (mSize >> Log2PageSize) + 1) {
470  tbb::spin_mutex::scoped_lock lock(mGrowthMutex);
471  const size_t pageCount = (mSize >> Log2PageSize) + 1;
472  if (mPageTable.size() > pageCount) {
473  delete mPageTable.back();
474  mPageTable.pop_back();
475  mCapacity -= Page::Size;
476  }
477  }
478 }
479 
480 template <typename ValueT, size_t Log2PageSize>
482 {
483  if (&other != this && !other.isEmpty()) {
484  tbb::spin_mutex::scoped_lock lock(mGrowthMutex);
485  // extract last partially full page if it exists
486  Page* page = nullptr;
487  const size_t size = mSize & Page::Mask; //number of elements in the last page
488  if ( size > 0 ) {
489  page = mPageTable.back();
490  mPageTable.pop_back();
491  mSize -= size;
492  }
493  // transfer all pages from the other page table
494  mPageTable.insert(mPageTable.end(), other.mPageTable.begin(), other.mPageTable.end());
495  mSize += other.mSize;
496  mCapacity = Page::Size*mPageTable.size();
497  other.mSize = 0;
498  other.mCapacity = 0;
499  PageTableT().swap(other.mPageTable);
500  // add back last partially full page
501  if (page) this->add_partially_full(page, size);
502  }
503 }
504 
505 template <typename ValueT, size_t Log2PageSize>
506 void PagedArray<ValueT, Log2PageSize>::add_full(Page*& page, size_t size)
507 {
508  assert(size == Page::Size);//page must be full
509  if (mSize & Page::Mask) {//page-table is partially full
510  Page*& tmp = mPageTable.back();
511  std::swap(tmp, page);//swap last table entry with page
512  }
513  mPageTable.push_back(page);
514  mCapacity += Page::Size;
515  mSize += size;
516  page = nullptr;
517 }
518 
519 template <typename ValueT, size_t Log2PageSize>
521 {
522  assert(size > 0 && size < Page::Size);//page must be partially full
523  if (size_t m = mSize & Page::Mask) {//page table is also partially full
524  ValueT *s = page->data(), *t = mPageTable.back()->data() + m;
525  for (size_t i=std::min(mSize+size, mCapacity)-mSize; i; --i) *t++ = *s++;
526  if (mSize+size > mCapacity) {//grow page table
527  mPageTable.push_back( new Page() );
528  t = mPageTable.back()->data();
529  for (size_t i=mSize+size-mCapacity; i; --i) *t++ = *s++;
530  mCapacity += Page::Size;
531  }
532  } else {//page table is full so simply append page
533  mPageTable.push_back( page );
534  mCapacity += Page::Size;
535  page = nullptr;
536  }
537  mSize += size;
538 }
539 
540 ////////////////////////////////////////////////////////////////////////////////
541 
542 // Public member-class of PagedArray
543 template <typename ValueT, size_t Log2PageSize>
544 class PagedArray<ValueT, Log2PageSize>::
546 {
547 public:
549  /// @brief Constructor from a PageArray
550  ValueBuffer(PagedArray& parent) : mParent(&parent), mPage(new Page()), mSize(0) {}
551  /// @warning This copy-constructor is shallow in the sense that no
552  /// elements are copied, i.e. size = 0.
553  ValueBuffer(const ValueBuffer& other) : mParent(other.mParent), mPage(new Page()), mSize(0) {}
554  /// @brief Destructor that transfers an buffered values to the parent PagedArray.
555  ~ValueBuffer() { mParent->add(mPage, mSize); delete mPage; }
556 
557  ValueBuffer& operator=(const ValueBuffer&) = delete;// disallow copy assignment
558 
559  /// @brief Add a value to the buffer and increment the size.
560  ///
561  /// @details If the internal memory page is full it will
562  /// automaically flush the page to the parent PagedArray.
563  void push_back(const ValueT& v) {
564  (*mPage)[mSize++] = v;
565  if (mSize == Page::Size) this->flush();
566  }
567  /// @brief Manually transfers the values in this buffer to the parent PagedArray.
568  ///
569  /// @note This method is also called by the destructor and
570  /// push_back so it should only be called if one manually wants to
571  /// sync up the buffer with the array, e.g. during debugging.
572  void flush() {
573  mParent->add(mPage, mSize);
574  if (mPage == nullptr) mPage = new Page();
575  mSize = 0;
576  }
577  /// @brief Return a reference to the parent PagedArray
578  PagedArrayType& parent() const { return *mParent; }
579  /// @brief Return the current number of elements cached in this buffer.
580  size_t size() const { return mSize; }
581  static size_t pageSize() { return 1UL << Log2PageSize; }
582 private:
583  PagedArray* mParent;
584  Page* mPage;
585  size_t mSize;
586 };// Public class PagedArray::ValueBuffer
587 
588 ////////////////////////////////////////////////////////////////////////////////
589 
590 // Const std-compliant iterator
591 // Public member-class of PagedArray
592 template <typename ValueT, size_t Log2PageSize>
593 class PagedArray<ValueT, Log2PageSize>::
594 ConstIterator : public std::iterator<std::random_access_iterator_tag, ValueT>
595 {
596 public:
597  using BaseT = std::iterator<std::random_access_iterator_tag, ValueT>;
598  using difference_type = typename BaseT::difference_type;
599  // constructors and assignment
600  ConstIterator() : mPos(0), mParent(nullptr) {}
601  ConstIterator(const PagedArray& parent, size_t pos=0) : mPos(pos), mParent(&parent) {}
602  ConstIterator(const ConstIterator& other) : mPos(other.mPos), mParent(other.mParent) {}
604  mPos=other.mPos;
605  mParent=other.mParent;
606  return *this;
607  }
608  // prefix
609  ConstIterator& operator++() { ++mPos; return *this; }
610  ConstIterator& operator--() { --mPos; return *this; }
611  // postfix
612  ConstIterator operator++(int) { ConstIterator tmp(*this); ++mPos; return tmp; }
613  ConstIterator operator--(int) { ConstIterator tmp(*this); --mPos; return tmp; }
614  // value access
615  const ValueT& operator*() const { return (*mParent)[mPos]; }
616  const ValueT* operator->() const { return &(this->operator*()); }
617  const ValueT& operator[](const difference_type& pos) const { return (*mParent)[mPos+pos]; }
618  // offset
619  ConstIterator& operator+=(const difference_type& pos) { mPos += pos; return *this; }
620  ConstIterator& operator-=(const difference_type& pos) { mPos -= pos; return *this; }
621  ConstIterator operator+(const difference_type &pos) const { return Iterator(*mParent,mPos+pos); }
622  ConstIterator operator-(const difference_type &pos) const { return Iterator(*mParent,mPos-pos); }
623  difference_type operator-(const ConstIterator& other) const { return mPos - other.pos(); }
624  // comparisons
625  bool operator==(const ConstIterator& other) const { return mPos == other.mPos; }
626  bool operator!=(const ConstIterator& other) const { return mPos != other.mPos; }
627  bool operator>=(const ConstIterator& other) const { return mPos >= other.mPos; }
628  bool operator<=(const ConstIterator& other) const { return mPos <= other.mPos; }
629  bool operator< (const ConstIterator& other) const { return mPos < other.mPos; }
630  bool operator> (const ConstIterator& other) const { return mPos > other.mPos; }
631  // non-std methods
632  bool isValid() const { return mParent != nullptr && mPos < mParent->size(); }
633  size_t pos() const { return mPos; }
634 private:
635  size_t mPos;
636  const PagedArray* mParent;
637 };// Public class PagedArray::ConstIterator
638 
639 ////////////////////////////////////////////////////////////////////////////////
640 
641 // Non-const std-compliant iterator
642 // Public member-class of PagedArray
643 template <typename ValueT, size_t Log2PageSize>
644 class PagedArray<ValueT, Log2PageSize>::
645 Iterator : public std::iterator<std::random_access_iterator_tag, ValueT>
646 {
647 public:
648  using BaseT = std::iterator<std::random_access_iterator_tag, ValueT>;
649  using difference_type = typename BaseT::difference_type;
650  // constructors and assignment
651  Iterator() : mPos(0), mParent(nullptr) {}
652  Iterator(PagedArray& parent, size_t pos=0) : mPos(pos), mParent(&parent) {}
653  Iterator(const Iterator& other) : mPos(other.mPos), mParent(other.mParent) {}
654  Iterator& operator=(const Iterator& other) {
655  mPos=other.mPos;
656  mParent=other.mParent;
657  return *this;
658  }
659  // prefix
660  Iterator& operator++() { ++mPos; return *this; }
661  Iterator& operator--() { --mPos; return *this; }
662  // postfix
663  Iterator operator++(int) { Iterator tmp(*this); ++mPos; return tmp; }
664  Iterator operator--(int) { Iterator tmp(*this); --mPos; return tmp; }
665  // value access
666  ValueT& operator*() const { return (*mParent)[mPos]; }
667  ValueT* operator->() const { return &(this->operator*()); }
668  ValueT& operator[](const difference_type& pos) const { return (*mParent)[mPos+pos]; }
669  // offset
670  Iterator& operator+=(const difference_type& pos) { mPos += pos; return *this; }
671  Iterator& operator-=(const difference_type& pos) { mPos -= pos; return *this; }
672  Iterator operator+(const difference_type &pos) const { return Iterator(*mParent, mPos+pos); }
673  Iterator operator-(const difference_type &pos) const { return Iterator(*mParent, mPos-pos); }
674  difference_type operator-(const Iterator& other) const { return mPos - other.pos(); }
675  // comparisons
676  bool operator==(const Iterator& other) const { return mPos == other.mPos; }
677  bool operator!=(const Iterator& other) const { return mPos != other.mPos; }
678  bool operator>=(const Iterator& other) const { return mPos >= other.mPos; }
679  bool operator<=(const Iterator& other) const { return mPos <= other.mPos; }
680  bool operator< (const Iterator& other) const { return mPos < other.mPos; }
681  bool operator> (const Iterator& other) const { return mPos > other.mPos; }
682  // non-std methods
683  bool isValid() const { return mParent != nullptr && mPos < mParent->size(); }
684  size_t pos() const { return mPos; }
685  private:
686  size_t mPos;
687  PagedArray* mParent;
688 };// Public class PagedArray::Iterator
689 
690 ////////////////////////////////////////////////////////////////////////////////
691 
692 // Private member-class of PagedArray implementing a memory page
693 template <typename ValueT, size_t Log2PageSize>
694 class PagedArray<ValueT, Log2PageSize>::
695 Page
696 {
697 public:
698  static const size_t Size = 1UL << Log2PageSize;
699  static const size_t Mask = Size - 1UL;
700  static size_t memUsage() { return sizeof(ValueT)*Size; }
701  // Raw memory allocation without any initialization
702  Page() : mData(reinterpret_cast<ValueT*>(new char[sizeof(ValueT)*Size])) {}
703  ~Page() { delete [] mData; }
704  Page(const Page&) = delete;//copy construction is not implemented
705  Page& operator=(const Page&) = delete;//copy assignment is not implemented
706  ValueT& operator[](const size_t i) { return mData[i & Mask]; }
707  const ValueT& operator[](const size_t i) const { return mData[i & Mask]; }
708  void fill(const ValueT& v) {
709  ValueT* dst = mData;
710  for (size_t i=Size; i; --i) *dst++ = v;
711  }
712  ValueT* data() { return mData; }
713  // Copy the first n elements of this Page to dst (which is assumed to large
714  // enough to hold the n elements).
715  void copy(ValueType *dst, size_t n) const {
716  const ValueT* src = mData;
717  for (size_t i=n; i; --i) *dst++ = *src++;
718  }
719 protected:
720  ValueT* mData;
721 };// Private class PagedArray::Page
722 
723 ////////////////////////////////////////////////////////////////////////////////
724 
725 } // namespace util
726 } // namespace OPENVDB_VERSION_NAME
727 } // namespace openvdb
728 
729 #endif // OPENVDB_UTIL_PAGED_ARRAY_HAS_BEEN_INCLUDED
ValueT & operator[](const size_t i)
Definition: PagedArray.h:706
Iterator & operator=(const Iterator &other)
Definition: PagedArray.h:654
bool isValid() const
Definition: PagedArray.h:683
ConstIterator & operator=(const ConstIterator &other)
Definition: PagedArray.h:603
void copy(ValueType *dst, size_t n) const
Definition: PagedArray.h:715
#define OPENVDB_DEPRECATED
Definition: Platform.h:122
Index64 memUsage(const TreeT &tree, bool threaded=true)
Return the total amount of memory in bytes occupied by this tree.
Definition: Count.h:493
SharedPtr< PagedArray > Ptr
Definition: PagedArray.h:153
bool operator>=(const Iterator &other) const
Definition: PagedArray.h:678
ValueBuffer getBuffer()
Definition: PagedArray.h:179
Iterator(PagedArray &parent, size_t pos=0)
Definition: PagedArray.h:652
static size_t memUsage()
Definition: PagedArray.h:700
Iterator(const Iterator &other)
Definition: PagedArray.h:653
~ValueBuffer()
Destructor that transfers an buffered values to the parent PagedArray.
Definition: PagedArray.h:555
ConstIterator & operator--()
Definition: PagedArray.h:610
ConstIterator begin() const
Return a const iterator pointing to the first element.
Definition: PagedArray.h:385
Iterator operator+(const difference_type &pos) const
Definition: PagedArray.h:672
Iterator & operator+=(const difference_type &pos)
Definition: PagedArray.h:670
void clear()
Removes all elements from the array and delete all pages.
Definition: PagedArray.h:364
Definition: PagedArray.h:644
size_t pos() const
Definition: PagedArray.h:633
ConstIterator end() const
Return a const iterator pointing to the past-the-last element.
Definition: PagedArray.h:395
size_t pos() const
Definition: PagedArray.h:684
bool operator<=(const ConstIterator &other) const
Definition: PagedArray.h:628
static size_t log2PageSize()
Return log2 of the number of elements per memory page.
Definition: PagedArray.h:342
typename BaseT::difference_type difference_type
Definition: PagedArray.h:598
PagedArray()
Default constructor.
Definition: PagedArray.h:156
Iterator end()
Return a non-const iterator pointing to the past-the-last element.
Definition: PagedArray.h:380
static size_t pageSize()
Definition: PagedArray.h:581
Iterator operator++(int)
Definition: PagedArray.h:663
void sort(Functor func)
Parallel sort of all the elements based on a custom functor with the api:
Definition: PagedArray.h:410
Iterator & operator--()
Definition: PagedArray.h:661
ConstIterator & operator-=(const difference_type &pos)
Definition: PagedArray.h:620
void resize(size_t size, const ValueType &v)
Resize this array to the specified size and initialize all values to v.
Definition: PagedArray.h:318
const ValueT & operator[](const difference_type &pos) const
Definition: PagedArray.h:617
void resize(size_t size)
Resize this array to the specified size.
Definition: PagedArray.h:293
void sort()
Parallel sort of all the elements in ascending order.
Definition: PagedArray.h:399
ValueBuffer(const ValueBuffer &other)
Definition: PagedArray.h:553
bool isValid() const
Definition: PagedArray.h:632
bool operator>(const Tuple< SIZE, T0 > &t0, const Tuple< SIZE, T1 > &t1)
Definition: Tuple.h:212
const ValueType & operator[](size_t i) const
Return a const-reference to the value at the specified offset.
Definition: PagedArray.h:234
std::iterator< std::random_access_iterator_tag, ValueT > BaseT
Definition: PagedArray.h:648
bool operator==(const Iterator &other) const
Definition: PagedArray.h:676
size_t push_back(const ValueType &value)
This method is deprecated and will be removed shortly!
Definition: PagedArray.h:188
void fill(const ValueType &v)
Set all elements in the page table to the specified value.
Definition: PagedArray.h:245
bool operator!=(const ConstIterator &other) const
Definition: PagedArray.h:626
size_t size() const
Return the number of elements in this array.
Definition: PagedArray.h:325
typename BaseT::difference_type difference_type
Definition: PagedArray.h:649
bool operator<(const Tuple< SIZE, T0 > &t0, const Tuple< SIZE, T1 > &t1)
Definition: Tuple.h:200
Concurrent, page-based, dynamically-sized linear data structure with O(1) random access and STL-compl...
Definition: PagedArray.h:142
ConstIterator operator+(const difference_type &pos) const
Definition: PagedArray.h:621
bool operator==(const ConstIterator &other) const
Definition: PagedArray.h:625
ConstIterator cend() const
Return a const iterator pointing to the past-the-last element.
Definition: PagedArray.h:394
ConstIterator(const ConstIterator &other)
Definition: PagedArray.h:602
Definition: PagedArray.h:694
bool operator!=(const Iterator &other) const
Definition: PagedArray.h:677
difference_type operator-(const ConstIterator &other) const
Definition: PagedArray.h:623
std::iterator< std::random_access_iterator_tag, ValueT > BaseT
Definition: PagedArray.h:597
PagedArrayType & parent() const
Return a reference to the parent PagedArray.
Definition: PagedArray.h:578
ConstIterator operator--(int)
Definition: PagedArray.h:613
Definition: Exceptions.h:13
const std::enable_if<!VecTraits< T >::IsVec, T >::type & min(const T &a, const T &b)
Definition: Composite.h:103
const ValueT * operator->() const
Definition: PagedArray.h:616
ValueT value
Definition: GridBuilder.h:1287
void copy(ValueType *p) const
Definition: PagedArray.h:277
size_t push_back_unsafe(const ValueType &value)
Definition: PagedArray.h:198
size_t memUsage() const
Return the memory footprint of this array in bytes.
Definition: PagedArray.h:345
ConstIterator & operator++()
Definition: PagedArray.h:609
ValueT * operator->() const
Definition: PagedArray.h:667
Iterator operator--(int)
Definition: PagedArray.h:664
Iterator begin()
Return a non-const iterator pointing to the first element.
Definition: PagedArray.h:373
ConstIterator()
Definition: PagedArray.h:600
Iterator & operator-=(const difference_type &pos)
Definition: PagedArray.h:671
ValueBuffer(PagedArray &parent)
Constructor from a PageArray.
Definition: PagedArray.h:550
Iterator()
Definition: PagedArray.h:651
ConstIterator(const PagedArray &parent, size_t pos=0)
Definition: PagedArray.h:601
bool isEmpty() const
Return true if the container contains no elements.
Definition: PagedArray.h:351
size_t capacity() const
Return the maximum number of elements that this array can contain without allocating more memory page...
Definition: PagedArray.h:329
void push_back(const ValueT &v)
Add a value to the buffer and increment the size.
Definition: PagedArray.h:563
ValueT ValueType
Definition: PagedArray.h:152
~Page()
Definition: PagedArray.h:703
Page()
Definition: PagedArray.h:702
void invSort()
Parallel sort of all the elements in descending order.
Definition: PagedArray.h:402
Iterator operator-(const difference_type &pos) const
Definition: PagedArray.h:673
size_t size() const
Return the current number of elements cached in this buffer.
Definition: PagedArray.h:580
ValueT & operator*() const
Definition: PagedArray.h:666
ConstIterator cbegin() const
Return a const iterator pointing to the first element.
Definition: PagedArray.h:384
const ValueT & operator*() const
Definition: PagedArray.h:615
const ValueT & operator[](const size_t i) const
Definition: PagedArray.h:707
bool operator>=(const ConstIterator &other) const
Definition: PagedArray.h:627
void fill(const ValueT &v)
Definition: PagedArray.h:708
bool copy(ValueType *p, size_t count) const
Copy the first count values in this PageArray into a raw c-style array, assuming it to be at least co...
Definition: PagedArray.h:260
ValueT * mData
Definition: PagedArray.h:720
ValueType & operator[](size_t i)
Return a reference to the value at the specified offset.
Definition: PagedArray.h:221
difference_type operator-(const Iterator &other) const
Definition: PagedArray.h:674
static size_t pageSize()
Return the number of elements per memory page.
Definition: PagedArray.h:339
void flush()
Manually transfers the values in this buffer to the parent PagedArray.
Definition: PagedArray.h:572
std::shared_ptr< T > SharedPtr
Definition: Types.h:114
size_t freeCount() const
Return the number of additional elements that can be added to this array without allocating more memo...
Definition: PagedArray.h:333
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h.in:116
ConstIterator & operator+=(const difference_type &pos)
Definition: PagedArray.h:619
static Ptr create()
Return a shared pointer to a new instance of this class.
Definition: PagedArray.h:166
ValueT & operator[](const difference_type &pos) const
Definition: PagedArray.h:668
~PagedArray()
Destructor removed all allocated pages.
Definition: PagedArray.h:159
bool operator<=(const Iterator &other) const
Definition: PagedArray.h:679
ValueT * data()
Definition: PagedArray.h:712
bool isPartiallyFull() const
Return true if the page table is partially full, i.e. the last non-empty page contains less than page...
Definition: PagedArray.h:359
void print(std::ostream &os=std::cout) const
Print information for debugging.
Definition: PagedArray.h:423
ConstIterator operator++(int)
Definition: PagedArray.h:612
Iterator & operator++()
Definition: PagedArray.h:660
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h.in:202
Mat3< typename promote< T0, T1 >::type > operator*(const Mat3< T0 > &m0, const Mat3< T1 > &m1)
Multiply m0 by m1 and return the resulting matrix.
Definition: Mat3.h:611
size_t pageCount() const
Return the number of allocated memory pages.
Definition: PagedArray.h:336
ConstIterator operator-(const difference_type &pos) const
Definition: PagedArray.h:622