Field3D
SparseFile.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <vector>
50 #include <list>
51 
52 #include <hdf5.h>
53 
54 #include "Exception.h"
55 #include "Hdf5Util.h"
56 #include "SparseDataReader.h"
57 #include "Traits.h"
58 
59 //----------------------------------------------------------------------------//
60 
61 #include "ns.h"
62 
64 
65 //----------------------------------------------------------------------------//
66 // Forward declarations
67 //----------------------------------------------------------------------------//
68 
69 namespace Sparse {
70 
71  template <typename Data_T>
72  struct SparseBlock;
73 
74 }
75 
76 template <typename Data_T>
77 class SparseField;
78 
79 //----------------------------------------------------------------------------//
80 
81 namespace SparseFile {
82 
83 //----------------------------------------------------------------------------//
84 // Reference
85 //----------------------------------------------------------------------------//
86 
92 //----------------------------------------------------------------------------//
93 
94 template <class Data_T>
95 class Reference
96 {
97 public:
98 
99  // Typedefs ------------------------------------------------------------------
100 
101  typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
102 
103  // Public data members -------------------------------------------------------
104 
105  std::string filename;
106  std::string layerPath;
109 
111  std::vector<int> fileBlockIndices;
114  std::vector<int> blockLoaded;
117  BlockPtrs blocks;
121  std::vector<bool> blockUsed;
124  std::vector<int> loadCounts;
128  std::vector<int> refCounts;
132  boost::mutex *blockMutex;
133 
134  // Ctors, dtor ---------------------------------------------------------------
135 
137  Reference(const std::string filename, const std::string layerPath);
138  ~Reference();
139 
141  Reference(const Reference &o);
142 
144  Reference & operator=(const Reference &o);
145 
146  // Main methods --------------------------------------------------------------
147 
149  bool fileIsOpen();
151  void setNumBlocks(int numBlocks);
155  void openFile();
159  void loadBlock(int blockIdx);
161  void unloadBlock(int blockIdx);
164  void incBlockRef(int blockIdx);
166  void decBlockRef(int blockIdx);
168  int blockSize(int blockIdx) const;
171  int totalLoads() const;
174  int numLoadedBlocks() const;
177  int totalLoadedBlocks() const;
180  float averageLoads() const;
182  void resetCacheStatistics();
183 
184 private:
185 
188 
192 
196 
198  boost::mutex m_mutex;
199 
200 };
201 
202 //----------------------------------------------------------------------------//
203 // References
204 //----------------------------------------------------------------------------//
205 
207 {
208 public:
209 
210  // Main methods --------------------------------------------------------------
211 
214  template <class Data_T>
215  Reference<Data_T>& ref(int idx);
216 
219  template <class Data_T>
220  int append(const Reference<Data_T>& ref);
221 
223  template <class Data_T>
224  int numRefs() const;
225 
226 private:
227 
228  // Data members --------------------------------------------------------------
229 
230  std::vector<Reference<half> > m_hRefs;
231  std::vector<Reference<V3h> > m_vhRefs;
232  std::vector<Reference<float> > m_fRefs;
233  std::vector<Reference<V3f> > m_vfRefs;
234  std::vector<Reference<double> > m_dRefs;
235  std::vector<Reference<V3d> > m_vdRefs;
236 
237 };
238 
239 //----------------------------------------------------------------------------//
240 
241 class CacheBlock {
242 public:
244  int refIdx;
245  int blockIdx;
246  CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
247  blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
248  { }
249 };
250 
251 //----------------------------------------------------------------------------//
252 
253 } // namespace SparseFile
254 
255 //----------------------------------------------------------------------------//
256 // SparseFileManager
257 //----------------------------------------------------------------------------//
258 
314 //----------------------------------------------------------------------------//
315 
317 {
318 
319 public:
320 
321  template <class Data_T>
322  friend class SparseField;
323 
324  // typedefs ------------------------------------------------------------------
325 
326  typedef std::list<SparseFile::CacheBlock> CacheList;
327 
328  // Main methods --------------------------------------------------------------
329 
331  static SparseFileManager &singleton();
332 
335  void setLimitMemUse(bool enabled);
336 
339  bool doLimitMemUse() const;
340 
342  void setMaxMemUse(float maxMemUse);
343 
346  void flushCache();
347 
349  long long totalLoads();
350 
352  long long numLoadedBlocks();
353 
355  long long totalLoadedBlocks();
356 
359  float cacheFractionLoaded();
360 
362  float cacheLoadsPerBlock();
363 
366  float cacheEfficiency();
367 
369  void resetCacheStatistics();
370 
371  //--------------------------------------------------------------------------//
372  // Utility functions
373 
378  template <class Data_T>
379  void incBlockRef(int fileId, int blockIdx);
380 
385  template <class Data_T>
386  void decBlockRef(int fileId, int blockIdx);
387 
391  template <class Data_T>
392  void activateBlock(int fileId, int blockIdx);
393 
394 protected:
395 
397  template <class Data_T>
399 
402  template <class Data_T>
403  int getNextId(const std::string filename, const std::string layerPath);
404 
405  template <class Data_T>
406  void removeFieldFromCache(int refIdx);
407 
408 private:
409 
412 
415 
417  void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
418 
421  void deallocateBlocks(int bytesNeeded);
422 
425  template <class Data_T>
427 
429  template <class Data_T>
430  void deallocateBlock(CacheList::iterator &it);
431 
433  float m_maxMemUse;
434 
437 
439  int m_memUse;
440 
444 
448 
454  CacheList m_blockCacheList;
455 
458  CacheList::iterator m_nextBlock;
459 
462  boost::mutex m_mutex;
463 
464 };
465 
466 //----------------------------------------------------------------------------//
467 // Reference implementations
468 //----------------------------------------------------------------------------//
469 
470 namespace SparseFile {
471 
472 //----------------------------------------------------------------------------//
473 
474 template <class Data_T>
475 Reference<Data_T>::Reference(const std::string a_filename,
476  const std::string a_layerPath)
477  : filename(a_filename), layerPath(a_layerPath),
478  valuesPerBlock(-1), occupiedBlocks(-1),
479  blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) {
480  /* Empty */
481 }
482 
483 //----------------------------------------------------------------------------//
484 
485 template <class Data_T>
487 {
488  if (m_reader)
489  delete m_reader;
490 
491  if (blockMutex)
492  delete [] blockMutex;
493 }
494 
495 //----------------------------------------------------------------------------//
496 
497 template <class Data_T>
499 {
500  m_reader = NULL;
501  blockMutex = NULL;
502  *this = o;
503 }
504 
505 //----------------------------------------------------------------------------//
506 
507 template <class Data_T>
510 {
511  // Copy public member variables (where appropriate)
512  filename = o.filename;
513  layerPath = o.layerPath;
514  valuesPerBlock = o.valuesPerBlock;
515  occupiedBlocks = o.occupiedBlocks;
516  fileBlockIndices = o.fileBlockIndices;
517  blockLoaded = o.blockLoaded;
518  blocks = o.blocks;
519  blockUsed = o.blockUsed;
520  loadCounts = o.loadCounts;
521  refCounts = o.refCounts;
522  if (blockMutex)
523  delete[] blockMutex;
524  blockMutex = new boost::mutex[blocks.size()];
525 
526  // Copy private member variables (where appropriate)
527  m_fileHandle = o.m_fileHandle;
528  // Don't copy id, let hdf5 generate a new one.
529  if (m_fileHandle >= 0) {
530  m_layerGroup.open(m_fileHandle, layerPath.c_str());
531  }
532 
533  if (m_reader)
534  delete m_reader;
535  m_reader = NULL;
536 
537  return *this;
538 }
539 
540 //----------------------------------------------------------------------------//
541 
542 template <class Data_T>
544 {
545  return m_fileHandle >= 0;
546 }
547 
548 //----------------------------------------------------------------------------//
549 
550 template <class Data_T>
552 {
553  boost::mutex::scoped_lock lock(m_mutex);
554 
555  fileBlockIndices.resize(numBlocks);
556  blockLoaded.resize(numBlocks, 0);
557  blocks.resize(numBlocks, 0);
558  blockUsed.resize(numBlocks, false);
559  loadCounts.resize(numBlocks, 0);
560  refCounts.resize(numBlocks, 0);
561  if (blockMutex)
562  delete[] blockMutex;
563  blockMutex = new boost::mutex[numBlocks];
564 }
565 
566 //----------------------------------------------------------------------------//
567 
568 template <class Data_T>
570 {
571  using namespace Exc;
572  using namespace Hdf5Util;
573 
574  boost::mutex::scoped_lock lock_A(m_mutex);
575 
576  // check that the file wasn't already opened before obtaining the lock
577  if (fileIsOpen()) {
578  return;
579  }
580 
581  m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
582  if (m_fileHandle < 0)
583  throw NoSuchFileException(filename);
584 
585  m_layerGroup.open(m_fileHandle, layerPath.c_str());
586  if (m_layerGroup.id() < 0) {
587  Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
588  "Couldn't find layer group " + layerPath +
589  " in .f3d file ");
590  throw FileIntegrityException(filename);
591  }
592 
593  m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(), valuesPerBlock,
594  occupiedBlocks);
595 }
596 
597 //----------------------------------------------------------------------------//
598 
599 template <class Data_T>
601 {
602  boost::mutex::scoped_lock lock(m_mutex);
603 
604  // Allocate the block
605  blocks[blockIdx]->resize(valuesPerBlock);
606  assert(blocks[blockIdx]->data.size() > 0);
607  // Read the data
608  assert(m_reader);
609  m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef());
610  // Mark block as loaded
611  blockLoaded[blockIdx] = 1;
612 }
613 
614 //----------------------------------------------------------------------------//
615 
616 template <class Data_T>
618 {
619  // Deallocate the block
620  blocks[blockIdx]->clear();
621 
622  // Mark block as unloaded
623  blockLoaded[blockIdx] = 0;
624 }
625 
626 //----------------------------------------------------------------------------//
627 
628 template <class Data_T>
630 {
631  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
632  ++refCounts[blockIdx];
633 }
634 
635 //----------------------------------------------------------------------------//
636 
637 template <class Data_T>
639 {
640  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
641  --refCounts[blockIdx];
642 }
643 
644 //----------------------------------------------------------------------------//
645 
646 template <class Data_T>
647 int Reference<Data_T>::blockSize(int /* blockIdx */) const
648 {
649  return valuesPerBlock * sizeof(Data_T);
650 }
651 
652 //----------------------------------------------------------------------------//
653 
654 template <class Data_T>
656 {
657  std::vector<int>::const_iterator i = loadCounts.begin();
658  std::vector<int>::const_iterator end = loadCounts.end();
659  int numLoads = 0;
660  for (; i != end; ++i)
661  numLoads += *i;
662 
663  return numLoads;
664 }
665 
666 //----------------------------------------------------------------------------//
667 
668 template <class Data_T>
670 {
671  std::vector<int>::const_iterator i = blockLoaded.begin();
672  std::vector<int>::const_iterator end = blockLoaded.end();
673  int numBlocks = 0;
674  for (; i != end; ++i)
675  if (*i)
676  numBlocks++;
677 
678  return numBlocks;
679 }
680 
681 //----------------------------------------------------------------------------//
682 
683 template <class Data_T>
685 {
686  std::vector<int>::const_iterator i = loadCounts.begin();
687  std::vector<int>::const_iterator li = blockLoaded.begin();
688  std::vector<int>::const_iterator end = loadCounts.end();
689  int numBlocks = 0;
690 
691  if (blockLoaded.size() == 0) {
692  for (; i != end; ++i)
693  if (*i)
694  numBlocks++;
695  } else {
696  assert(loadCounts.size() == blockLoaded.size());
697 
698  for (; i != end; ++i, ++li)
699  if (*i || *li)
700  numBlocks++;
701  }
702 
703  return numBlocks;
704 }
705 
706 //----------------------------------------------------------------------------//
707 
708 template <class Data_T>
710 {
711  std::vector<int>::const_iterator i = loadCounts.begin();
712  std::vector<int>::const_iterator end = loadCounts.end();
713  int numLoads = 0, numBlocks = 0;
714  for (; i != end; ++i) {
715  if (*i) {
716  numLoads += *i;
717  numBlocks++;
718  }
719  }
720 
721  return (float)numLoads / std::max(1, numBlocks);
722 }
723 
724 //----------------------------------------------------------------------------//
725 
726 template <class Data_T>
728 {
729  std::vector<int>::iterator li = loadCounts.begin();
730  std::vector<int>::iterator lend = loadCounts.end();
731  for (; li != lend; ++li)
732  *li = 0;
733 }
734 
735 //----------------------------------------------------------------------------//
736 
737 } // namespace SparseFile
738 
739 //----------------------------------------------------------------------------//
740 // Specializations for FileReferences
741 //----------------------------------------------------------------------------//
742 
743 namespace SparseFile {
744 
745 //----------------------------------------------------------------------------//
746 
747 template <>
748 inline Reference<half>&
750 {
751  return m_hRefs[idx];
752 }
753 
754 //----------------------------------------------------------------------------//
755 
756 template <>
757 inline Reference<V3h>&
759 {
760  return m_vhRefs[idx];
761 }
762 
763 //----------------------------------------------------------------------------//
764 
765 template <>
766 inline Reference<float>&
768 {
769  return m_fRefs[idx];
770 }
771 
772 //----------------------------------------------------------------------------//
773 
774 template <>
775 inline Reference<V3f>&
777 {
778  return m_vfRefs[idx];
779 }
780 
781 //----------------------------------------------------------------------------//
782 
783 template <>
784 inline Reference<double>&
786 {
787  return m_dRefs[idx];
788 }
789 
790 //----------------------------------------------------------------------------//
791 
792 template <>
793 inline Reference<V3d>&
795 {
796  return m_vdRefs[idx];
797 }
798 
799 //----------------------------------------------------------------------------//
800 
801 template <>
803 {
804  m_hRefs.push_back(ref);
805  return m_hRefs.size() - 1;
806 }
807 
808 //----------------------------------------------------------------------------//
809 
810 template <>
812 {
813  m_vhRefs.push_back(ref);
814  return m_vhRefs.size() - 1;
815 }
816 
817 //----------------------------------------------------------------------------//
818 
819 template <>
821 {
822  m_fRefs.push_back(ref);
823  return m_fRefs.size() - 1;
824 }
825 
826 //----------------------------------------------------------------------------//
827 
828 template <>
830 {
831  m_vfRefs.push_back(ref);
832  return m_vfRefs.size() - 1;
833 }
834 
835 //----------------------------------------------------------------------------//
836 
837 template <>
839 {
840  m_dRefs.push_back(ref);
841  return m_dRefs.size() - 1;
842 }
843 
844 //----------------------------------------------------------------------------//
845 
846 template <>
848 {
849  m_vdRefs.push_back(ref);
850  return m_vdRefs.size() - 1;
851 }
852 
853 //----------------------------------------------------------------------------//
854 
855 template <>
856 inline int FileReferences::numRefs<half>() const
857 {
858  return m_hRefs.size();
859 }
860 
861 //----------------------------------------------------------------------------//
862 
863 template <>
864 inline int FileReferences::numRefs<V3h>() const
865 {
866  return m_vhRefs.size();
867 }
868 
869 //----------------------------------------------------------------------------//
870 
871 template <>
872 inline int FileReferences::numRefs<float>() const
873 {
874  return m_fRefs.size();
875 }
876 
877 //----------------------------------------------------------------------------//
878 
879 template <>
880 inline int FileReferences::numRefs<V3f>() const
881 {
882  return m_vfRefs.size();
883 }
884 
885 //----------------------------------------------------------------------------//
886 
887 template <>
888 inline int FileReferences::numRefs<double>() const
889 {
890  return m_dRefs.size();
891 }
892 
893 //----------------------------------------------------------------------------//
894 
895 template <>
896 inline int FileReferences::numRefs<V3d>() const
897 {
898  return m_vdRefs.size();
899 }
900 
901 //----------------------------------------------------------------------------//
902 // Implementations for FileReferences
903 //----------------------------------------------------------------------------//
904 
905 template <class Data_T>
907 {
908  assert(false && "Do not use memory limiting on sparse fields that aren't "
909  "simple scalars or vectors!");
911  "FileReferences::ref(): Do not use memory limiting on sparse "
912  "fields that aren't simple scalars or vectors!");
913  static Reference<Data_T> dummy("", "");
914  return dummy;
915 }
916 
917 //----------------------------------------------------------------------------//
918 
919 template <class Data_T>
921 {
922  assert(false && "Do not use memory limiting on sparse fields that aren't "
923  "simple scalars or vectors!");
925  "FileReferences::append(): Do not use memory limiting on sparse "
926  "fields that aren't simple scalars or vectors!");
927  return -1;
928 }
929 
930 //----------------------------------------------------------------------------//
931 
932 template <class Data_T>
934 {
935  assert(false && "Do not use memory limiting on sparse fields that aren't "
936  "simple scalars or vectors!");
938  "FileReferences::numRefs(): "
939  "Do not use memory limiting on sparse "
940  "fields that aren't "
941  "simple scalars or vectors!");
942  return -1;
943 }
944 
945 //----------------------------------------------------------------------------//
946 
947 } // namespace SparseFile
948 
949 //----------------------------------------------------------------------------//
950 // SparseFileManager implementations
951 //----------------------------------------------------------------------------//
952 
953 template <class Data_T>
954 int
955 SparseFileManager::getNextId(const std::string filename,
956  const std::string layerPath)
957 {
958  using namespace SparseFile;
959 
960  int id = m_fileData.append(Reference<Data_T>(filename, layerPath));
961  return id;
962 }
963 
964 //----------------------------------------------------------------------------//
965 
966 template <class Data_T>
967 void
969 {
970  boost::mutex::scoped_lock lock(m_mutex);
971 
974 
975  CacheList::iterator it = m_blockCacheList.begin();
976  CacheList::iterator end = m_blockCacheList.end();
977  CacheList::iterator next;
978 
979  int bytesFreed = 0;
980 
981  while (it != end) {
982  if (it->blockType == blockType && it->refIdx == refIdx) {
983  if (it == m_nextBlock) {
984  ++m_nextBlock;
985  }
986  next = it;
987  ++next;
988  bytesFreed += reference.blockSize(it->blockIdx);
989  m_blockCacheList.erase(it);
990  it = next;
991  } else {
992  ++it;
993  }
994  }
995  m_memUse -= bytesFreed;
996 
997  // reset the block indices to -1, to ensure that the cache manager
998  // won't try to activate a block
999  reference.fileBlockIndices.clear();
1000  reference.fileBlockIndices.resize(reference.blocks.size(), -1);
1001  // clear the reference's pointers into the field, and relevant
1002  reference.blocks.clear();
1003  reference.blockLoaded.clear();
1004  reference.blockUsed.clear();
1005 }
1006 
1007 //----------------------------------------------------------------------------//
1008 
1009 template <class Data_T>
1012 {
1013  return m_fileData.ref<Data_T>(index);
1014 }
1015 
1016 //----------------------------------------------------------------------------//
1017 
1018 template <class Data_T>
1019 void
1020 SparseFileManager::activateBlock(int fileId, int blockIdx)
1021 {
1023 
1024  if (reference.fileBlockIndices[blockIdx] >= 0) {
1025  if (!reference.blockLoaded[blockIdx]) {
1026  int blockSize = reference.blockSize(blockIdx);
1027  if (m_limitMemUse) {
1028  // if we already have enough free memory, deallocateBlocks()
1029  // will just return
1030  deallocateBlocks(blockSize);
1031  }
1032 
1033  if (!reference.fileIsOpen()) {
1034  reference.openFile();
1035  }
1036 
1037  boost::mutex::scoped_lock lock_A(m_mutex);
1038  boost::mutex::scoped_lock lock_B(reference.blockMutex[blockIdx]);
1039  // check to see if it was loaded between when the function
1040  // started and we got the lock on the block
1041  if (!reference.blockLoaded[blockIdx]) {
1042  reference.loadBlock(blockIdx);
1043  reference.loadCounts[blockIdx]++;
1045  m_memUse += blockSize;
1046  }
1047  }
1048  }
1049  reference.blockUsed[blockIdx] = true;
1050 }
1051 
1052 //----------------------------------------------------------------------------//
1053 
1054 template <class Data_T>
1055 void
1056 SparseFileManager::incBlockRef(int fileId, int blockIdx)
1057 {
1059 
1060  if (reference.fileBlockIndices[blockIdx] >= 0) {
1061  reference.incBlockRef(blockIdx);
1062  }
1063 }
1064 
1065 //----------------------------------------------------------------------------//
1066 
1067 template <class Data_T>
1068 void
1069 SparseFileManager::decBlockRef(int fileId, int blockIdx)
1070 {
1072 
1073  if (reference.fileBlockIndices[blockIdx] >= 0) {
1074  reference.decBlockRef(blockIdx);
1075  }
1076 }
1077 
1078 //----------------------------------------------------------------------------//
1079 
1081 
1082 //----------------------------------------------------------------------------//
1083 
1084 #endif
void removeFieldFromCache(int refIdx)
Definition: SparseFile.h:968
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
long long totalLoads()
Returns the total number of block loads in the cache.
Definition: SparseFile.cpp:284
void resetCacheStatistics()
Resets block load.
Definition: SparseFile.cpp:404
Contains utility functions and classes for Hdf5 files.
Definition: Hdf5Util.h:76
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition: SparseFile.h:187
int deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
Definition: SparseFile.cpp:96
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Definition: SparseFile.h:454
Namespace for Exception objects.
Definition: Exception.h:57
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition: SparseFile.h:647
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user...
Definition: SparseFile.h:1020
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector is weird.
Definition: SparseFile.h:114
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
Definition: SparseFile.h:1056
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition: SparseFile.h:629
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads...
Definition: SparseFile.cpp:397
int m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:439
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition: SparseFile.h:121
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition: SparseFile.h:246
std::vector< Sparse::SparseBlock< Data_T > * > BlockPtrs
Definition: SparseFile.h:101
void deallocateBlocks(int bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks...
Definition: SparseFile.cpp:152
Reference< Data_T > & ref(int idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
Definition: SparseFile.h:906
void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
Definition: Log.cpp:62
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
Definition: SparseFile.cpp:350
std::vector< int > fileBlockIndices
Index in file for each block.
Definition: SparseFile.h:111
int m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:436
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
Definition: SparseFile.h:551
std::string layerPath
Definition: SparseFile.h:106
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition: SparseFile.h:638
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Definition: SparseFile.h:128
Contains various utility functions for Hdf5.
std::vector< Reference< double > > m_dRefs
Definition: SparseFile.h:234
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition: SparseFile.h:569
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:433
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition: SparseFile.h:195
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:462
std::vector< Reference< V3f > > m_vfRefs
Definition: SparseFile.h:233
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:443
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition: SparseFile.h:709
std::vector< Reference< V3h > > m_vhRefs
Definition: SparseFile.h:231
static SparseFileManager * ms_singleton
Pointer to singleton.
Definition: SparseFile.h:414
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:73
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
std::vector< Reference< half > > m_hRefs
Definition: SparseFile.h:230
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
Definition: SparseFile.cpp:383
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
Definition: SparseFile.h:117
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
Definition: SparseFile.cpp:317
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:458
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition: SparseFile.h:617
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:447
Namespace for sparse field specifics.
Definition: SparseField.h:79
std::list< SparseFile::CacheBlock > CacheList
Definition: SparseFile.h:326
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Definition: SparseFile.h:684
Contains functions controlling the loading of sparse fields.
boost::mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition: SparseFile.h:198
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition: SparseFile.h:655
void resetCacheStatistics()
Resets counts of total block loads.
Definition: SparseFile.h:727
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition: SparseFile.h:191
SparseFile::Reference< Data_T > & reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1011
std::vector< Reference< float > > m_fRefs
Definition: SparseFile.h:232
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Definition: SparseFile.h:955
int numRefs() const
Returns the number of file references of the corresponding collection.
Definition: SparseFile.h:933
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition: SparseFile.cpp:252
SparseFileManager()
Private to prevent instantiation.
Definition: SparseFile.cpp:274
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:80
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
Definition: SparseFile.h:1069
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition: SparseFile.h:600
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition: SparseFile.h:509
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
Definition: SparseFile.cpp:215
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Definition: SparseFile.cpp:390
std::string filename
Definition: SparseFile.h:105
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition: SparseFile.h:475
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition: SparseFile.h:543
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition: SparseFile.h:124
DataTypeEnum blockType
Definition: SparseFile.h:243
This Field subclass stores voxel data in block-allocated arrays.
Definition: SparseField.h:160
DataTypeEnum
Definition: Traits.h:66
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition: SparseFile.cpp:63
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Definition: SparseFile.cpp:87
Scoped object - opens a group on creation and closes it on destruction.
Definition: Hdf5Util.h:176
Contains Exception base class.
std::vector< Reference< V3d > > m_vdRefs
Definition: SparseFile.h:235
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition: SparseFile.h:132
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition: SparseFile.h:669
int append(const Reference< Data_T > &ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
Definition: SparseFile.h:920