44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
49 #include <boost/scoped_ptr.hpp>
64 #define F3D_SHORT_MUTEX_ARRAY 1
65 #define F3D_MUTEX_ARRAY_SIZE 1000
66 #define F3D_NO_BLOCKS_ARRAY 1
78 template <
typename Data_T>
83 template <
typename Data_T>
86 template <
typename Data_T>
104 template <
class Data_T>
114 typedef boost::shared_ptr<Reference>
Ptr;
119 #if F3D_NO_BLOCKS_ARRAY
122 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
157 #if F3D_SHORT_MUTEX_ARRAY
171 static Ptr
create(
const std::string a_filename,
172 const std::string a_layerPath);
220 Reference(
const std::string filename,
const std::string layerPath);
286 template <
class Data_T>
291 template <
class Data_T>
295 template <
class Data_T>
328 blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
402 template <
class Data_T>
416 void setLimitMemUse(
bool enabled);
420 bool doLimitMemUse()
const;
423 void setMaxMemUse(
float maxMemUse);
430 long long totalLoads();
433 long long numLoadedBlocks();
436 long long totalLoadedBlocks();
440 float cacheFractionLoaded();
443 float cacheLoadsPerBlock();
447 float cacheEfficiency();
450 void resetCacheStatistics();
462 template <
class Data_T>
469 template <
class Data_T>
475 template <
class Data_T>
481 template <
class Data_T>
486 template <
class Data_T>
487 int getNextId(
const std::string filename,
const std::string layerPath);
489 template <
class Data_T>
490 void removeFieldFromCache(
int refIdx);
501 void addBlockToCache(
DataTypeEnum blockType,
int fileId,
int blockIdx);
505 void deallocateBlocks(int64_t bytesNeeded);
509 template <
class Data_T>
513 template <
class Data_T>
514 void deallocateBlock(CacheList::iterator &it);
558 template <
class Data_T>
560 const std::string a_layerPath)
561 : filename(a_filename), layerPath(a_layerPath),
562 valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
563 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
571 template <
class Data_T>
581 delete [] blockMutex;
586 template <
class Data_T>
589 m_ogReaderPtr.reset();
598 template <
class Data_T>
620 #if F3D_SHORT_MUTEX_ARRAY
621 # if F3D_NO_BLOCKS_ARRAY
624 blockMutexSize =
std::min(static_cast<int>(blocks.size()),
627 blockMutex =
new boost::mutex[blockMutexSize];
629 blockMutex =
new boost::mutex[blocks.size()];
638 if (m_fileHandle >= 0) {
639 m_layerGroup.open(m_fileHandle, layerPath.c_str());
650 m_ogReaderPtr.reset();
658 template <
class Data_T>
661 const std::string a_layerPath)
668 template <
class Data_T>
671 return m_fileHandle >= 0;
676 template <
class Data_T>
679 boost::mutex::scoped_lock lock(m_mutex);
682 numBlocks = a_numBlocks;
684 fileBlockIndices.resize(numBlocks);
685 blockLoaded.resize(numBlocks, 0);
686 #if !F3D_NO_BLOCKS_ARRAY
687 blocks.resize(numBlocks, 0);
689 blockUsed.resize(numBlocks,
false);
690 loadCounts.resize(numBlocks, 0);
691 refCounts.resize(numBlocks, 0);
694 #if F3D_SHORT_MUTEX_ARRAY
695 # if F3D_NO_BLOCKS_ARRAY
698 blockMutexSize =
std::min(blocks.size(),
701 blockMutex =
new boost::mutex[blockMutexSize];
703 blockMutex =
new boost::mutex[numBlocks];
709 template <
class Data_T>
712 if (m_fileHandle >= 0) {
713 if (H5Fclose(m_fileHandle) < 0) {
714 Msg::print(
"In ~Reference(): Error closing file " + filename);
721 template <
class Data_T>
725 #if F3D_NO_BLOCKS_ARRAY
726 blocks[blockIdx].clear();
728 blocks[blockIdx]->clear();
731 blockLoaded[blockIdx] = 0;
736 if (m_numActiveBlocks == 0) {
744 template <
class Data_T>
747 #if F3D_SHORT_MUTEX_ARRAY
748 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
750 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
752 ++refCounts[blockIdx];
757 template <
class Data_T>
760 #if F3D_SHORT_MUTEX_ARRAY
761 boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
763 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
765 --refCounts[blockIdx];
770 template <
class Data_T>
773 return numVoxels *
sizeof(Data_T);
778 template <
class Data_T>
781 std::vector<int>::const_iterator i = loadCounts.begin();
782 std::vector<int>::const_iterator end = loadCounts.end();
784 for (; i != end; ++i)
792 template <
class Data_T>
795 std::vector<int>::const_iterator i = blockLoaded.begin();
796 std::vector<int>::const_iterator end = blockLoaded.end();
797 int numBlockCounter = 0;
798 for (; i != end; ++i)
802 return numBlockCounter;
807 template <
class Data_T>
810 std::vector<int>::const_iterator i = loadCounts.begin();
811 std::vector<int>::const_iterator li = blockLoaded.begin();
812 std::vector<int>::const_iterator end = loadCounts.end();
813 int numBlockCounter = 0;
815 if (blockLoaded.size() == 0) {
816 for (; i != end; ++i)
820 assert(loadCounts.size() == blockLoaded.size());
822 for (; i != end; ++i, ++li)
827 return numBlockCounter;
832 template <
class Data_T>
835 std::vector<int>::const_iterator i = loadCounts.begin();
836 std::vector<int>::const_iterator end = loadCounts.end();
837 int numLoads = 0, numBlockCounter = 0;
838 for (; i != end; ++i) {
845 return (
float)numLoads /
std::max(1, numBlockCounter);
850 template <
class Data_T>
853 std::vector<int>::iterator li = loadCounts.begin();
854 std::vector<int>::iterator lend = loadCounts.end();
855 for (; li != lend; ++li)
861 template <
class Data_T>
865 boost::mutex::scoped_lock lock(m_mutex);
867 return sizeof(*this) +
868 fileBlockIndices.capacity() *
sizeof(int) +
869 blockLoaded.capacity() *
sizeof(int) +
873 blockUsed.capacity() *
sizeof(bool) +
874 loadCounts.capacity() *
sizeof(int) +
875 refCounts.capacity() *
sizeof(int) +
877 blockMutexSize *
sizeof(boost::mutex) +
879 numBlocks *
sizeof(boost::mutex) +
899 for (
size_t i = 0, end =
m_hRefs.size(); i < end; ++i) {
902 for (
size_t i = 0, end =
m_fRefs.size(); i < end; ++i) {
905 for (
size_t i = 0, end =
m_dRefs.size(); i < end; ++i) {
908 for (
size_t i = 0, end =
m_vhRefs.size(); i < end; ++i) {
911 for (
size_t i = 0, end =
m_vfRefs.size(); i < end; ++i) {
914 for (
size_t i = 0, end =
m_vdRefs.size(); i < end; ++i) {
1003 Mutex::scoped_lock lock(m_mutex);
1005 m_hRefs.push_back(ref);
1006 return m_hRefs.size() - 1;
1014 Mutex::scoped_lock lock(m_mutex);
1016 m_vhRefs.push_back(ref);
1017 return m_vhRefs.size() - 1;
1025 Mutex::scoped_lock lock(m_mutex);
1027 m_fRefs.push_back(ref);
1028 return m_fRefs.size() - 1;
1036 Mutex::scoped_lock lock(m_mutex);
1038 m_vfRefs.push_back(ref);
1039 return m_vfRefs.size() - 1;
1047 Mutex::scoped_lock lock(m_mutex);
1049 m_dRefs.push_back(ref);
1050 return m_dRefs.size() - 1;
1058 Mutex::scoped_lock lock(m_mutex);
1060 m_vdRefs.push_back(ref);
1061 return m_vdRefs.size() - 1;
1067 inline size_t FileReferences::numRefs<half>()
const
1069 Mutex::scoped_lock lock(m_mutex);
1071 return m_hRefs.size();
1077 inline size_t FileReferences::numRefs<V3h>()
const
1079 Mutex::scoped_lock lock(m_mutex);
1081 return m_vhRefs.size();
1087 inline size_t FileReferences::numRefs<float>()
const
1089 Mutex::scoped_lock lock(m_mutex);
1091 return m_fRefs.size();
1097 inline size_t FileReferences::numRefs<V3f>()
const
1099 Mutex::scoped_lock lock(m_mutex);
1101 return m_vfRefs.size();
1107 inline size_t FileReferences::numRefs<double>()
const
1109 Mutex::scoped_lock lock(m_mutex);
1111 return m_dRefs.size();
1117 inline size_t FileReferences::numRefs<V3d>()
const
1119 Mutex::scoped_lock lock(m_mutex);
1121 return m_vdRefs.size();
1130 template <
class Data_T>
1133 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1134 "simple scalars or vectors!");
1136 "FileReferences::ref(): Do not use memory limiting on sparse "
1137 "fields that aren't simple scalars or vectors!");
1144 template <
class Data_T>
1147 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1148 "simple scalars or vectors!");
1150 "FileReferences::append(): Do not use memory limiting on sparse "
1151 "fields that aren't simple scalars or vectors!");
1157 template <
class Data_T>
1160 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
1161 "simple scalars or vectors!");
1163 "FileReferences::numRefs(): "
1164 "Do not use memory limiting on sparse "
1165 "fields that aren't "
1166 "simple scalars or vectors!");
1180 template <
class Data_T>
1183 const std::string layerPath)
1188 boost::mutex::scoped_lock lock(
m_mutex);
1197 template <
class Data_T>
1201 boost::mutex::scoped_lock lock(
m_mutex);
1208 CacheList::iterator next;
1210 int64_t bytesFreed = 0;
1213 if (it->blockType == blockType && it->refIdx == refIdx) {
1219 bytesFreed += reference->
blockSize(it->blockIdx);
1229 #if F3D_NO_BLOCKS_ARRAY
1234 BlockPtrs().swap(reference->
blocks);
1237 std::vector<bool>().swap(reference->
blockUsed);
1238 std::vector<int>().swap(reference->
loadCounts);
1239 std::vector<int>().swap(reference->
refCounts);
1246 template <
class Data_T>
1255 template <
class Data_T>
1263 int blockSize = reference->
blockSize(blockIdx);
1274 boost::mutex::scoped_lock lock_A(
m_mutex);
1275 #if F3D_SHORT_MUTEX_ARRAY
1276 boost::mutex::scoped_lock
1279 boost::mutex::scoped_lock lock_B(reference->
blockMutex[blockIdx]);
1296 template <
class Data_T>
1309 template <
class Data_T>
void removeFieldFromCache(int refIdx)
#define FIELD3D_NAMESPACE_HEADER_CLOSE
size_t m_numActiveBlocks
Number of currently active blocks.
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
IArchivePtr m_ogArchive
Ogawa archive.
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user...
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector is weird.
#define F3D_SHORT_MUTEX_ARRAY
Contains forward declarations for Ogawa classes.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
long long int memSize() const
Memory use for the Reference.
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
boost::shared_ptr< OgIGroup > OgIGroupPtr
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
std::vector< int > fileBlockIndices
Index in file for each block.
void activateBlock(const int blockId) const
Activates a given block.
std::deque< Reference< V3f >::Ptr > m_vfRefs
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Contains various utility functions for Hdf5.
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
std::deque< Reference< V3d >::Ptr > m_vdRefs
float m_maxMemUse
Max amount om memory to use in megabytes.
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
T max(const T a, const T2 b)
Max operation on mixed types.
int blockMutexSize
Size of the mutex array. Used as modulus base.
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
OgIGroupPtr m_ogRoot
Ogawa archive root.
Namespace for sparse field specifics.
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
std::list< SparseFile::CacheBlock > CacheList
std::deque< Reference< float >::Ptr > m_fRefs
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Sparse::SparseBlock< Data_T > * BlockPtrs
Contains functions controlling the loading of sparse fields.
int64_t m_memUse
Current amount of memory in use in bytes.
#define F3D_MUTEX_ARRAY_SIZE
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
void resetCacheStatistics()
Resets counts of total block loads.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
long long int memSize() const
Returns the memory use for the refs.
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
virtual long long int memSize() const
Returns the memory usage (in bytes)
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Storage for one individual block of a SparseField.
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
size_t numRefs() const
Returns the number of file references of the corresponding collection.
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks...
#define F3D_NO_BLOCKS_ARRAY
std::deque< Reference< double >::Ptr > m_dRefs
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
bool fileIsOpen()
Checks if the file used by this reference is open already.
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
This Field subclass stores voxel data in block-allocated arrays.
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
std::deque< Reference< V3h >::Ptr > m_vhRefs
T min(const T a, const T2 b)
Min operation on mixed types.
void closeFile()
Closes the file. Will be re-opened as needed.
Scoped object - opens a group on creation and closes it on destruction.
Contains Exception base class.
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
std::deque< Reference< half >::Ptr > m_hRefs