44 #include <boost/intrusive_ptr.hpp>
46 #include <boost/thread/thread.hpp>
47 #include <boost/thread/mutex.hpp>
50 #include "SparseFieldIO.h"
55 using namespace boost;
77 template <
typename Data_T>
78 struct ReadThreadingState
80 ReadThreadingState(
const OgIGroup &i_location,
82 const size_t i_numVoxels,
83 const size_t i_numBlocks,
84 const size_t i_numOccupiedBlocks,
85 const bool i_isCompressed,
86 const std::vector<size_t> &i_blockIdxToDatasetIdx)
87 : location(i_location),
89 numVoxels(i_numVoxels),
90 numBlocks(i_numBlocks),
91 numOccupiedBlocks(i_numOccupiedBlocks),
92 isCompressed(i_isCompressed),
93 blockIdxToDatasetIdx(i_blockIdxToDatasetIdx),
97 const OgIGroup &location;
99 const size_t numVoxels;
100 const size_t numBlocks;
101 const size_t numOccupiedBlocks;
102 const bool isCompressed;
103 const std::vector<size_t> &blockIdxToDatasetIdx;
104 size_t nextBlockToRead;
106 boost::mutex readMutex;
111 template <
typename Data_T>
115 ReadBlockOp(ReadThreadingState<Data_T> &state,
const size_t threadId)
119 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
120 const uLong cmpLenBound = compressBound(srcLen);
121 m_cache.resize(cmpLenBound);
125 m_state.numOccupiedBlocks,
126 m_state.isCompressed));
127 m_reader = m_readerPtr.get();
129 m_reader->setThreadId(threadId);
136 boost::mutex::scoped_lock lock(m_state.readMutex);
137 blockIdx = m_state.nextBlockToRead;
138 m_state.nextBlockToRead++;
141 while (blockIdx < m_state.numBlocks) {
142 if (m_state.blocks[blockIdx].isAllocated) {
143 const size_t datasetIdx = m_state.blockIdxToDatasetIdx[blockIdx];
144 m_reader->readBlock(datasetIdx, m_state.blocks[blockIdx].data);
148 boost::mutex::scoped_lock lock(m_state.readMutex);
149 blockIdx = m_state.nextBlockToRead;
150 m_state.nextBlockToRead++;
156 ReadThreadingState<Data_T> &m_state;
157 std::vector<uint8_t> m_cache;
158 boost::shared_ptr<OgSparseDataReader<Data_T> > m_readerPtr;
164 template <
typename Data_T>
165 struct ThreadingState
167 ThreadingState(OgOCDataset<Data_T> &i_data,
169 const size_t i_numVoxels,
170 const size_t i_numBlocks,
171 const std::vector<uint8_t> &i_isAllocated)
174 numVoxels(i_numVoxels),
175 numBlocks(i_numBlocks),
176 isAllocated(i_isAllocated),
177 nextBlockToCompress(0),
181 for (
size_t i = 0; i < numBlocks; ++i) {
182 if (blocks[i].isAllocated) {
183 nextBlockToCompress = i;
184 nextBlockToWrite = i;
189 nextBlockToCompress = numBlocks;
190 nextBlockToWrite = numBlocks;
193 OgOCDataset<Data_T> &data;
195 const size_t numVoxels;
196 const size_t numBlocks;
197 const std::vector<uint8_t> isAllocated;
198 size_t nextBlockToCompress;
199 size_t nextBlockToWrite;
201 boost::mutex compressMutex;
206 template <
typename Data_T>
210 WriteBlockOp(ThreadingState<Data_T> &state,
const size_t threadId)
211 : m_state(state), m_threadId(threadId)
213 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
214 const uLong cmpLenBound = compressBound(srcLen);
215 m_cache.resize(cmpLenBound);
223 boost::mutex::scoped_lock lock(m_state.compressMutex);
224 blockIdx = m_state.nextBlockToCompress++;
226 while (m_state.nextBlockToCompress < m_state.numBlocks) {
227 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
230 m_state.nextBlockToCompress++;
234 while (blockIdx < m_state.numBlocks) {
235 if (m_state.blocks[blockIdx].isAllocated) {
237 const uint8_t *srcData =
238 reinterpret_cast<const uint8_t *
>(m_state.blocks[blockIdx].data);
240 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
241 const uLong cmpLenBound = compressBound(srcLen);
242 uLong cmpLen = cmpLenBound;
244 const int status = compress2(&m_cache[0], &cmpLen,
245 srcData, srcLen, level);
247 if (status != Z_OK) {
248 std::cout <<
"ERROR: Couldn't compress in SparseFieldIO." << std::endl
249 <<
" Level: " << level << std::endl
250 <<
" Status: " << status << std::endl
251 <<
" srcLen: " << srcLen << std::endl
252 <<
" cmpLenBound: " << cmpLenBound << std::endl
253 <<
" cmpLen: " << cmpLen << std::endl;
257 while (m_state.nextBlockToWrite != blockIdx) {
259 boost::this_thread::sleep(boost::posix_time::microseconds(1));
262 m_state.data.addData(cmpLen, &m_cache[0]);
264 m_state.nextBlockToWrite++;
265 while (m_state.nextBlockToWrite < m_state.numBlocks){
267 if (m_state.blocks[m_state.nextBlockToWrite].isAllocated) {
270 m_state.nextBlockToWrite++;
275 boost::mutex::scoped_lock lock(m_state.compressMutex);
276 blockIdx = m_state.nextBlockToCompress++;
278 while (m_state.nextBlockToCompress < m_state.numBlocks) {
279 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
282 m_state.nextBlockToCompress++;
289 ThreadingState<Data_T> &m_state;
290 std::vector<uint8_t> m_cache;
291 const size_t m_threadId;
302 const int SparseFieldIO::k_versionNumber(1);
303 const std::string SparseFieldIO::k_versionAttrName(
"version");
304 const std::string SparseFieldIO::k_extentsStr(
"extents");
305 const std::string SparseFieldIO::k_extentsMinStr(
"extents_min");
306 const std::string SparseFieldIO::k_extentsMaxStr(
"extents_max");
307 const std::string SparseFieldIO::k_dataWindowStr(
"data_window");
308 const std::string SparseFieldIO::k_dataWindowMinStr(
"data_window_min");
309 const std::string SparseFieldIO::k_dataWindowMaxStr(
"data_window_max");
310 const std::string SparseFieldIO::k_componentsStr(
"components");
311 const std::string SparseFieldIO::k_dataStr(
"data");
312 const std::string SparseFieldIO::k_blockOrderStr(
"block_order");
313 const std::string SparseFieldIO::k_numBlocksStr(
"num_blocks");
314 const std::string SparseFieldIO::k_blockResStr(
"block_res");
315 const std::string SparseFieldIO::k_bitsPerComponentStr(
"bits_per_component");
316 const std::string SparseFieldIO::k_numOccupiedBlocksStr(
"num_occupied_blocks");
317 const std::string SparseFieldIO::k_isCompressed(
"data_is_compressed");
322 SparseFieldIO::read(hid_t layerGroup,
const std::string &filename,
323 const std::string &layerPath,
326 Box3i extents, dataW;
332 if (layerGroup == -1) {
338 if (!
readAttribute(layerGroup, k_versionAttrName, 1, version))
339 throw MissingAttributeException(
"Couldn't find attribute: " +
342 if (version != k_versionNumber)
343 throw UnsupportedVersionException(
"SparseField version not supported: " +
344 lexical_cast<std::string>(version));
346 if (!
readAttribute(layerGroup, k_extentsStr, 6, extents.min.x))
347 throw MissingAttributeException(
"Couldn't find attribute: " +
350 if (!
readAttribute(layerGroup, k_dataWindowStr, 6, dataW.min.x))
351 throw MissingAttributeException(
"Couldn't find attribute: " +
354 if (!
readAttribute(layerGroup, k_componentsStr, 1, components))
355 throw MissingAttributeException(
"Couldn't find attribute: " +
359 if (!
readAttribute(layerGroup, k_blockOrderStr, 1, blockOrder))
360 throw MissingAttributeException(
"Couldn't find attribute: " +
364 if (!
readAttribute(layerGroup, k_numBlocksStr, 1, numBlocks))
365 throw MissingAttributeException(
"Couldn't find attribute: " +
369 if (!
readAttribute(layerGroup, k_blockResStr, 3, blockRes.x))
370 throw MissingAttributeException(
"Couldn't find attribute: " +
375 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
376 if (numCalculatedBlocks != numBlocks)
377 throw FileIntegrityException(
"Incorrect block count in SparseFieldIO::read");
384 if (!
readAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks))
385 throw MissingAttributeException(
"Couldn't find attribute: " +
386 k_numOccupiedBlocksStr);
391 if (!
readAttribute(layerGroup, k_bitsPerComponentStr, 1, bits))
392 throw MissingAttributeException(
"Couldn't find attribute: " +
393 k_bitsPerComponentStr);
396 bool isFloat =
false;
397 bool isDouble =
false;
413 if (components == 1) {
416 field->setSize(extents, dataW);
417 field->setBlockOrder(blockOrder);
418 readData<half>(layerGroup, numBlocks, filename, layerPath, field);
422 field->setSize(extents, dataW);
423 field->setBlockOrder(blockOrder);
424 readData<float>(layerGroup, numBlocks, filename, layerPath, field);
428 field->setSize(extents, dataW);
429 field->setBlockOrder(blockOrder);
430 readData<double>(layerGroup, numBlocks, filename, layerPath, field);
433 }
else if (components == 3) {
436 field->setSize(extents, dataW);
437 field->setBlockOrder(blockOrder);
438 readData<V3h>(layerGroup, numBlocks, filename, layerPath, field);
442 field->setSize(extents, dataW);
443 field->setBlockOrder(blockOrder);
444 readData<V3f>(layerGroup, numBlocks, filename, layerPath, field);
448 field->setSize(extents, dataW);
449 field->setBlockOrder(blockOrder);
450 readData<V3d>(layerGroup, numBlocks, filename, layerPath, field);
461 SparseFieldIO::read(
const OgIGroup &layerGroup,
const std::string &filename,
462 const std::string &layerPath,
OgDataType typeEnum)
464 Box3i extents, dataW;
469 if (!layerGroup.isValid()) {
470 throw MissingGroupException(
"Invalid group in SparseFieldIO::read()");
476 layerGroup.findAttribute<
int>(k_versionAttrName);
477 if (!versionAttr.isValid()) {
478 throw MissingAttributeException(
"Couldn't find attribute: " +
481 const int version = versionAttr.value();
483 if (version != k_versionNumber) {
484 throw UnsupportedVersionException(
"SparseField version not supported: " +
485 lexical_cast<std::string>(version));
491 layerGroup.findAttribute<
veci32_t>(k_extentsMinStr);
493 layerGroup.findAttribute<
veci32_t>(k_extentsMaxStr);
494 if (!extMinAttr.isValid()) {
495 throw MissingAttributeException(
"Couldn't find attribute " +
498 if (!extMaxAttr.isValid()) {
499 throw MissingAttributeException(
"Couldn't find attribute " +
503 extents.min = extMinAttr.value();
504 extents.max = extMaxAttr.value();
509 layerGroup.findAttribute<
veci32_t>(k_dataWindowMinStr);
511 layerGroup.findAttribute<
veci32_t>(k_dataWindowMaxStr);
512 if (!dwMinAttr.isValid()) {
513 throw MissingAttributeException(
"Couldn't find attribute " +
516 if (!dwMaxAttr.isValid()) {
517 throw MissingAttributeException(
"Couldn't find attribute " +
521 dataW.min = dwMinAttr.value();
522 dataW.max = dwMaxAttr.value();
527 layerGroup.findAttribute<uint8_t>(k_componentsStr);
528 if (!numComponentsAttr.isValid()) {
529 throw MissingAttributeException(
"Couldn't find attribute " +
536 layerGroup.findAttribute<uint8_t>(k_blockOrderStr);
537 if (!blockOrderAttr.isValid()) {
538 throw MissingAttributeException(
"Couldn't find attribute: " +
541 blockOrder = blockOrderAttr.value();
546 layerGroup.findAttribute<uint32_t>(k_numBlocksStr);
547 if (!numBlocksAttr.isValid()) {
548 throw MissingAttributeException(
"Couldn't find attribute: " +
551 numBlocks = numBlocksAttr.value();
556 layerGroup.findAttribute<
veci32_t>(k_blockResStr);
557 if (!blockResAttr.isValid()) {
558 throw MissingAttributeException(
"Couldn't find attribute: " +
561 blockRes = blockResAttr.value();
565 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
566 if (numCalculatedBlocks != numBlocks) {
567 throw FileIntegrityException(
"Incorrect block count in "
568 "SparseFieldIO::read()");
574 layerGroup.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
575 if (!occupiedBlocksAttr.isValid()) {
576 throw MissingAttributeException(
"Couldn't find attribute: " +
577 k_numOccupiedBlocksStr);
583 layerGroup.findAttribute<uint8_t>(k_isCompressed);
584 if (!isCompressedAttr.isValid()) {
585 throw MissingAttributeException(
"Couldn't find attribute: " +
595 if (isCompressedAttr.value() == 0) {
596 typeOnDisk = layerGroup.datasetType(k_dataStr);
598 typeOnDisk = layerGroup.compressedDatasetType(k_dataStr);
601 if (typeEnum == typeOnDisk) {
603 result = readData<float16_t>(layerGroup, extents, dataW, blockOrder,
604 numBlocks, filename, layerPath);
606 result = readData<float32_t>(layerGroup, extents, dataW, blockOrder,
607 numBlocks, filename, layerPath);
609 result = readData<float64_t>(layerGroup, extents, dataW, blockOrder,
610 numBlocks, filename, layerPath);
612 result = readData<vec16_t>(layerGroup, extents, dataW, blockOrder,
613 numBlocks, filename, layerPath);
615 result = readData<vec32_t>(layerGroup, extents, dataW, blockOrder,
616 numBlocks, filename, layerPath);
618 result = readData<vec64_t>(layerGroup, extents, dataW, blockOrder,
619 numBlocks, filename, layerPath);
631 if (layerGroup == -1) {
638 1, k_versionNumber)) {
658 success = writeInternal<half>(layerGroup, halfField);
659 }
else if (floatField) {
660 success = writeInternal<float>(layerGroup, floatField);
661 }
else if (doubleField) {
662 success = writeInternal<double>(layerGroup, doubleField);
663 }
else if (vecHalfField) {
664 success = writeInternal<V3h>(layerGroup, vecHalfField);
665 }
else if (vecFloatField) {
666 success = writeInternal<V3f>(layerGroup, vecFloatField);
667 }
else if (vecDoubleField) {
668 success = writeInternal<V3d>(layerGroup, vecDoubleField);
670 throw WriteLayerException(
"SparseFieldIO::write does not support the given "
671 "SparseField template parameter");
703 success = writeInternal<float>(layerGroup, floatField);
705 else if (halfField) {
706 success = writeInternal<half>(layerGroup, halfField);
708 else if (doubleField) {
709 success = writeInternal<double>(layerGroup, doubleField);
711 else if (vecFloatField) {
712 success = writeInternal<V3f>(layerGroup, vecFloatField);
714 else if (vecHalfField) {
715 success = writeInternal<V3h>(layerGroup, vecHalfField);
717 else if (vecDoubleField) {
718 success = writeInternal<V3d>(layerGroup, vecDoubleField);
721 throw WriteLayerException(
"SparseFieldIO does not support the given "
722 "SparseField template parameter");
730 template <
class Data_T>
732 SparseFieldIO::readData(
const OgIGroup &location,
const Box3i &extents,
733 const Box3i &dataW,
const size_t blockOrder,
734 const size_t numBlocks,
const std::string &filename,
735 const std::string &layerPath)
742 result->setSize(extents, dataW);
743 result->setBlockOrder(blockOrder);
747 const size_t numVoxels = (1 << (result->m_blockOrder * 3));
748 const int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
753 location.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
754 if (!occupiedBlocksAttr.isValid()) {
755 throw MissingAttributeException(
"Couldn't find attribute: " +
756 k_numOccupiedBlocksStr);
758 const size_t occupiedBlocks = occupiedBlocksAttr.value();
762 if (dynamicLoading) {
765 result->addReference(filename, layerPath, valuesPerBlock, numVoxels,
774 std::vector<size_t> blockIdxToDatasetIdx(numBlocks);
778 vector<uint8_t> isAllocated(numBlocks);
780 location.findDataset<uint8_t>(
"block_is_allocated_data");
781 if (!isAllocatedData.isValid()) {
782 throw MissingGroupException(
"Couldn't find block_is_allocated_data: ");
784 isAllocatedData.getData(0, &isAllocated[0], OGAWA_THREAD);
786 for (
size_t i = 0, nextBlockOnDisk = 0; i < numBlocks; ++i) {
788 if (!dynamicLoading && isAllocated[i]) {
789 blocks[i].
resize(numVoxels);
791 blockIdxToDatasetIdx[i] = nextBlockOnDisk;
801 vector<Data_T> emptyValue(numBlocks);
803 location.findDataset<Data_T>(
"block_empty_value_data");
804 if (!emptyValueData.isValid()) {
805 throw MissingGroupException(
"Couldn't find block_empty_value_data: ");
807 emptyValueData.getData(0, &emptyValue[0], OGAWA_THREAD);
809 for (
size_t i = 0; i < numBlocks; ++i) {
818 location.findAttribute<uint8_t>(k_isCompressed);
819 const bool isCompressed = isCompressedAttr.value() != 0;
821 if (occupiedBlocks > 0) {
822 if (dynamicLoading) {
824 result->setupReferenceBlocks();
827 ReadThreadingState<Data_T> state(location, blocks, numVoxels, numBlocks,
828 occupiedBlocks, isCompressed,
829 blockIdxToDatasetIdx);
833 boost::thread_group threads;
834 for (
size_t i = 0; i < numThreads; ++i) {
835 threads.create_thread(ReadBlockOp<Data_T>(state, i));
849 template <
class Data_T>
850 bool SparseFieldIO::writeInternal(hid_t layerGroup,
862 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
867 { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
877 { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
879 if (!
writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
886 if (!
writeAttribute(layerGroup, k_componentsStr, 1, components)) {
895 if (!
writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
903 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
920 if (!
writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
931 vector<char> isAllocated(numBlocks);
932 for (
int i = 0; i < numBlocks; ++i) {
933 isAllocated[i] =
static_cast<char>(blocks[i].
isAllocated);
935 writeSimpleData<char>(layerGroup,
"block_is_allocated_data", isAllocated);
940 vector<Data_T> emptyValue(numBlocks);
941 for (
int i = 0; i < numBlocks; ++i) {
942 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
944 writeSimpleData<Data_T>(layerGroup,
"block_empty_value_data", emptyValue);
948 int occupiedBlocks = 0;
949 for (
int i = 0; i < numBlocks; ++i) {
950 if (blocks[i].isAllocated) {
955 if (!
writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
956 throw WriteAttributeException(
"Couldn't add attribute " +
957 k_numOccupiedBlocksStr);
960 if (occupiedBlocks > 0) {
964 memDims[0] = valuesPerBlock;
966 H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
970 fileDims[0] = occupiedBlocks;
971 fileDims[1] = valuesPerBlock;
973 H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
977 hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
978 hsize_t chunkSize[2];
980 chunkSize[1] = valuesPerBlock;
982 herr_t status = H5Pset_deflate(dcpl, 9);
986 status = H5Pset_chunk(dcpl, 2, chunkSize);
996 H5P_DEFAULT, dcpl, H5P_DEFAULT);
997 if (dataSet.id() < 0)
998 throw CreateDataSetException(
"Couldn't create data set in "
999 "SparseFieldIO::writeInternal");
1003 int nextBlockIdx = 0;
1008 for (
int i = 0; i < numBlocks; ++i) {
1009 if (blocks[i].isAllocated) {
1010 offset[0] = nextBlockIdx;
1013 count[1] = valuesPerBlock;
1014 status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
1015 offset, NULL, count, NULL);
1017 throw WriteHyperSlabException(
1018 "Couldn't select slab " +
1019 boost::lexical_cast<std::string>(nextBlockIdx));
1024 fileDataSpace.id(), H5P_DEFAULT, data);
1026 throw WriteHyperSlabException(
1027 "Couldn't write slab " +
1028 boost::lexical_cast<std::string>(nextBlockIdx));
1043 template <
class Data_T>
1044 bool SparseFieldIO::writeInternal(OgOGroup &layerGroup,
1047 using namespace Exc;
1055 const size_t numBlocks = blockRes.x * blockRes.y * blockRes.z;
1056 const size_t numVoxels = (1 << (field->
m_blockOrder * 3));
1082 std::vector<uint8_t> isAllocated(numBlocks);
1083 for (
size_t i = 0; i < numBlocks; ++i) {
1084 isAllocated[i] =
static_cast<uint8_t
>(blocks[i].
isAllocated);
1087 isAllocatedData.addData(numBlocks, &isAllocated[0]);
1090 std::vector<Data_T> emptyValue(numBlocks);
1091 for (
size_t i = 0; i < numBlocks; ++i) {
1092 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
1095 emptyValueData.addData(numBlocks, &emptyValue[0]);
1098 int occupiedBlocks = 0;
1099 for (
size_t i = 0; i < numBlocks; ++i) {
1100 if (blocks[i].isAllocated) {
1105 k_numOccupiedBlocksStr,
1112 OgOCDataset<Data_T> data(layerGroup, k_dataStr);
1114 if (occupiedBlocks > 0) {
1116 ThreadingState<Data_T> state(data, blocks, numVoxels, numBlocks,
1121 boost::thread_group threads;
1122 for (
size_t i = 0; i < numThreads; ++i) {
1123 threads.create_thread(WriteBlockOp<Data_T>(state, i));
1133 template <
class Data_T>
1134 bool SparseFieldIO::readData(hid_t location,
1136 const std::string &filename,
1137 const std::string &layerPath,
1140 using namespace std;
1141 using namespace Exc;
1151 int valuesPerBlock = numVoxels * components;
1155 if (!
readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
1156 throw MissingAttributeException(
"Couldn't find attribute: " +
1157 k_numOccupiedBlocksStr);
1161 if (dynamicLoading) {
1164 valuesPerBlock, numVoxels,
1175 vector<char> isAllocated(numBlocks);
1176 readSimpleData<char>(location,
"block_is_allocated_data", isAllocated);
1177 for (
int i = 0; i < numBlocks; ++i) {
1179 if (!dynamicLoading && isAllocated[i]) {
1180 blocks[i].
resize(numVoxels);
1188 vector<Data_T> emptyValue(numBlocks);
1189 readSimpleData<Data_T>(location,
"block_empty_value_data", emptyValue);
1190 for (
int i = 0; i < numBlocks; ++i) {
1197 if (occupiedBlocks > 0) {
1199 if (dynamicLoading) {
1205 size_t b = 0, bend = b + numBlocks;
1210 static const long maxMemPerPass = 50*1024*1024;
1212 for (
int nextBlockIdx = 0;;) {
1215 std::vector<Data_T*> memoryList;
1217 for (; b != bend && mem < maxMemPerPass; ++b) {
1218 if (blocks[b].isAllocated) {
1219 mem +=
sizeof(Data_T)*numVoxels;
1220 memoryList.push_back(blocks[b].data);
1225 if (!memoryList.size()) {
1229 reader.readBlockList(nextBlockIdx, memoryList);
1230 nextBlockIdx += memoryList.size();
Field_T::Ptr field_dynamic_cast(RefBase::Ptr field)
Dynamic cast that uses string-comparison in order to be safe even after an object crosses a shared li...
V3i m_blockRes
Block array resolution.
Contains utility functions and classes for Hdf5 files.
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
Contains typedefs for the commonly used types in Field3D.
FIELD3D_API bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
#define FIELD3D_NAMESPACE_SOURCE_CLOSE
Namespace for Exception objects.
FIELD3D_API bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
void setupReferenceBlocks()
Internal function to setup the Reference's block pointers, for use with dynamic reading.
boost::intrusive_ptr< FieldBase > Ptr
Data_T * data
Pointer to data. Null if block is unallocated.
FIELD3D_API size_t numIOThreads()
Returns the number of I/O threads to use.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
int m_blockOrder
Block order (size = 2^blockOrder)
boost::intrusive_ptr< SparseField > Ptr
OgDataType
Enumerates the various uses for Ogawa-level groups.
Data_T emptyValue
The value to use if the block isn't allocated. We allow setting this per block so that we for example...
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int numVoxels, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
FIELD3D_API bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
Namespace for sparse field specifics.
void resize(int n)
Alloc data.
bool isAllocated
Whether the block is allocated or not.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Storage for one individual block of a SparseField.
Scoped object - creates a dataspace on creation and closes it on destruction.
Contains the initIO function.
Scoped object - creates a dataset on creation and closes it on destruction.
This Field subclass stores voxel data in block-allocated arrays.
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
Block * m_blocks
Array of blocks. Not using std::vector since SparseBlock is noncopyable.