Go to the documentation of this file.
50 #include "OgSparseDataReader.h"
98 template <
class Data_T>
101 int64_t bytesFreed = 0;
116 #if F3D_SHORT_MUTEX_ARRAY
117 boost::mutex::scoped_lock
147 template <
class Data_T>
162 boost::mutex::scoped_lock lock_A(
m_mutex);
174 int64_t bytesFreed = 0;
178 bytesFreed = deallocateBlock<half>(cb);
179 if (bytesFreed > 0) {
184 bytesFreed = deallocateBlock<float>(cb);
185 if (bytesFreed > 0) {
190 bytesFreed = deallocateBlock<double>(cb);
191 if (bytesFreed > 0) {
196 bytesFreed = deallocateBlock<V3h>(cb);
197 if (bytesFreed > 0) {
202 bytesFreed = deallocateBlock<V3f>(cb);
203 if (bytesFreed > 0) {
208 bytesFreed = deallocateBlock<V3d>(cb);
209 if (bytesFreed > 0) {
225 boost::mutex::scoped_lock lock(
m_mutex);
233 deallocateBlock<half>(it);
236 deallocateBlock<float>(it);
239 deallocateBlock<double>(it);
242 deallocateBlock<V3h>(it);
245 deallocateBlock<V3f>(it);
248 deallocateBlock<V3d>(it);
261 int fileId,
int blockIdx)
295 long long int numLoads = 0;
328 long long int numBlocks = 0;
361 long long int numBlocks = 0;
444 boost::mutex::scoped_lock lock(
m_mutex);
454 Mutex::scoped_lock lock(
m_mutex);
456 long long int size = 0;
467 for (
size_t i = 0, end =
m_hRefs.size(); i < end; ++i) {
470 for (
size_t i = 0, end =
m_vhRefs.size(); i < end; ++i) {
473 for (
size_t i = 0, end =
m_fRefs.size(); i < end; ++i) {
476 for (
size_t i = 0, end =
m_vfRefs.size(); i < end; ++i) {
479 for (
size_t i = 0, end =
m_dRefs.size(); i < end; ++i) {
482 for (
size_t i = 0, end =
m_vdRefs.size(); i < end; ++i) {
497 template <
class Data_T>
500 boost::mutex::scoped_lock lock(m_mutex);
503 #if F3D_NO_BLOCKS_ARRAY
504 blocks[blockIdx].resize(numVoxels);
505 assert(blocks[blockIdx].data != NULL);
507 assert(m_reader || m_ogReader);
509 m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx].data);
511 m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx].data);
514 blockLoaded[blockIdx] = 1;
518 blocks[blockIdx]->resize(numVoxels);
519 assert(blocks[blockIdx]->data != NULL);
521 assert(m_reader || m_ogReader);
523 m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx]->data);
525 m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->data);
528 blockLoaded[blockIdx] = 1;
536 template <
class Data_T>
542 boost::mutex::scoped_lock lock_A(m_mutex);
551 m_ogArchive.reset(
new Alembic::Ogawa::IArchive(filename));
552 if (m_ogArchive->isValid()) {
553 m_ogRoot.reset(
new OgIGroup(*m_ogArchive));
554 m_ogLayerGroup.reset(
new OgIGroup(m_ogRoot->findGroup(layerPath)));
555 if (m_ogLayerGroup->isValid()) {
561 m_ogReader = m_ogReaderPtr.get();
573 m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
574 if (m_fileHandle >= 0) {
576 m_layerGroup.open(m_fileHandle, layerPath.c_str());
577 if (m_layerGroup.id() < 0) {
579 "Couldn't find layer group " + layerPath +
581 throw FileIntegrityException(filename);
585 "Couldn't open HDF5 file ");
586 throw NoSuchFileException(filename);
601 #define FIELD3D_INSTANTIATION_LOADBLOCK(type) \
603 void Reference<type>::loadBlock(int blockIdx); \
614 #define FIELD3D_INSTANTIATION_OPENFILE(type) \
616 void Reference<type>::openFile(); \
#define FIELD3D_NAMESPACE_HEADER_CLOSE
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
size_t numRefs() const
Returns the number of file references of the corresponding collection.
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
std::deque< Reference< V3h >::Ptr > m_vhRefs
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads....
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
std::deque< Reference< half >::Ptr > m_hRefs
static SparseFileManager * ms_singleton
Pointer to singleton.
FIELD3D_NAMESPACE_OPEN typedef ::half half
std::deque< Reference< double >::Ptr > m_dRefs
long long int memSize() const
Returns the number of bytes used by the SparseFileManager itself.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
int64_t m_memUse
Current amount of memory in use in bytes.
T max(const T a, const T2 b)
Max operation on mixed types.
float m_maxMemUse
Max amount om memory to use in megabytes.
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
SparseFileManager()
Private to prevent instantiation.
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
FIELD3D_INSTANTIATION_OPENFILE(float16_t)
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Namespace for Exception objects.
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
boost::recursive_mutex::scoped_lock GlobalLock
Contains utility functions and classes for Hdf5 files.
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
std::deque< Reference< V3f >::Ptr > m_vfRefs
long long int memSize() const
Returns the memory use for the refs.
std::deque< Reference< V3d >::Ptr > m_vdRefs
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
FIELD3D_INSTANTIATION_LOADBLOCK(float16_t)
Contains the SparseField class.
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
std::deque< Reference< float >::Ptr > m_fRefs
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
long long totalLoads()
Returns the total number of block loads in the cache.
void resetCacheStatistics()
Resets block load.