Field3D
SparseFile.cpp
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 // SparseField.h includes SparseFile.h, but we need the definition of
45 // SparseBlock from SparseField.h, so just include that to get both
46 // files
47 #include "SparseField.h"
48 
49 #include "OgIO.h"
50 #include "OgSparseDataReader.h"
51 
52 //----------------------------------------------------------------------------//
53 
55 
56 //----------------------------------------------------------------------------//
57 // Static instances
58 //----------------------------------------------------------------------------//
59 
61 
62 //----------------------------------------------------------------------------//
63 // SparseFileManager
64 //----------------------------------------------------------------------------//
65 
67 {
68  if (!ms_singleton) {
70  }
71  return *ms_singleton;
72 }
73 
74 //----------------------------------------------------------------------------//
75 
77 {
78  m_limitMemUse = enabled;
79 }
80 
81 //----------------------------------------------------------------------------//
82 
84 {
85  return m_limitMemUse;
86 }
87 
88 //----------------------------------------------------------------------------//
89 
90 void SparseFileManager::setMaxMemUse(float maxMemUse)
91 {
92  m_maxMemUse = maxMemUse;
93  m_maxMemUseInBytes = static_cast<int64_t>(m_maxMemUse * 1024 * 1024);
94 }
95 
96 //----------------------------------------------------------------------------//
97 
98 template <class Data_T>
100 {
101  int64_t bytesFreed = 0;
103 
104  // Note: we don't need to lock the block's mutex because
105  // deallocateBlock() is only called while the SparseFileManager's
106  // mutex is also locked (in flushCache() or deallocateBlocks()).
107  // Don't lock the block, to make sure we don't have a deadlock by
108  // holding two locks at the same time. (Because addBlockToCache()
109  // locks the manager but is also in a block-specific lock.)
110 
111  // lock the current block to make sure its blockUsed flag and ref
112  // counts don't change
113  // Note: this lock order is made consistent w/ allocate to prevent
114  // deadlocks and crashes.
115 
116 #if F3D_SHORT_MUTEX_ARRAY
117  boost::mutex::scoped_lock
118  lock_B(reference->blockMutex[cb.blockIdx % reference->blockMutexSize]);
119 #else
120  boost::mutex::scoped_lock lock_B(reference->blockMutex[cb.blockIdx]);
121 #endif
122 
123  // check whether the block is still in use
124  if (reference->refCounts[cb.blockIdx] > 0)
125  return bytesFreed;
126 
127  if (reference->blockUsed[cb.blockIdx]) {
128  // the block was recently used according to Second-chance paging
129  // algorithm, so skip it
130  reference->blockUsed[cb.blockIdx] = false;
131  }
132  else {
133 
134  // the block wasn't in use, so free it
135  reference->unloadBlock(cb.blockIdx);
136  bytesFreed = reference->blockSize(cb.blockIdx);
137  m_memUse -= bytesFreed;
138  CacheList::iterator toRemove = m_nextBlock;
139  ++m_nextBlock;
140  m_blockCacheList.erase(toRemove);
141  }
142  return bytesFreed;
143 }
144 
145 //----------------------------------------------------------------------------//
146 
147 template <class Data_T>
148 void SparseFileManager::deallocateBlock(CacheList::iterator &it)
149 {
150  SparseFile::CacheBlock &cb = *it;
152  int64_t bytesFreed = reference->blockSize(cb.blockIdx);
153  m_memUse -= bytesFreed;
154  reference->unloadBlock(cb.blockIdx);
155  it = m_blockCacheList.erase(it);
156 }
157 
158 //----------------------------------------------------------------------------//
159 
160 void SparseFileManager::deallocateBlocks(int64_t bytesNeeded)
161 {
162  boost::mutex::scoped_lock lock_A(m_mutex);
163 
164  while (m_blockCacheList.begin() != m_blockCacheList.end() &&
165  m_maxMemUseInBytes-m_memUse < bytesNeeded) {
166 
167  if (m_nextBlock == m_blockCacheList.end())
168  m_nextBlock = m_blockCacheList.begin();
169 
171 
172  // if bytesFreed is set to >0, then we've already freed a block
173  // and advanced the "clock hand" iterator
174  int64_t bytesFreed = 0;
175 
176  switch(cb.blockType) {
177  case DataTypeHalf:
178  bytesFreed = deallocateBlock<half>(cb);
179  if (bytesFreed > 0) {
180  continue;
181  }
182  break;
183  case DataTypeFloat:
184  bytesFreed = deallocateBlock<float>(cb);
185  if (bytesFreed > 0) {
186  continue;
187  }
188  break;
189  case DataTypeDouble:
190  bytesFreed = deallocateBlock<double>(cb);
191  if (bytesFreed > 0) {
192  continue;
193  }
194  break;
195  case DataTypeVecHalf:
196  bytesFreed = deallocateBlock<V3h>(cb);
197  if (bytesFreed > 0) {
198  continue;
199  }
200  break;
201  case DataTypeVecFloat:
202  bytesFreed = deallocateBlock<V3f>(cb);
203  if (bytesFreed > 0) {
204  continue;
205  }
206  break;
207  case DataTypeVecDouble:
208  bytesFreed = deallocateBlock<V3d>(cb);
209  if (bytesFreed > 0) {
210  continue;
211  }
212  break;
213  case DataTypeUnknown:
214  default:
215  break;
216  }
217  ++m_nextBlock;
218  }
219 }
220 
221 //----------------------------------------------------------------------------//
222 
224 {
225  boost::mutex::scoped_lock lock(m_mutex);
226 
227  CacheList::iterator it = m_blockCacheList.begin();
228  while (it != m_blockCacheList.end()) {
229  SparseFile::CacheBlock &cb = *it;
230 
231  switch(cb.blockType) {
232  case DataTypeHalf:
233  deallocateBlock<half>(it);
234  break;
235  case DataTypeFloat:
236  deallocateBlock<float>(it);
237  break;
238  case DataTypeDouble:
239  deallocateBlock<double>(it);
240  break;
241  case DataTypeVecHalf:
242  deallocateBlock<V3h>(it);
243  break;
244  case DataTypeVecFloat:
245  deallocateBlock<V3f>(it);
246  break;
247  case DataTypeVecDouble:
248  deallocateBlock<V3d>(it);
249  break;
250  case DataTypeUnknown:
251  default:
252  break;
253  }
254  }
255  m_nextBlock = m_blockCacheList.begin();
256 }
257 
258 //----------------------------------------------------------------------------//
259 
261  int fileId, int blockIdx)
262 {
263  // Note: this lock is obtained while we also have a lock on the
264  // specific block (in activateBlock()), so we should make sure we
265  // never lock the SparseFileManager and *then* a block, to ensure we
266  // don't have a deadlock.
267  //
268  // Note: this was changed so the order was consistent w/ dealloc
269  // again, see activateBlock()
270  // boost::mutex::scoped_lock lock(m_mutex);
271 
272  SparseFile::CacheBlock block(blockType, fileId, blockIdx);
273  if (m_nextBlock == m_blockCacheList.end()) {
274  m_blockCacheList.push_back(block);
275  } else {
276  m_blockCacheList.insert(m_nextBlock, block);
277  }
278 }
279 
280 //----------------------------------------------------------------------------//
281 
283  : m_memUse(0),
284  m_limitMemUse(false)
285 {
286  setMaxMemUse(1000.0);
287  m_nextBlock = m_blockCacheList.begin();
288 }
289 
290 //----------------------------------------------------------------------------//
291 
293 {
294 
295  long long int numLoads = 0;
296 
297  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
298  numLoads += m_fileData.ref<half>(i)->totalLoads();
299  }
300 
301  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
302  numLoads += m_fileData.ref<V3h>(i)->totalLoads();
303  }
304 
305  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
306  numLoads += m_fileData.ref<float>(i)->totalLoads();
307  }
308 
309  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
310  numLoads += m_fileData.ref<V3f>(i)->totalLoads();
311  }
312 
313  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
314  numLoads += m_fileData.ref<double>(i)->totalLoads();
315  }
316 
317  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
318  numLoads += m_fileData.ref<V3d>(i)->totalLoads();
319  }
320  return numLoads;
321 }
322 
323 //----------------------------------------------------------------------------//
324 
326 {
327 
328  long long int numBlocks = 0;
329 
330  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
331  numBlocks += m_fileData.ref<half>(i)->numLoadedBlocks();
332  }
333 
334  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
335  numBlocks += m_fileData.ref<V3h>(i)->numLoadedBlocks();
336  }
337 
338  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
339  numBlocks += m_fileData.ref<float>(i)->numLoadedBlocks();
340  }
341 
342  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
343  numBlocks += m_fileData.ref<V3f>(i)->numLoadedBlocks();
344  }
345 
346  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
347  numBlocks += m_fileData.ref<double>(i)->numLoadedBlocks();
348  }
349 
350  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
351  numBlocks += m_fileData.ref<V3d>(i)->numLoadedBlocks();
352  }
353  return numBlocks;
354 }
355 
356 //----------------------------------------------------------------------------//
357 
359 {
360 
361  long long int numBlocks = 0;
362 
363  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
364  numBlocks += m_fileData.ref<half>(i)->totalLoadedBlocks();
365  }
366 
367  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
368  numBlocks += m_fileData.ref<V3h>(i)->totalLoadedBlocks();
369  }
370 
371  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
372  numBlocks += m_fileData.ref<float>(i)->totalLoadedBlocks();
373  }
374 
375  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
376  numBlocks += m_fileData.ref<V3f>(i)->totalLoadedBlocks();
377  }
378 
379  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
380  numBlocks += m_fileData.ref<double>(i)->totalLoadedBlocks();
381  }
382 
383  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
384  numBlocks += m_fileData.ref<V3d>(i)->totalLoadedBlocks();
385  }
386  return numBlocks;
387 }
388 
389 //----------------------------------------------------------------------------//
390 
392 {
393  return ((double)numLoadedBlocks())/std::max(1.0, ((double)totalLoadedBlocks()));
394 }
395 
396 //----------------------------------------------------------------------------//
397 
399 {
400  return ((double)totalLoads())/std::max(1.0, ((double)totalLoadedBlocks()));
401 }
402 
403 //----------------------------------------------------------------------------//
404 
406 {
407  return ((double)totalLoadedBlocks())/std::max(1.0, ((double)totalLoads()));
408 }
409 
410 //----------------------------------------------------------------------------//
411 
413 {
414 
415  for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
417  }
418 
419  for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
421  }
422 
423  for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
424  m_fileData.ref<float>(i)->resetCacheStatistics();
425  }
426 
427  for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
429  }
430 
431  for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
432  m_fileData.ref<double>(i)->resetCacheStatistics();
433  }
434 
435  for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
437  }
438 }
439 
440 //----------------------------------------------------------------------------//
441 
442 long long int SparseFileManager::memSize() const
443 {
444  boost::mutex::scoped_lock lock(m_mutex);
445 
446  return sizeof(*this) + m_fileData.memSize() +
447  m_blockCacheList.size() * sizeof(SparseFile::CacheBlock);
448 }
449 
450 //----------------------------------------------------------------------------//
451 
453 {
454  Mutex::scoped_lock lock(m_mutex);
455 
456  long long int size = 0;
457 
458  // Size of the std::deque's
459  size += m_hRefs.size() * sizeof(Reference<half>::Ptr);
460  size += m_vhRefs.size() * sizeof(Reference<V3h>::Ptr);
461  size += m_fRefs.size() * sizeof(Reference<float>::Ptr);
462  size += m_vfRefs.size() * sizeof(Reference<V3f>::Ptr);
463  size += m_dRefs.size() * sizeof(Reference<double>::Ptr);
464  size += m_vdRefs.size() * sizeof(Reference<V3d>::Ptr);
465 
466  // Size of the references themselves
467  for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
468  size += m_hRefs[i]->memSize();
469  }
470  for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
471  size += m_vhRefs[i]->memSize();
472  }
473  for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
474  size += m_fRefs[i]->memSize();
475  }
476  for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
477  size += m_vfRefs[i]->memSize();
478  }
479  for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
480  size += m_dRefs[i]->memSize();
481  }
482  for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
483  size += m_vdRefs[i]->memSize();
484  }
485 
486  return size;
487 }
488 
489 //----------------------------------------------------------------------------//
490 // Template implementations
491 //----------------------------------------------------------------------------//
492 
493 namespace SparseFile {
494 
495 //----------------------------------------------------------------------------//
496 
497 template <class Data_T>
499 {
500  boost::mutex::scoped_lock lock(m_mutex);
501 
502  // Allocate the block
503 #if F3D_NO_BLOCKS_ARRAY
504  blocks[blockIdx].resize(numVoxels);
505  assert(blocks[blockIdx].data != NULL);
506  // Read the data
507  assert(m_reader || m_ogReader);
508  if (m_reader) {
509  m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx].data);
510  } else {
511  m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx].data);
512  }
513  // Mark block as loaded
514  blockLoaded[blockIdx] = 1;
515  // Track count
516  m_numActiveBlocks++;
517 #else
518  blocks[blockIdx]->resize(numVoxels);
519  assert(blocks[blockIdx]->data != NULL);
520  // Read the data
521  assert(m_reader || m_ogReader);
522  if (m_reader) {
523  m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx]->data);
524  } else {
525  m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->data);
526  }
527  // Mark block as loaded
528  blockLoaded[blockIdx] = 1;
529  // Track count
530  m_numActiveBlocks++;
531 #endif
532 }
533 
534 //----------------------------------------------------------------------------//
535 
536 template <class Data_T>
538 {
539  using namespace Exc;
540  using namespace Hdf5Util;
541 
542  boost::mutex::scoped_lock lock_A(m_mutex);
543 
544  // check that the file wasn't already opened before obtaining the lock
545  if (fileIsOpen()) {
546  return;
547  }
548 
549  // First try Ogawa ---
550 
551  m_ogArchive.reset(new Alembic::Ogawa::IArchive(filename));
552  if (m_ogArchive->isValid()) {
553  m_ogRoot.reset(new OgIGroup(*m_ogArchive));
554  m_ogLayerGroup.reset(new OgIGroup(m_ogRoot->findGroup(layerPath)));
555  if (m_ogLayerGroup->isValid()) {
556  // Allocate the reader
557  m_ogReaderPtr.reset(new OgSparseDataReader<Data_T>(*m_ogLayerGroup,
558  numVoxels,
559  occupiedBlocks,
560  true));
561  m_ogReader = m_ogReaderPtr.get();
562  // Done
563  return;
564  }
565  }
566 
567  // Then, try HDF5 ---
568 
569  {
570  // Hold the global lock
571  GlobalLock lock(g_hdf5Mutex);
572  // Open the file
573  m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
574  if (m_fileHandle >= 0) {
575  // Open the layer group
576  m_layerGroup.open(m_fileHandle, layerPath.c_str());
577  if (m_layerGroup.id() < 0) {
578  Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
579  "Couldn't find layer group " + layerPath +
580  " in .f3d file ");
581  throw FileIntegrityException(filename);
582  }
583  } else {
584  Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
585  "Couldn't open HDF5 file ");
586  throw NoSuchFileException(filename);
587  }
588  }
589 
590  // Re-allocate reader
591  if (m_reader) {
592  delete m_reader;
593  }
594  m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(),
595  valuesPerBlock,
596  occupiedBlocks);
597 }
598 
599 //----------------------------------------------------------------------------//
600 
601 #define FIELD3D_INSTANTIATION_LOADBLOCK(type) \
602  template \
603  void Reference<type>::loadBlock(int blockIdx); \
604 
611 
612 //----------------------------------------------------------------------------//
613 
614 #define FIELD3D_INSTANTIATION_OPENFILE(type) \
615  template \
616  void Reference<type>::openFile(); \
617 
624 
625 } // namespace SparseFile
626 
627 //----------------------------------------------------------------------------//
628 
630 
631 //----------------------------------------------------------------------------//
632 
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseFileManager::reference
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1247
SparseFileManager::setMaxMemUse
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Definition: SparseFile.cpp:90
SparseFile::CacheBlock::refIdx
int refIdx
Definition: SparseFile.h:324
OgSparseDataReader
Definition: SparseFile.h:86
SparseFile::Reference::openFile
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition: SparseFile.cpp:537
V3d
Imath::V3d V3d
Definition: SpiMathLib.h:74
Msg::SevWarning
Definition: Log.h:68
SparseFile::FileReferences::numRefs
size_t numRefs() const
Returns the number of file references of the corresponding collection.
SparseFileManager::m_blockCacheList
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition: SparseFile.h:537
SparseFileManager::doLimitMemUse
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:83
SparseFileManager::totalLoadedBlocks
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
Definition: SparseFile.cpp:358
DataTypeFloat
Definition: Traits.h:112
float64_t
double float64_t
Definition: Traits.h:88
DataTypeEnum
DataTypeEnum
Definition: Traits.h:108
SparseFile::FileReferences::m_vhRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition: SparseFile.h:305
DataTypeUnknown
Definition: Traits.h:117
V3f
Imath::V3f V3f
Definition: SpiMathLib.h:73
SparseFile
Definition: SparseFile.h:90
SparseFileManager::cacheEfficiency
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads....
Definition: SparseFile.cpp:405
SparseFileManager::m_maxMemUseInBytes
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:519
SparseFile::FileReferences::ref
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
DataTypeVecHalf
Definition: Traits.h:114
SparseFile::CacheBlock::blockIdx
int blockIdx
Definition: SparseFile.h:325
SparseFileManager::m_limitMemUse
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:526
SparseFile::FileReferences::m_hRefs
std::deque< Reference< half >::Ptr > m_hRefs
Definition: SparseFile.h:304
SparseFileManager::ms_singleton
static SparseFileManager * ms_singleton
Pointer to singleton.
Definition: SparseFile.h:497
DataTypeHalf
Definition: Traits.h:109
half
FIELD3D_NAMESPACE_OPEN typedef ::half half
Definition: SpiMathLib.h:64
SparseFile::FileReferences::m_dRefs
std::deque< Reference< double >::Ptr > m_dRefs
Definition: SparseFile.h:308
SparseFile::CacheBlock::blockType
DataTypeEnum blockType
Definition: SparseFile.h:323
SparseFileManager::memSize
long long int memSize() const
Returns the number of bytes used by the SparseFileManager itself.
Definition: SparseFile.cpp:442
vec32_t
Field3D::V3f vec32_t
Definition: Traits.h:92
SparseFileManager::m_fileData
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:530
SparseFileManager::m_memUse
int64_t m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:522
detail::max
T max(const T a, const T2 b)
Max operation on mixed types.
Definition: FieldSampler.h:32
SparseFileManager::m_maxMemUse
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:516
SparseFileManager::m_mutex
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:545
SparseFileManager::cacheFractionLoaded
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
Definition: SparseFile.cpp:391
SparseFileManager::addBlockToCache
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition: SparseFile.cpp:260
vec64_t
Field3D::V3d vec64_t
Definition: Traits.h:93
SparseFileManager::SparseFileManager
SparseFileManager()
Private to prevent instantiation.
Definition: SparseFile.cpp:282
SparseFileManager::flushCache
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
Definition: SparseFile.cpp:223
SparseFileManager::cacheLoadsPerBlock
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Definition: SparseFile.cpp:398
SparseFileManager
Definition: SparseFile.h:396
SparseFile::FIELD3D_INSTANTIATION_OPENFILE
FIELD3D_INSTANTIATION_OPENFILE(float16_t)
SparseFileManager::setLimitMemUse
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:76
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition: SparseDataReader.h:70
DataTypeVecDouble
Definition: Traits.h:116
g_hdf5Mutex
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition: Hdf5Util.cpp:67
float32_t
float float32_t
Definition: Traits.h:87
DataTypeVecFloat
Definition: Traits.h:115
Exc
Namespace for Exception objects.
Definition: Exception.h:57
SparseFileManager::numLoadedBlocks
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
Definition: SparseFile.cpp:325
SparseFileManager::m_nextBlock
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:541
SparseFile::Reference
Definition: SparseFile.h:104
GlobalLock
boost::recursive_mutex::scoped_lock GlobalLock
Definition: Hdf5Util.h:78
DataTypeDouble
Definition: Traits.h:113
V3h
Imath::Vec3< half > V3h
Definition: SpiMathLib.h:72
Hdf5Util
Contains utility functions and classes for Hdf5 files.
Definition: Hdf5Util.h:86
SparseFileManager::singleton
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition: SparseFile.cpp:66
SparseFile::FileReferences::m_vfRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition: SparseFile.h:307
SparseFile::FileReferences::m_mutex
Mutex m_mutex
Definition: SparseFile.h:315
SparseFile::FileReferences::memSize
long long int memSize() const
Returns the memory use for the refs.
Definition: SparseFile.cpp:452
SparseFile::FileReferences::m_vdRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition: SparseFile.h:309
FIELD3D_NAMESPACE_OPEN
Definition: FieldMapping.cpp:74
SparseFileManager::deallocateBlocks
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
Definition: SparseFile.cpp:160
Msg::print
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition: Log.cpp:70
SparseFile::FIELD3D_INSTANTIATION_LOADBLOCK
FIELD3D_INSTANTIATION_LOADBLOCK(float16_t)
SparseField.h
Contains the SparseField class.
SparseFileManager::deallocateBlock
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
Definition: SparseFile.cpp:99
SparseFile::FileReferences::m_fRefs
std::deque< Reference< float >::Ptr > m_fRefs
Definition: SparseFile.h:306
vec16_t
Field3D::V3h vec16_t
Definition: Traits.h:91
SparseFile::Reference::loadBlock
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition: SparseFile.cpp:498
float16_t
half float16_t
Definition: Traits.h:86
SparseFile::CacheBlock
Definition: SparseFile.h:321
SparseFileManager::totalLoads
long long totalLoads()
Returns the total number of block loads in the cache.
Definition: SparseFile.cpp:292
SparseFileManager::resetCacheStatistics
void resetCacheStatistics()
Resets block load.
Definition: SparseFile.cpp:412