Field3D
SparseFile.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <deque>
50 #include <list>
51 #include <vector>
52 
53 #include <hdf5.h>
54 
55 #include "Exception.h"
56 #include "Hdf5Util.h"
57 #include "OgawaFwd.h"
58 #include "SparseDataReader.h"
59 #include "Traits.h"
60 
61 //----------------------------------------------------------------------------//
62 
63 #define F3D_SHORT_MUTEX_ARRAY 1
64 #define F3D_MUTEX_ARRAY_SIZE 1000
65 #define F3D_NO_BLOCKS_ARRAY 1
66 
67 #include "ns.h"
68 
70 
71 //----------------------------------------------------------------------------//
72 // Forward declarations
73 //----------------------------------------------------------------------------//
74 
75 namespace Sparse {
76 
77  template <typename Data_T>
78  struct SparseBlock;
79 
80 }
81 
82 template <typename Data_T>
83 class SparseField;
84 
85 template <typename Data_T>
87 
88 //----------------------------------------------------------------------------//
89 
90 namespace SparseFile {
91 
92 //----------------------------------------------------------------------------//
93 // Reference
94 //----------------------------------------------------------------------------//
95 
101 //----------------------------------------------------------------------------//
102 
103 template <class Data_T>
104 class Reference
105 {
106 public:
107 
108  // Typedefs ------------------------------------------------------------------
109 
110 #define USE_SHPTR 0
111 
112 #if USE_SHPTR
113  typedef boost::shared_ptr<Reference> Ptr;
114 #else
115  typedef Reference* Ptr;
116 #endif
117 
118 #if F3D_NO_BLOCKS_ARRAY
120 #else
121  typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
122 #endif
123 
124  // Public data members -------------------------------------------------------
125 
126  std::string filename;
127  std::string layerPath;
132 
134  std::vector<int> fileBlockIndices;
137  std::vector<int> blockLoaded;
144  std::vector<bool> blockUsed;
147  std::vector<int> loadCounts;
151  std::vector<int> refCounts;
155  boost::mutex *blockMutex;
156 #if F3D_SHORT_MUTEX_ARRAY
157  int blockMutexSize;
159 #endif
160 
161  // Ctors, dtor ---------------------------------------------------------------
162 
164  ~Reference();
165 
166  // Main methods --------------------------------------------------------------
167 
170  static Ptr create(const std::string a_filename,
171  const std::string a_layerPath);
173  bool fileIsOpen();
175  void setNumBlocks(int numBlocks);
179  void openFile();
181  void closeFile();
185  void loadBlock(int blockIdx);
187  void unloadBlock(int blockIdx);
190  void incBlockRef(int blockIdx);
192  void decBlockRef(int blockIdx);
194  int blockSize(int blockIdx) const;
197  int totalLoads() const;
200  int numLoadedBlocks() const;
203  int totalLoadedBlocks() const;
206  float averageLoads() const;
208  void resetCacheStatistics();
210  long long int memSize() const;
211 
212 private:
213 
214  typedef boost::mutex Mutex;
215 
216  // Private constructors ---
217 
219  Reference(const std::string filename, const std::string layerPath);
220 
222  Reference(const Reference &o);
223 
225  Reference & operator=(const Reference &o);
226 
227  // Data members ---
228 
231 
235 
239 
241  boost::shared_ptr<OgSparseDataReader<Data_T> > m_ogReaderPtr;
251 
253  mutable Mutex m_mutex;
254 
257 
258 };
259 
260 //----------------------------------------------------------------------------//
261 // FileReferences
262 //----------------------------------------------------------------------------//
263 
265 {
266 public:
267 
268  /* A note on thread safety
269 
270  FileReferences uses a std::deque to store the References, which means
271  that already-added references 'stay put' in memory, even as more elements
272  are added. For threading purposes, we only need to lock during append()
273  and numRefs(), but ref() is safe, since we never remove references.
274 
275  */
276 
277  // Ctors, dtor ---------------------------------------------------------------
278 
279  ~FileReferences();
280 
281  // Main methods --------------------------------------------------------------
282 
285  template <class Data_T>
286  Reference<Data_T>* ref(size_t idx);
287 
290  template <class Data_T>
291  size_t append(typename Reference<Data_T>::Ptr ref);
292 
294  template <class Data_T>
295  size_t numRefs() const;
296 
298  long long int memSize() const;
299 
300 private:
301 
302  // Data members --------------------------------------------------------------
303 
304  std::deque<Reference<half>::Ptr> m_hRefs;
305  std::deque<Reference<V3h>::Ptr> m_vhRefs;
306  std::deque<Reference<float>::Ptr> m_fRefs;
307  std::deque<Reference<V3f>::Ptr> m_vfRefs;
308  std::deque<Reference<double>::Ptr> m_dRefs;
309  std::deque<Reference<V3d>::Ptr> m_vdRefs;
310 
311  // Mutexes -------------------------------------------------------------------
312 
313  typedef boost::mutex Mutex;
314 
315  mutable Mutex m_mutex;
316 
317 };
318 
319 //----------------------------------------------------------------------------//
320 
321 class CacheBlock {
322 public:
324  int refIdx;
325  int blockIdx;
326  CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
327  blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
328  { }
329 };
330 
331 //----------------------------------------------------------------------------//
332 
333 } // namespace SparseFile
334 
335 //----------------------------------------------------------------------------//
336 // SparseFileManager
337 //----------------------------------------------------------------------------//
338 
394 //----------------------------------------------------------------------------//
395 
397 {
398 
399 public:
400 
401  template <class Data_T>
402  friend class SparseField;
403 
404  // typedefs ------------------------------------------------------------------
405 
406  typedef std::list<SparseFile::CacheBlock> CacheList;
407 
408  // Main methods --------------------------------------------------------------
409 
411  static SparseFileManager &singleton();
412 
415  void setLimitMemUse(bool enabled);
416 
419  bool doLimitMemUse() const;
420 
422  void setMaxMemUse(float maxMemUse);
423 
426  void flushCache();
427 
429  long long totalLoads();
430 
432  long long numLoadedBlocks();
433 
435  long long totalLoadedBlocks();
436 
439  float cacheFractionLoaded();
440 
442  float cacheLoadsPerBlock();
443 
446  float cacheEfficiency();
447 
449  void resetCacheStatistics();
450 
452  long long int memSize() const;
453 
454  //--------------------------------------------------------------------------//
455  // Utility functions
456 
461  template <class Data_T>
462  void incBlockRef(int fileId, int blockIdx);
463 
468  template <class Data_T>
469  void decBlockRef(int fileId, int blockIdx);
470 
474  template <class Data_T>
475  void activateBlock(int fileId, int blockIdx);
476 
477 protected:
478 
480  template <class Data_T>
481  SparseFile::Reference<Data_T>* reference(int index);
482 
485  template <class Data_T>
486  int getNextId(const std::string filename, const std::string layerPath);
487 
488  template <class Data_T>
489  void removeFieldFromCache(int refIdx);
490 
491 private:
492 
495 
498 
500  void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
501 
504  void deallocateBlocks(int64_t bytesNeeded);
505 
508  template <class Data_T>
509  int64_t deallocateBlock(const SparseFile::CacheBlock &cb);
510 
512  template <class Data_T>
513  void deallocateBlock(CacheList::iterator &it);
514 
516  float m_maxMemUse;
517 
520 
522  int64_t m_memUse;
523 
527 
531 
538 
541  CacheList::iterator m_nextBlock;
542 
545  mutable boost::mutex m_mutex;
546 
547 };
548 
549 //----------------------------------------------------------------------------//
550 // Reference implementations
551 //----------------------------------------------------------------------------//
552 
553 namespace SparseFile {
554 
555 //----------------------------------------------------------------------------//
556 
557 template <class Data_T>
558 Reference<Data_T>::Reference(const std::string a_filename,
559  const std::string a_layerPath)
560  : filename(a_filename), layerPath(a_layerPath),
561  valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
562  blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
563  m_numActiveBlocks(0)
564 {
565  /* Empty */
566 }
567 
568 //----------------------------------------------------------------------------//
569 
570 template <class Data_T>
572 {
573  closeFile();
574 
575  if (m_reader) {
576  delete m_reader;
577  }
578 
579  if (blockMutex)
580  delete [] blockMutex;
581 }
582 
583 //----------------------------------------------------------------------------//
584 
585 template <class Data_T>
587 {
588  m_ogReaderPtr.reset();
589  m_ogReader = NULL;
590  m_reader = NULL;
591  blockMutex = NULL;
592  *this = o;
593 }
594 
595 //----------------------------------------------------------------------------//
596 
597 template <class Data_T>
600 {
601  if (this == &o) {
602  return *this;
603  }
604 
605  // Copy public member variables (where appropriate)
606  filename = o.filename;
607  layerPath = o.layerPath;
608  valuesPerBlock = o.valuesPerBlock;
609  numVoxels = o.numVoxels;
610  occupiedBlocks = o.occupiedBlocks;
611  fileBlockIndices = o.fileBlockIndices;
612  blockLoaded = o.blockLoaded;
613  blocks = o.blocks;
614  blockUsed = o.blockUsed;
615  loadCounts = o.loadCounts;
616  refCounts = o.refCounts;
617  if (blockMutex)
618  delete[] blockMutex;
619 #if F3D_SHORT_MUTEX_ARRAY
620 # if F3D_NO_BLOCKS_ARRAY
621  blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
622 # else
623  blockMutexSize = std::min(static_cast<int>(blocks.size()),
625 # endif
626  blockMutex = new boost::mutex[blockMutexSize];
627 #else
628  blockMutex = new boost::mutex[blocks.size()];
629 #endif
630 
631 #if 0
632  // MW: Should this be copying the file handle? Never seems to happen,
633  // but it also seems bad. Ifdef'ing out for now.
634  // Copy private member variables (where appropriate)
635  m_fileHandle = o.m_fileHandle;
636  // Don't copy id, let hdf5 generate a new one.
637  if (m_fileHandle >= 0) {
638  m_layerGroup.open(m_fileHandle, layerPath.c_str());
639  }
640 #else
641  m_fileHandle = -1;
642 #endif
643 
644  // Re-allocate reader
645  if (m_reader)
646  delete m_reader;
647  m_reader = NULL;
648 
649  m_ogReaderPtr.reset();
650  m_ogReader = NULL;
651 
652  return *this;
653 }
654 
655 //----------------------------------------------------------------------------//
656 
657 template <class Data_T>
658 typename Reference<Data_T>::Ptr
659 Reference<Data_T>::create(const std::string a_filename,
660  const std::string a_layerPath)
661 {
662  return Ptr(new Reference(a_filename, a_layerPath));
663 }
664 
665 //----------------------------------------------------------------------------//
666 
667 template <class Data_T>
669 {
670  return m_fileHandle >= 0;
671 }
672 
673 //----------------------------------------------------------------------------//
674 
675 template <class Data_T>
676 void Reference<Data_T>::setNumBlocks(int a_numBlocks)
677 {
678  boost::mutex::scoped_lock lock(m_mutex);
679 
680  // Store number of blocks in member variable
681  numBlocks = a_numBlocks;
682 
683  fileBlockIndices.resize(numBlocks);
684  blockLoaded.resize(numBlocks, 0);
685 #if !F3D_NO_BLOCKS_ARRAY
686  blocks.resize(numBlocks, 0);
687 #endif
688  blockUsed.resize(numBlocks, false);
689  loadCounts.resize(numBlocks, 0);
690  refCounts.resize(numBlocks, 0);
691  if (blockMutex)
692  delete[] blockMutex;
693 #if F3D_SHORT_MUTEX_ARRAY
694 # if F3D_NO_BLOCKS_ARRAY
695  blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
696 # else
697  blockMutexSize = std::min(blocks.size(),
698  static_cast<size_t>(F3D_MUTEX_ARRAY_SIZE));
699 # endif
700  blockMutex = new boost::mutex[blockMutexSize];
701 #else
702  blockMutex = new boost::mutex[numBlocks];
703 #endif
704 }
705 
706 //----------------------------------------------------------------------------//
707 
708 template <class Data_T>
710 {
711  if (m_fileHandle >= 0) {
712  if (H5Fclose(m_fileHandle) < 0) {
713  Msg::print("In ~Reference(): Error closing file " + filename);
714  }
715  }
716 }
717 
718 //----------------------------------------------------------------------------//
719 
720 template <class Data_T>
722 {
723  // Deallocate the block
724 #if F3D_NO_BLOCKS_ARRAY
725  blocks[blockIdx].clear();
726 #else
727  blocks[blockIdx]->clear();
728 #endif
729  // Mark block as unloaded
730  blockLoaded[blockIdx] = 0;
731  // Track count
732  m_numActiveBlocks--;
733 #if 0
734  // If no active blocks, close the file. De-activate for now.
735  if (m_numActiveBlocks == 0) {
736  closeFile();
737  }
738 #endif
739 }
740 
741 //----------------------------------------------------------------------------//
742 
743 template <class Data_T>
745 {
746 #if F3D_SHORT_MUTEX_ARRAY
747  boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
748 #else
749  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
750 #endif
751  ++refCounts[blockIdx];
752 }
753 
754 //----------------------------------------------------------------------------//
755 
756 template <class Data_T>
758 {
759 #if F3D_SHORT_MUTEX_ARRAY
760  boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
761 #else
762  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
763 #endif
764  --refCounts[blockIdx];
765 }
766 
767 //----------------------------------------------------------------------------//
768 
769 template <class Data_T>
770 int Reference<Data_T>::blockSize(int /* blockIdx */) const
771 {
772  return numVoxels * sizeof(Data_T);
773 }
774 
775 //----------------------------------------------------------------------------//
776 
777 template <class Data_T>
779 {
780  std::vector<int>::const_iterator i = loadCounts.begin();
781  std::vector<int>::const_iterator end = loadCounts.end();
782  int numLoads = 0;
783  for (; i != end; ++i)
784  numLoads += *i;
785 
786  return numLoads;
787 }
788 
789 //----------------------------------------------------------------------------//
790 
791 template <class Data_T>
793 {
794  std::vector<int>::const_iterator i = blockLoaded.begin();
795  std::vector<int>::const_iterator end = blockLoaded.end();
796  int numBlockCounter = 0;
797  for (; i != end; ++i)
798  if (*i)
799  numBlockCounter++;
800 
801  return numBlockCounter;
802 }
803 
804 //----------------------------------------------------------------------------//
805 
806 template <class Data_T>
808 {
809  std::vector<int>::const_iterator i = loadCounts.begin();
810  std::vector<int>::const_iterator li = blockLoaded.begin();
811  std::vector<int>::const_iterator end = loadCounts.end();
812  int numBlockCounter = 0;
813 
814  if (blockLoaded.size() == 0) {
815  for (; i != end; ++i)
816  if (*i)
817  numBlockCounter++;
818  } else {
819  assert(loadCounts.size() == blockLoaded.size());
820 
821  for (; i != end; ++i, ++li)
822  if (*i || *li)
823  numBlockCounter++;
824  }
825 
826  return numBlockCounter;
827 }
828 
829 //----------------------------------------------------------------------------//
830 
831 template <class Data_T>
833 {
834  std::vector<int>::const_iterator i = loadCounts.begin();
835  std::vector<int>::const_iterator end = loadCounts.end();
836  int numLoads = 0, numBlockCounter = 0;
837  for (; i != end; ++i) {
838  if (*i) {
839  numLoads += *i;
840  numBlockCounter++;
841  }
842  }
843 
844  return (float)numLoads / std::max(1, numBlockCounter);
845 }
846 
847 //----------------------------------------------------------------------------//
848 
849 template <class Data_T>
851 {
852  std::vector<int>::iterator li = loadCounts.begin();
853  std::vector<int>::iterator lend = loadCounts.end();
854  for (; li != lend; ++li)
855  *li = 0;
856 }
857 
858 //----------------------------------------------------------------------------//
859 
860 template <class Data_T>
861 long long int
863 {
864  boost::mutex::scoped_lock lock(m_mutex);
865 
866  return sizeof(*this) +
867  fileBlockIndices.capacity() * sizeof(int) +
868  blockLoaded.capacity() * sizeof(int) +
870  blocks.capacity() * sizeof(Sparse::SparseBlock<Data_T>*) +
871 #endif
872  blockUsed.capacity() * sizeof(bool) +
873  loadCounts.capacity() * sizeof(int) +
874  refCounts.capacity() * sizeof(int) +
876  blockMutexSize * sizeof(boost::mutex) +
877 #else
878  numBlocks * sizeof(boost::mutex) +
879 #endif
880  sizeof(SparseDataReader<Data_T>);
881 }
882 
883 //----------------------------------------------------------------------------//
884 
885 } // namespace SparseFile
886 
887 //----------------------------------------------------------------------------//
888 // Specializations for FileReferences
889 //----------------------------------------------------------------------------//
890 
891 namespace SparseFile {
892 
893 //----------------------------------------------------------------------------//
894 
896 {
897 #if !USE_SHPTR
898  for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
899  delete m_hRefs[i];
900  }
901  for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
902  delete m_fRefs[i];
903  }
904  for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
905  delete m_dRefs[i];
906  }
907  for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
908  delete m_vhRefs[i];
909  }
910  for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
911  delete m_vfRefs[i];
912  }
913  for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
914  delete m_vdRefs[i];
915  }
916 #endif
917 }
918 
919 //----------------------------------------------------------------------------//
920 
921 template <>
922 inline Reference<half>*
924 {
925 #if USE_SHPTR
926  return m_hRefs[idx].get();
927 #else
928  return m_hRefs[idx];
929 #endif
930 }
931 
932 //----------------------------------------------------------------------------//
933 
934 template <>
935 inline Reference<V3h>*
937 {
938 #if USE_SHPTR
939  return m_vhRefs[idx].get();
940 #else
941  return m_vhRefs[idx];
942 #endif
943 }
944 
945 //----------------------------------------------------------------------------//
946 
947 template <>
948 inline Reference<float>*
950 {
951 #if USE_SHPTR
952  return m_fRefs[idx].get();
953 #else
954  return m_fRefs[idx];
955 #endif
956 }
957 
958 //----------------------------------------------------------------------------//
959 
960 template <>
961 inline Reference<V3f>*
963 {
964 #if USE_SHPTR
965  return m_vfRefs[idx].get();
966 #else
967  return m_vfRefs[idx];
968 #endif
969 }
970 
971 //----------------------------------------------------------------------------//
972 
973 template <>
974 inline Reference<double>*
976 {
977 #if USE_SHPTR
978  return m_dRefs[idx].get();
979 #else
980  return m_dRefs[idx];
981 #endif
982 }
983 
984 //----------------------------------------------------------------------------//
985 
986 template <>
987 inline Reference<V3d>*
989 {
990 #if USE_SHPTR
991  return m_vdRefs[idx].get();
992 #else
993  return m_vdRefs[idx];
994 #endif
995 }
996 
997 //----------------------------------------------------------------------------//
998 
999 template <>
1000 inline size_t FileReferences::append<half>(Reference<half>::Ptr ref)
1001 {
1002  Mutex::scoped_lock lock(m_mutex);
1003 
1004  m_hRefs.push_back(ref);
1005  return m_hRefs.size() - 1;
1006 }
1007 
1008 //----------------------------------------------------------------------------//
1009 
1010 template <>
1011 inline size_t FileReferences::append<V3h>(Reference<V3h>::Ptr ref)
1012 {
1013  Mutex::scoped_lock lock(m_mutex);
1014 
1015  m_vhRefs.push_back(ref);
1016  return m_vhRefs.size() - 1;
1017 }
1018 
1019 //----------------------------------------------------------------------------//
1020 
1021 template <>
1022 inline size_t FileReferences::append<float>(Reference<float>::Ptr ref)
1023 {
1024  Mutex::scoped_lock lock(m_mutex);
1025 
1026  m_fRefs.push_back(ref);
1027  return m_fRefs.size() - 1;
1028 }
1029 
1030 //----------------------------------------------------------------------------//
1031 
1032 template <>
1033 inline size_t FileReferences::append<V3f>(Reference<V3f>::Ptr ref)
1034 {
1035  Mutex::scoped_lock lock(m_mutex);
1036 
1037  m_vfRefs.push_back(ref);
1038  return m_vfRefs.size() - 1;
1039 }
1040 
1041 //----------------------------------------------------------------------------//
1042 
1043 template <>
1044 inline size_t FileReferences::append<double>(Reference<double>::Ptr ref)
1045 {
1046  Mutex::scoped_lock lock(m_mutex);
1047 
1048  m_dRefs.push_back(ref);
1049  return m_dRefs.size() - 1;
1050 }
1051 
1052 //----------------------------------------------------------------------------//
1053 
1054 template <>
1055 inline size_t FileReferences::append<V3d>(Reference<V3d>::Ptr ref)
1056 {
1057  Mutex::scoped_lock lock(m_mutex);
1058 
1059  m_vdRefs.push_back(ref);
1060  return m_vdRefs.size() - 1;
1061 }
1062 
1063 //----------------------------------------------------------------------------//
1064 
1065 template <>
1066 inline size_t FileReferences::numRefs<half>() const
1067 {
1068  Mutex::scoped_lock lock(m_mutex);
1069 
1070  return m_hRefs.size();
1071 }
1072 
1073 //----------------------------------------------------------------------------//
1074 
1075 template <>
1076 inline size_t FileReferences::numRefs<V3h>() const
1077 {
1078  Mutex::scoped_lock lock(m_mutex);
1079 
1080  return m_vhRefs.size();
1081 }
1082 
1083 //----------------------------------------------------------------------------//
1084 
1085 template <>
1086 inline size_t FileReferences::numRefs<float>() const
1087 {
1088  Mutex::scoped_lock lock(m_mutex);
1089 
1090  return m_fRefs.size();
1091 }
1092 
1093 //----------------------------------------------------------------------------//
1094 
1095 template <>
1096 inline size_t FileReferences::numRefs<V3f>() const
1097 {
1098  Mutex::scoped_lock lock(m_mutex);
1099 
1100  return m_vfRefs.size();
1101 }
1102 
1103 //----------------------------------------------------------------------------//
1104 
1105 template <>
1106 inline size_t FileReferences::numRefs<double>() const
1107 {
1108  Mutex::scoped_lock lock(m_mutex);
1109 
1110  return m_dRefs.size();
1111 }
1112 
1113 //----------------------------------------------------------------------------//
1114 
1115 template <>
1116 inline size_t FileReferences::numRefs<V3d>() const
1117 {
1118  Mutex::scoped_lock lock(m_mutex);
1119 
1120  return m_vdRefs.size();
1121 }
1122 
1123 //----------------------------------------------------------------------------//
1124 // Implementations for FileReferences
1125 //----------------------------------------------------------------------------//
1126 
1127 #if 0
1128 
1129 template <class Data_T>
1131 {
1132  assert(false && "Do not use memory limiting on sparse fields that aren't "
1133  "simple scalars or vectors!");
1135  "FileReferences::ref(): Do not use memory limiting on sparse "
1136  "fields that aren't simple scalars or vectors!");
1137  static Reference<Data_T> dummy("", "");
1138  return dummy;
1139 }
1140 
1141 //----------------------------------------------------------------------------//
1142 
1143 template <class Data_T>
1144 size_t FileReferences::append(const Reference<Data_T>&)
1145 {
1146  assert(false && "Do not use memory limiting on sparse fields that aren't "
1147  "simple scalars or vectors!");
1149  "FileReferences::append(): Do not use memory limiting on sparse "
1150  "fields that aren't simple scalars or vectors!");
1151  return -1;
1152 }
1153 
1154 //----------------------------------------------------------------------------//
1155 
1156 template <class Data_T>
1157 size_t FileReferences::numRefs() const
1158 {
1159  assert(false && "Do not use memory limiting on sparse fields that aren't "
1160  "simple scalars or vectors!");
1162  "FileReferences::numRefs(): "
1163  "Do not use memory limiting on sparse "
1164  "fields that aren't "
1165  "simple scalars or vectors!");
1166  return -1;
1167 }
1168 
1169 #endif
1170 
1171 //----------------------------------------------------------------------------//
1172 
1173 } // namespace SparseFile
1174 
1175 //----------------------------------------------------------------------------//
1176 // SparseFileManager implementations
1177 //----------------------------------------------------------------------------//
1178 
1179 template <class Data_T>
1180 int
1181 SparseFileManager::getNextId(const std::string filename,
1182  const std::string layerPath)
1183 {
1184  using namespace SparseFile;
1185 
1186  // Must hold a mutex while appending to m_fileData
1187  boost::mutex::scoped_lock lock(m_mutex);
1188 
1189  int id = m_fileData.append<Data_T>(Reference<Data_T>::create(filename,
1190  layerPath));
1191  return id;
1192 }
1193 
1194 //----------------------------------------------------------------------------//
1195 
1196 template <class Data_T>
1197 void
1199 {
1200  boost::mutex::scoped_lock lock(m_mutex);
1201 
1204 
1205  CacheList::iterator it = m_blockCacheList.begin();
1206  CacheList::iterator end = m_blockCacheList.end();
1207  CacheList::iterator next;
1208 
1209  int64_t bytesFreed = 0;
1210 
1211  while (it != end) {
1212  if (it->blockType == blockType && it->refIdx == refIdx) {
1213  if (it == m_nextBlock) {
1214  ++m_nextBlock;
1215  }
1216  next = it;
1217  ++next;
1218  bytesFreed += reference->blockSize(it->blockIdx);
1219  m_blockCacheList.erase(it);
1220  it = next;
1221  } else {
1222  ++it;
1223  }
1224  }
1225  m_memUse -= bytesFreed;
1226 
1227  std::vector<int>().swap(reference->fileBlockIndices);
1228 #if F3D_NO_BLOCKS_ARRAY
1229  reference->fileBlockIndices.resize(reference->numBlocks, -1);
1230 #else
1231  reference->fileBlockIndices.resize(reference->blocks.size(), -1);
1232  typedef typename SparseFile::Reference<Data_T>::BlockPtrs BlockPtrs;
1233  BlockPtrs().swap(reference->blocks);
1234 #endif
1235  std::vector<int>().swap(reference->blockLoaded);
1236  std::vector<bool>().swap(reference->blockUsed);
1237  std::vector<int>().swap(reference->loadCounts);
1238  std::vector<int>().swap(reference->refCounts);
1239  delete[] reference->blockMutex;
1240  reference->blockMutex = NULL;
1241 }
1242 
1243 //----------------------------------------------------------------------------//
1244 
1245 template <class Data_T>
1248 {
1249  return m_fileData.ref<Data_T>(index);
1250 }
1251 
1252 //----------------------------------------------------------------------------//
1253 
1254 template <class Data_T>
1255 void
1256 SparseFileManager::activateBlock(int fileId, int blockIdx)
1257 {
1259 
1260  if (reference->fileBlockIndices[blockIdx] >= 0) {
1261  if (!reference->blockLoaded[blockIdx]) {
1262  int blockSize = reference->blockSize(blockIdx);
1263  if (m_limitMemUse) {
1264  // if we already have enough free memory, deallocateBlocks()
1265  // will just return
1266  deallocateBlocks(blockSize);
1267  }
1268 
1269  if (!reference->fileIsOpen()) {
1270  reference->openFile();
1271  }
1272 
1273  boost::mutex::scoped_lock lock_A(m_mutex);
1274 #if F3D_SHORT_MUTEX_ARRAY
1275  boost::mutex::scoped_lock
1276  lock_B(reference->blockMutex[blockIdx % reference->blockMutexSize]);
1277 #else
1278  boost::mutex::scoped_lock lock_B(reference->blockMutex[blockIdx]);
1279 #endif
1280  // check to see if it was loaded between when the function
1281  // started and we got the lock on the block
1282  if (!reference->blockLoaded[blockIdx]) {
1283  reference->loadBlock(blockIdx);
1284  reference->loadCounts[blockIdx]++;
1286  m_memUse += blockSize;
1287  }
1288  }
1289  }
1290  reference->blockUsed[blockIdx] = true;
1291 }
1292 
1293 //----------------------------------------------------------------------------//
1294 
1295 template <class Data_T>
1296 void
1297 SparseFileManager::incBlockRef(int fileId, int blockIdx)
1298 {
1300 
1301  if (reference->fileBlockIndices[blockIdx] >= 0) {
1302  reference->incBlockRef(blockIdx);
1303  }
1304 }
1305 
1306 //----------------------------------------------------------------------------//
1307 
1308 template <class Data_T>
1309 void
1310 SparseFileManager::decBlockRef(int fileId, int blockIdx)
1311 {
1313 
1314  if (reference->fileBlockIndices[blockIdx] >= 0) {
1315  reference->decBlockRef(blockIdx);
1316  }
1317 }
1318 
1319 //----------------------------------------------------------------------------//
1320 
1322 
1323 //----------------------------------------------------------------------------//
1324 
1325 #endif
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseFileManager::reference
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1247
SparseFile::Reference::blockUsed
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition: SparseFile.h:144
SparseFile::Reference::layerPath
std::string layerPath
Definition: SparseFile.h:127
SparseFile::CacheBlock::refIdx
int refIdx
Definition: SparseFile.h:324
SparseFile::Reference::averageLoads
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition: SparseFile.h:832
OgSparseDataReader
Definition: SparseFile.h:86
SparseFileManager::removeFieldFromCache
void removeFieldFromCache(int refIdx)
Definition: SparseFile.h:1198
SparseFile::Reference::m_mutex
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition: SparseFile.h:253
SparseFile::Reference::closeFile
void closeFile()
Closes the file. Will be re-opened as needed.
Definition: SparseFile.h:709
SparseFile::Reference::openFile
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition: SparseFile.cpp:537
Msg::SevWarning
Definition: Log.h:68
SparseFile::Reference::fileIsOpen
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition: SparseFile.h:668
SparseFile::Reference::loadCounts
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition: SparseFile.h:147
SparseFile::CacheBlock::CacheBlock
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition: SparseFile.h:326
SparseFile::FileReferences::numRefs
size_t numRefs() const
Returns the number of file references of the corresponding collection.
SparseFile::Reference::memSize
long long int memSize() const
Memory use for the Reference.
Definition: SparseFile.h:862
Sparse::SparseBlock
Definition: SparseField.h:227
SparseFileManager::m_blockCacheList
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition: SparseFile.h:537
SparseFileManager::activateBlock
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user,...
Definition: SparseFile.h:1256
SparseDataReader.h
Contains functions controlling the loading of sparse fields.
SparseFileManager::incBlockRef
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
Definition: SparseFile.h:1297
SparseField::memSize
virtual long long int memSize() const
Returns the memory usage (in bytes)
Definition: SparseField.h:1708
F3D_SHORT_MUTEX_ARRAY
#define F3D_SHORT_MUTEX_ARRAY
Definition: SparseFile.h:63
SparseFile::Reference::Ptr
Reference * Ptr
Definition: SparseFile.h:115
SparseFile::Reference::m_numActiveBlocks
size_t m_numActiveBlocks
Number of currently active blocks.
Definition: SparseFile.h:256
SparseFile::Reference::m_fileHandle
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition: SparseFile.h:230
SparseFile::Reference::blockMutexSize
int blockMutexSize
Size of the mutex array. Used as modulus base.
Definition: SparseFile.h:158
DataTypeEnum
DataTypeEnum
Definition: Traits.h:108
SparseFile::FileReferences::m_vhRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition: SparseFile.h:305
SparseFile::Reference::unloadBlock
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition: SparseFile.h:721
SparseFile::Reference::m_ogRoot
OgIGroupPtr m_ogRoot
Ogawa archive root.
Definition: SparseFile.h:248
SparseFile::Reference::m_ogReaderPtr
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
Definition: SparseFile.h:241
SparseFile
Definition: SparseFile.h:90
SparseFileManager::m_maxMemUseInBytes
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:519
SparseFile::FileReferences
Definition: SparseFile.h:264
SparseFile::Reference::m_ogArchive
IArchivePtr m_ogArchive
Ogawa archive.
Definition: SparseFile.h:246
Hdf5Util::H5ScopedGopen
Scoped object - opens a group on creation and closes it on destruction.
Definition: Hdf5Util.h:194
SparseFile::FileReferences::ref
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
SparseFile::Reference::blocks
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition: SparseFile.h:140
SparseFile::Reference::create
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
Definition: SparseFile.h:659
SparseFile::CacheBlock::blockIdx
int blockIdx
Definition: SparseFile.h:325
SparseField
This Field subclass stores voxel data in block-allocated arrays.
Definition: SparseField.h:350
SparseFileManager::m_limitMemUse
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:526
SparseFile::FileReferences::m_hRefs
std::deque< Reference< half >::Ptr > m_hRefs
Definition: SparseFile.h:304
SparseFileManager::ms_singleton
static SparseFileManager * ms_singleton
Pointer to singleton.
Definition: SparseFile.h:497
OgawaFwd.h
Contains forward declarations for Ogawa classes.
SparseFile::FileReferences::append
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
SparseFile::Reference::incBlockRef
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition: SparseFile.h:744
ns.h
Exception.h
Contains Exception base class.
SparseFile::Reference::Mutex
boost::mutex Mutex
Definition: SparseFile.h:214
SparseFile::Reference::setNumBlocks
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
Definition: SparseFile.h:676
SparseFile::FileReferences::m_dRefs
std::deque< Reference< double >::Ptr > m_dRefs
Definition: SparseFile.h:308
SparseFile::CacheBlock::blockType
DataTypeEnum blockType
Definition: SparseFile.h:323
SparseFileManager::m_fileData
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:530
SparseFile::Reference::operator=
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition: SparseFile.h:599
Hdf5Util.h
Contains various utility functions for Hdf5.
SparseFileManager::m_memUse
int64_t m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:522
IArchivePtr
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
Definition: OgawaFwd.h:87
detail::max
T max(const T a, const T2 b)
Max operation on mixed types.
Definition: FieldSampler.h:32
SparseFileManager::m_maxMemUse
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:516
SparseFile::Reference::m_ogReader
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
Definition: SparseFile.h:244
SparseFile::Reference::m_reader
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition: SparseFile.h:238
SparseFileManager::m_mutex
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:545
SparseFile::Reference::fileBlockIndices
std::vector< int > fileBlockIndices
Index in file for each block.
Definition: SparseFile.h:134
SparseFileManager::getNextId
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Definition: SparseFile.h:1181
SparseFile::Reference::blockSize
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition: SparseFile.h:770
DataTypeTraits::typeEnum
static DataTypeEnum typeEnum()
SparseFileManager::addBlockToCache
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition: SparseFile.cpp:260
SparseFile::Reference::m_ogLayerGroup
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
Definition: SparseFile.h:250
SparseFile::Reference::decBlockRef
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition: SparseFile.h:757
SparseFile::Reference::refCounts
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Definition: SparseFile.h:151
Sparse
Namespace for sparse field specifics.
Definition: SparseField.h:221
SparseField::activateBlock
void activateBlock(const int blockId) const
Activates a given block.
Definition: SparseField.h:1927
SparseFileManager
Definition: SparseFile.h:396
SparseFileManager::CacheList
std::list< SparseFile::CacheBlock > CacheList
Definition: SparseFile.h:406
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition: SparseDataReader.h:70
SparseFile::Reference::Reference
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition: SparseFile.h:558
OgIGroupPtr
boost::shared_ptr< OgIGroup > OgIGroupPtr
Definition: OgawaFwd.h:69
SparseField::decBlockRef
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
Definition: SparseField.h:1935
SparseFileManager::m_nextBlock
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:541
SparseFile::Reference
Definition: SparseFile.h:104
SparseFile::Reference::occupiedBlocks
int occupiedBlocks
Definition: SparseFile.h:131
SparseFile::FileReferences::m_vfRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition: SparseFile.h:307
SparseFile::Reference::numBlocks
int numBlocks
Definition: SparseFile.h:130
SparseFile::FileReferences::m_mutex
Mutex m_mutex
Definition: SparseFile.h:315
Traits.h
SparseFile::FileReferences::memSize
long long int memSize() const
Returns the memory use for the refs.
Definition: SparseFile.cpp:452
SparseFile::Reference::numLoadedBlocks
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition: SparseFile.h:792
SparseFile::Reference::~Reference
~Reference()
Destructor.
Definition: SparseFile.h:571
SparseFileManager::decBlockRef
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
Definition: SparseFile.h:1310
SparseField::incBlockRef
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
Definition: SparseField.h:1919
SparseFile::FileReferences::~FileReferences
~FileReferences()
Definition: SparseFile.h:895
SparseFile::Reference::blockMutex
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition: SparseFile.h:155
SparseFile::Reference::resetCacheStatistics
void resetCacheStatistics()
Resets counts of total block loads.
Definition: SparseFile.h:850
SparseFile::Reference::m_layerGroup
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition: SparseFile.h:234
DataTypeTraits
Definition: Traits.h:266
SparseFile::Reference::totalLoads
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition: SparseFile.h:778
SparseFile::FileReferences::m_vdRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition: SparseFile.h:309
FIELD3D_API
#define FIELD3D_API
Definition: ns.h:77
FIELD3D_NAMESPACE_OPEN
Definition: FieldMapping.cpp:74
SparseFileManager::deallocateBlocks
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
Definition: SparseFile.cpp:160
SparseFile::Reference::blockLoaded
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition: SparseFile.h:137
Msg::print
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition: Log.cpp:70
SparseFile::Reference::BlockPtrs
Sparse::SparseBlock< Data_T > * BlockPtrs
Definition: SparseFile.h:119
detail::min
T min(const T a, const T2 b)
Min operation on mixed types.
Definition: FieldSampler.h:25
SparseFile::FileReferences::m_fRefs
std::deque< Reference< float >::Ptr > m_fRefs
Definition: SparseFile.h:306
F3D_MUTEX_ARRAY_SIZE
#define F3D_MUTEX_ARRAY_SIZE
Definition: SparseFile.h:64
SparseFile::Reference::numVoxels
int numVoxels
Definition: SparseFile.h:129
SparseFile::FileReferences::Mutex
boost::mutex Mutex
Definition: SparseFile.h:313
SparseFile::Reference::loadBlock
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition: SparseFile.cpp:498
SparseFile::Reference::totalLoadedBlocks
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks),...
Definition: SparseFile.h:807
SparseFile::Reference::filename
std::string filename
Definition: SparseFile.h:126
SparseFile::CacheBlock
Definition: SparseFile.h:321
SparseFile::Reference::valuesPerBlock
int valuesPerBlock
Definition: SparseFile.h:128
F3D_NO_BLOCKS_ARRAY
#define F3D_NO_BLOCKS_ARRAY
Definition: SparseFile.h:65