Merge pull request #9391 from gem/less-memory

Using less memory in multiFaultSources

debian/changelog

@@ -1,3 +1,7 @@
+  [Michele Simionato]
+  * Reduced memory consumption in multiFaultSources
+  * Implemented batch processing for AELO calculations
+
   [Anirudh Rao]
   * Improved the performance of non-parametric liquefaction models
 

openquake/baselib/general.py

@@ -1378,6 +1378,9 @@ def getsizeof(o, ids=None):
     if id(o) in ids:
         return 0
 
+    if hasattr(o, 'nbytes'):
+        return o.nbytes
+
     nbytes = sys.getsizeof(o)
     ids.add(id(o))
 

openquake/hazardlib/source/multi_fault.py

@@ -52,7 +52,7 @@ class MultiFaultSource(BaseSeismicSource):
     :param tectonic_region_type:
         A string that defines the TRT of the fault source
     :param rupture_idxs:
-        A list of lists. Each element contains the IDs of the sections
+        A list of arrays. Each element contains the IDs of the sections
         participating to a rupture. The cardinality of this list is N.
         The IDs are integers.
     :param occurrence_probs:
@@ -107,7 +107,7 @@ def get_sections(self):
         if self.hdf5path == '':  # in the tests
             return self.sections
         with hdf5.File(self.hdf5path, 'r') as f:
-            geoms = f['multi_fault_sections'][:]
+            geoms = f['multi_fault_sections'][:]  # small
         sections = [geom_to_kite(geom) for geom in geoms]
         for idx, sec in enumerate(sections):
             sec.suid = idx

openquake/hazardlib/source_reader.py

@@ -31,6 +31,7 @@
 from openquake.hazardlib.lt import apply_uncertainties
 from openquake.hazardlib.geo.surface.kite_fault import kite_to_geom
 
+U16 = numpy.uint16
 TWO16 = 2 ** 16  # 65,536
 TWO24 = 2 ** 24  # 16,777,216
 TWO30 = 2 ** 30  # 1,073,741,24
@@ -347,7 +348,7 @@ def fix_geometry_sections(smdict, dstore):
                         raise RuntimeError('Missing geometryModel files!')
                     if dstore:
                         src.hdf5path = dstore.tempname
-                    src.rupture_idxs = [tuple(s2i[idx] for idx in idxs)
+                    src.rupture_idxs = [U16([s2i[idx] for idx in idxs])
                                         for idxs in src.rupture_idxs]
                     for idxs in src.rupture_idxs:
                         section_idxs.extend(idxs)

openquake/hazardlib/sourceconverter.py

@@ -37,6 +37,7 @@
 U32 = numpy.uint32
 F32 = numpy.float32
 F64 = numpy.float64
+TWO16 = 2**16
 EPSILON = 1E-12
 source_dt = numpy.dtype([('source_id', U32), ('num_ruptures', U32),
                          ('pik', hdf5.vuint8)])
@@ -1144,6 +1145,8 @@ def convert_multiFaultSource(self, node):
             with context(self.fname, node):
                 idxs = [x.decode('utf8').split() for x in dic['rupture_idxs']]
                 mags = rounded_unique(dic['mag'], idxs)
+                for idx in idxs:
+                    assert U32(idx).max() < TWO16, idx
             # NB: the sections will be fixed later on, in source_reader
             mfs = MultiFaultSource(sid, name, trt, idxs,
                                    dic['probs_occur'],