Skip to content

Commit 44ba82d

Browse files
michelesmicheles
committed
Merge pull request gem#41 from gem/smarter-risklib
Make the risklib able to read csv inputs
2 parents 1370e3c + a5ed176 commit 44ba82d

File tree

6 files changed

+316
-25
lines changed

6 files changed

+316
-25
lines changed

‎openquake/risklib/api.py

+23-25
Original file line numberDiff line numberDiff line change
@@ -34,29 +34,10 @@ def __init__(self, vulnerability_function, steps=10):
3434

3535
def __call__(self, hazard_curves):
3636
return [scientific.classical(
37-
self.vulnerability_function,
38-
self.loss_ratio_exceedance_matrix,
39-
hazard_curve,
40-
self.steps)
41-
for hazard_curve in hazard_curves]
42-
43-
44-
class ScenarioDamage(object):
45-
"""
46-
Scenario damage calculator producing a damage distribution for each asset,
47-
i.e. a matrix NxM where N is the number of realizations of the ground
48-
motion field and M is the numbers of damage states. Take in input a
49-
FragilityFunctionSequence object.
50-
"""
51-
def __init__(self, ffs):
52-
self.ffs = ffs
53-
54-
def __call__(self, ground_motion_fields):
55-
return [
56-
numpy.array([
57-
self.ffs.ground_motion_value_fractions(gmv)
58-
for gmv in ground_motion_field])
59-
for ground_motion_field in ground_motion_fields]
37+
self.vulnerability_function,
38+
self.loss_ratio_exceedance_matrix,
39+
hazard_curve,
40+
self.steps) for hazard_curve in hazard_curves]
6041

6142

6243
class ProbabilisticEventBased(object):
@@ -117,5 +98,22 @@ def __call__(self, ground_motion_fields):
11798
len(ground_motion_fields), len(ground_motion_fields[0]),
11899
self.seed, self.correlation)
119100

120-
return [self.vulnerability_function(ground_motion_values)
121-
for ground_motion_values in ground_motion_fields]
101+
return map(self.vulnerability_function, ground_motion_fields)
102+
103+
104+
class ScenarioDamage(object):
105+
"""
106+
Scenario damage calculator producing a damage distribution for each asset,
107+
i.e. a matrix NxM where N is the number of realizations of the ground
108+
motion field and M is the numbers of damage states. Take in input a
109+
FragilityFunctionSequence object.
110+
"""
111+
def __init__(self, ffs):
112+
self.ffs = ffs
113+
114+
def __call__(self, ground_motion_fields):
115+
"""
116+
The ground motion field is a list of ground motion values
117+
(one array for each site). Returns a list of arrays (one per site).
118+
"""
119+
return map(self.ffs.ground_motion_fractions, ground_motion_fields)

‎openquake/risklib/models/input.py

+8
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,14 @@ def ground_motion_value_fractions(self, gmv):
150150
damage_state_values[n_limit_states] = self[n_limit_states - 1].poe(gmv)
151151
return damage_state_values
152152

153+
def ground_motion_fractions(self, gmvs):
154+
"""
155+
Compute the damage state fractions for the given array of ground
156+
motion values. Returns an NxM matrix where N is the number of
157+
realizations and M is the numbers of damage states.
158+
"""
159+
return numpy.array(map(self.ground_motion_value_fractions, gmvs))
160+
153161

154162
class FragilityModel(Mapping):
155163
"""

‎openquake/risklib/readers.py

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
# -*- coding: utf-8 -*-
2+
# vim: tabstop=4 shiftwidth=4 softtabstop=4
3+
4+
# Copyright (c) 2013, GEM foundation
5+
6+
# OpenQuake is free software: you can redistribute it and/or modify it
7+
# under the terms of the GNU Affero General Public License as published
8+
# by the Free Software Foundation, either version 3 of the License, or
9+
# (at your option) any later version.
10+
11+
# OpenQuake is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU General Public License for more details.
15+
16+
# You should have received a copy of the GNU Affero General Public License
17+
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
18+
19+
"""
20+
Routines for reading csv files for the risklib calculators
21+
"""
22+
23+
import os
24+
import ast
25+
import csv
26+
import gzip
27+
import zipfile
28+
import ConfigParser
29+
import collections
30+
31+
import numpy
32+
33+
from openquake.risklib.models.input import FragilityModel
34+
from openquake.risklib.scientific import VulnerabilityFunction
35+
from openquake.risklib.utils import Register
36+
37+
38+
read = Register() # dictionary {filetype: function}
39+
40+
41+
def may_be_gz(name, fileobj):
42+
"""
43+
If fileobj is a regular file returns it unchanged, if it refers to a
44+
gzipped file returns a :class:`gzip.GzipFile` object. The selection
45+
is made according to the extension in `name`.
46+
"""
47+
if name.endswith('.gz'):
48+
return gzip.GzipFile(name, fileobj=fileobj)
49+
else:
50+
return fileobj
51+
52+
53+
class Archive(object):
54+
"""
55+
A class to read/write files from a directory or a zipfile
56+
archive. The directory/zipfile can contain .gz files.
57+
"""
58+
def __init__(self, path):
59+
self.path = path
60+
self.zfile = None
61+
if not os.path.isdir(path):
62+
assert zipfile.is_zipfile(path), '%r is not a zipfile' % path
63+
self.zfile = zipfile.ZipFile(path)
64+
65+
def open(self, name, mode='r'):
66+
assert name, 'Expected a file name, got %r' % name
67+
if self.zfile:
68+
return may_be_gz(name, self.zfile.open(name, mode))
69+
else:
70+
return may_be_gz(name, open(os.path.join(self.path, name), mode))
71+
72+
73+
Asset = collections.namedtuple(
74+
'Asset', 'site asset_id number_of_units taxonomy')
75+
76+
77+
@read.add('fragility')
78+
def read_fragility(rows):
79+
"""
80+
Given a list of string lists returns a FragilityModel
81+
"""
82+
irows = iter(rows)
83+
fmt, iml, limit_states = map(ast.literal_eval, irows.next())
84+
args = [map(ast.literal_eval, row) for row in irows]
85+
return FragilityModel(fmt, iml['IMT'], iml['imls'], limit_states, *args)
86+
87+
88+
@read.add('vulnerability')
89+
def read_vulnerability(rows):
90+
"""
91+
Given a list of string lists returns a dictionary of VulnerabilityFunctions
92+
"""
93+
d = {}
94+
for row in rows:
95+
id_, imt, iml, loss_ratio, coefficients, distribution = map(
96+
ast.literal_eval, row)
97+
d[id_] = VulnerabilityFunction(
98+
iml, loss_ratio, coefficients, distribution)
99+
return d
100+
101+
102+
@read.add('exposure')
103+
def read_exposure(rows):
104+
"""
105+
Given a list of string lists returns a list of assets ordered by taxonomy.
106+
"""
107+
assetlist = []
108+
by_taxonomy = sorted(rows, key=lambda row: row[-1])
109+
for lon, lat, asset_id, number_of_units, taxonomy in by_taxonomy:
110+
assetlist.append(Asset((float(lon), float(lat)), asset_id,
111+
float(number_of_units), taxonomy))
112+
return assetlist
113+
114+
115+
@read.add('gmf')
116+
def read_gmf(rows):
117+
"""
118+
Given a list of string lists of the form
119+
[(lon, lat, gmv1, ... , gmvN), ...]
120+
returns a list of the form
121+
[(lon, lat, gmvs), ...]
122+
"""
123+
out = []
124+
for row in rows:
125+
row = map(float, row)
126+
out.append([row[0], row[1], numpy.array(row[2:])])
127+
return out
128+
129+
130+
def read_calculator_input(path, config='job.ini', delimiter='\t'):
131+
"""
132+
Read a .ini configuration file from a directory or a zip archive
133+
and then extract the inner files. Keep everything in memory.
134+
135+
:param path: a filename
136+
:param config: a .ini file name (non-absolute)
137+
:param delimiter: the separator of the csv files in the archive
138+
(default tab)
139+
"""
140+
archive = Archive(path)
141+
conf = archive.open(config)
142+
try:
143+
cfp = ConfigParser.RawConfigParser()
144+
cfp.readfp(conf)
145+
finally:
146+
conf.close()
147+
inputdic = {}
148+
for parname, parvalue in cfp.items('general'):
149+
inputdic[parname] = parvalue
150+
if parname.endswith('_file'):
151+
filetype = parname[:-5] # strip _file suffix
152+
with archive.open(parvalue) as fileobj:
153+
reader = csv.reader(fileobj, delimiter=delimiter)
154+
inputdic[filetype] = read[filetype](reader)
155+
return inputdic

‎openquake/risklib/tests/readers_test.py

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
import os
2+
import unittest
3+
import tempfile
4+
import shutil
5+
import zipfile
6+
from openquake.risklib.readers import read_calculator_input, Archive
7+
8+
9+
TEST_FRAGILITY = '''\
10+
'continuous' {'IMT': 'PGA', 'imls': None} ['slight', 'moderate', 'extensive', 'complete']
11+
'W' [(0.147, 0.414), (0.236, 0.666), (0.416, 1.172), (0.627, 1.77)] None
12+
'A' [(0.122, 0.345), (0.171, 0.483), (0.236, 0.666), (0.383, 1.08)] None
13+
'DS' [(0.081, 0.23), (0.122, 0.345), (0.228, 0.643), (0.326, 0.919)] None
14+
'UFB' [(0.114, 0.322), (0.171, 0.483), (0.326, 0.919), (0.489, 1.379)] None
15+
'RC' [(0.13, 0.368), (0.187, 0.529), (0.334, 0.942), (0.627, 1.77)] None
16+
'''
17+
18+
TEST_VULNERABILITY = '''\
19+
'W' 'PGA' [0.01, 0.06, 0.11, 0.16, 0.21, 0.26, 0.31, 0.36, 0.41, 0.46, 0.51, 0.56, 0.61, 0.66, 0.71, 0.76, 0.81, 0.86, 0.91, 0.96, 1.01, 1.06, 1.11, 1.16, 1.21, 1.26, 1.31, 1.36, 1.41, 1.46] [6.88403e-08, 0.001595496, 0.013508641, 0.040191158, 0.080174564, 0.12987209, 0.185478238, 0.243763275, 0.302283223, 0.359329841, 0.413793327, 0.465015667, 0.512663741, 0.556629259, 0.596953791, 0.63377442, 0.667285292, 0.697710983, 0.725288448, 0.750255091, 0.772841156, 0.793265105, 0.811731071, 0.828427693, 0.843527877, 0.857189145, 0.869554353, 0.880752611, 0.890900313, 0.900102185] [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3] 'LN'
20+
'A' 'PGA' [0.01, 0.06, 0.11, 0.16, 0.21, 0.26, 0.31, 0.36, 0.41, 0.46, 0.51, 0.56, 0.61, 0.66, 0.71, 0.76, 0.81, 0.86, 0.91, 0.96, 1.01, 1.06, 1.11, 1.16, 1.21, 1.26, 1.31, 1.36, 1.41, 1.46] [3.39926e-07, 0.006623843, 0.050557647, 0.131254324, 0.228841325, 0.328190116, 0.421201301, 0.50444515, 0.577064937, 0.639478485, 0.692657584, 0.737750901, 0.775897604, 0.808143263, 0.835409033, 0.858487537, 0.878051212, 0.894665685, 0.908804349, 0.920862323, 0.931169015, 0.93999901, 0.94758135, 0.954107302, 0.959736836, 0.964603981, 0.968821258, 0.972483321, 0.975669973, 0.978448638] [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3] 'LN'
21+
'''
22+
23+
TEST_EXPOSURE = '''\
24+
83.313823 29.461172 a1 1.0 W
25+
83.313823 29.236172 a2 1.0 W
26+
83.538823 29.086172 a3 1.0 W
27+
80.688823 28.936172 a4 1.0 W
28+
83.538823 29.011172 a5 1.0 W
29+
81.138823 28.786172 a6 1.0 W
30+
83.988823 28.486172 a7 1.0 W
31+
83.238823 29.386172 a8 1.0 W
32+
83.013823 29.086172 a9 1.0 W
33+
83.313823 28.711172 a10 1.0 W
34+
86.913823 27.736172 a11 1.0 W
35+
83.163823 29.311172 a12 1.0 W
36+
80.613823 28.936172 a13 1.0 W
37+
83.913823 29.011172 a14 1.0 W
38+
82.038823 30.286172 a15 1.0 W
39+
83.388823 29.311172 a16 1.0 W
40+
80.688823 28.861172 a17 1.0 W
41+
83.463823 28.711172 a18 1.0 W
42+
84.138823 28.411172 a19 1.0 W
43+
83.088823 29.161172 a20 1.0 W
44+
84.138823 28.786172 a21 1.0 W
45+
85.113823 28.336172 a22 1.0 W
46+
84.063823 29.011172 a23 1.0 W
47+
83.013823 29.611172 a24 1.0 W
48+
86.838823 27.736172 a25 2.0 W
49+
84.363823 28.786172 a26 2.0 W
50+
84.138823 28.561172 a27 2.0 W
51+
83.163823 29.011172 a28 2.0 W
52+
83.013823 29.236172 a29 2.0 W
53+
82.863823 28.861172 a30 2.0 W
54+
85.038823 28.561172 a31 2.0 W
55+
86.088823 28.036172 a32 2.0 W
56+
83.313823 28.786172 a33 2.0 W
57+
81.888823 28.186172 a34 2.0 W
58+
83.238823 29.311172 a35 2.0 W
59+
84.138823 29.161172 a36 2.0 W
60+
84.213823 28.786172 a37 2.0 W
61+
'''
62+
63+
64+
# 12 sites, 10 realizations each
65+
TEST_GMF = '''\
66+
81.1360288452 31.601922511 0.0155109509698 0.00281034507007 0.006137162745 0.00454256869266 0.00841536691411 0.00815489110943 0.00254942300302 0.0101655147721 0.00944116473873 0.0094607789499
67+
81.1677059042 31.601922511 0.00899057046441 0.00405841972037 0.00441534148331 0.00529987234583 0.00334456852911 0.0221581127201 0.00660507453023 0.00840619504425 0.0105270002691 0.0116465013536
68+
81.1993829633 31.701922511 0.0105041752935 0.0108706808919 0.0037678053978 0.0105597140166 0.00442298626614 0.0153968883449 0.00489369708777 0.0174061610838 0.0103710312376 0.0254641145336
69+
81.2310600223 31.801922511 0.0038314949803 0.00701340019108 0.00401934628066 0.00276358630792 0.00624378179609 0.0335800686421 0.0172930286757 0.0206501711555 0.00753954561238 0.0072787269004
70+
81.2627370814 31.901922511 0.0128910667848 0.00580351154962 0.00388071555435 0.00238070724146 0.00546143484763 0.0104924694544 0.00532817103564 0.0170757540609 0.0124151013776 0.00377333270407
71+
81.2944141404 30.601922511 0.012519733433 0.0057127539192 0.00453065872893 0.00949023614394 0.00636330038807 0.00735163083445 0.00925537617681 0.0275380124595 0.0114083376694 0.00465955611959
72+
81.3260911994 30.601922511 0.0045877936891 0.0147964518058 0.00322050382136 0.00814347439692 0.00716796830064 0.0103024384277 0.00981195819997 0.0177419573727 0.0125670163171 0.0101488439452
73+
81.3577682585 30.701922511 0.0102443105402 0.00606207129316 0.00823714945475 0.00524593011434 0.00540979277623 0.010082922739 0.00784231206331 0.0101236254072 0.0190497345128 0.00805915211349
74+
81.3894453175 30.801922511 0.00654663652265 0.0095964929733 0.0122152562482 0.00834771864269 0.00570107753851 0.0126915283663 0.0109467946063 0.0143932147114 0.0121319515459 0.00539985982664
75+
81.4211223766 30.901922511 0.012959412241 0.00354379598578 0.00646797082495 0.0108914758967 0.00545189598765 0.00799026400705 0.01864942028 0.0323645600851 0.00352523429059 0.00610965297776
76+
81.4527994356 31.101922511 0.0187101169056 0.00840790036664 0.00623415340343 0.010885256269 0.00439819502041 0.0206557263556 0.0132246632429 0.014605474028 0.00519319743559 0.0164983082137
77+
81.4844764947 31.201922511 0.0111324218367 0.00665166335463 0.00481592358942 0.0151723816152 0.0092773187225 0.0171059289499 0.00777314631674 0.0122995131714 0.0131377292775 0.00887701181102
78+
'''
79+
80+
81+
class TestIO(unittest.TestCase):
82+
FILETYPES = 'FRAGILITY VULNERABILITY EXPOSURE GMF'.split()
83+
84+
def setUp(self):
85+
# create a temporary directory with different input files
86+
self.path = tempfile.mkdtemp()
87+
self.archive = Archive(self.path)
88+
89+
def writefiles(self, ext):
90+
with self.archive.open('job.ini', 'w') as ini:
91+
ini.write('[general]\n')
92+
for ftype in self.FILETYPES:
93+
ini.write('%s_file=%s%s\n' % (ftype, ftype, ext))
94+
with self.archive.open(ftype + ext, 'w') as f:
95+
f.write(globals()['TEST_%s' % ftype])
96+
97+
def test_read_csv_files(self):
98+
self.writefiles('.csv')
99+
inp = read_calculator_input(self.path)
100+
self.assertEqual(len(inp['fragility']), 5)
101+
self.assertEqual(len(inp['vulnerability']), 2)
102+
self.assertEqual(len(inp['exposure']), 37)
103+
self.assertEqual(len(inp['gmf']), 12)
104+
105+
def test_read_csv_gz_files(self):
106+
self.writefiles('.csv.gz')
107+
inp = read_calculator_input(self.path)
108+
self.assertEqual(len(inp['fragility']), 5)
109+
self.assertEqual(len(inp['vulnerability']), 2)
110+
self.assertEqual(len(inp['exposure']), 37)
111+
self.assertEqual(len(inp['gmf']), 12)
112+
113+
def test_read_zip_file(self):
114+
self.writefiles('.csv')
115+
path = self.path + '.zip'
116+
zfile = zipfile.ZipFile(path, 'w')
117+
zfile.write(os.path.join(self.path, 'job.ini'), 'job.ini')
118+
for ftype in self.FILETYPES:
119+
zfile.write(os.path.join(self.path, ftype + '.csv'),
120+
ftype + '.csv')
121+
zfile.close()
122+
inp = read_calculator_input(path)
123+
self.assertEqual(len(inp['fragility']), 5)
124+
self.assertEqual(len(inp['vulnerability']), 2)
125+
self.assertEqual(len(inp['exposure']), 37)
126+
self.assertEqual(len(inp['gmf']), 12)
127+
128+
def tearDown(self):
129+
# remove the directory
130+
shutil.rmtree(self.path)

‎qa_tests/data/exposure_model.csv.gz

69 KB
Binary file not shown.

‎qa_tests/data/gmf.csv.gz

6.37 MB
Binary file not shown.

0 commit comments

Comments
 (0)