Skip to content

Commit c25245c

Browse files
yoachimyoachim
committed
make it possible to convert schemas
1 parent 4a88e16 commit c25245c

File tree

4 files changed

+288
-2
lines changed

4 files changed

+288
-2
lines changed

‎configs/README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,7 @@ A spot to keep track of my OpSim experiments
22

33
| Filename | run name | Notes |
44
|---------- | ----------- | ------ |
5-
| blob_baseline.py | 5928523c54f7_2020 | Looks good. Excess on GP in z nad y? |
5+
| blob_baseline.py | 5928523c54f7_2030 | 1-year. Looks pretty good! (73% OSF), 76 hr (scaled up) |
6+
| new_wfd.py | 5928523c54f7_2029 | testing a new potential WFD footprint. (73% OSF), 78.9 hr |
7+
| r_band_only.py | 5928523c54f7_2032 | time test with only greedy algo 1-filter (76% OSF) (30.87) (scaled up)|
8+
| r_band_andBlob.py | 5928523c54f7_2033 | time test with blob and greedy (74% OSF), 24.4 hr (scaled up) |

‎configs/blob_baseline.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,33 @@ def generate_target_maps(nside):
1919

2020
target_maps['u'] = fs.generate_goal_map(NES_fraction=0.,
2121
WFD_fraction=0.31, SCP_fraction=0.15,
22-
GP_fraction=0.15, nside=nside,
22+
GP_fraction=0.15,
23+
WFD_upper_edge_fraction = 0,
24+
nside=nside,
2325
generate_id_map=True)
2426
target_maps['g'] = fs.generate_goal_map(NES_fraction=0.2,
2527
WFD_fraction=0.44, SCP_fraction=0.15,
28+
WFD_upper_edge_fraction = 0,
2629
GP_fraction=0.15, nside=nside,
2730
generate_id_map=True)
2831
target_maps['r'] = fs.generate_goal_map(NES_fraction=0.46,
2932
WFD_fraction=1.0, SCP_fraction=0.15,
33+
WFD_upper_edge_fraction = 0,
3034
GP_fraction=0.15, nside=nside,
3135
generate_id_map=True)
3236
target_maps['i'] = fs.generate_goal_map(NES_fraction=0.46,
3337
WFD_fraction=1.0, SCP_fraction=0.15,
38+
WFD_upper_edge_fraction = 0,
3439
GP_fraction=0.15, nside=nside,
3540
generate_id_map=True)
3641
target_maps['z'] = fs.generate_goal_map(NES_fraction=0.4,
3742
WFD_fraction=0.9, SCP_fraction=0.15,
43+
WFD_upper_edge_fraction = 0,
3844
GP_fraction=0.15, nside=nside,
3945
generate_id_map=True)
4046
target_maps['y'] = fs.generate_goal_map(NES_fraction=0.,
4147
WFD_fraction=0.9, SCP_fraction=0.15,
48+
WFD_upper_edge_fraction = 0,
4249
GP_fraction=0.15, nside=nside,
4350
generate_id_map=True)
4451

‎configs/r_band_only.py

+239
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
"""
2+
This is a configuration for a baseline simulation the Feature Based Scheduler. The if statement is used to bypass the
3+
configuration expect when instantiated by the Feature Scheduler Driver. Note that we import and fill in a
4+
SurveyTopology class that informs the (S)OCS about the projects defined on the configuration.
5+
6+
The only things that cannot be changed here are the names of the variables survey_topoly and scheduler. It is possible
7+
as those are expected by the Driver. The way those objects are configured are entirely up to the user though.
8+
9+
09/07/2018 - Ribeiro, T.
10+
"""
11+
import numpy as np
12+
import healpy as hp
13+
import lsst.sims.featureScheduler as fs
14+
from lsst.ts.scheduler.kernel import SurveyTopology
15+
16+
17+
def generate_target_maps(nside):
18+
target_maps = {}
19+
20+
target_maps['u'] = fs.generate_goal_map(NES_fraction=0.,
21+
WFD_fraction=0.31, SCP_fraction=0.15,
22+
GP_fraction=0.15,
23+
WFD_upper_edge_fraction = 0,
24+
nside=nside,
25+
generate_id_map=True)
26+
target_maps['g'] = fs.generate_goal_map(NES_fraction=0.2,
27+
WFD_fraction=0.44, SCP_fraction=0.15,
28+
WFD_upper_edge_fraction = 0,
29+
GP_fraction=0.15, nside=nside,
30+
generate_id_map=True)
31+
target_maps['r'] = fs.generate_goal_map(NES_fraction=0.46,
32+
WFD_fraction=1.0, SCP_fraction=0.15,
33+
WFD_upper_edge_fraction = 0,
34+
GP_fraction=0.15, nside=nside,
35+
generate_id_map=True)
36+
target_maps['i'] = fs.generate_goal_map(NES_fraction=0.46,
37+
WFD_fraction=1.0, SCP_fraction=0.15,
38+
WFD_upper_edge_fraction = 0,
39+
GP_fraction=0.15, nside=nside,
40+
generate_id_map=True)
41+
target_maps['z'] = fs.generate_goal_map(NES_fraction=0.4,
42+
WFD_fraction=0.9, SCP_fraction=0.15,
43+
WFD_upper_edge_fraction = 0,
44+
GP_fraction=0.15, nside=nside,
45+
generate_id_map=True)
46+
target_maps['y'] = fs.generate_goal_map(NES_fraction=0.,
47+
WFD_fraction=0.9, SCP_fraction=0.15,
48+
WFD_upper_edge_fraction = 0,
49+
GP_fraction=0.15, nside=nside,
50+
generate_id_map=True)
51+
52+
just_targets = {}
53+
54+
for key in target_maps:
55+
just_targets[key] = target_maps[key][0]
56+
norm_factor = fs.calc_norm_factor(just_targets)
57+
58+
return target_maps, norm_factor
59+
60+
61+
def generate_blob_surveys(nside):
62+
# Define what we want the final visit ratio map to look like
63+
64+
target_maps, norm_factor = generate_target_maps(nside)
65+
66+
# set up a cloud map
67+
cloud_map = target_maps['r'][0]*0 + 0.7
68+
69+
# Set up observations to be taken in blocks
70+
filter1s = ['u', 'g', 'r', 'i', 'z', 'y']
71+
filter2s = [None, 'g', 'r', 'i', None, None]
72+
73+
pair_surveys = []
74+
for filtername, filtername2 in zip(filter1s, filter2s):
75+
bfs = []
76+
bfs.append(fs.M5_diff_basis_function(filtername=filtername, nside=nside))
77+
if filtername2 is not None:
78+
bfs.append(fs.M5_diff_basis_function(filtername=filtername2, nside=nside))
79+
bfs.append(fs.Target_map_basis_function(filtername=filtername,
80+
target_map=target_maps[filtername][0],
81+
out_of_bounds_val=hp.UNSEEN, nside=nside,
82+
norm_factor=norm_factor))
83+
if filtername2 is not None:
84+
bfs.append(fs.Target_map_basis_function(filtername=filtername2,
85+
target_map=target_maps[filtername2][0],
86+
out_of_bounds_val=hp.UNSEEN, nside=nside,
87+
norm_factor=norm_factor))
88+
bfs.append(fs.Slewtime_basis_function(filtername=filtername, nside=nside))
89+
bfs.append(fs.Strict_filter_basis_function(filtername=filtername))
90+
# Masks, give these 0 weight
91+
bfs.append(fs.Zenith_shadow_mask_basis_function(nside=nside, shadow_minutes=60., max_alt=76.))
92+
bfs.append(fs.Moon_avoidance_basis_function(nside=nside, moon_distance=40.))
93+
bfs.append(fs.Bulk_cloud_basis_function(max_cloud_map=cloud_map, nside=nside))
94+
weights = np.array([3.0, 3.0, .3, .3, 3., 3., 0., 0., 0.])
95+
if filtername2 is None:
96+
# Need to scale weights up so filter balancing still works properly.
97+
weights = np.array([6.0, 0.6, 3., 3., 0., 0., 0.])
98+
if filtername2 is None:
99+
survey_name = 'blob, %s' % filtername
100+
else:
101+
survey_name = 'blob, %s%s' % (filtername, filtername2)
102+
pair_surveys.append(fs.Blob_survey(bfs, weights, filtername=filtername, filter2=filtername2,
103+
survey_note=survey_name, ignore_obs='DD', tag_fields=True,
104+
tag_map=target_maps[filtername][1],
105+
tag_names=target_maps[filtername][2]))
106+
107+
return pair_surveys
108+
109+
110+
def generate_greedy(nside):
111+
target_maps, norm_factor = generate_target_maps(nside)
112+
113+
cloud_map = fs.generate_cloud_map(target_maps, filtername='i',
114+
wfd_cloud_max=0.7,
115+
scp_cloud_max=0.7,
116+
gp_cloud_max=0.7,
117+
nes_cloud_max=0.7)
118+
119+
# filters = ['u', 'g', 'r', 'i', 'z', 'y']
120+
filters = ['r']
121+
surveys = []
122+
123+
for filtername in filters:
124+
bfs = list()
125+
bfs.append(fs.M5_diff_basis_function(filtername=filtername, nside=nside))
126+
bfs.append(fs.Target_map_basis_function(filtername=filtername,
127+
target_map=target_maps[filtername][0],
128+
out_of_bounds_val=hp.UNSEEN, nside=nside,
129+
norm_factor=norm_factor))
130+
bfs.append(fs.Aggressive_Slewtime_basis_function(filtername=filtername, nside=nside, order=6., hard_max=120.))
131+
bfs.append(fs.Strict_filter_basis_function(filtername=filtername))
132+
bfs.append(fs.Zenith_shadow_mask_basis_function(nside=nside, shadow_minutes=0., max_alt=76.))
133+
bfs.append(fs.Bulk_cloud_basis_function(max_cloud_map=cloud_map, nside=nside))
134+
bfs.append(fs.Moon_avoidance_basis_function(nside=nside, moon_distance=40.))
135+
weights = np.array([3., 1., 3., 3., 0., 0, 0])
136+
surveys.append(fs.Greedy_survey_fields(bfs, weights, block_size=1,
137+
filtername=filtername, dither=True,
138+
nside=nside,
139+
tag_fields=True,
140+
tag_map=target_maps[filtername][1],
141+
tag_names=target_maps[filtername][2],
142+
ignore_obs='DD'))
143+
return surveys
144+
145+
146+
def generate_dd(nside):
147+
148+
# XXX-TODO: Not sure what this should actually be.
149+
filter_prop = None
150+
151+
# Set up the DD
152+
# ELAIS S1
153+
dd_surveys = list()
154+
dd_surveys.append(fs.Deep_drilling_survey(9.45, -44., sequence='rgizy',
155+
nvis=[20, 10, 20, 26, 20],
156+
survey_name='DD:ELAISS1', reward_value=100, moon_up=None,
157+
fraction_limit=0.148, ha_limits=([0., 0.5], [23.5, 24.]),
158+
nside=nside,
159+
avoid_same_day=True,
160+
filter_goals=filter_prop))
161+
dd_surveys.append(fs.Deep_drilling_survey(9.45, -44., sequence='u',
162+
nvis=[7],
163+
survey_name='DD:u,ELAISS1', reward_value=100, moon_up=False,
164+
fraction_limit=0.0012, ha_limits=([0., 0.5], [23.5, 24.]),
165+
nside=nside))
166+
167+
# XMM-LSS
168+
dd_surveys.append(fs.Deep_drilling_survey(35.708333, -4 - 45 / 60., sequence='rgizy',
169+
nvis=[20, 10, 20, 26, 20],
170+
survey_name='DD:XMM-LSS', reward_value=100, moon_up=None,
171+
fraction_limit=0.148, ha_limits=([0., 0.5], [23.5, 24.]),
172+
nside=nside,
173+
avoid_same_day=True,
174+
filter_goals=filter_prop))
175+
dd_surveys.append(fs.Deep_drilling_survey(35.708333, -4 - 45 / 60., sequence='u',
176+
nvis=[7],
177+
survey_name='DD:u,XMM-LSS', reward_value=100, moon_up=False,
178+
fraction_limit=0.0012, ha_limits=([0., 0.5], [23.5, 24.]),
179+
nside=nside))
180+
181+
# Extended Chandra Deep Field South
182+
# XXX--Note, this one can pass near zenith. Should go back and add better planning on this.
183+
dd_surveys.append(fs.Deep_drilling_survey(53.125, -28. - 6 / 60., sequence='rgizy',
184+
nvis=[20, 10, 20, 26, 20],
185+
survey_name='DD:ECDFS', reward_value=100, moon_up=None,
186+
fraction_limit=0.148, ha_limits=[[0.5, 1.0], [23., 22.5]],
187+
nside=nside,
188+
avoid_same_day=True,
189+
filter_goals=filter_prop))
190+
dd_surveys.append(fs.Deep_drilling_survey(53.125, -28. - 6 / 60., sequence='u',
191+
nvis=[7],
192+
survey_name='DD:u,ECDFS', reward_value=100, moon_up=False,
193+
fraction_limit=0.0012, ha_limits=[[0.5, 1.0], [23., 22.5]],
194+
nside=nside))
195+
# COSMOS
196+
dd_surveys.append(fs.Deep_drilling_survey(150.1, 2. + 10. / 60. + 55 / 3600., sequence='rgizy',
197+
nvis=[20, 10, 20, 26, 20],
198+
survey_name='DD:COSMOS', reward_value=100, moon_up=None,
199+
fraction_limit=0.148, ha_limits=([0., 0.5], [23.5, 24.]),
200+
nside=nside,
201+
avoid_same_day=True,
202+
filter_goals=filter_prop))
203+
dd_surveys.append(fs.Deep_drilling_survey(150.1, 2. + 10. / 60. + 55 / 3600., sequence='u',
204+
nvis=[7], ha_limits=([0., .5], [23.5, 24.]),
205+
survey_name='DD:u,COSMOS', reward_value=100, moon_up=False,
206+
fraction_limit=0.0012,
207+
nside=nside))
208+
209+
# Extra DD Field, just to get to 5. Still not closed on this one
210+
dd_surveys.append(fs.Deep_drilling_survey(349.386443, -63.321004, sequence='rgizy',
211+
nvis=[20, 10, 20, 26, 20],
212+
survey_name='DD:290', reward_value=100, moon_up=None,
213+
fraction_limit=0.148, ha_limits=([0., 0.5], [23.5, 24.]),
214+
nside=nside,
215+
avoid_same_day=True,
216+
filter_goals=filter_prop))
217+
dd_surveys.append(fs.Deep_drilling_survey(349.386443, -63.321004, sequence='u',
218+
nvis=[7],
219+
survey_name='DD:u,290', reward_value=100, moon_up=False,
220+
fraction_limit=0.0012, ha_limits=([0., 0.5], [23.5, 24.]),
221+
nside=nside,
222+
filter_goals=filter_prop))
223+
return dd_surveys
224+
225+
if __name__ == 'config':
226+
survey_topology = SurveyTopology()
227+
survey_topology.num_general_props = 4
228+
survey_topology.general_propos = ["NorthEclipticSpur", "SouthCelestialPole", "WideFastDeep", "GalacticPlane"]
229+
survey_topology.num_seq_props = 1
230+
survey_topology.sequence_propos = ["DeepDrillingCosmology1"]
231+
232+
nside = fs.set_default_nside(nside=32) # Required
233+
234+
greedy_surveys = generate_greedy(nside)
235+
blob_surveys = generate_blob_surveys(nside)
236+
dd_surveys = generate_dd(nside)
237+
238+
239+
scheduler = fs.Core_scheduler([greedy_surveys], nside=nside) # Required

‎runs/convert_schema.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import numpy as np
2+
import pandas as pd
3+
import sqlite3 as db
4+
5+
6+
def convert_schema(filename, fileout):
7+
8+
9+
remap_dict ={'RA':'fieldRA', 'dec':'fieldDec', 'mjd':'observationStartMJD', 'exptime':'visitExposureTime',
10+
'filter':'filter', 'rotSkyPos':'rotSkyPos', 'nexp':'numExposures',
11+
'airmass':'airmass', 'FWHMeff':'seeingFwhmEff', 'FWHM_geometric':'seeingFwhmGeom',
12+
'skybrightness':'skyBrightness', 'night': 'night', 'slewtime':'slewTime', 'fivesigmadepth':'fiveSigmaDepth',
13+
'alt':'altitude', 'az':'azimuth', 'clouds':'clouds', 'moonAlt':'moonAlt', 'sunAlt':'sunAlt', 'note':'note',
14+
'field_id':'fieldId', 'survey_id':'proposalId', 'block_id':'block_id'}
15+
16+
17+
conn = db.connect(filename)
18+
df = pd.read_sql('select * from observations;', conn)
19+
df = df.rename(index=str, columns=remap_dict)
20+
# Kludge on the visitTime
21+
df['visitTime'] = 2.*df['numExposures'].values + df['visitExposureTime'].values
22+
# Dummy column
23+
df['slewDistance'] = 0.*df['numExposures'].values
24+
25+
conn.close()
26+
conn = db.connect(fileout)
27+
df.to_sql('SummaryAllProps', conn, index=False, if_exists='replace')
28+
29+
30+
if __name__ == '__main__':
31+
32+
files = {'rolling/rolling_10yrs.db':'rolling/rolling_10yrs_opsim.db',
33+
'roll_mix/rolling_mix_10yrs.db': 'roll_mix/rolling_mix_10yrs_opsim.db'}
34+
35+
for infile in files:
36+
convert_schema(infile, files[infile])
37+

0 commit comments

Comments
 (0)