Skip to content

Commit 7896f78

Browse files
Sean-Morrisonhavok2063
Sean-Morrison
and
havok2063
authored
update boss paths for v6_2+ (#65)
* update boss paths for v6_2+ * Fixed bug in sptypefolder special function * Update path.py * Update path.py * Update python/sdss_access/path/path.py * Update path.py * Update test_sdss5.py * Update path.py * Update path.py * Update conftest.py * Update test_sdss5.py * Update test_sdss5.py * Update test_sdss5.py * restore tilegrp --------- Co-authored-by: Brian Cherinka <[email protected]>
1 parent cf47a46 commit 7896f78

File tree

3 files changed

+233
-14
lines changed

3 files changed

+233
-14
lines changed

‎python/sdss_access/path/path.py

+216-10
Original file line numberDiff line numberDiff line change
@@ -325,20 +325,32 @@ def extract(self, name, example):
325325
# handle special functions; perform a drop in replacement
326326
if re.match('@spectrodir[|]', template):
327327
template = re.sub('@spectrodir[|]', os.environ['BOSS_SPECTRO_REDUX'], template)
328-
elif re.search('@platedir[|]', template):
328+
if re.search('@platedir[|]', template):
329329
template = re.sub('@platedir[|]', r'(.*)/{plateid:0>6}', template)
330-
elif re.search('@definitiondir[|]', template):
330+
if re.search('@definitiondir[|]', template):
331331
template = re.sub('@definitiondir[|]', '{designid:0>6}', template)
332-
elif re.search('@apgprefix[|]', template):
332+
if re.search('@apgprefix[|]', template):
333333
template = re.sub('@apgprefix[|]', '{prefix}', template)
334-
elif re.search('@healpixgrp[|]', template):
334+
if re.search('@healpixgrp[|]', template):
335335
template = re.sub('@healpixgrp[|]', '{healpixgrp}', template)
336-
elif re.search('@configgrp[|]', template):
336+
if re.search('@configgrp[|]', template):
337337
template = re.sub('@configgrp[|]', '{configgrp}', template)
338-
elif re.search('@isplate[|]', template):
338+
if re.search('@isplate[|]', template):
339339
template = re.sub('@isplate[|]', '{isplate}', template)
340-
elif re.search('@pad_fieldid[|]', template):
340+
if re.search('@pad_fieldid[|]', template):
341341
template = re.sub('@pad_fieldid[|]', '{fieldid}', template)
342+
if re.search('@spcoaddfolder[|]',template):
343+
template = re.sub('@spcoaddfolder[|]', '{spcoaddfolder}', template)
344+
if re.search('@sptypefolder[|]',template):
345+
template = re.sub('@sptypefolder[|]', '{sptypefolder}', template)
346+
if re.search('@epochflag[|]', template):
347+
template = re.sub('@epochflag[|]', '{epochflag}', template)
348+
if re.search('@spcoaddobs[|]',template):
349+
template = re.sub('@spcoaddobs[|]', '{obs}', template)
350+
if re.search('@spcoaddgrp[|]',template):
351+
template = re.sub('@spcoaddgrp[|]', '{spcoaddgrp}', template)
352+
if re.search('@fieldgrp[|]',template):
353+
template = re.sub('@fieldgrp[|]', '{fieldgrp}', template)
342354
if re.search('@plateid6[|]', template):
343355
template = re.sub('@plateid6[|]', '{plateid:0>6}', template)
344356
if re.search('@component_default[|]', template):
@@ -347,6 +359,8 @@ def extract(self, name, example):
347359
template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template)
348360
if re.search('@sdss_id_groups[|]', template):
349361
template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template)
362+
if re.search('@tilegrp[|]', template):
363+
template = re.sub('@tilegrp[|]', '{tilegrp}', template)
350364

351365
# check if template has any brackets
352366
haskwargs = re.search('[{}]', template)
@@ -386,12 +400,28 @@ def extract(self, name, example):
386400
drval = re.match('^DR[1-9][0-9]', value).group(0)
387401
otherval = value.split(drval)[-1]
388402
pdict = {keys[0]: drval, keys[1]: otherval}
403+
elif keys == ['fieldid','isplate']:
404+
# for {fieldid}{isplate} as isplate is calculated automatically
405+
if value.endswith('p') and value[:-1].isdigit():
406+
value = value[:-1]
407+
pdict = {keys[0]:value}
408+
elif keys == ['run2d','epochflag']:
409+
# for {run2d}{epochflag} as epochflag is calculated automatically
410+
pdict = {keys[0]:value.replace('-epoch','')}
411+
elif keys == ['coadd', 'obs']:
412+
value = value.split('_')
413+
if len(value) == 1:
414+
value.append('')
415+
pdict = {keys[0]: value[0], keys[1]: value[1]}
389416
elif keys[0] in ['rc', 'br', 'filter', 'camrow']:
390417
# for {camrow}{camcol}, {filter}{camcol}, {br}{id}, etc
391418
pdict = {keys[0]: value[0], keys[1]: value[1:]}
392419
else:
393420
raise ValueError('This case has not yet been accounted for.')
394421
path_dict.update(pdict)
422+
elif keys[0] in ['sptypefolder','fieldgrp','spcoaddfolder','spcoaddgrp']:
423+
# supress the keys since they are automatically calculated
424+
continue
395425
else:
396426
path_dict[keys[0]] = value
397427
return path_dict
@@ -678,7 +708,7 @@ def full(self, filetype, **kwargs):
678708
if not skip_tag_check:
679709
template = re.sub(r'tags/(v?[0-9._]+)', r'\1', template, count=1)
680710

681-
return self._check_compression(template)
711+
return self._check_compression(os.path.normpath(template))
682712

683713
@staticmethod
684714
def check_modules(template, permanent=None):
@@ -1531,8 +1561,184 @@ def pad_fieldid(self, filetype, **kwargs):
15311561
return str(fieldid)
15321562
if fieldid.isnumeric():
15331563
return str(fieldid).zfill(6)
1534-
else:
1535-
return fieldid
1564+
return fieldid
1565+
1566+
def spcoaddfolder(self, filetype, **kwargs):
1567+
''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
1568+
1569+
Parameters
1570+
---------
1571+
filetype : str
1572+
File type parameter
1573+
run2d : str
1574+
BOSS idlspec2d run2d version
1575+
coadd : str
1576+
Name of the custom coadd schema
1577+
Returns
1578+
-------
1579+
sptypefolder : str
1580+
'''
1581+
1582+
run2d = kwargs.get('run2d', None)
1583+
coaddname = kwargs.get('coadd', None)
1584+
1585+
if (not run2d):
1586+
return ''
1587+
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
1588+
('v6_0' in run2d) or ('v6_1' in run2d)):
1589+
return ''
1590+
if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']:
1591+
return 'summary'
1592+
if filetype.lower() in ['speclite_coadd','specfull_coadd',
1593+
'spallfield_coadd','spalllinefield_coadd']:
1594+
return coaddname
1595+
return 'fields'
1596+
1597+
1598+
def spcoaddgrp(self, filetype, **kwargs):
1599+
''' Returns the coadd group (field group analog) subfolder structure for the BOSS idlspec2d run2d version
1600+
1601+
Parameters
1602+
---------
1603+
filetype : str
1604+
File type parameter
1605+
run2d : str
1606+
BOSS idlspec2d run2d version
1607+
coadd : str
1608+
Name of the custom coadd schema
1609+
Returns
1610+
-------
1611+
spcoaddgrp : str
1612+
'''
1613+
run2d = kwargs.get('run2d', None)
1614+
coaddname = kwargs.get('coadd', None)
1615+
1616+
if (not run2d):
1617+
return ''
1618+
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
1619+
('v6_0' in run2d) or ('v6_1' in run2d)):
1620+
return ''
1621+
return coaddname
1622+
1623+
def sptypefolder(self, filetype, **kwargs):
1624+
''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
1625+
1626+
Parameters
1627+
---------
1628+
filetype : str
1629+
File type parameter
1630+
run2d : str
1631+
BOSS idlspec2d run2d version
1632+
Returns
1633+
-------
1634+
sptypefolder : str
1635+
'''
1636+
1637+
run2d = kwargs.get('run2d', None)
1638+
1639+
if (not run2d) or ('v5' in run2d) or (str(run2d) in ['26','103','104']):
1640+
return ''
1641+
if ('v6_0' in run2d) or ('v6_1' in run2d):
1642+
if filetype.lower() in ['speclite_epoch','specfull_epoch',
1643+
'spallfield_epoch','spalllinefield_epoch']:
1644+
return 'epoch/spectra'
1645+
if 'epoch' in filetype.lower():
1646+
return 'epoch'
1647+
return ''
1648+
if filetype.lower() in ['fieldlist_epoch','spall_epoch',
1649+
'spall-lite_epoch','spallline_epoch']:
1650+
return 'summary/epoch'
1651+
if filetype.lower() in ['speclite_epoch','specfull_epoch',
1652+
'spallfield_epoch','spalllinefield_epoch']:
1653+
return 'spectra/epoch'
1654+
if filetype.lower() in ['conflist','fieldlist','spall',
1655+
'spall-lite','spallline']:
1656+
return 'summary/daily'
1657+
if filetype.lower() in ['speclite','specfull',
1658+
'spallfield','spalllinefield']:
1659+
return 'daily'
1660+
return 'fields'
1661+
1662+
def spcoaddobs(self, filetype, **kwargs):
1663+
''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d
1664+
1665+
Parameters
1666+
----------
1667+
filetype : str
1668+
File type parameter
1669+
run2d : str
1670+
BOSS idlspec2d run2d version
1671+
obs : str
1672+
Observatory of observations: LCO, APO, ''(for merged)
1673+
1674+
Returns
1675+
-------
1676+
obs : str
1677+
'''
1678+
1679+
obs = kwargs.get('obs', None)
1680+
run2d = kwargs.get('run2d', None)
1681+
1682+
if not obs or obs == '':
1683+
return ''
1684+
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
1685+
(('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower()) == 'apo':
1686+
return ''
1687+
if obs == '*':
1688+
return obs
1689+
return '_{}'.format(obs.lower())
1690+
1691+
def epochflag(self, filetype, **kwargs):
1692+
''' Returns the flag for epoch coadds for the BOSS idlspec2d
1693+
1694+
Parameters
1695+
----------
1696+
filetype : str
1697+
File type parameter
1698+
run2d : str
1699+
BOSS idlspec2d run2d version
1700+
1701+
Returns
1702+
-------
1703+
epochflag : str
1704+
'''
1705+
1706+
run2d = kwargs.get('run2d', None)
1707+
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
1708+
('v6_0' in run2d) or ('v6_1' in run2d)):
1709+
return ''
1710+
return '-epoch'
1711+
1712+
def fieldgrp(self, filetype, **kwargs):
1713+
''' Returns the fieldid group for the BOSS idlspec2d run2d version
1714+
1715+
Parameters
1716+
---------
1717+
filetype : str
1718+
File type parameter
1719+
run2d : str
1720+
BOSS idlspec2d run2d version
1721+
fieldid : str or int
1722+
Field ID number. Will be converted to str internally.
1723+
1724+
Returns
1725+
-------
1726+
fieldgrp : str
1727+
'''
1728+
1729+
fieldid = kwargs.get('fieldid', None)
1730+
run2d = kwargs.get('run2d', None)
1731+
1732+
if (not fieldid):
1733+
return ''
1734+
1735+
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
1736+
('v6_0' in run2d) or ('v6_1' in run2d)):
1737+
return ''
1738+
fieldid = str(fieldid)
1739+
if fieldid.isnumeric():
1740+
return '{:0>3d}XXX'.format(int(fieldid) // 1000)
1741+
return fieldid
15361742

15371743
def tilegrp(self, filetype, **kwargs):
15381744
''' Returns LVM tile id group subdirectory

‎tests/conftest.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ def expdata(datapath):
8080
# full source file location
8181
source = os.path.join(base, sas_module, location)
8282
# full final file location
83-
destination = os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location)
83+
destination = os.path.normpath(os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location))
8484
# combined dict
8585
result = {'name': datapath['name'], 'params': datapath['params'], 'base': base,
8686
'sas_module': sas_module, 'location': location, 'source': source,

‎tests/path/test_sdss5.py

+16-3
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,28 @@ def test_apogee_paths(self, path, name, special, keys, exp):
5959
'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'),
6060
('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'},
6161
'v6_1_1/*/spField-*-59630.fits'),
62+
('spCOADDS','@spcoaddfolder',{'run2d':'v6_2_0','survey':'BHM','coadd':'allepoch','phase':'SDSSV'},
63+
'v6_2_0/fields/SDSSV_BHM_COADDS.par'),
64+
('spField','@sptypefolder',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
65+
'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
66+
('spField','@fieldgrp',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
67+
'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
68+
('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'},
69+
'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'),
70+
('spAll_epoch','@epochflag',{'run2d':'v6_2_0'},
71+
'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'),
72+
('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'},
73+
'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits'),
6274
('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235,
6375
'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817},
6476
'1055XX/1055360/60235/lvmCFrame-00006817.fits')],
6577
ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins',
66-
'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp',
67-
'pad_fieldid-*', 'lvm-tileid'])
78+
'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad',
79+
'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder',
80+
'fieldgrp','spcoaddobs','epochflag','spcoaddgrp','lvm-tileid'])
6881
def test_special_function(self, path, name, special, keys, exp):
6982
assert special in path.templates[name]
70-
full = path.full(name, **keys)
83+
full = os.path.normpath(path.full(name, **keys))
7184
assert exp in full
7285

7386
@pytest.mark.parametrize('name, keys', [('specLite', ['fieldid', 'catalogid', 'run2d', 'mjd']),

0 commit comments

Comments
 (0)