Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Additional test and related fixes stemming from Platform group PR #1058 [all tests ci] #1061

Merged
merged 5 commits into from
Jun 11, 2023
Merged

Additional test and related fixes stemming from Platform group PR #1058 [all tests ci] #1061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dc1cd3d
Fix processing level L1A test to account for AZFP open_raw now creati…
emiliom Jun 11, 2023
47202ed
Remove obsolete environmental parameters tests
emiliom Jun 11, 2023
909448d
Fix test_plot tests and visualize api involving water_level > vertica…
emiliom Jun 11, 2023
cdd790b
Cosmetic change simply to trigger all ci tests
emiliom Jun 11, 2023
a9215a9
Remove remnant test case for ek60 vertical_offset channel dim; correc…
emiliom Jun 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 0 additions & 132 deletions echopype/tests/calibrate/test_calibrate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,135 +263,3 @@ def test_compute_Sv_ek80_CW_complex_BB_complex(ek80_cal_path, ek80_path):
ed, waveform_mode="BB", encode_mode="complex"
)
assert isinstance(ds_Sv, xr.Dataset)


def test_env_params(ek60_path):
"""
Tests EnvParams interpolation
"""

ed = ep.open_raw(ek60_path / "ncei-wcsd" / "Summer2017-D20170620-T011027.raw", "EK60")

# stationary
# since the raw ping_times go up to 1:43 but the env_params ping_time stops at 1:25,
# values after 1:25 will be extrapolated
env_params_data = xr.Dataset(
data_vars={
"pressure": ("time3", np.arange(50)),
"salinity": ("time3", np.arange(50)),
"temperature": ("time3", np.arange(50)),
},
coords={
"time3": np.arange("2017-06-20T01:00", "2017-06-20T01:25", np.timedelta64(30, "s"), dtype="datetime64[ns]")
}
)
env_params = EnvParams(env_params_data, "stationary")
converted_env_params = env_params._apply(ed)
for var in converted_env_params.values():
assert np.all(np.diff(var) > 0)
assert np.all(0 <= var)
assert np.all(var < 100)
# TODO: substitute ping_time and input values of the env variables
# so that interpolation gives nice outputs
known_values = {
"temperature": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.03483333973334,
},
"salinity": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.0348333397333,
},
"pressure": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.03483333973334,
}
}
for var, values in known_values.items():
for time, value in values.items():
assert np.isclose(converted_env_params[var].sel(time1=time), value)

# mobile
rng = np.random.default_rng(0)
env_params_data = xr.Dataset(
data_vars={
"pressure": ("time", np.arange(100)),
"salinity": ("time", np.arange(100)),
"temperature": ("time", np.arange(100)),
},
coords={
"latitude": ("time", rng.random(size=100) + 44),
"longitude": ("time", rng.random(size=100) - 125),
}
)
env_params = EnvParams(env_params_data, "mobile")
converted_env_params = env_params._apply(ed)
for var in converted_env_params.values():
assert np.all(0 <= var[~np.isnan(var)])
assert np.all(var[~np.isnan(var)] < 100)
known_values = {
"temperature": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
"salinity": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
"pressure": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
}
for var, values in known_values.items():
for time, value in values.items():
print(var, time, value)
assert np.isnan(value) or np.isclose(converted_env_params[var].sel(time1=time), value)
11 changes: 9 additions & 2 deletions echopype/tests/utils/test_processinglevels_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,14 @@ def _absence_test(test_ds):
assert "processing_level_url" not in test_ds.attrs

# ---- Convert raw file and update_platform
def _var_presence_notnan_test(name):
if name in ed['Platform'].data_vars and not ed["Platform"][name].isnull().all():
return True
else:
return False

ed = ep.open_raw(raw_path, xml_path=xml_path, sonar_model=sonar_model)
if "longitude" in ed['Platform'].data_vars and "latitude" in ed['Platform'].data_vars:
if _var_presence_notnan_test("longitude") and _var_presence_notnan_test("latitude"):
_presence_test(ed["Top-level"], "Level 1A")
elif "longitude" in extras and "latitude" in extras:
_absence_test(ed["Top-level"])
Expand Down Expand Up @@ -120,7 +126,8 @@ def _freqdiff_applymask(test_ds):

# ---- Compute MVBS
# compute_MVBS expects a variable named "Sv"
# No product level is assigned because at present compute_MVBS drops the lat/lon data associated with the input Sv dataset
# No product level is assigned because at present compute_MVBS drops the lat/lon data
# associated with the input Sv dataset
# ds = ds.rename_vars(name_dict={"Sv": "Sv_unmasked", "Sv_ch0": "Sv"})
mvbs_ds = ep.commongrid.compute_MVBS(ds, range_meter_bin=30, ping_time_bin='1min')
_absence_test(mvbs_ds)
68 changes: 31 additions & 37 deletions echopype/tests/visualize/test_plot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,20 +197,20 @@ def test_plot_mvbs(


@pytest.mark.parametrize(
("water_level", "expect_warning"),
("vertical_offset", "expect_warning"),
[
(True, False),
([True], True),
(False, True),
(xr.DataArray(np.array(50.0)).expand_dims({'channel': 3}), False),
(xr.DataArray(np.array(50.0)), False),
([10, 30.5], False),
(10, False),
(30.5, False),
],
)
def test_water_level_echodata(water_level, expect_warning, caplog):
def test_vertical_offset_echodata(vertical_offset, expect_warning, caplog):
from echopype.echodata import EchoData
from echopype.visualize.api import _add_water_level
from echopype.visualize.api import _add_vertical_offset
echopype.verbose()

filepath = ek60_path / "ncei-wcsd" / "Summer2017-D20170719-T211347.raw"
Expand All @@ -231,45 +231,41 @@ def test_water_level_echodata(water_level, expect_warning, caplog):

single_array = range_in_meter.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
ping_time='2017-07-19T21:13:47.984999936').values
no_input_water_level = False

if isinstance(water_level, list):
water_level = water_level[0]
echodata["Platform"] = echodata["Platform"].drop_vars('water_level')
no_input_water_level = True

if isinstance(water_level, xr.DataArray):
if 'channel' in water_level.dims:
original_array = single_array + water_level.isel(channel=0).values
elif isinstance(water_level, bool) and water_level is True:
if no_input_water_level is False:
no_input_vertical_offset = False

if isinstance(vertical_offset, list):
vertical_offset = vertical_offset[0]
echodata["Platform"] = echodata["Platform"].drop_vars('vertical_offset')
no_input_vertical_offset = True

if isinstance(vertical_offset, xr.DataArray):
original_array = single_array + vertical_offset.values
elif isinstance(vertical_offset, bool) and vertical_offset is True:
if not no_input_vertical_offset:
original_array = (
single_array
+ echodata["Platform"].water_level.sel(
channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
time3='2017-07-19T21:13:47.984999936'
).values
+ echodata["Platform"].vertical_offset.sel(time2='2017-07-19T21:13:47.984999936').values
)
else:
original_array = single_array
elif water_level is not False and isinstance(water_level, (int, float)):
original_array = single_array + water_level
elif vertical_offset is not False and isinstance(vertical_offset, (int, float)):
original_array = single_array + vertical_offset
else:
original_array = single_array

results = None
try:
results = _add_water_level(
results = _add_vertical_offset(
range_in_meter=range_in_meter,
water_level=water_level,
vertical_offset=vertical_offset,
data_type=EchoData,
platform_data=echodata["Platform"],
)
if expect_warning:
assert 'WARNING' in caplog.text
except Exception as e:
assert isinstance(e, ValueError)
assert str(e) == 'Water level must have any of these dimensions: channel, ping_time, range_sample' # noqa
assert str(e) == 'vertical_offset must have any of these dimensions: ping_time, range_sample' # noqa

if isinstance(results, xr.DataArray):
final_array = results.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
Expand All @@ -280,18 +276,17 @@ def test_water_level_echodata(water_level, expect_warning, caplog):


@pytest.mark.parametrize(
("water_level", "expect_warning"),
("vertical_offset", "expect_warning"),
[
(True, True),
(False, True),
(xr.DataArray(np.array(50.0)).expand_dims({'channel': 3}), False),
(xr.DataArray(np.array(50.0)), False),
(10, False),
(30.5, False),
],
)
def test_water_level_Sv_dataset(water_level, expect_warning, caplog):
from echopype.visualize.api import _add_water_level
def test_vertical_offset_Sv_dataset(vertical_offset, expect_warning, caplog):
from echopype.visualize.api import _add_vertical_offset
echopype.verbose()

filepath = ek60_path / "ncei-wcsd" / "Summer2017-D20170719-T211347.raw"
Expand All @@ -308,27 +303,26 @@ def test_water_level_Sv_dataset(water_level, expect_warning, caplog):
single_array = range_in_meter.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
ping_time='2017-07-19T21:13:47.984999936').values

if isinstance(water_level, xr.DataArray):
if 'channel' in water_level.dims:
original_array = single_array + water_level.isel(channel=0).values
elif not isinstance(water_level, bool) and isinstance(water_level, (int, float)):
original_array = single_array + water_level
if isinstance(vertical_offset, xr.DataArray):
original_array = single_array + vertical_offset.values
elif not isinstance(vertical_offset, bool) and isinstance(vertical_offset, (int, float)):
original_array = single_array + vertical_offset
else:
original_array = single_array

results = None
try:
results = _add_water_level(
results = _add_vertical_offset(
range_in_meter=range_in_meter,
water_level=water_level,
vertical_offset=vertical_offset,
data_type=xr.Dataset,
)

if expect_warning:
assert 'WARNING' in caplog.text
except Exception as e:
assert isinstance(e, ValueError)
assert str(e) == 'Water level must have any of these dimensions: channel, ping_time, range_sample' # noqa
assert str(e) == 'vertical_offset must have any of these dimensions: ping_time, range_sample' # noqa

if isinstance(results, xr.DataArray):
final_array = results.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
Expand Down
Loading