Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

JP-3734: Making OLS_C Default for CI Tests #289

Closed
wants to merge 5 commits into from
Closed

JP-3734: Making OLS_C Default for CI Tests #289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7d626b5
Making OLS_C default for testing and making sure it works correctly w…
kmacdonald-stsci Sep 20, 2024
788b303
Adding PR 289 fragment for the changelog.
kmacdonald-stsci Sep 23, 2024
a7f72d4
Removing unneeded comment.
kmacdonald-stsci Sep 24, 2024
6ccf181
Updating change log.
kmacdonald-stsci Sep 30, 2024
2be0844
Merge branch 'main' into jp_3734
tapastro Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changes/289.general.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[ramp_fitting] Changing the default algorithm for ramp fitting to be ``OLS_C`` for CI tests.
melanieclarke marked this conversation as resolved.
Show resolved Hide resolved
It also fixes a bug related to ``RampData.run_chargeloss`` defaulting to
``True``. That parameter now defaults to ``False``.
3 changes: 3 additions & 0 deletions src/stcal/ramp_fitting/ramp_fit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@
wh_chargeloss = np.where(np.bitwise_and(model.groupdq.astype(np.uint32), dqflags['CHARGELOSS']))
if len(wh_chargeloss[0]) > 0:
orig_gdq = model.groupdq.copy()
ramp_data.run_chargeloss = True

Check warning on line 67 in src/stcal/ramp_fitting/ramp_fit.py

View check run for this annotation

Codecov / codecov/patch

src/stcal/ramp_fitting/ramp_fit.py#L67 

Added line #L67 was not covered by tests
else:
ramp_data.run_chargeloss = False

Check warning on line 69 in src/stcal/ramp_fitting/ramp_fit.py

View check run for this annotation

Codecov / codecov/patch

src/stcal/ramp_fitting/ramp_fit.py#L69 

Added line #L69 was not covered by tests
del wh_chargeloss

if isinstance(model.data, u.Quantity):
Expand Down
5 changes: 2 additions & 3 deletions src/stcal/ramp_fitting/ramp_fit_class.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,8 @@ def __init__(self):
self.suppress_one_group_ramps = False

# C code debugging switch.
self.run_c_code = False
self.run_chargeloss = True
# self.run_chargeloss = False
self.run_c_code = True
self.run_chargeloss = False

self.one_groups_locs = None # One good group locations.
self.one_groups_time = None # Time to use for one good group ramps.
Expand Down
42 changes: 23 additions & 19 deletions tests/test_ramp_fitting.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
JUMP = dqflags["JUMP_DET"]
CHRGL = dqflags["CHARGELOSS"]

DEFAULT_ALGO = "OLS_C"


# -----------------------------------------------------------------------------
# Test Suite
Expand All @@ -52,7 +54,7 @@ def test_long_integration():

ramp_data.data[0, 291:, 0, 0] = 320 * 3
# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise_array, gain_array,
algo, wt, ncores, dqflags)
Expand Down Expand Up @@ -80,7 +82,7 @@ def base_neg_med_rates_single_integration():
ramp_data.data[0, :, 0, 0] = neg_ramp

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -163,7 +165,7 @@ def base_neg_med_rates_multi_integrations():
ramp_data.data[k, :, 0, 0] = neg_ramp * n

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -249,7 +251,7 @@ def base_neg_med_rates_single_integration_multi_segment():
ramp_data.groupdq[0, 10, 1, 0] = dqflags["JUMP_DET"]

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -296,7 +298,7 @@ def test_neg_with_avgdark():
ramp_data.average_dark_current[:] = 1.0

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -345,7 +347,7 @@ def test_utils_dq_compress_final():
ramp_data.groupdq[0, :, 0, 1] = np.array([dqflags["SATURATED"]] * ngroups)

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, False, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, False, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -424,7 +426,7 @@ def test_miri_ramp_dnu_at_ramp_beginning():
ramp_data.groupdq[0, 1, 0, 0] = dqflags["DO_NOT_USE"]

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes1, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand All @@ -446,7 +448,7 @@ def test_miri_ramp_dnu_and_jump_at_ramp_beginning():
ramp_data.groupdq[0, 1, 0, 0] = dqflags["JUMP_DET"]

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes2, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -519,7 +521,7 @@ def test_2_group_cases():
ramp_data.set_dqflags(dqflags)

# Run ramp fit on RampData
buffsize, save_opt, algo, wt, ncores = 512, True, "OLS", "optimal", "none"
buffsize, save_opt, algo, wt, ncores = 512, True, DEFAULT_ALGO, "optimal", "none"
slopes, cube, optional, gls_dummy = ramp_fit_data(
ramp_data, buffsize, save_opt, rnoise, gain, algo, wt, ncores, dqflags
)
Expand Down Expand Up @@ -593,7 +595,7 @@ def run_one_group_ramp_suppression(nints, suppress):

ramp_data.suppress_one_group_ramps = suppress

algo = "OLS"
algo = DEFAULT_ALGO
save_opt, ncores, bufsize = False, "none", 1024 * 30000
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp_data, bufsize, save_opt, rnoise2d, gain2d, algo, "optimal", ncores, dqflags
Expand Down Expand Up @@ -870,7 +872,7 @@ def test_zeroframe():
"""
ramp_data, gain, rnoise = create_zero_frame_data()

algo, save_opt, ncores, bufsize = "OLS", False, "none", 1024 * 30000
algo, save_opt, ncores, bufsize = DEFAULT_ALGO, False, "none", 1024 * 30000
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp_data, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -989,7 +991,7 @@ def test_only_good_0th_group():
# Dimensions are (1, 5, 1, 3)
ramp_data, gain, rnoise = create_only_good_0th_group_data()

algo, save_opt, ncores, bufsize = "OLS", False, "none", 1024 * 30000
algo, save_opt, ncores, bufsize = DEFAULT_ALGO, False, "none", 1024 * 30000
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp_data, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1038,7 +1040,7 @@ def test_all_sat():
ramp, gain, rnoise = create_blank_ramp_data(dims, var, tm)
ramp.groupdq[:, 0, :, :] = ramp.flags_saturated

algo, save_opt, ncores, bufsize = "OLS", False, "none", 1024 * 30000
algo, save_opt, ncores, bufsize = DEFAULT_ALGO, False, "none", 1024 * 30000
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1068,7 +1070,7 @@ def test_dq_multi_int_dnu():
ramp.data[1, :, 0, 0] = np.array(base_arr)
ramp.groupdq[0, :, 0, 0] = np.array(dq_arr)

algo, save_opt, ncores, bufsize = "OLS", False, "none", 1024 * 30000
algo, save_opt, ncores, bufsize = DEFAULT_ALGO, False, "none", 1024 * 30000
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1165,7 +1167,7 @@ def test_multi_more_cores_than_rows():
ramp.data[integ, :, row, col] = bramp
bramp = bramp * factor

bufsize, algo, save_opt, ncores = 512, "OLS", False, "all"
bufsize, algo, save_opt, ncores = 512, DEFAULT_ALGO, False, "all"
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1316,7 +1318,7 @@ def test_new_saturation():
"""
ramp, gain, rnoise = get_new_saturation()

save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, "OLS"
save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, DEFAULT_ALGO
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1408,7 +1410,7 @@ def test_invalid_integrations():

ramp.suppress_one_group_ramps = True

save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, "OLS"
save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, DEFAULT_ALGO
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1472,7 +1474,7 @@ def test_one_group():

ramp.data[0, 0, 0, 0] = 105.31459

save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, "OLS"
save_opt, ncores, bufsize, algo = False, "none", 1024 * 30000, DEFAULT_ALGO
slopes, cube, ols_opt, gls_opt = ramp_fit_data(
ramp, bufsize, save_opt, rnoise, gain, algo, "optimal", ncores, dqflags
)
Expand Down Expand Up @@ -1590,7 +1592,8 @@ def test_cext_chargeloss():
tm = frame_time, nframes, groupgap
ramp, gain, rnoise = create_blank_ramp_data(dims, var, tm)

ramp.run_c_code = True # Need to make this default in future
ramp.run_chargeloss = True
ramp.run_c_code = True
base = 15.
arr = [(k+1) * base for k in range(ngroups)]

Expand Down Expand Up @@ -1671,6 +1674,7 @@ def create_blank_ramp_data(dims, var, tm):
drop_frames1=None,
)
ramp_data.set_dqflags(dqflags)
ramp_data.run_chargeloss = False

gain = np.ones(shape=(nrows, ncols), dtype=np.float32) * gval
rnoise = np.ones(shape=(nrows, ncols), dtype=np.float32) * rnval
Expand Down
Loading