diff --git a/meta_analytics/dataframes/glucose_endpoints/constants.py b/meta_analytics/dataframes/glucose_endpoints/constants.py index b300d03..83357c2 100644 --- a/meta_analytics/dataframes/glucose_endpoints/constants.py +++ b/meta_analytics/dataframes/glucose_endpoints/constants.py @@ -1,8 +1,10 @@ -OGTT_THRESHOLD_MET = "OGTT >= 11.1" -EOS_DM_MET = "EOS - Patient developed diabetes" -CASE_OGTT = 1 CASE_EOS = 7 +CASE_FBGS_WITH_FIRST_OGTT = 2 +CASE_FBGS_WITH_SECOND_OGTT = 3 CASE_FBG_ONLY = 4 +CASE_OGTT = 1 +EOS_DM_MET = "EOS - Patient developed diabetes" +OGTT_THRESHOLD_MET = "OGTT >= 11.1" endpoint_columns = [ "subject_identifier", @@ -24,8 +26,8 @@ endpoint_cases = { CASE_OGTT: OGTT_THRESHOLD_MET, - 2: "FBG >= 7 x 2, first OGTT<=11.1", - 3: "FBG >= 7 x 2, second OGTT<=11.1", - 4: "FBG >= 7 x 2, OGTT not considered", + CASE_FBGS_WITH_FIRST_OGTT: "FBG >= 7 x 2, first OGTT<=11.1", + CASE_FBGS_WITH_SECOND_OGTT: "FBG >= 7 x 2, second OGTT<=11.1", + CASE_FBG_ONLY: "FBG >= 7 x 2, OGTT not considered", CASE_EOS: EOS_DM_MET, } diff --git a/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py b/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py index 7f5d228..ce6db26 100644 --- a/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py +++ b/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py @@ -2,19 +2,14 @@ import pandas as pd from django.apps import apps as django_apps from edc_constants.constants import NO, YES -from edc_pdutils.dataframes import ( - get_crf, - get_eos, - get_subject_consent, - get_subject_visit, -) +from edc_pdutils.dataframes import get_crf, get_eos, get_subject_consent from edc_utils import get_utcnow -from meta_reports.models import Endpoints - from .constants import ( CASE_EOS, CASE_FBG_ONLY, + CASE_FBGS_WITH_FIRST_OGTT, + CASE_FBGS_WITH_SECOND_OGTT, CASE_OGTT, endpoint_cases, endpoint_columns, @@ -28,13 +23,31 @@ ) +def normalize_date_columns(df: pd.DataFrame, cols: list[str] = None) -> pd.DataFrame: + """Normalize date columns by flooring""" + for col in cols: + if not df[col].empty: + df[col] = df[col].dt.floor("d") + else: + df[col] = pd.NaT + return df + + +def calculate_fasting_hrs(df: pd.DataFrame): + df.loc[(df["fasting"] == NO), "fasting_duration_delta"] = pd.NaT + if df.empty: + df["fasting_hrs"] = np.nan + else: + df["fasting_hrs"] = df["fasting_duration_delta"].dt.total_seconds() / 3600 + return df + + class GlucoseEndpointsByDate: fbg_threshhold = 7.0 ogtt_threshhold = 11.1 endpoint_cls = EndpointByDate keep_cols = [ - "subject_visit_id", "fasting", "fasting_hrs", "fbg_value", @@ -45,9 +58,6 @@ class GlucoseEndpointsByDate: "ogtt_datetime", "source", "report_datetime", - ] - - visit_cols = [ "subject_visit_id", "subject_identifier", "visit_code", @@ -59,45 +69,31 @@ class GlucoseEndpointsByDate: def __init__( self, subject_identifiers: list[str] | None = None, case_list: list[int] | None = None ): - self.subject_identifiers = subject_identifiers or [] - if len(self.subject_identifiers) == Endpoints.objects.all().count(): - self.subject_identifiers = [] - self.case_list = case_list or [CASE_OGTT, 2, 3, CASE_EOS] - self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list} + self._glucose_fbg_df = pd.DataFrame() + self._glucose_fbg_ogtt_df = pd.DataFrame() self.endpoint_only_df = pd.DataFrame() - self.fbg_only_df = self.get_fbg_only_df() - - self.df = get_crf( - model="meta_subject.glucose", - subject_identifiers=self.subject_identifiers, - ) - self.df["source"] = "meta_subject.glucose" - - self.calculate_fasting_hrs() - - self.fbg_only_df = self.fbg_only_df[ - [col for col in self.keep_cols if not col.startswith("ogtt")] + self.subject_identifiers = subject_identifiers or [] + self.case_list = case_list or [ + CASE_OGTT, + CASE_FBGS_WITH_FIRST_OGTT, + CASE_FBGS_WITH_SECOND_OGTT, + CASE_EOS, ] - self.df = self.df[self.keep_cols] - self.df.reset_index(drop=True) - self.df = self.df.copy() - - self.normalize_dates() + self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list} - # same shape but fbg_only_df ogtt columns are null + # merge two model DFs self.df = pd.merge( - self.df, - self.fbg_only_df, + self.glucose_fbg_ogtt_df, + self.glucose_fbg_df, on=["subject_visit_id", "fbg_datetime", "fbg_value"], how="outer", indicator=True, suffixes=("", "2"), ) - self.df.reset_index(drop=True, inplace=True) - self.df_merged = self.df.copy() + self.df = self.df.reset_index(drop=True) - # right_only + # pivot right_only cols cols = { "fasting": None, "fasting_hrs": np.nan, @@ -107,77 +103,25 @@ def __init__( } for col, null_value in cols.items(): self.df.loc[self.df["_merge"] == "right_only", col] = self.df[f"{col}2"] + cols = [col for col in self.df.columns if col.endswith("2")] + cols.append("_merge") + self.df = self.df.drop(columns=cols) + self.df = self.df.reset_index(drop=True) - df_subject_visit = get_subject_visit( - "meta_subject.subjectvisit", subject_identifiers=subject_identifiers - ) - self.df = pd.merge( - df_subject_visit[self.visit_cols], self.df, on="subject_visit_id", how="left" - ) - self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) - self.df.reset_index(drop=True, inplace=True) - - df_consent = get_subject_consent( - "meta_consent.subjectconsent", subject_identifiers=subject_identifiers - ) - self.df = pd.merge(self.df, df_consent, on="subject_identifier", how="left") - self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) - self.df.reset_index(drop=True, inplace=True) - - df_eos = get_eos("meta_prn.endofstudy", subject_identifiers=subject_identifiers) - self.df = pd.merge(self.df, df_eos, on="subject_identifier", how="left") - self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) - self.df.reset_index(drop=True, inplace=True) - - if not self.df.loc[self.df["visit_datetime"].notna()].empty: - self.df["visit_days"] = ( - self.df["baseline_datetime"].rsub(self.df["visit_datetime"]).dt.days - ) - if not self.df.loc[self.df["fbg_datetime"].notna()].empty: - self.df["fgb_days"] = ( - self.df.loc[self.df["fbg_datetime"].notna()]["baseline_datetime"] - .rsub(self.df["fbg_datetime"]) - .dt.days - ) - else: - self.df["fgb_days"] = np.nan - if not self.df.loc[self.df["ogtt_datetime"].notna()].empty: - self.df["ogtt_days"] = ( - self.df.loc[self.df["ogtt_datetime"].notna()]["baseline_datetime"] - .rsub(self.df["ogtt_datetime"]) - .dt.days - ) - else: - self.df["ogtt_days"] = np.nan - - if self.df.empty: - self.df["visit_days"] = np.nan - self.df["fgb_days"] = np.nan - self.df["ogtt_days"] = np.nan - self.df["test"] = np.nan - else: - self.df["visit_days"] = pd.to_numeric(self.df["visit_days"], downcast="integer") - self.df["fgb_days"] = pd.to_numeric(self.df["fgb_days"], downcast="integer") - self.df["ogtt_days"] = pd.to_numeric(self.df["ogtt_days"], downcast="integer") - - # label rows by type of glu tests (ones with value) - self.df["test"] = self.df.apply(get_test_string, axis=1) + self.merge_with_consent() + self.merge_with_eos() + self.add_calculated_days_from_baseline_to_event_columns() - self.df = self.df.sort_values(by=["subject_identifier", "visit_code"]) + # label rows by type of glu tests (ones with value) + self.df["test"] = self.df.apply(get_test_string, axis=1) self.df = self.df.reset_index(drop=True) - self.df = self.df[ - self.df["offstudy_reason"] - != ( - "Patient fulfilled late exclusion criteria (due to abnormal blood " - "values or raised blood pressure at enrolment" - ) - ] + self.visit_codes_df = get_unique_visit_codes(self.df) + self.subject_identifiers_df = get_unique_subject_identifiers(self.df) self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) self.df = self.df.reset_index(drop=True) - self.visit_codes = get_unique_visit_codes(self.df) - self.subject_identifiers_df = get_unique_subject_identifiers(self.df) + self.working_df = self.df.copy() self.working_df["endpoint"] = 0 self.endpoint_df = get_empty_endpoint_df() @@ -186,16 +130,19 @@ def run(self): self.pre_check_endpoint() for index, row in self.subject_identifiers_df.iterrows(): subject_df = self.get_subject_df(row["subject_identifier"]) - subject_df = self.check_endpoint_by_fbg_for_subject(subject_df, case_list=[2, 3]) + subject_df = self.check_endpoint_by_fbg_for_subject( + subject_df, case_list=[CASE_FBGS_WITH_FIRST_OGTT, CASE_FBGS_WITH_SECOND_OGTT] + ) if len(subject_df.loc[subject_df["endpoint"] == 1]) == 1: self.append_subject_to_endpoint_df(subject_df) self.remove_subject_from_working_df(row) if CASE_FBG_ONLY in self.endpoint_cases: - # go back and rerun for case 5 for index, row in self.subject_identifiers_df.iterrows(): subject_df = self.get_subject_df(row["subject_identifier"]) - subject_df = self.check_endpoint_by_fbg_for_subject(subject_df, case_list=[4]) + subject_df = self.check_endpoint_by_fbg_for_subject( + subject_df, case_list=[CASE_FBG_ONLY] + ) if len(subject_df.loc[subject_df["endpoint"] == 1]) == 1: self.append_subject_to_endpoint_df(subject_df) self.remove_subject_from_working_df(row) @@ -203,60 +150,161 @@ def run(self): self.post_check_endpoint() self.merge_with_final_endpoints() + @property + def glucose_fbg_df(self) -> pd.DataFrame: + """Returns a prepared Dataframe of CRF + meta_subject.glucosefbg. + + Note: meta_subject.glucosefbg has only FBG measures. + """ + if self._glucose_fbg_df.empty: + df = get_crf( + model="meta_subject.glucosefbg", + subject_identifiers=self.subject_identifiers, + subject_visit_model="meta_subject.subjectvisit", + ) + df["source"] = "meta_subject.glucosefbg" + df.rename(columns={"fbg_fasting": "fasting"}, inplace=True) + df.loc[(df["fasting"] == "fasting"), "fasting"] = YES + df.loc[(df["fasting"] == "non_fasting"), "fasting"] = NO + df = calculate_fasting_hrs(df) + df = df[[col for col in self.keep_cols if not col.startswith("ogtt")]] + df = df.reset_index(drop=True) + df = normalize_date_columns( + df, cols=["fbg_datetime", "report_datetime", "visit_datetime"] + ) + self._glucose_fbg_df = df + return self._glucose_fbg_df + + @property + def glucose_fbg_ogtt_df(self): + """Returns a prepared Dataframe of CRF meta_subject.glucose. + + Note: meta_subject.glucose has FBG and OGTT measures. + """ + if self._glucose_fbg_ogtt_df.empty: + df = get_crf( + model="meta_subject.glucose", + subject_identifiers=self.subject_identifiers, + subject_visit_model="meta_subject.subjectvisit", + ) + df["source"] = "meta_subject.glucose" + df = calculate_fasting_hrs(df) + df = df[self.keep_cols] + df = df.reset_index(drop=True) + df = normalize_date_columns( + df, cols=["fbg_datetime", "ogtt_datetime", "report_datetime", "visit_datetime"] + ) + self._glucose_fbg_ogtt_df = df + return self._glucose_fbg_ogtt_df + + def merge_with_consent(self): + """Merge in consent DF.""" + df_consent = get_subject_consent("meta_consent.subjectconsent") + self.df = pd.merge(self.df, df_consent, on="subject_identifier", how="left") + self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) + self.df = self.df.reset_index(drop=True) + + def merge_with_eos(self): + """Merge in EoS DF. + + Drops patients who were taken off study by late exclusion. + """ + df_eos = get_eos("meta_prn.endofstudy") + df_eos = df_eos[ + df_eos["offstudy_reason"] + != ( + "Patient fulfilled late exclusion criteria (due to abnormal blood " + "values or raised blood pressure at enrolment" + ) + ] + self.df = pd.merge(self.df, df_eos, on="subject_identifier", how="left") + self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) + self.df = self.df.reset_index(drop=True) + + def add_calculated_days_from_baseline_to_event_columns(self): + """Add columns that calculate number of days from + baseline to visit, fbg, and ogtt. + """ + self.df["visit_days"] = np.nan + self.df["fbg_days"] = np.nan + self.df["ogtt_days"] = np.nan + self.df["test"] = np.nan + self.df["visit_days"] = ( + self.df["visit_datetime"] - self.df["baseline_datetime"] + ).dt.days + if not self.df["fbg_datetime"].empty: + self.df["fbg_days"] = ( + self.df["fbg_datetime"] - self.df["baseline_datetime"] + ).dt.days + if not self.df["ogtt_datetime"].empty: + self.df["ogtt_days"] = ( + self.df["ogtt_datetime"] - self.df["baseline_datetime"] + ).dt.days + self.df["visit_days"] = pd.to_numeric(self.df["visit_days"], downcast="integer") + self.df["fbg_days"] = pd.to_numeric(self.df["fbg_days"], downcast="integer") + self.df["ogtt_days"] = pd.to_numeric(self.df["ogtt_days"], downcast="integer") + self.df = self.df.reset_index(drop=True) + def pre_check_endpoint(self): - "Case 1: flag and remove all OGTT that met threshold" - subjects_df = self.working_df.loc[ + """Flag subjects that met endpoint by hitting the OGTT + threshold. + + Add them to the endpoint_df and remove them from the + working_df. + + Subject must have fasted at the timepoint. + + The OGTT must have an FBG measure at the same timepoint. + The value of the FBG is not considered. + + Most of these where taken off study for the OGTT. We are + using the OGTT as the reason/date instead of the offstudy + reason/date. + + See `merge_with_final_endpoints` where we pick the date of + the first OGTT. + """ + subject_endpoint_df = self.working_df.loc[ (self.working_df["ogtt_value"] >= self.ogtt_threshhold) + & (self.working_df["fasting"] == YES) & (self.working_df["fbg_value"].notna()) ].copy() - subjects_df["endpoint"] = 1 - subjects_df["endpoint_label"] = self.endpoint_cases[CASE_OGTT] - subjects_df["endpoint_type"] = CASE_OGTT - subjects_df["interval_in_days"] = np.nan - subjects_df = subjects_df.reset_index(drop=True) - self.append_subject_to_endpoint_df(subjects_df[endpoint_columns]) - self.working_df = self.working_df.drop( - index=self.working_df.loc[ - self.working_df["subject_identifier"].isin(subjects_df["subject_identifier"]) - ].index - ) - - def get_fbg_only_df(self) -> pd.DataFrame: - fbg_only_df = get_crf( - model="meta_subject.glucosefbg", subject_identifiers=self.subject_identifiers - ) - fbg_only_df["source"] = "meta_subject.glucosefbg" - fbg_only_df.rename( - columns={"fbg_fasting": "fasting", "subject_visit": "subject_visit_id"}, - inplace=True, - ) - fbg_only_df.loc[(fbg_only_df["fasting"] == "fasting"), "fasting"] = YES - fbg_only_df.loc[(fbg_only_df["fasting"] == "non_fasting"), "fasting"] = NO - return fbg_only_df - - def normalize_dates(self): - """Normalize dates""" - for col in ["fbg_datetime", "report_datetime"]: - if not self.fbg_only_df[col].empty: - self.fbg_only_df[col] = self.fbg_only_df[col].dt.floor("d") - if not self.df[col].empty: - self.df[col] = self.df[col].dt.floor("d") - if not self.df["ogtt_datetime"].empty: - self.df["ogtt_datetime"] = self.df["ogtt_datetime"].dt.floor("d") - else: - self.df["ogtt_datetime"] = pd.NaT - - def calculate_fasting_hrs(self): - for dftmp in [self.fbg_only_df, self.df]: - dftmp.loc[(dftmp["fasting"] == NO), "fasting_duration_delta"] = pd.NaT - if dftmp.empty: - dftmp["fasting_hrs"] = np.nan - else: - dftmp["fasting_hrs"] = ( - dftmp["fasting_duration_delta"].dt.total_seconds() / 3600 + if not subject_endpoint_df.empty: + # flag the selected endpoint rows as endpoints + subject_endpoint_df["endpoint"] = 1 + subject_endpoint_df["endpoint_label"] = self.endpoint_cases[CASE_OGTT] + subject_endpoint_df["endpoint_type"] = CASE_OGTT + subject_endpoint_df["interval_in_days"] = np.nan + + # add back the others rows for these subjects + subjects_df = self.working_df.loc[ + ( + self.working_df["subject_identifier"].isin( + subject_endpoint_df["subject_identifier"] + ) + & ~( + self.working_df["fbg_datetime"].isin( + subject_endpoint_df["fbg_datetime"] + ) + ) ) + ].copy() + subjects_df = subjects_df.reset_index(drop=True) + subjects_df["endpoint"] = np.nan + subjects_df["endpoint_label"] = None + subjects_df["endpoint_type"] = None + subjects_df["interval_in_days"] = np.nan + subjects_df = pd.concat([subjects_df, subject_endpoint_df]) + subjects_df = subjects_df.reset_index(drop=True) + + self.append_subject_to_endpoint_df(subjects_df[endpoint_columns]) + self.remove_subjects_from_working_df(subjects_df) def append_subject_to_endpoint_df(self, subject_df: pd.DataFrame) -> None: + """Appends all rows of a subject, or subjects, to the + Endpoints DF. + """ if self.endpoint_df.empty: self.endpoint_df = subject_df.copy() else: @@ -267,6 +315,9 @@ def append_subject_to_endpoint_df(self, subject_df: pd.DataFrame) -> None: self.endpoint_df = self.endpoint_df.reset_index(drop=True) def remove_subject_from_working_df(self, row: pd.Series) -> None: + """Removes one subject from the working DF given a Series with + value `subject_identifier`. + """ self.working_df = self.working_df.drop( index=self.working_df[ self.working_df["subject_identifier"] == row["subject_identifier"] @@ -274,6 +325,17 @@ def remove_subject_from_working_df(self, row: pd.Series) -> None: ) self.working_df = self.working_df.reset_index(drop=True) + def remove_subjects_from_working_df(self, rows: pd.DataFrame) -> None: + """Removes subjects from the working DF given a DF with + column `subject_identifier`. + """ + self.working_df = self.working_df.drop( + index=self.working_df.loc[ + self.working_df["subject_identifier"].isin(rows["subject_identifier"]) + ].index + ) + self.working_df = self.working_df.reset_index(drop=True) + def get_subject_df(self, subject_identifier: str) -> pd.DataFrame: subject_df = self.working_df.loc[ self.working_df["subject_identifier"] == subject_identifier @@ -286,7 +348,7 @@ def get_subject_df(self, subject_identifier: str) -> pd.DataFrame: subject_df = subject_df.reset_index(drop=True) subject_df = subject_df[endpoint_columns] subject_df = subject_df.merge( - self.visit_codes, + self.visit_codes_df, on="visit_code", how="outer", indicator=False, @@ -317,7 +379,7 @@ def post_check_endpoint(self): df_eos["endpoint_label"] = self.endpoint_cases[CASE_EOS] df_eos["endpoint_type"] = CASE_EOS df_eos["interval_in_days"] = np.nan - df_eos.reset_index(drop=True, inplace=True) + df_eos = df_eos.reset_index(drop=True) self.append_subject_to_endpoint_df(df_eos[endpoint_columns]) self.working_df = self.working_df.drop( index=self.working_df.loc[ @@ -326,7 +388,7 @@ def post_check_endpoint(self): ) def merge_with_final_endpoints(self): - # merge endpoint_df with original df + """Merge endpoint_df with original df""" if self.endpoint_df.empty: self.df = self.df[~(self.df["subject_identifier"].isin(self.subject_identifiers))] else: @@ -335,27 +397,34 @@ def merge_with_final_endpoints(self): self.endpoint_df["fbg_datetime"] - self.endpoint_df["baseline_datetime"] ).dt.days - # print(f"Before dedup = {len(self.endpoint_df)}") - + # Create DF of subjects taken offstudy (EOS) where endpoint==1. + # Keep the last record for the subject by fbg_datetime. df1 = self.endpoint_df.copy() - df1 = df1[(df1["endpoint_type"] == CASE_EOS) & (df1["endpoint"] == 1)] + df1 = df1[ + (df1["endpoint_type"].isin([CASE_EOS, CASE_OGTT])) & (df1["endpoint"] == 1) + ] df1 = df1.sort_values(["subject_identifier", "fbg_datetime"]) df1 = df1.reset_index(drop=True) df1 = df1.set_index(["subject_identifier"]) df1 = df1[~df1.index.duplicated(keep="last")] df1 = df1.reset_index(drop=False) + # Create DF of subjects still on-study where endpoint==1. + # Keep the first record for the subject by fbg_datetime. df2 = self.endpoint_df.copy() - df2 = df2[(df2["endpoint_type"] != CASE_EOS) & (df2["endpoint"] == 1)] + df2 = df2[ + ~(df2["endpoint_type"].isin([CASE_EOS, CASE_OGTT])) & (df2["endpoint"] == 1) + ] df2 = df2.sort_values(["subject_identifier", "fbg_datetime"]) df2 = df2.reset_index(drop=True) df2 = df2.set_index(["subject_identifier"]) df2 = df2[~df2.index.duplicated(keep="first")] df2 = df2.reset_index(drop=False) + # create new DF with ONE row per subject for those that reached + # the endpoint (endpoint=1) by merging two DFs above. self.endpoint_only_df = pd.concat([df1, df2]) self.endpoint_only_df = self.endpoint_only_df.reset_index(drop=True) - # print(f"After dedup = {len(self.endpoint_df)}") self.df = pd.merge( self.df, @@ -367,52 +436,8 @@ def merge_with_final_endpoints(self): self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"]) self.df = self.df.reset_index(drop=True) - def summarize( - self, - fasting: str | list | None = None, - interval_in_days_min: int | None = None, - ): - days_min = interval_in_days_min or 7 - - fasting = fasting or [YES, NO, pd.NA] - fasting = fasting if type(fasting) in [list, tuple] else [fasting] - - endpoint_df = self.endpoint_df.copy() - - # endpoint by eos with dm subjects - df7 = endpoint_df[ - (endpoint_df["endpoint_type"] == CASE_EOS) & (endpoint_df["endpoint"] == 1) - ] - df7.reset_index(drop=True, inplace=True) - # endpoint by glucose subjects - df = endpoint_df[ - (endpoint_df["endpoint_type"] != CASE_EOS) - & (endpoint_df["endpoint"] == 1) - & (endpoint_df["fasting"].isin(fasting)) - & ( - (endpoint_df["interval_in_days"] >= days_min) - | (endpoint_df["interval_in_days"].isna()) - ) - ] - df.reset_index(drop=True, inplace=True) - df = pd.concat([df, df7]) - df.reset_index(drop=True, inplace=True) - df_counts = df[["endpoint_type", "endpoint_label"]].value_counts().to_frame() - df_counts.sort_values(by=["endpoint_type"], inplace=True) - df_counts.reset_index(inplace=True) - - sums = { - "endpoint_type": [np.nan], - "endpoint_label": ["Total"], - "count": [ - df_counts["count"].sum(), - ], - } - sums_df = pd.DataFrame.from_dict(sums) - df_counts = pd.concat([df_counts, sums_df], ignore_index=True) - return df_counts - def to_model(self, model: str | None = None, subject_identifiers: list[str] | None = None): + """Write endpoint_only_df to the Endpoints model""" df = self.endpoint_only_df model = model or "meta_reports.endpoints" now = get_utcnow() diff --git a/meta_analytics/dataframes/glucose_endpoints/utils.py b/meta_analytics/dataframes/glucose_endpoints/utils.py index 7964ee5..1cbd692 100644 --- a/meta_analytics/dataframes/glucose_endpoints/utils.py +++ b/meta_analytics/dataframes/glucose_endpoints/utils.py @@ -49,18 +49,17 @@ def get_empty_endpoint_df() -> pd.DataFrame: return endpoint_df -def get_unique_visit_codes(source_df: pd.DataFrame) -> pd.DataFrame: - codes = source_df[source_df["visit_code"] % 1 == 0]["visit_code"].value_counts().to_frame() - codes = codes.reset_index() - codes["visit_code"] = codes["visit_code"].astype(float) - codes = codes.sort_values(["visit_code"]) - # visit_codes = visit_codes[visit_codes["visit_code"] > self.after_visit_code] - codes = codes.reset_index(drop=True) - return codes +def get_unique_visit_codes(df: pd.DataFrame) -> pd.DataFrame: + stats_df = df[df["visit_code"] % 1 == 0]["visit_code"].value_counts().to_frame() + stats_df = stats_df.reset_index() + stats_df["visit_code"] = stats_df["visit_code"].astype(float) + stats_df = stats_df.sort_values(["visit_code"]) + stats_df = stats_df.reset_index(drop=True) + return stats_df -def get_unique_subject_identifiers(source_df) -> pd.DataFrame: - df = pd.DataFrame(source_df["subject_identifier"].unique(), columns=["subject_identifier"]) - df = df.sort_values(["subject_identifier"]) - df = df.reset_index() - return df +def get_unique_subject_identifiers(df: pd.DataFrame) -> pd.DataFrame: + values_df = pd.DataFrame(df["subject_identifier"].unique(), columns=["subject_identifier"]) + values_df = values_df.sort_values(["subject_identifier"]) + values_df = values_df.reset_index() + return values_df diff --git a/meta_reports/admin/__init__.py b/meta_reports/admin/__init__.py index ea6590e..6cae27f 100644 --- a/meta_reports/admin/__init__.py +++ b/meta_reports/admin/__init__.py @@ -1,4 +1,5 @@ -from .endpoints_admin import EndpointAdmin +from .endpoints_admin import EndpointsAdmin +from .endpoints_all_admin import EndpointsAllAdmin from .unmanaged import ( GlucoseSummaryAdmin, MissingScreeningOgttAdmin, diff --git a/meta_reports/admin/endpoints_admin.py b/meta_reports/admin/endpoints_admin.py index 0c8067f..5312bd6 100644 --- a/meta_reports/admin/endpoints_admin.py +++ b/meta_reports/admin/endpoints_admin.py @@ -1,121 +1,14 @@ -from typing import Type - -from django.conf import settings from django.contrib import admin -from django.contrib.admin import SimpleListFilter -from django.db.models import QuerySet from django.template.loader import render_to_string -from django.urls import reverse -from edc_model_admin.dashboard import ModelAdminDashboardMixin -from edc_model_admin.mixins import ( - ModelAdminFormInstructionsMixin, - TemplatesModelAdminMixin, -) -from edc_qareports.modeladmin_mixins import QaReportModelAdminMixin -from edc_sites.admin import SiteModelAdminMixin -from edc_sites.admin.list_filters import SiteListFilter -from edc_visit_schedule.admin import ScheduleStatusListFilter from ..admin_site import meta_reports_admin from ..models import Endpoints -from ..tasks import update_endpoints_table - - -def update_endpoints_table_action(modeladmin, request, queryset): - subject_identifiers = [] - if queryset.count() != modeladmin.model.objects.count(): - subject_identifiers = [o.subject_identifier for o in queryset] - if settings.CELERY_ENABLED: - return update_endpoints_table.delay(subject_identifiers) - return update_endpoints_table(subject_identifiers) - - -update_endpoints_table_action.short_description = "Regenerate report for selected subjects" +from .modeladmin_mixins import EndpointsModelAdminMixin @admin.register(Endpoints, site=meta_reports_admin) -class EndpointAdmin( - QaReportModelAdminMixin, - SiteModelAdminMixin, - ModelAdminDashboardMixin, - ModelAdminFormInstructionsMixin, - TemplatesModelAdminMixin, - admin.ModelAdmin, -): - - change_list_note = render_to_string("meta_reports/endpoints_changelist_note.html") - actions = [update_endpoints_table_action] - qa_report_list_display_insert_pos = 2 - ordering = ["-fbg_datetime"] - list_display = [ - "dashboard", - "subject", - "visit", - "fbg_date", - "fast", - "fbg", - "ogtt", - "endpoint", - "last_updated", - "offstudy_datetime", - "offstudy_reason", - ] - - list_filter = [ - "endpoint_label", - ScheduleStatusListFilter, - SiteListFilter, - ] - - search_fields = ["subject_identifier"] - - def get_queryset(self, request) -> QuerySet: - qs = super().get_queryset(request) - qs = qs.filter(offstudy_datetime__isnull=True) - return qs - - def get_list_filter(self, request) -> tuple[str | Type[SimpleListFilter], ...]: - list_filter = super().get_list_filter(request) - list_filter = list_filter + (SiteListFilter,) - return list_filter - - @admin.display(description="subject", ordering="subject_identifier") - def subject(self, obj=None): - return obj.subject_identifier - - @admin.display(description="visit", ordering="visit_code") - def visit(self, obj=None): - return obj.visit_code - - @admin.display(description="FBG DATE", ordering="fbg_datetime") - def fbg_date(self, obj=None): - return obj.fbg_datetime.date() if obj.fbg_datetime else None - - @admin.display(description="FAST", ordering="fasting") - def fast(self, obj=None): - return obj.fasting - - @admin.display(description="FBG", ordering="fbg_value") - def fbg(self, obj=None): - return obj.fbg_value - - @admin.display(description="OGTT", ordering="ogtt_value") - def ogtt(self, obj=None): - return obj.ogtt_value - - @admin.display(description="endpoint", ordering="endpoint_label") - def endpoint(self, obj=None): - url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist") - return render_to_string( - "meta_reports/columns/subject_identifier_column.html", - { - "subject_identifier": obj.subject_identifier, - "url": url, - "label": obj.endpoint_label, - }, - ) - return +class EndpointsAdmin(EndpointsModelAdminMixin, admin.ModelAdmin): + queryset_filter = dict(offstudy_datetime__isnull=True) - @admin.display(description="last_updated", ordering="created") - def last_updated(self, obj=None): - return obj.created + def rendered_change_list_note(self): + return render_to_string("meta_reports/endpoints_change_list_note.html") diff --git a/meta_reports/admin/endpoints_all_admin.py b/meta_reports/admin/endpoints_all_admin.py new file mode 100644 index 0000000..fc69a07 --- /dev/null +++ b/meta_reports/admin/endpoints_all_admin.py @@ -0,0 +1,13 @@ +from django.contrib import admin +from django.template.loader import render_to_string + +from ..admin_site import meta_reports_admin +from ..models import EndpointsProxy +from .modeladmin_mixins import EndpointsModelAdminMixin + + +@admin.register(EndpointsProxy, site=meta_reports_admin) +class EndpointsAllAdmin(EndpointsModelAdminMixin, admin.ModelAdmin): + + def rendered_change_list_note(self): + return render_to_string("meta_reports/endpoints_all_change_list_note.html") diff --git a/meta_reports/admin/modeladmin_mixins.py b/meta_reports/admin/modeladmin_mixins.py new file mode 100644 index 0000000..f8890e0 --- /dev/null +++ b/meta_reports/admin/modeladmin_mixins.py @@ -0,0 +1,116 @@ +from typing import Type + +from django.conf import settings +from django.contrib import admin +from django.contrib.admin import SimpleListFilter +from django.db.models import QuerySet +from django.template.loader import render_to_string +from django.urls import reverse +from edc_model_admin.dashboard import ModelAdminDashboardMixin +from edc_model_admin.mixins import ( + ModelAdminFormInstructionsMixin, + TemplatesModelAdminMixin, +) +from edc_qareports.modeladmin_mixins import QaReportModelAdminMixin +from edc_sites.admin import SiteModelAdminMixin +from edc_sites.admin.list_filters import SiteListFilter +from edc_visit_schedule.admin import ScheduleStatusListFilter + +from ..tasks import update_endpoints_table + + +def update_endpoints_table_action(modeladmin, request, queryset): + subject_identifiers = [] + if queryset.count() != modeladmin.model.objects.count(): + subject_identifiers = [o.subject_identifier for o in queryset] + if settings.CELERY_ENABLED: + return update_endpoints_table.delay(subject_identifiers) + return update_endpoints_table(subject_identifiers) + + +update_endpoints_table_action.short_description = "Regenerate report for selected subjects" + + +class EndpointsModelAdminMixin( + QaReportModelAdminMixin, + SiteModelAdminMixin, + ModelAdminDashboardMixin, + ModelAdminFormInstructionsMixin, + TemplatesModelAdminMixin, +): + queryset_filter: dict | None = None + actions = [update_endpoints_table_action] + qa_report_list_display_insert_pos = 2 + ordering = ["-fbg_datetime"] + list_display = [ + "dashboard", + "subject", + "visit", + "fbg_date", + "fast", + "fbg", + "ogtt", + "endpoint", + "last_updated", + "offstudy_datetime", + "offstudy_reason", + ] + + list_filter = [ + "endpoint_label", + ScheduleStatusListFilter, + SiteListFilter, + ] + + search_fields = ["subject_identifier"] + + def get_queryset(self, request) -> QuerySet: + qs = super().get_queryset(request) + if self.queryset_filter: + qs = qs.filter(**self.queryset_filter) + return qs + + def get_list_filter(self, request) -> tuple[str | Type[SimpleListFilter], ...]: + list_filter = super().get_list_filter(request) + list_filter = list_filter + (SiteListFilter,) + return list_filter + + @admin.display(description="subject", ordering="subject_identifier") + def subject(self, obj=None): + return obj.subject_identifier + + @admin.display(description="visit", ordering="visit_code") + def visit(self, obj=None): + return obj.visit_code + + @admin.display(description="FBG DATE", ordering="fbg_datetime") + def fbg_date(self, obj=None): + return obj.fbg_datetime.date() if obj.fbg_datetime else None + + @admin.display(description="FAST", ordering="fasting") + def fast(self, obj=None): + return obj.fasting + + @admin.display(description="FBG", ordering="fbg_value") + def fbg(self, obj=None): + return obj.fbg_value + + @admin.display(description="OGTT", ordering="ogtt_value") + def ogtt(self, obj=None): + return obj.ogtt_value + + @admin.display(description="endpoint", ordering="endpoint_label") + def endpoint(self, obj=None): + url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist") + return render_to_string( + "meta_reports/columns/subject_identifier_column.html", + { + "subject_identifier": obj.subject_identifier, + "url": url, + "label": obj.endpoint_label, + }, + ) + + @admin.display(description="last_updated", ordering="created") + def last_updated(self, obj=None): + return obj.created diff --git a/meta_reports/admin/unmanaged/glucose_summary_admin.py b/meta_reports/admin/unmanaged/glucose_summary_admin.py index 3658bbf..3fd02a9 100644 --- a/meta_reports/admin/unmanaged/glucose_summary_admin.py +++ b/meta_reports/admin/unmanaged/glucose_summary_admin.py @@ -1,4 +1,5 @@ from django.contrib import admin +from django.core.exceptions import ObjectDoesNotExist from django.template.loader import render_to_string from django.urls import reverse from edc_constants.constants import YES @@ -10,7 +11,7 @@ from edc_visit_schedule.admin import ScheduleStatusListFilter from ...admin_site import meta_reports_admin -from ...models import Endpoints, GlucoseSummary +from ...models import Endpoints, EndpointsProxy, GlucoseSummary from ..list_filters import EndpointListFilter @@ -29,15 +30,18 @@ class GlucoseSummaryAdmin( "subject_identifier_link", "site", "visit", - "fbg_datetime", + "fasted", + "fbg_date", "fbg_value", "ogtt_value", - "ogtt_datetime", + "ogtt_date", "endpoint", + "offstudy_date", ] list_filter = [ ScheduleStatusListFilter, + "fasted", FbgListFilter, OgttListFilter, "fbg_datetime", @@ -53,20 +57,38 @@ def visit(self, obj=None): @admin.display(description="Endpoint") def endpoint(self, obj=None): - if Endpoints.objects.filter(subject_identifier=obj.subject_identifier).exists(): - url = reverse("meta_reports_admin:meta_reports_endpoints_changelist") - return render_to_string( + try: + endpoint_obj = Endpoints.objects.get(subject_identifier=obj.subject_identifier) + except ObjectDoesNotExist: + value = None + else: + if endpoint_obj.offstudy_datetime: + url = reverse("meta_reports_admin:meta_reports_endpointsproxy_changelist") + title = f"Go to {EndpointsProxy._meta.verbose_name}" + else: + url = reverse("meta_reports_admin:meta_reports_endpoints_changelist") + title = f"Go to {Endpoints._meta.verbose_name}" + value = render_to_string( "meta_reports/columns/subject_identifier_column.html", - {"subject_identifier": obj.subject_identifier, "url": url, "label": YES}, + { + "subject_identifier": obj.subject_identifier, + "url": url, + "label": YES, + "title": title, + }, ) - return None + return value @admin.display(description="Subject Idenfifier", ordering="subject_identifier") def subject_identifier_link(self, obj=None): url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist") return render_to_string( "meta_reports/columns/subject_identifier_column.html", - {"subject_identifier": obj.subject_identifier, "url": url}, + { + "subject_identifier": obj.subject_identifier, + "url": url, + "title": "Click to filter for this subject only", + }, ) def get_subject_dashboard_url_kwargs(self, obj) -> dict: @@ -74,3 +96,21 @@ def get_subject_dashboard_url_kwargs(self, obj) -> dict: subject_identifier=obj.subject_identifier, appointment=obj.appointment_id, ) + + @admin.display(description="Fbg date", ordering="fbg_datetime") + def fbg_date(self, obj): + if obj.fbg_datetime: + return obj.fbg_datetime.date() + return None + + @admin.display(description="OGTT date", ordering="ogtt_datetime") + def ogtt_date(self, obj): + if obj.ogtt_datetime: + return obj.ogtt_datetime.date() + return None + + @admin.display(description="Offstudy date", ordering="offstudy_datetime") + def offstudy_date(self, obj): + if obj.offstudy_datetime: + return obj.offstudy_datetime.date() + return None diff --git a/meta_reports/migrations/0030_auto_20240822_1637.py b/meta_reports/migrations/0030_auto_20240822_1637.py new file mode 100644 index 0000000..4cac776 --- /dev/null +++ b/meta_reports/migrations/0030_auto_20240822_1637.py @@ -0,0 +1,54 @@ +# Generated by Django 5.0.8 on 2024-08-22 13:37 + +import django_db_views.migration_functions +import django_db_views.operations +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("meta_reports", "0029_auto_20240822_0149"), + ] + + operations = [ + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.mysql", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + "# noqa\nselect *, uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime", + "glucose_summary_view", + engine="django.db.backends.mysql", + ), + atomic=False, + ), + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.postgresql", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + "# noqa\nselect *, get_random_uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime", + "glucose_summary_view", + engine="django.db.backends.postgresql", + ), + atomic=False, + ), + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.sqlite3", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + "# noqa\nSELECT *, lower(\n hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-' || '4' ||\n substr(hex( randomblob(2)), 2) || '-' ||\n substr('AB89', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || '-' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime", + "glucose_summary_view", + engine="django.db.backends.sqlite3", + ), + atomic=False, + ), + ] diff --git a/meta_reports/migrations/0031_endpointsproxy.py b/meta_reports/migrations/0031_endpointsproxy.py new file mode 100644 index 0000000..afce5c7 --- /dev/null +++ b/meta_reports/migrations/0031_endpointsproxy.py @@ -0,0 +1,25 @@ +# Generated by Django 5.0.8 on 2024-08-22 19:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("meta_reports", "0030_auto_20240822_1637"), + ] + + operations = [ + migrations.CreateModel( + name="EndpointsProxy", + fields=[], + options={ + "verbose_name": "Endpoints (DM): All", + "verbose_name_plural": "Endpoints (DM): All", + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("meta_reports.endpoints",), + ), + ] diff --git a/meta_reports/migrations/0032_alter_endpointsproxy_options.py b/meta_reports/migrations/0032_alter_endpointsproxy_options.py new file mode 100644 index 0000000..5cffae9 --- /dev/null +++ b/meta_reports/migrations/0032_alter_endpointsproxy_options.py @@ -0,0 +1,21 @@ +# Generated by Django 5.0.8 on 2024-08-22 19:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("meta_reports", "0031_endpointsproxy"), + ] + + operations = [ + migrations.AlterModelOptions( + name="endpointsproxy", + options={ + "default_permissions": ("view", "export", "viewallsites"), + "verbose_name": "Endpoints (DM): All", + "verbose_name_plural": "Endpoints (DM): All", + }, + ), + ] diff --git a/meta_reports/migrations/0033_auto_20240823_0012.py b/meta_reports/migrations/0033_auto_20240823_0012.py new file mode 100644 index 0000000..8c30c23 --- /dev/null +++ b/meta_reports/migrations/0033_auto_20240823_0012.py @@ -0,0 +1,54 @@ +# Generated by Django 5.0.8 on 2024-08-22 21:12 + +import django_db_views.migration_functions +import django_db_views.operations +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("meta_reports", "0032_alter_endpointsproxy_options"), + ] + + operations = [ + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.mysql", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.mysql", + ), + atomic=False, + ), + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.postgresql", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.postgresql", + ), + atomic=False, + ), + django_db_views.operations.ViewRunPython( + code=django_db_views.migration_functions.ForwardViewMigration( + '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier \n ) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.sqlite3", + ), + reverse_code=django_db_views.migration_functions.BackwardViewMigration( + '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime', + "glucose_summary_view", + engine="django.db.backends.sqlite3", + ), + atomic=False, + ), + ] diff --git a/meta_reports/models/__init__.py b/meta_reports/models/__init__.py index f89873a..59a5cf9 100644 --- a/meta_reports/models/__init__.py +++ b/meta_reports/models/__init__.py @@ -7,3 +7,4 @@ UnattendedTwoInRow, ) from .endpoints import Endpoints +from .endpoints_proxy import EndpointsProxy diff --git a/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py b/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py index 25a2143..d32ac37 100644 --- a/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py +++ b/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py @@ -15,12 +15,16 @@ class GlucoseSummary(QaReportModelMixin, DBView): ogtt_datetime = models.DateTimeField(null=True) + fasted = models.CharField(max_length=15, null=True) + visit_code = models.CharField(max_length=25) visit_code_sequence = models.IntegerField() appointment_id = models.UUIDField(null=True) + offstudy_datetime = models.DateTimeField(null=True) + view_definition = get_view_definition() class Meta: diff --git a/meta_reports/models/dbviews/glucose_summary/view_definition.py b/meta_reports/models/dbviews/glucose_summary/view_definition.py index a1b5d82..3fe2889 100644 --- a/meta_reports/models/dbviews/glucose_summary/view_definition.py +++ b/meta_reports/models/dbviews/glucose_summary/view_definition.py @@ -1,30 +1,40 @@ mysql_view: str = """ # noqa select *, uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model from ( - select subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime', - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime', + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as 'fasted', + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucosefbg as fbg left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier UNION - select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as 'fasted', + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucose as fbg - left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A + left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier +) as A order by subject_identifier, fbg_datetime; """ pg_view: str = """ # noqa select *, get_random_uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model from ( - select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime, - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime, + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted, + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucosefbg as fbg left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier UNION - select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted, + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucose as fbg - left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A + left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier +) as A order by subject_identifier, fbg_datetime; """ @@ -37,15 +47,20 @@ hex(randomblob(6)) ) as id, datetime() as `created`, 'meta_reports.glucose_summary_view' as report_model from ( - select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime, - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime, + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted, + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucosefbg as fbg left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier UNION - select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, - fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id + select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime, + case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted, + fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime from meta_subject_glucose as fbg - left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A + left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id + left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier + ) as A order by subject_identifier, fbg_datetime; """ diff --git a/meta_reports/models/endpoints_proxy.py b/meta_reports/models/endpoints_proxy.py new file mode 100644 index 0000000..7fee579 --- /dev/null +++ b/meta_reports/models/endpoints_proxy.py @@ -0,0 +1,11 @@ +from edc_qareports.model_mixins import qa_reports_permissions + +from .endpoints import Endpoints + + +class EndpointsProxy(Endpoints): + class Meta: + proxy = True + verbose_name = "Endpoints (DM): All" + verbose_name_plural = "Endpoints (DM): All" + default_permissions = qa_reports_permissions diff --git a/meta_reports/tasks.py b/meta_reports/tasks.py index 290e0db..fb1c5a5 100644 --- a/meta_reports/tasks.py +++ b/meta_reports/tasks.py @@ -5,6 +5,8 @@ def update_endpoints_table(subject_identifiers: list[str] | None = None): from meta_analytics.dataframes import GlucoseEndpointsByDate + if len(subject_identifiers) > 5: + subject_identifiers = [] cls = GlucoseEndpointsByDate(subject_identifiers=subject_identifiers) cls.run() return cls.to_model(subject_identifiers=subject_identifiers) diff --git a/meta_reports/templates/meta_reports/columns/subject_identifier_column.html b/meta_reports/templates/meta_reports/columns/subject_identifier_column.html index 726f589..b8ecdfb 100644 --- a/meta_reports/templates/meta_reports/columns/subject_identifier_column.html +++ b/meta_reports/templates/meta_reports/columns/subject_identifier_column.html @@ -1 +1 @@ -{{ label|default:subject_identifier }} \ No newline at end of file +{{ label|default:subject_identifier }} \ No newline at end of file diff --git a/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html b/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html new file mode 100644 index 0000000..aeee9da --- /dev/null +++ b/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html @@ -0,0 +1,12 @@ +Endpoints report shows subjects who have reached the protocol endpoint. +
+This report INCLUDES subjects who are no longer on-study. See also the Endpoints (DM) report. +
+Pay attention to the "LAST_UPDATED" column. This tells you the last time the report was updated per subject. +
+
+Unlike other reports, this report does not automatically update, but you can update it : + \ No newline at end of file diff --git a/meta_reports/templates/meta_reports/endpoints_changelist_note.html b/meta_reports/templates/meta_reports/endpoints_change_list_note.html similarity index 50% rename from meta_reports/templates/meta_reports/endpoints_changelist_note.html rename to meta_reports/templates/meta_reports/endpoints_change_list_note.html index 9116fcf..3aa7b38 100644 --- a/meta_reports/templates/meta_reports/endpoints_changelist_note.html +++ b/meta_reports/templates/meta_reports/endpoints_change_list_note.html @@ -1,10 +1,12 @@ Endpoints report shows subjects who have reached the protocol endpoint and should enroll to the DM Referral Schedule.
+The report EXCLUDES subjects who are no longer on-study. See also the Endpoints (DM): All report.
Pay attention to the "LAST_UPDATED" column. This tells you the last time the report was updated per subject.

-Unlike other reports, this report does not automatically update but you can update it : +Unlike other reports, this report does not automatically update, but you can update it : \ No newline at end of file