diff --git a/meta_analytics/dataframes/glucose_endpoints/constants.py b/meta_analytics/dataframes/glucose_endpoints/constants.py
index b300d03..83357c2 100644
--- a/meta_analytics/dataframes/glucose_endpoints/constants.py
+++ b/meta_analytics/dataframes/glucose_endpoints/constants.py
@@ -1,8 +1,10 @@
-OGTT_THRESHOLD_MET = "OGTT >= 11.1"
-EOS_DM_MET = "EOS - Patient developed diabetes"
-CASE_OGTT = 1
CASE_EOS = 7
+CASE_FBGS_WITH_FIRST_OGTT = 2
+CASE_FBGS_WITH_SECOND_OGTT = 3
CASE_FBG_ONLY = 4
+CASE_OGTT = 1
+EOS_DM_MET = "EOS - Patient developed diabetes"
+OGTT_THRESHOLD_MET = "OGTT >= 11.1"
endpoint_columns = [
"subject_identifier",
@@ -24,8 +26,8 @@
endpoint_cases = {
CASE_OGTT: OGTT_THRESHOLD_MET,
- 2: "FBG >= 7 x 2, first OGTT<=11.1",
- 3: "FBG >= 7 x 2, second OGTT<=11.1",
- 4: "FBG >= 7 x 2, OGTT not considered",
+ CASE_FBGS_WITH_FIRST_OGTT: "FBG >= 7 x 2, first OGTT<=11.1",
+ CASE_FBGS_WITH_SECOND_OGTT: "FBG >= 7 x 2, second OGTT<=11.1",
+ CASE_FBG_ONLY: "FBG >= 7 x 2, OGTT not considered",
CASE_EOS: EOS_DM_MET,
}
diff --git a/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py b/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py
index 7f5d228..ce6db26 100644
--- a/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py
+++ b/meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py
@@ -2,19 +2,14 @@
import pandas as pd
from django.apps import apps as django_apps
from edc_constants.constants import NO, YES
-from edc_pdutils.dataframes import (
- get_crf,
- get_eos,
- get_subject_consent,
- get_subject_visit,
-)
+from edc_pdutils.dataframes import get_crf, get_eos, get_subject_consent
from edc_utils import get_utcnow
-from meta_reports.models import Endpoints
-
from .constants import (
CASE_EOS,
CASE_FBG_ONLY,
+ CASE_FBGS_WITH_FIRST_OGTT,
+ CASE_FBGS_WITH_SECOND_OGTT,
CASE_OGTT,
endpoint_cases,
endpoint_columns,
@@ -28,13 +23,31 @@
)
+def normalize_date_columns(df: pd.DataFrame, cols: list[str] = None) -> pd.DataFrame:
+ """Normalize date columns by flooring"""
+ for col in cols:
+ if not df[col].empty:
+ df[col] = df[col].dt.floor("d")
+ else:
+ df[col] = pd.NaT
+ return df
+
+
+def calculate_fasting_hrs(df: pd.DataFrame):
+ df.loc[(df["fasting"] == NO), "fasting_duration_delta"] = pd.NaT
+ if df.empty:
+ df["fasting_hrs"] = np.nan
+ else:
+ df["fasting_hrs"] = df["fasting_duration_delta"].dt.total_seconds() / 3600
+ return df
+
+
class GlucoseEndpointsByDate:
fbg_threshhold = 7.0
ogtt_threshhold = 11.1
endpoint_cls = EndpointByDate
keep_cols = [
- "subject_visit_id",
"fasting",
"fasting_hrs",
"fbg_value",
@@ -45,9 +58,6 @@ class GlucoseEndpointsByDate:
"ogtt_datetime",
"source",
"report_datetime",
- ]
-
- visit_cols = [
"subject_visit_id",
"subject_identifier",
"visit_code",
@@ -59,45 +69,31 @@ class GlucoseEndpointsByDate:
def __init__(
self, subject_identifiers: list[str] | None = None, case_list: list[int] | None = None
):
- self.subject_identifiers = subject_identifiers or []
- if len(self.subject_identifiers) == Endpoints.objects.all().count():
- self.subject_identifiers = []
- self.case_list = case_list or [CASE_OGTT, 2, 3, CASE_EOS]
- self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list}
+ self._glucose_fbg_df = pd.DataFrame()
+ self._glucose_fbg_ogtt_df = pd.DataFrame()
self.endpoint_only_df = pd.DataFrame()
- self.fbg_only_df = self.get_fbg_only_df()
-
- self.df = get_crf(
- model="meta_subject.glucose",
- subject_identifiers=self.subject_identifiers,
- )
- self.df["source"] = "meta_subject.glucose"
-
- self.calculate_fasting_hrs()
-
- self.fbg_only_df = self.fbg_only_df[
- [col for col in self.keep_cols if not col.startswith("ogtt")]
+ self.subject_identifiers = subject_identifiers or []
+ self.case_list = case_list or [
+ CASE_OGTT,
+ CASE_FBGS_WITH_FIRST_OGTT,
+ CASE_FBGS_WITH_SECOND_OGTT,
+ CASE_EOS,
]
- self.df = self.df[self.keep_cols]
- self.df.reset_index(drop=True)
- self.df = self.df.copy()
-
- self.normalize_dates()
+ self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list}
- # same shape but fbg_only_df ogtt columns are null
+ # merge two model DFs
self.df = pd.merge(
- self.df,
- self.fbg_only_df,
+ self.glucose_fbg_ogtt_df,
+ self.glucose_fbg_df,
on=["subject_visit_id", "fbg_datetime", "fbg_value"],
how="outer",
indicator=True,
suffixes=("", "2"),
)
- self.df.reset_index(drop=True, inplace=True)
- self.df_merged = self.df.copy()
+ self.df = self.df.reset_index(drop=True)
- # right_only
+ # pivot right_only cols
cols = {
"fasting": None,
"fasting_hrs": np.nan,
@@ -107,77 +103,25 @@ def __init__(
}
for col, null_value in cols.items():
self.df.loc[self.df["_merge"] == "right_only", col] = self.df[f"{col}2"]
+ cols = [col for col in self.df.columns if col.endswith("2")]
+ cols.append("_merge")
+ self.df = self.df.drop(columns=cols)
+ self.df = self.df.reset_index(drop=True)
- df_subject_visit = get_subject_visit(
- "meta_subject.subjectvisit", subject_identifiers=subject_identifiers
- )
- self.df = pd.merge(
- df_subject_visit[self.visit_cols], self.df, on="subject_visit_id", how="left"
- )
- self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
- self.df.reset_index(drop=True, inplace=True)
-
- df_consent = get_subject_consent(
- "meta_consent.subjectconsent", subject_identifiers=subject_identifiers
- )
- self.df = pd.merge(self.df, df_consent, on="subject_identifier", how="left")
- self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
- self.df.reset_index(drop=True, inplace=True)
-
- df_eos = get_eos("meta_prn.endofstudy", subject_identifiers=subject_identifiers)
- self.df = pd.merge(self.df, df_eos, on="subject_identifier", how="left")
- self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
- self.df.reset_index(drop=True, inplace=True)
-
- if not self.df.loc[self.df["visit_datetime"].notna()].empty:
- self.df["visit_days"] = (
- self.df["baseline_datetime"].rsub(self.df["visit_datetime"]).dt.days
- )
- if not self.df.loc[self.df["fbg_datetime"].notna()].empty:
- self.df["fgb_days"] = (
- self.df.loc[self.df["fbg_datetime"].notna()]["baseline_datetime"]
- .rsub(self.df["fbg_datetime"])
- .dt.days
- )
- else:
- self.df["fgb_days"] = np.nan
- if not self.df.loc[self.df["ogtt_datetime"].notna()].empty:
- self.df["ogtt_days"] = (
- self.df.loc[self.df["ogtt_datetime"].notna()]["baseline_datetime"]
- .rsub(self.df["ogtt_datetime"])
- .dt.days
- )
- else:
- self.df["ogtt_days"] = np.nan
-
- if self.df.empty:
- self.df["visit_days"] = np.nan
- self.df["fgb_days"] = np.nan
- self.df["ogtt_days"] = np.nan
- self.df["test"] = np.nan
- else:
- self.df["visit_days"] = pd.to_numeric(self.df["visit_days"], downcast="integer")
- self.df["fgb_days"] = pd.to_numeric(self.df["fgb_days"], downcast="integer")
- self.df["ogtt_days"] = pd.to_numeric(self.df["ogtt_days"], downcast="integer")
-
- # label rows by type of glu tests (ones with value)
- self.df["test"] = self.df.apply(get_test_string, axis=1)
+ self.merge_with_consent()
+ self.merge_with_eos()
+ self.add_calculated_days_from_baseline_to_event_columns()
- self.df = self.df.sort_values(by=["subject_identifier", "visit_code"])
+ # label rows by type of glu tests (ones with value)
+ self.df["test"] = self.df.apply(get_test_string, axis=1)
self.df = self.df.reset_index(drop=True)
- self.df = self.df[
- self.df["offstudy_reason"]
- != (
- "Patient fulfilled late exclusion criteria (due to abnormal blood "
- "values or raised blood pressure at enrolment"
- )
- ]
+ self.visit_codes_df = get_unique_visit_codes(self.df)
+ self.subject_identifiers_df = get_unique_subject_identifiers(self.df)
self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
self.df = self.df.reset_index(drop=True)
- self.visit_codes = get_unique_visit_codes(self.df)
- self.subject_identifiers_df = get_unique_subject_identifiers(self.df)
+
self.working_df = self.df.copy()
self.working_df["endpoint"] = 0
self.endpoint_df = get_empty_endpoint_df()
@@ -186,16 +130,19 @@ def run(self):
self.pre_check_endpoint()
for index, row in self.subject_identifiers_df.iterrows():
subject_df = self.get_subject_df(row["subject_identifier"])
- subject_df = self.check_endpoint_by_fbg_for_subject(subject_df, case_list=[2, 3])
+ subject_df = self.check_endpoint_by_fbg_for_subject(
+ subject_df, case_list=[CASE_FBGS_WITH_FIRST_OGTT, CASE_FBGS_WITH_SECOND_OGTT]
+ )
if len(subject_df.loc[subject_df["endpoint"] == 1]) == 1:
self.append_subject_to_endpoint_df(subject_df)
self.remove_subject_from_working_df(row)
if CASE_FBG_ONLY in self.endpoint_cases:
- # go back and rerun for case 5
for index, row in self.subject_identifiers_df.iterrows():
subject_df = self.get_subject_df(row["subject_identifier"])
- subject_df = self.check_endpoint_by_fbg_for_subject(subject_df, case_list=[4])
+ subject_df = self.check_endpoint_by_fbg_for_subject(
+ subject_df, case_list=[CASE_FBG_ONLY]
+ )
if len(subject_df.loc[subject_df["endpoint"] == 1]) == 1:
self.append_subject_to_endpoint_df(subject_df)
self.remove_subject_from_working_df(row)
@@ -203,60 +150,161 @@ def run(self):
self.post_check_endpoint()
self.merge_with_final_endpoints()
+ @property
+ def glucose_fbg_df(self) -> pd.DataFrame:
+ """Returns a prepared Dataframe of CRF
+ meta_subject.glucosefbg.
+
+ Note: meta_subject.glucosefbg has only FBG measures.
+ """
+ if self._glucose_fbg_df.empty:
+ df = get_crf(
+ model="meta_subject.glucosefbg",
+ subject_identifiers=self.subject_identifiers,
+ subject_visit_model="meta_subject.subjectvisit",
+ )
+ df["source"] = "meta_subject.glucosefbg"
+ df.rename(columns={"fbg_fasting": "fasting"}, inplace=True)
+ df.loc[(df["fasting"] == "fasting"), "fasting"] = YES
+ df.loc[(df["fasting"] == "non_fasting"), "fasting"] = NO
+ df = calculate_fasting_hrs(df)
+ df = df[[col for col in self.keep_cols if not col.startswith("ogtt")]]
+ df = df.reset_index(drop=True)
+ df = normalize_date_columns(
+ df, cols=["fbg_datetime", "report_datetime", "visit_datetime"]
+ )
+ self._glucose_fbg_df = df
+ return self._glucose_fbg_df
+
+ @property
+ def glucose_fbg_ogtt_df(self):
+ """Returns a prepared Dataframe of CRF meta_subject.glucose.
+
+ Note: meta_subject.glucose has FBG and OGTT measures.
+ """
+ if self._glucose_fbg_ogtt_df.empty:
+ df = get_crf(
+ model="meta_subject.glucose",
+ subject_identifiers=self.subject_identifiers,
+ subject_visit_model="meta_subject.subjectvisit",
+ )
+ df["source"] = "meta_subject.glucose"
+ df = calculate_fasting_hrs(df)
+ df = df[self.keep_cols]
+ df = df.reset_index(drop=True)
+ df = normalize_date_columns(
+ df, cols=["fbg_datetime", "ogtt_datetime", "report_datetime", "visit_datetime"]
+ )
+ self._glucose_fbg_ogtt_df = df
+ return self._glucose_fbg_ogtt_df
+
+ def merge_with_consent(self):
+ """Merge in consent DF."""
+ df_consent = get_subject_consent("meta_consent.subjectconsent")
+ self.df = pd.merge(self.df, df_consent, on="subject_identifier", how="left")
+ self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
+ self.df = self.df.reset_index(drop=True)
+
+ def merge_with_eos(self):
+ """Merge in EoS DF.
+
+ Drops patients who were taken off study by late exclusion.
+ """
+ df_eos = get_eos("meta_prn.endofstudy")
+ df_eos = df_eos[
+ df_eos["offstudy_reason"]
+ != (
+ "Patient fulfilled late exclusion criteria (due to abnormal blood "
+ "values or raised blood pressure at enrolment"
+ )
+ ]
+ self.df = pd.merge(self.df, df_eos, on="subject_identifier", how="left")
+ self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
+ self.df = self.df.reset_index(drop=True)
+
+ def add_calculated_days_from_baseline_to_event_columns(self):
+ """Add columns that calculate number of days from
+ baseline to visit, fbg, and ogtt.
+ """
+ self.df["visit_days"] = np.nan
+ self.df["fbg_days"] = np.nan
+ self.df["ogtt_days"] = np.nan
+ self.df["test"] = np.nan
+ self.df["visit_days"] = (
+ self.df["visit_datetime"] - self.df["baseline_datetime"]
+ ).dt.days
+ if not self.df["fbg_datetime"].empty:
+ self.df["fbg_days"] = (
+ self.df["fbg_datetime"] - self.df["baseline_datetime"]
+ ).dt.days
+ if not self.df["ogtt_datetime"].empty:
+ self.df["ogtt_days"] = (
+ self.df["ogtt_datetime"] - self.df["baseline_datetime"]
+ ).dt.days
+ self.df["visit_days"] = pd.to_numeric(self.df["visit_days"], downcast="integer")
+ self.df["fbg_days"] = pd.to_numeric(self.df["fbg_days"], downcast="integer")
+ self.df["ogtt_days"] = pd.to_numeric(self.df["ogtt_days"], downcast="integer")
+ self.df = self.df.reset_index(drop=True)
+
def pre_check_endpoint(self):
- "Case 1: flag and remove all OGTT that met threshold"
- subjects_df = self.working_df.loc[
+ """Flag subjects that met endpoint by hitting the OGTT
+ threshold.
+
+ Add them to the endpoint_df and remove them from the
+ working_df.
+
+ Subject must have fasted at the timepoint.
+
+ The OGTT must have an FBG measure at the same timepoint.
+ The value of the FBG is not considered.
+
+ Most of these where taken off study for the OGTT. We are
+ using the OGTT as the reason/date instead of the offstudy
+ reason/date.
+
+ See `merge_with_final_endpoints` where we pick the date of
+ the first OGTT.
+ """
+ subject_endpoint_df = self.working_df.loc[
(self.working_df["ogtt_value"] >= self.ogtt_threshhold)
+ & (self.working_df["fasting"] == YES)
& (self.working_df["fbg_value"].notna())
].copy()
- subjects_df["endpoint"] = 1
- subjects_df["endpoint_label"] = self.endpoint_cases[CASE_OGTT]
- subjects_df["endpoint_type"] = CASE_OGTT
- subjects_df["interval_in_days"] = np.nan
- subjects_df = subjects_df.reset_index(drop=True)
- self.append_subject_to_endpoint_df(subjects_df[endpoint_columns])
- self.working_df = self.working_df.drop(
- index=self.working_df.loc[
- self.working_df["subject_identifier"].isin(subjects_df["subject_identifier"])
- ].index
- )
-
- def get_fbg_only_df(self) -> pd.DataFrame:
- fbg_only_df = get_crf(
- model="meta_subject.glucosefbg", subject_identifiers=self.subject_identifiers
- )
- fbg_only_df["source"] = "meta_subject.glucosefbg"
- fbg_only_df.rename(
- columns={"fbg_fasting": "fasting", "subject_visit": "subject_visit_id"},
- inplace=True,
- )
- fbg_only_df.loc[(fbg_only_df["fasting"] == "fasting"), "fasting"] = YES
- fbg_only_df.loc[(fbg_only_df["fasting"] == "non_fasting"), "fasting"] = NO
- return fbg_only_df
-
- def normalize_dates(self):
- """Normalize dates"""
- for col in ["fbg_datetime", "report_datetime"]:
- if not self.fbg_only_df[col].empty:
- self.fbg_only_df[col] = self.fbg_only_df[col].dt.floor("d")
- if not self.df[col].empty:
- self.df[col] = self.df[col].dt.floor("d")
- if not self.df["ogtt_datetime"].empty:
- self.df["ogtt_datetime"] = self.df["ogtt_datetime"].dt.floor("d")
- else:
- self.df["ogtt_datetime"] = pd.NaT
-
- def calculate_fasting_hrs(self):
- for dftmp in [self.fbg_only_df, self.df]:
- dftmp.loc[(dftmp["fasting"] == NO), "fasting_duration_delta"] = pd.NaT
- if dftmp.empty:
- dftmp["fasting_hrs"] = np.nan
- else:
- dftmp["fasting_hrs"] = (
- dftmp["fasting_duration_delta"].dt.total_seconds() / 3600
+ if not subject_endpoint_df.empty:
+ # flag the selected endpoint rows as endpoints
+ subject_endpoint_df["endpoint"] = 1
+ subject_endpoint_df["endpoint_label"] = self.endpoint_cases[CASE_OGTT]
+ subject_endpoint_df["endpoint_type"] = CASE_OGTT
+ subject_endpoint_df["interval_in_days"] = np.nan
+
+ # add back the others rows for these subjects
+ subjects_df = self.working_df.loc[
+ (
+ self.working_df["subject_identifier"].isin(
+ subject_endpoint_df["subject_identifier"]
+ )
+ & ~(
+ self.working_df["fbg_datetime"].isin(
+ subject_endpoint_df["fbg_datetime"]
+ )
+ )
)
+ ].copy()
+ subjects_df = subjects_df.reset_index(drop=True)
+ subjects_df["endpoint"] = np.nan
+ subjects_df["endpoint_label"] = None
+ subjects_df["endpoint_type"] = None
+ subjects_df["interval_in_days"] = np.nan
+ subjects_df = pd.concat([subjects_df, subject_endpoint_df])
+ subjects_df = subjects_df.reset_index(drop=True)
+
+ self.append_subject_to_endpoint_df(subjects_df[endpoint_columns])
+ self.remove_subjects_from_working_df(subjects_df)
def append_subject_to_endpoint_df(self, subject_df: pd.DataFrame) -> None:
+ """Appends all rows of a subject, or subjects, to the
+ Endpoints DF.
+ """
if self.endpoint_df.empty:
self.endpoint_df = subject_df.copy()
else:
@@ -267,6 +315,9 @@ def append_subject_to_endpoint_df(self, subject_df: pd.DataFrame) -> None:
self.endpoint_df = self.endpoint_df.reset_index(drop=True)
def remove_subject_from_working_df(self, row: pd.Series) -> None:
+ """Removes one subject from the working DF given a Series with
+ value `subject_identifier`.
+ """
self.working_df = self.working_df.drop(
index=self.working_df[
self.working_df["subject_identifier"] == row["subject_identifier"]
@@ -274,6 +325,17 @@ def remove_subject_from_working_df(self, row: pd.Series) -> None:
)
self.working_df = self.working_df.reset_index(drop=True)
+ def remove_subjects_from_working_df(self, rows: pd.DataFrame) -> None:
+ """Removes subjects from the working DF given a DF with
+ column `subject_identifier`.
+ """
+ self.working_df = self.working_df.drop(
+ index=self.working_df.loc[
+ self.working_df["subject_identifier"].isin(rows["subject_identifier"])
+ ].index
+ )
+ self.working_df = self.working_df.reset_index(drop=True)
+
def get_subject_df(self, subject_identifier: str) -> pd.DataFrame:
subject_df = self.working_df.loc[
self.working_df["subject_identifier"] == subject_identifier
@@ -286,7 +348,7 @@ def get_subject_df(self, subject_identifier: str) -> pd.DataFrame:
subject_df = subject_df.reset_index(drop=True)
subject_df = subject_df[endpoint_columns]
subject_df = subject_df.merge(
- self.visit_codes,
+ self.visit_codes_df,
on="visit_code",
how="outer",
indicator=False,
@@ -317,7 +379,7 @@ def post_check_endpoint(self):
df_eos["endpoint_label"] = self.endpoint_cases[CASE_EOS]
df_eos["endpoint_type"] = CASE_EOS
df_eos["interval_in_days"] = np.nan
- df_eos.reset_index(drop=True, inplace=True)
+ df_eos = df_eos.reset_index(drop=True)
self.append_subject_to_endpoint_df(df_eos[endpoint_columns])
self.working_df = self.working_df.drop(
index=self.working_df.loc[
@@ -326,7 +388,7 @@ def post_check_endpoint(self):
)
def merge_with_final_endpoints(self):
- # merge endpoint_df with original df
+ """Merge endpoint_df with original df"""
if self.endpoint_df.empty:
self.df = self.df[~(self.df["subject_identifier"].isin(self.subject_identifiers))]
else:
@@ -335,27 +397,34 @@ def merge_with_final_endpoints(self):
self.endpoint_df["fbg_datetime"] - self.endpoint_df["baseline_datetime"]
).dt.days
- # print(f"Before dedup = {len(self.endpoint_df)}")
-
+ # Create DF of subjects taken offstudy (EOS) where endpoint==1.
+ # Keep the last record for the subject by fbg_datetime.
df1 = self.endpoint_df.copy()
- df1 = df1[(df1["endpoint_type"] == CASE_EOS) & (df1["endpoint"] == 1)]
+ df1 = df1[
+ (df1["endpoint_type"].isin([CASE_EOS, CASE_OGTT])) & (df1["endpoint"] == 1)
+ ]
df1 = df1.sort_values(["subject_identifier", "fbg_datetime"])
df1 = df1.reset_index(drop=True)
df1 = df1.set_index(["subject_identifier"])
df1 = df1[~df1.index.duplicated(keep="last")]
df1 = df1.reset_index(drop=False)
+ # Create DF of subjects still on-study where endpoint==1.
+ # Keep the first record for the subject by fbg_datetime.
df2 = self.endpoint_df.copy()
- df2 = df2[(df2["endpoint_type"] != CASE_EOS) & (df2["endpoint"] == 1)]
+ df2 = df2[
+ ~(df2["endpoint_type"].isin([CASE_EOS, CASE_OGTT])) & (df2["endpoint"] == 1)
+ ]
df2 = df2.sort_values(["subject_identifier", "fbg_datetime"])
df2 = df2.reset_index(drop=True)
df2 = df2.set_index(["subject_identifier"])
df2 = df2[~df2.index.duplicated(keep="first")]
df2 = df2.reset_index(drop=False)
+ # create new DF with ONE row per subject for those that reached
+ # the endpoint (endpoint=1) by merging two DFs above.
self.endpoint_only_df = pd.concat([df1, df2])
self.endpoint_only_df = self.endpoint_only_df.reset_index(drop=True)
- # print(f"After dedup = {len(self.endpoint_df)}")
self.df = pd.merge(
self.df,
@@ -367,52 +436,8 @@ def merge_with_final_endpoints(self):
self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
self.df = self.df.reset_index(drop=True)
- def summarize(
- self,
- fasting: str | list | None = None,
- interval_in_days_min: int | None = None,
- ):
- days_min = interval_in_days_min or 7
-
- fasting = fasting or [YES, NO, pd.NA]
- fasting = fasting if type(fasting) in [list, tuple] else [fasting]
-
- endpoint_df = self.endpoint_df.copy()
-
- # endpoint by eos with dm subjects
- df7 = endpoint_df[
- (endpoint_df["endpoint_type"] == CASE_EOS) & (endpoint_df["endpoint"] == 1)
- ]
- df7.reset_index(drop=True, inplace=True)
- # endpoint by glucose subjects
- df = endpoint_df[
- (endpoint_df["endpoint_type"] != CASE_EOS)
- & (endpoint_df["endpoint"] == 1)
- & (endpoint_df["fasting"].isin(fasting))
- & (
- (endpoint_df["interval_in_days"] >= days_min)
- | (endpoint_df["interval_in_days"].isna())
- )
- ]
- df.reset_index(drop=True, inplace=True)
- df = pd.concat([df, df7])
- df.reset_index(drop=True, inplace=True)
- df_counts = df[["endpoint_type", "endpoint_label"]].value_counts().to_frame()
- df_counts.sort_values(by=["endpoint_type"], inplace=True)
- df_counts.reset_index(inplace=True)
-
- sums = {
- "endpoint_type": [np.nan],
- "endpoint_label": ["Total"],
- "count": [
- df_counts["count"].sum(),
- ],
- }
- sums_df = pd.DataFrame.from_dict(sums)
- df_counts = pd.concat([df_counts, sums_df], ignore_index=True)
- return df_counts
-
def to_model(self, model: str | None = None, subject_identifiers: list[str] | None = None):
+ """Write endpoint_only_df to the Endpoints model"""
df = self.endpoint_only_df
model = model or "meta_reports.endpoints"
now = get_utcnow()
diff --git a/meta_analytics/dataframes/glucose_endpoints/utils.py b/meta_analytics/dataframes/glucose_endpoints/utils.py
index 7964ee5..1cbd692 100644
--- a/meta_analytics/dataframes/glucose_endpoints/utils.py
+++ b/meta_analytics/dataframes/glucose_endpoints/utils.py
@@ -49,18 +49,17 @@ def get_empty_endpoint_df() -> pd.DataFrame:
return endpoint_df
-def get_unique_visit_codes(source_df: pd.DataFrame) -> pd.DataFrame:
- codes = source_df[source_df["visit_code"] % 1 == 0]["visit_code"].value_counts().to_frame()
- codes = codes.reset_index()
- codes["visit_code"] = codes["visit_code"].astype(float)
- codes = codes.sort_values(["visit_code"])
- # visit_codes = visit_codes[visit_codes["visit_code"] > self.after_visit_code]
- codes = codes.reset_index(drop=True)
- return codes
+def get_unique_visit_codes(df: pd.DataFrame) -> pd.DataFrame:
+ stats_df = df[df["visit_code"] % 1 == 0]["visit_code"].value_counts().to_frame()
+ stats_df = stats_df.reset_index()
+ stats_df["visit_code"] = stats_df["visit_code"].astype(float)
+ stats_df = stats_df.sort_values(["visit_code"])
+ stats_df = stats_df.reset_index(drop=True)
+ return stats_df
-def get_unique_subject_identifiers(source_df) -> pd.DataFrame:
- df = pd.DataFrame(source_df["subject_identifier"].unique(), columns=["subject_identifier"])
- df = df.sort_values(["subject_identifier"])
- df = df.reset_index()
- return df
+def get_unique_subject_identifiers(df: pd.DataFrame) -> pd.DataFrame:
+ values_df = pd.DataFrame(df["subject_identifier"].unique(), columns=["subject_identifier"])
+ values_df = values_df.sort_values(["subject_identifier"])
+ values_df = values_df.reset_index()
+ return values_df
diff --git a/meta_reports/admin/__init__.py b/meta_reports/admin/__init__.py
index ea6590e..6cae27f 100644
--- a/meta_reports/admin/__init__.py
+++ b/meta_reports/admin/__init__.py
@@ -1,4 +1,5 @@
-from .endpoints_admin import EndpointAdmin
+from .endpoints_admin import EndpointsAdmin
+from .endpoints_all_admin import EndpointsAllAdmin
from .unmanaged import (
GlucoseSummaryAdmin,
MissingScreeningOgttAdmin,
diff --git a/meta_reports/admin/endpoints_admin.py b/meta_reports/admin/endpoints_admin.py
index 0c8067f..5312bd6 100644
--- a/meta_reports/admin/endpoints_admin.py
+++ b/meta_reports/admin/endpoints_admin.py
@@ -1,121 +1,14 @@
-from typing import Type
-
-from django.conf import settings
from django.contrib import admin
-from django.contrib.admin import SimpleListFilter
-from django.db.models import QuerySet
from django.template.loader import render_to_string
-from django.urls import reverse
-from edc_model_admin.dashboard import ModelAdminDashboardMixin
-from edc_model_admin.mixins import (
- ModelAdminFormInstructionsMixin,
- TemplatesModelAdminMixin,
-)
-from edc_qareports.modeladmin_mixins import QaReportModelAdminMixin
-from edc_sites.admin import SiteModelAdminMixin
-from edc_sites.admin.list_filters import SiteListFilter
-from edc_visit_schedule.admin import ScheduleStatusListFilter
from ..admin_site import meta_reports_admin
from ..models import Endpoints
-from ..tasks import update_endpoints_table
-
-
-def update_endpoints_table_action(modeladmin, request, queryset):
- subject_identifiers = []
- if queryset.count() != modeladmin.model.objects.count():
- subject_identifiers = [o.subject_identifier for o in queryset]
- if settings.CELERY_ENABLED:
- return update_endpoints_table.delay(subject_identifiers)
- return update_endpoints_table(subject_identifiers)
-
-
-update_endpoints_table_action.short_description = "Regenerate report for selected subjects"
+from .modeladmin_mixins import EndpointsModelAdminMixin
@admin.register(Endpoints, site=meta_reports_admin)
-class EndpointAdmin(
- QaReportModelAdminMixin,
- SiteModelAdminMixin,
- ModelAdminDashboardMixin,
- ModelAdminFormInstructionsMixin,
- TemplatesModelAdminMixin,
- admin.ModelAdmin,
-):
-
- change_list_note = render_to_string("meta_reports/endpoints_changelist_note.html")
- actions = [update_endpoints_table_action]
- qa_report_list_display_insert_pos = 2
- ordering = ["-fbg_datetime"]
- list_display = [
- "dashboard",
- "subject",
- "visit",
- "fbg_date",
- "fast",
- "fbg",
- "ogtt",
- "endpoint",
- "last_updated",
- "offstudy_datetime",
- "offstudy_reason",
- ]
-
- list_filter = [
- "endpoint_label",
- ScheduleStatusListFilter,
- SiteListFilter,
- ]
-
- search_fields = ["subject_identifier"]
-
- def get_queryset(self, request) -> QuerySet:
- qs = super().get_queryset(request)
- qs = qs.filter(offstudy_datetime__isnull=True)
- return qs
-
- def get_list_filter(self, request) -> tuple[str | Type[SimpleListFilter], ...]:
- list_filter = super().get_list_filter(request)
- list_filter = list_filter + (SiteListFilter,)
- return list_filter
-
- @admin.display(description="subject", ordering="subject_identifier")
- def subject(self, obj=None):
- return obj.subject_identifier
-
- @admin.display(description="visit", ordering="visit_code")
- def visit(self, obj=None):
- return obj.visit_code
-
- @admin.display(description="FBG DATE", ordering="fbg_datetime")
- def fbg_date(self, obj=None):
- return obj.fbg_datetime.date() if obj.fbg_datetime else None
-
- @admin.display(description="FAST", ordering="fasting")
- def fast(self, obj=None):
- return obj.fasting
-
- @admin.display(description="FBG", ordering="fbg_value")
- def fbg(self, obj=None):
- return obj.fbg_value
-
- @admin.display(description="OGTT", ordering="ogtt_value")
- def ogtt(self, obj=None):
- return obj.ogtt_value
-
- @admin.display(description="endpoint", ordering="endpoint_label")
- def endpoint(self, obj=None):
- url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist")
- return render_to_string(
- "meta_reports/columns/subject_identifier_column.html",
- {
- "subject_identifier": obj.subject_identifier,
- "url": url,
- "label": obj.endpoint_label,
- },
- )
- return
+class EndpointsAdmin(EndpointsModelAdminMixin, admin.ModelAdmin):
+ queryset_filter = dict(offstudy_datetime__isnull=True)
- @admin.display(description="last_updated", ordering="created")
- def last_updated(self, obj=None):
- return obj.created
+ def rendered_change_list_note(self):
+ return render_to_string("meta_reports/endpoints_change_list_note.html")
diff --git a/meta_reports/admin/endpoints_all_admin.py b/meta_reports/admin/endpoints_all_admin.py
new file mode 100644
index 0000000..fc69a07
--- /dev/null
+++ b/meta_reports/admin/endpoints_all_admin.py
@@ -0,0 +1,13 @@
+from django.contrib import admin
+from django.template.loader import render_to_string
+
+from ..admin_site import meta_reports_admin
+from ..models import EndpointsProxy
+from .modeladmin_mixins import EndpointsModelAdminMixin
+
+
+@admin.register(EndpointsProxy, site=meta_reports_admin)
+class EndpointsAllAdmin(EndpointsModelAdminMixin, admin.ModelAdmin):
+
+ def rendered_change_list_note(self):
+ return render_to_string("meta_reports/endpoints_all_change_list_note.html")
diff --git a/meta_reports/admin/modeladmin_mixins.py b/meta_reports/admin/modeladmin_mixins.py
new file mode 100644
index 0000000..f8890e0
--- /dev/null
+++ b/meta_reports/admin/modeladmin_mixins.py
@@ -0,0 +1,116 @@
+from typing import Type
+
+from django.conf import settings
+from django.contrib import admin
+from django.contrib.admin import SimpleListFilter
+from django.db.models import QuerySet
+from django.template.loader import render_to_string
+from django.urls import reverse
+from edc_model_admin.dashboard import ModelAdminDashboardMixin
+from edc_model_admin.mixins import (
+ ModelAdminFormInstructionsMixin,
+ TemplatesModelAdminMixin,
+)
+from edc_qareports.modeladmin_mixins import QaReportModelAdminMixin
+from edc_sites.admin import SiteModelAdminMixin
+from edc_sites.admin.list_filters import SiteListFilter
+from edc_visit_schedule.admin import ScheduleStatusListFilter
+
+from ..tasks import update_endpoints_table
+
+
+def update_endpoints_table_action(modeladmin, request, queryset):
+ subject_identifiers = []
+ if queryset.count() != modeladmin.model.objects.count():
+ subject_identifiers = [o.subject_identifier for o in queryset]
+ if settings.CELERY_ENABLED:
+ return update_endpoints_table.delay(subject_identifiers)
+ return update_endpoints_table(subject_identifiers)
+
+
+update_endpoints_table_action.short_description = "Regenerate report for selected subjects"
+
+
+class EndpointsModelAdminMixin(
+ QaReportModelAdminMixin,
+ SiteModelAdminMixin,
+ ModelAdminDashboardMixin,
+ ModelAdminFormInstructionsMixin,
+ TemplatesModelAdminMixin,
+):
+ queryset_filter: dict | None = None
+ actions = [update_endpoints_table_action]
+ qa_report_list_display_insert_pos = 2
+ ordering = ["-fbg_datetime"]
+ list_display = [
+ "dashboard",
+ "subject",
+ "visit",
+ "fbg_date",
+ "fast",
+ "fbg",
+ "ogtt",
+ "endpoint",
+ "last_updated",
+ "offstudy_datetime",
+ "offstudy_reason",
+ ]
+
+ list_filter = [
+ "endpoint_label",
+ ScheduleStatusListFilter,
+ SiteListFilter,
+ ]
+
+ search_fields = ["subject_identifier"]
+
+ def get_queryset(self, request) -> QuerySet:
+ qs = super().get_queryset(request)
+ if self.queryset_filter:
+ qs = qs.filter(**self.queryset_filter)
+ return qs
+
+ def get_list_filter(self, request) -> tuple[str | Type[SimpleListFilter], ...]:
+ list_filter = super().get_list_filter(request)
+ list_filter = list_filter + (SiteListFilter,)
+ return list_filter
+
+ @admin.display(description="subject", ordering="subject_identifier")
+ def subject(self, obj=None):
+ return obj.subject_identifier
+
+ @admin.display(description="visit", ordering="visit_code")
+ def visit(self, obj=None):
+ return obj.visit_code
+
+ @admin.display(description="FBG DATE", ordering="fbg_datetime")
+ def fbg_date(self, obj=None):
+ return obj.fbg_datetime.date() if obj.fbg_datetime else None
+
+ @admin.display(description="FAST", ordering="fasting")
+ def fast(self, obj=None):
+ return obj.fasting
+
+ @admin.display(description="FBG", ordering="fbg_value")
+ def fbg(self, obj=None):
+ return obj.fbg_value
+
+ @admin.display(description="OGTT", ordering="ogtt_value")
+ def ogtt(self, obj=None):
+ return obj.ogtt_value
+
+ @admin.display(description="endpoint", ordering="endpoint_label")
+ def endpoint(self, obj=None):
+ url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist")
+ return render_to_string(
+ "meta_reports/columns/subject_identifier_column.html",
+ {
+ "subject_identifier": obj.subject_identifier,
+ "url": url,
+ "label": obj.endpoint_label,
+ },
+ )
+
+ @admin.display(description="last_updated", ordering="created")
+ def last_updated(self, obj=None):
+ return obj.created
diff --git a/meta_reports/admin/unmanaged/glucose_summary_admin.py b/meta_reports/admin/unmanaged/glucose_summary_admin.py
index 3658bbf..3fd02a9 100644
--- a/meta_reports/admin/unmanaged/glucose_summary_admin.py
+++ b/meta_reports/admin/unmanaged/glucose_summary_admin.py
@@ -1,4 +1,5 @@
from django.contrib import admin
+from django.core.exceptions import ObjectDoesNotExist
from django.template.loader import render_to_string
from django.urls import reverse
from edc_constants.constants import YES
@@ -10,7 +11,7 @@
from edc_visit_schedule.admin import ScheduleStatusListFilter
from ...admin_site import meta_reports_admin
-from ...models import Endpoints, GlucoseSummary
+from ...models import Endpoints, EndpointsProxy, GlucoseSummary
from ..list_filters import EndpointListFilter
@@ -29,15 +30,18 @@ class GlucoseSummaryAdmin(
"subject_identifier_link",
"site",
"visit",
- "fbg_datetime",
+ "fasted",
+ "fbg_date",
"fbg_value",
"ogtt_value",
- "ogtt_datetime",
+ "ogtt_date",
"endpoint",
+ "offstudy_date",
]
list_filter = [
ScheduleStatusListFilter,
+ "fasted",
FbgListFilter,
OgttListFilter,
"fbg_datetime",
@@ -53,20 +57,38 @@ def visit(self, obj=None):
@admin.display(description="Endpoint")
def endpoint(self, obj=None):
- if Endpoints.objects.filter(subject_identifier=obj.subject_identifier).exists():
- url = reverse("meta_reports_admin:meta_reports_endpoints_changelist")
- return render_to_string(
+ try:
+ endpoint_obj = Endpoints.objects.get(subject_identifier=obj.subject_identifier)
+ except ObjectDoesNotExist:
+ value = None
+ else:
+ if endpoint_obj.offstudy_datetime:
+ url = reverse("meta_reports_admin:meta_reports_endpointsproxy_changelist")
+ title = f"Go to {EndpointsProxy._meta.verbose_name}"
+ else:
+ url = reverse("meta_reports_admin:meta_reports_endpoints_changelist")
+ title = f"Go to {Endpoints._meta.verbose_name}"
+ value = render_to_string(
"meta_reports/columns/subject_identifier_column.html",
- {"subject_identifier": obj.subject_identifier, "url": url, "label": YES},
+ {
+ "subject_identifier": obj.subject_identifier,
+ "url": url,
+ "label": YES,
+ "title": title,
+ },
)
- return None
+ return value
@admin.display(description="Subject Idenfifier", ordering="subject_identifier")
def subject_identifier_link(self, obj=None):
url = reverse("meta_reports_admin:meta_reports_glucosesummary_changelist")
return render_to_string(
"meta_reports/columns/subject_identifier_column.html",
- {"subject_identifier": obj.subject_identifier, "url": url},
+ {
+ "subject_identifier": obj.subject_identifier,
+ "url": url,
+ "title": "Click to filter for this subject only",
+ },
)
def get_subject_dashboard_url_kwargs(self, obj) -> dict:
@@ -74,3 +96,21 @@ def get_subject_dashboard_url_kwargs(self, obj) -> dict:
subject_identifier=obj.subject_identifier,
appointment=obj.appointment_id,
)
+
+ @admin.display(description="Fbg date", ordering="fbg_datetime")
+ def fbg_date(self, obj):
+ if obj.fbg_datetime:
+ return obj.fbg_datetime.date()
+ return None
+
+ @admin.display(description="OGTT date", ordering="ogtt_datetime")
+ def ogtt_date(self, obj):
+ if obj.ogtt_datetime:
+ return obj.ogtt_datetime.date()
+ return None
+
+ @admin.display(description="Offstudy date", ordering="offstudy_datetime")
+ def offstudy_date(self, obj):
+ if obj.offstudy_datetime:
+ return obj.offstudy_datetime.date()
+ return None
diff --git a/meta_reports/migrations/0030_auto_20240822_1637.py b/meta_reports/migrations/0030_auto_20240822_1637.py
new file mode 100644
index 0000000..4cac776
--- /dev/null
+++ b/meta_reports/migrations/0030_auto_20240822_1637.py
@@ -0,0 +1,54 @@
+# Generated by Django 5.0.8 on 2024-08-22 13:37
+
+import django_db_views.migration_functions
+import django_db_views.operations
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("meta_reports", "0029_auto_20240822_0149"),
+ ]
+
+ operations = [
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.mysql",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ "# noqa\nselect *, uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime",
+ "glucose_summary_view",
+ engine="django.db.backends.mysql",
+ ),
+ atomic=False,
+ ),
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.postgresql",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ "# noqa\nselect *, get_random_uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime",
+ "glucose_summary_view",
+ engine="django.db.backends.postgresql",
+ ),
+ atomic=False,
+ ),
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.sqlite3",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ "# noqa\nSELECT *, lower(\n hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-' || '4' ||\n substr(hex( randomblob(2)), 2) || '-' ||\n substr('AB89', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || '-' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, 'meta_reports.glucose_summary_view' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg \n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime",
+ "glucose_summary_view",
+ engine="django.db.backends.sqlite3",
+ ),
+ atomic=False,
+ ),
+ ]
diff --git a/meta_reports/migrations/0031_endpointsproxy.py b/meta_reports/migrations/0031_endpointsproxy.py
new file mode 100644
index 0000000..afce5c7
--- /dev/null
+++ b/meta_reports/migrations/0031_endpointsproxy.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.0.8 on 2024-08-22 19:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("meta_reports", "0030_auto_20240822_1637"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="EndpointsProxy",
+ fields=[],
+ options={
+ "verbose_name": "Endpoints (DM): All",
+ "verbose_name_plural": "Endpoints (DM): All",
+ "proxy": True,
+ "indexes": [],
+ "constraints": [],
+ },
+ bases=("meta_reports.endpoints",),
+ ),
+ ]
diff --git a/meta_reports/migrations/0032_alter_endpointsproxy_options.py b/meta_reports/migrations/0032_alter_endpointsproxy_options.py
new file mode 100644
index 0000000..5cffae9
--- /dev/null
+++ b/meta_reports/migrations/0032_alter_endpointsproxy_options.py
@@ -0,0 +1,21 @@
+# Generated by Django 5.0.8 on 2024-08-22 19:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("meta_reports", "0031_endpointsproxy"),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name="endpointsproxy",
+ options={
+ "default_permissions": ("view", "export", "viewallsites"),
+ "verbose_name": "Endpoints (DM): All",
+ "verbose_name_plural": "Endpoints (DM): All",
+ },
+ ),
+ ]
diff --git a/meta_reports/migrations/0033_auto_20240823_0012.py b/meta_reports/migrations/0033_auto_20240823_0012.py
new file mode 100644
index 0000000..8c30c23
--- /dev/null
+++ b/meta_reports/migrations/0033_auto_20240823_0012.py
@@ -0,0 +1,54 @@
+# Generated by Django 5.0.8 on 2024-08-22 21:12
+
+import django_db_views.migration_functions
+import django_db_views.operations
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("meta_reports", "0032_alter_endpointsproxy_options"),
+ ]
+
+ operations = [
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.mysql",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ '# noqa\nselect *, uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as \'ogtt_value\', null as \'ogtt_datetime\',\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as \'fasted\',\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.mysql",
+ ),
+ atomic=False,
+ ),
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.postgresql",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ '# noqa\nselect *, get_random_uuid() as id, now() as created, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.postgresql",
+ ),
+ atomic=False,
+ ),
+ django_db_views.operations.ViewRunPython(
+ code=django_db_views.migration_functions.ForwardViewMigration(
+ '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier\n UNION\n select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier \n ) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.sqlite3",
+ ),
+ reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+ '# noqa\nSELECT *, lower(\n hex(randomblob(4)) || \'-\' || hex(randomblob(2)) || \'-\' || \'4\' ||\n substr(hex( randomblob(2)), 2) || \'-\' ||\n substr(\'AB89\', 1 + (abs(random()) % 4) , 1) ||\n substr(hex(randomblob(2)), 2) || \'-\' ||\n hex(randomblob(6))\n ) as id, datetime() as `created`, \'meta_reports.glucose_summary_view\' as report_model\nfrom (\n select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucosefbg as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id\n UNION\n select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,\n case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,\n fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id\n from meta_subject_glucose as fbg\n left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A\norder by subject_identifier, fbg_datetime',
+ "glucose_summary_view",
+ engine="django.db.backends.sqlite3",
+ ),
+ atomic=False,
+ ),
+ ]
diff --git a/meta_reports/models/__init__.py b/meta_reports/models/__init__.py
index f89873a..59a5cf9 100644
--- a/meta_reports/models/__init__.py
+++ b/meta_reports/models/__init__.py
@@ -7,3 +7,4 @@
UnattendedTwoInRow,
)
from .endpoints import Endpoints
+from .endpoints_proxy import EndpointsProxy
diff --git a/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py b/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py
index 25a2143..d32ac37 100644
--- a/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py
+++ b/meta_reports/models/dbviews/glucose_summary/unmanaged_model.py
@@ -15,12 +15,16 @@ class GlucoseSummary(QaReportModelMixin, DBView):
ogtt_datetime = models.DateTimeField(null=True)
+ fasted = models.CharField(max_length=15, null=True)
+
visit_code = models.CharField(max_length=25)
visit_code_sequence = models.IntegerField()
appointment_id = models.UUIDField(null=True)
+ offstudy_datetime = models.DateTimeField(null=True)
+
view_definition = get_view_definition()
class Meta:
diff --git a/meta_reports/models/dbviews/glucose_summary/view_definition.py b/meta_reports/models/dbviews/glucose_summary/view_definition.py
index a1b5d82..3fe2889 100644
--- a/meta_reports/models/dbviews/glucose_summary/view_definition.py
+++ b/meta_reports/models/dbviews/glucose_summary/view_definition.py
@@ -1,30 +1,40 @@
mysql_view: str = """ # noqa
select *, uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model
from (
- select subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime',
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, null as 'ogtt_value', null as 'ogtt_datetime',
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as 'fasted',
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucosefbg as fbg
left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
UNION
- select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as 'fasted',
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucose as fbg
- left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A
+ left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
+) as A
order by subject_identifier, fbg_datetime;
"""
pg_view: str = """ # noqa
select *, get_random_uuid() as id, now() as created, 'meta_reports.glucose_summary_view' as report_model
from (
- select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucosefbg as fbg
left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
UNION
- select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucose as fbg
- left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A
+ left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
+) as A
order by subject_identifier, fbg_datetime;
"""
@@ -37,15 +47,20 @@
hex(randomblob(6))
) as id, datetime() as `created`, 'meta_reports.glucose_summary_view' as report_model
from (
- select subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, null as ogtt_value, null as ogtt_datetime,
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucosefbg as fbg
left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
UNION
- select subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
- fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id
+ select v.subject_identifier, fbg_value, fbg_datetime, ogtt_value, ogtt_datetime,
+ case when fasting="fasting" then "Yes" when fasting="non_fasting" then "No" else fasting end as fasted,
+ fbg.site_id, v.visit_code, v.visit_code_sequence, v.appointment_id, eos.offstudy_datetime
from meta_subject_glucose as fbg
- left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id) as A
+ left join meta_subject_subjectvisit as v on v.id=fbg.subject_visit_id
+ left join meta_prn_endofstudy as eos on v.subject_identifier=eos.subject_identifier
+ ) as A
order by subject_identifier, fbg_datetime;
"""
diff --git a/meta_reports/models/endpoints_proxy.py b/meta_reports/models/endpoints_proxy.py
new file mode 100644
index 0000000..7fee579
--- /dev/null
+++ b/meta_reports/models/endpoints_proxy.py
@@ -0,0 +1,11 @@
+from edc_qareports.model_mixins import qa_reports_permissions
+
+from .endpoints import Endpoints
+
+
+class EndpointsProxy(Endpoints):
+ class Meta:
+ proxy = True
+ verbose_name = "Endpoints (DM): All"
+ verbose_name_plural = "Endpoints (DM): All"
+ default_permissions = qa_reports_permissions
diff --git a/meta_reports/tasks.py b/meta_reports/tasks.py
index 290e0db..fb1c5a5 100644
--- a/meta_reports/tasks.py
+++ b/meta_reports/tasks.py
@@ -5,6 +5,8 @@
def update_endpoints_table(subject_identifiers: list[str] | None = None):
from meta_analytics.dataframes import GlucoseEndpointsByDate
+ if len(subject_identifiers) > 5:
+ subject_identifiers = []
cls = GlucoseEndpointsByDate(subject_identifiers=subject_identifiers)
cls.run()
return cls.to_model(subject_identifiers=subject_identifiers)
diff --git a/meta_reports/templates/meta_reports/columns/subject_identifier_column.html b/meta_reports/templates/meta_reports/columns/subject_identifier_column.html
index 726f589..b8ecdfb 100644
--- a/meta_reports/templates/meta_reports/columns/subject_identifier_column.html
+++ b/meta_reports/templates/meta_reports/columns/subject_identifier_column.html
@@ -1 +1 @@
-{{ label|default:subject_identifier }}
\ No newline at end of file
+{{ label|default:subject_identifier }}
\ No newline at end of file
diff --git a/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html b/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html
new file mode 100644
index 0000000..aeee9da
--- /dev/null
+++ b/meta_reports/templates/meta_reports/endpoints_all_change_list_note.html
@@ -0,0 +1,12 @@
+Endpoints report shows subjects who have reached the protocol endpoint.
+
+This report INCLUDES subjects who are no longer on-study. See also the Endpoints (DM) report.
+
+Pay attention to the "LAST_UPDATED" column. This tells you the last time the report was updated per subject.
+
+
+Unlike other reports, this report does not automatically update, but you can update it :
+