From d237d28216c5787727c6f97091ba419a2d928845 Mon Sep 17 00:00:00 2001 From: Lissa Pearson Date: Fri, 9 Aug 2024 14:50:27 -0500 Subject: [PATCH 1/4] Added date range filtering options. Still need to link up to date pipeline. --- src/app.py | 127 ++- src/assets/DataFreeze_1_022823.csv | 810 ++++++++++++++++ src/assets/DataFreeze_2_022924.csv | 1403 ++++++++++++++++++++++++++++ 3 files changed, 2329 insertions(+), 11 deletions(-) create mode 100644 src/assets/DataFreeze_1_022823.csv create mode 100644 src/assets/DataFreeze_2_022924.csv diff --git a/src/app.py b/src/app.py index a47269d..1374924 100644 --- a/src/app.py +++ b/src/app.py @@ -63,6 +63,40 @@ icols2 = list(scan_dict.values()) color_mapping_list = [(0.0, 'white'),(0.1, 'lightgrey'),(0.25, 'red'),(0.5, 'orange'),(0.75, 'yellow'),(1.0, 'green')] +# --------------------------------- +# Data ETL +# --------------------------------- + +release1_ids = list(pd.read_csv('assets/DataFreeze_1_022823.csv').record_id) +release2_ids = list(pd.read_csv('assets/DataFreeze_2_022924.csv').record_id) + +def relative_date(nDays): + today = datetime.today() + relativeDate = (today - pd.Timedelta(days=nDays)).date() + return relativeDate + +def filter_imaging_by_date(imaging_df, start_date = None, end_date = None): + '''Filter the imaging datatable using: + start_date: select imaging records acquired on or after this date + end_date: select imaging records acquired on or before this date''' + filtered_imaging = imaging_df.copy() + filtered_imaging['acquisition_week'] = pd.to_datetime(filtered_imaging['acquisition_week']).dt.date + + if start_date and isinstance(start_date, date): + filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] >= start_date] + + if end_date and isinstance(end_date, date): + filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] <= end_date] + + return filtered_imaging + + +def filter_by_release(imaging, release_list): + ''' Filter imaging list to only include the V1 visit for subjects from specific releases. ''' + filtered_imaging = imaging.copy() + filtered_imaging = filtered_imaging[(filtered_imaging['subject_id'].isin(release_list)) & (filtered_imaging['visit']=='V1') ] + + return filtered_imaging # ---------------------------------------------------------------------------- # APP Settings @@ -408,19 +442,34 @@ def create_content(source, data_date, sites): ) ], width=6), dbc.Col([ - html.P('Report Dates'), - html.Div( + html.Div([ dcc.Dropdown( - id='dropdown-date-range-selector', - options=[ - {'label': 'All records', 'value': 'all'}, - {'label': 'Initial ~100 subjects', 'value': 'initial'}, - {'label': 'Last Month', 'value': 'month'}, - {'label': 'Last Week', 'value': 'week'}, - ], - value='all' + id='dropdown-date-range', + options=[ + {'label': 'All records', 'value': 'all'}, + {'label': 'Custom Date Range', 'value': 'custom'}, + {'label': 'Data Release 1', 'value': 'release 1'}, + {'label': 'Data Release 2', 'value': 'release 2'}, + {'label': 'Recent (15 days)', 'value': '15'}, + {'label': '1 Month (30 days)', 'value': '30'}, + {'label': '6 Months (180 days)', 'value': '180'}, + ], + value='all' ), - ), + html.Div(id='report-dates'), + html.Div([ + dcc.DatePickerRange( + id='date-picker-range', + min_date_allowed=date(2021, 3, 29), + start_date=date(2021, 3, 29), + end_date = datetime.today().date(), + # style={ + # 'display': 'none' + # }, + ), + ]), + html.Button('Re-load Report', id='btn-selections', n_clicks=0), + ]), ], width=3) ], justify='end', align='center' ), @@ -529,6 +578,62 @@ def serve_layout(): # ---------------------------------------------------------------------------- # DATA CALLBACKS # ---------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- +# Date Range / Filter type +# ---------------------------------------------------------------------------- + +# @app.callback( +# Output("date-picker-range", "style"), +# Input("dropdown-date-range", "value"), +# State("date-picker-range", "start_date"), +# State("date-picker-range", "end_date"), +# ) +# def update_visibility(customValue, startDate, endDate): +# visible = {'display': 'block'} +# hidden = {'display': 'none'} +# if customValue == 'custom': +# return visible +# else: +# return hidden + +def relative_date(nDays): + today = datetime.today() + relativeDate = (today - pd.Timedelta(days=nDays)).date() + return relativeDate + +@app.callback( + Output("date-picker-range", "style"), + Output("report-dates", "style"), + Input("dropdown-date-range", "value"), + State("date-picker-range", "start_date"), + State("date-picker-range", "end_date"), + ) +def update_visibility(customValue, startDate, endDate): + visible = {'display': 'block'} + hidden = {'display': 'none'} + if customValue == 'custom': + return visible, hidden + else: + return hidden, visible + +@app.callback( + Output("date-picker-range", "start_date"), + Output("date-picker-range", "end_date"), + Output("report-dates", "children"), + Input("dropdown-date-range", "value"), + ) +def update_date_range(customValue): + full_range_options = ['all','custom'] + start_date=date(2021, 3, 29) + end_date = datetime.today().date() + + if customValue and customValue.isnumeric(): + start_date = relative_date(int(customValue)) + + print(start_date, end_date) + report_dates = 'Date Range: ' + start_date.strftime("%m/%d/%Y") + ' to ' + end_date.strftime("%m/%d/%Y") + return start_date, end_date, report_dates + @app.callback( Output('filtered_data', 'data'), Input('session_data', 'data') diff --git a/src/assets/DataFreeze_1_022823.csv b/src/assets/DataFreeze_1_022823.csv new file mode 100644 index 0000000..629cdd2 --- /dev/null +++ b/src/assets/DataFreeze_1_022823.csv @@ -0,0 +1,810 @@ +record_id +10008 +10007 +10005 +10011 +10003 +10010 +10012 +10016 +10001 +10017 +10020 +10006 +10014 +10022 +10013 +10027 +10030 +10015 +10018 +10035 +10031 +10026 +10028 +10032 +10038 +10037 +10023 +10024 +10043 +10042 +10033 +10051 +10034 +10036 +10053 +10041 +10054 +10044 +10045 +10050 +10047 +10052 +10061 +10059 +10068 +10060 +10070 +20004 +10056 +10071 +10073 +10046 +10048 +10067 +20002 +10058 +10062 +10063 +10077 +10074 +10075 +10084 +10072 +10076 +10079 +10080 +10085 +10092 +20011 +10055 +10083 +20007 +10066 +10094 +10049 +10078 +10086 +10089 +10090 +10064 +10095 +10100 +10029 +10081 +10087 +10040 +10091 +10093 +10082 +10088 +20013 +20008 +20016 +20014 +20015 +10105 +10101 +10116 +10117 +20006 +10098 +20009 +10099 +10102 +10104 +10118 +10121 +10106 +10133 +10109 +10111 +10112 +10113 +10125 +10122 +20012 +10107 +10123 +10127 +10130 +10132 +10136 +10128 +10134 +10138 +10141 +10124 +10142 +10152 +20018 +10137 +10139 +20017 +10145 +10151 +10135 +10148 +10129 +20023 +10119 +10157 +10160 +10108 +10143 +10162 +20021 +10149 +10155 +20019 +20024 +10156 +10161 +10163 +10164 +20022 +10004 +10165 +10175 +10103 +20025 +10110 +10126 +10150 +10158 +10168 +20020 +10174 +10146 +10167 +10173 +10183 +10097 +10144 +10147 +10159 +10166 +10169 +10190 +10197 +10179 +20026 +10184 +20028 +10189 +10171 +10177 +10180 +10191 +10203 +10170 +10195 +10201 +10206 +20027 +20029 +10208 +10185 +10200 +10181 +10204 +10120 +10182 +10192 +10199 +10196 +10198 +20035 +10194 +10205 +10211 +10216 +10207 +10215 +10202 +20030 +10217 +10219 +20034 +20036 +20038 +10153 +10220 +10221 +10231 +10218 +10212 +10224 +10230 +10222 +10238 +20031 +10226 +10232 +10225 +10233 +10236 +10209 +10246 +10223 +10234 +20041 +10239 +10247 +20039 +20043 +10227 +10256 +20033 +20042 +10251 +10237 +20050 +10255 +10243 +20047 +20054 +20056 +20040 +10268 +20051 +20046 +20049 +10154 +10265 +10266 +10272 +20001 +10213 +10270 +10249 +10261 +10273 +20045 +20062 +10275 +20037 +20057 +10269 +10271 +10278 +10280 +10250 +10057 +10276 +20066 +20067 +10257 +10241 +10274 +10279 +10281 +10277 +10284 +20064 +10285 +10288 +20055 +20065 +20058 +10252 +10282 +10258 +10287 +10289 +20071 +10293 +10294 +10283 +10303 +10297 +10295 +20061 +10235 +10244 +10301 +10302 +10307 +10310 +20059 +20075 +10298 +10299 +10305 +10292 +10315 +20076 +20078 +20068 +10312 +10313 +10317 +20072 +10245 +10311 +10314 +10309 +10319 +10318 +20089 +20083 +20085 +10323 +20073 +10325 +20081 +20087 +10188 +10320 +10321 +10326 +10341 +20090 +10328 +10329 +10331 +20077 +10330 +20092 +10336 +10337 +10333 +10334 +10332 +10340 +10327 +20084 +20091 +10335 +10339 +20096 +10338 +10342 +10345 +10352 +20098 +10344 +10350 +10324 +20094 +10267 +10349 +10353 +10362 +10363 +20095 +10348 +20086 +10308 +10357 +10361 +10367 +20097 +20101 +10347 +20093 +20103 +10354 +10360 +10376 +10343 +10366 +10375 +20100 +20088 +10371 +10384 +10358 +10364 +10346 +10382 +10355 +10378 +20105 +20104 +20107 +10389 +10391 +10394 +20102 +10380 +20116 +10385 +10386 +20111 +20108 +20110 +10383 +10395 +20099 +20106 +10396 +10390 +10398 +10387 +10397 +20115 +10372 +20118 +10401 +10402 +20120 +20121 +10400 +10403 +20112 +15001 +20117 +10417 +10370 +10404 +10415 +10416 +20119 +10411 +20130 +20123 +20124 +10420 +20127 +10405 +10406 +20132 +20133 +10407 +10408 +10393 +10426 +25003 +10399 +10428 +20134 +10414 +10423 +10214 +10413 +10427 +20128 +20136 +25004 +20135 +10419 +20125 +20137 +25005 +10409 +10432 +10438 +20138 +10424 +10433 +25007 +10392 +10430 +20113 +10421 +10431 +10356 +20142 +20145 +10429 +20140 +10444 +25009 +10437 +20129 +20143 +10434 +10445 +25010 +10418 +10436 +10447 +10377 +25011 +10443 +20131 +20139 +25012 +20141 +25013 +10449 +10452 +10446 +10439 +10453 +25017 +25020 +10442 +10457 +10460 +20152 +25018 +25019 +10451 +10463 +10468 +25014 +25015 +20146 +20149 +20155 +20158 +10425 +10458 +20153 +20160 +25021 +25022 +10471 +15004 +20154 +10459 +10469 +10472 +20161 +15003 +20156 +20163 +10474 +25024 +10464 +15002 +20165 +25028 +25016 +10475 +10477 +10479 +20109 +20169 +25023 +25035 +15005 +25027 +10467 +10470 +10473 +20151 +20167 +20171 +20159 +25029 +25038 +10476 +10485 +20166 +10480 +20177 +25031 +10448 +20164 +20172 +10488 +25040 +10489 +20170 +25032 +10482 +10486 +10487 +10490 +10491 +20179 +20182 +10495 +10497 +10500 +20175 +25039 +25042 +10450 +10493 +10494 +10496 +25041 +25044 +10502 +20174 +20187 +20189 +20176 +10501 +20185 +10503 +10505 +20191 +10492 +10481 +20162 +10483 +10504 +10510 +20178 +25046 +10507 +10515 +20192 +10513 +10478 +10506 +25049 +20197 +20181 +25048 +20194 +10519 +20198 +25047 +25054 +10509 +10530 +10499 +20202 +25058 +20180 +25055 +10521 +10523 +10517 +20173 +20183 +25057 +10516 +10531 +20193 +20201 +20203 +25052 +10291 +25059 +10514 +25053 +10512 +10532 +10541 +20209 +25060 +25034 +25061 +10540 +10542 +25062 +10524 +20207 +25056 +10518 +10528 +20195 +20212 +25065 +10552 +20213 +25050 +10555 +10520 +10522 +10534 +10544 +10549 +20214 +25068 +10546 +20215 +10547 +10548 +25064 +25051 +25067 +25071 +10561 +20205 +25072 +10529 +20206 +25043 +10545 +10566 +10569 +20220 +10498 +20144 +25074 +25077 +10550 +10564 +10567 +25076 +10575 +20208 +20216 +25073 +10557 +10560 +20227 +10565 +10551 +10556 +10559 +10563 +10571 +20219 +10538 +10539 +10578 +10574 +10576 +20224 +25070 +10562 +10572 +10583 +10589 +20221 +25081 +10553 +10573 +20230 +25084 +10580 +20228 +10581 +10584 +10585 +10586 +20223 +20225 +10582 +20232 +10587 +10606 +10595 +25087 +25088 +10591 +10602 +25086 +10594 +10603 +10616 +20233 +25089 +25091 +10592 +10596 +10597 +10604 +10607 +10609 +20211 +10441 +20239 +25093 +10593 +10605 +10610 +10615 +20235 +25092 +10588 +10600 +10611 +20229 +25090 +25095 +20236 +25094 +10614 +10623 +20241 \ No newline at end of file diff --git a/src/assets/DataFreeze_2_022924.csv b/src/assets/DataFreeze_2_022924.csv new file mode 100644 index 0000000..ca96392 --- /dev/null +++ b/src/assets/DataFreeze_2_022924.csv @@ -0,0 +1,1403 @@ +record_id +10001 +10003 +10004 +10005 +10006 +10007 +10008 +10010 +10011 +10012 +10013 +10014 +10015 +10016 +10017 +10018 +10020 +10022 +10023 +10024 +10026 +10027 +10028 +10029 +10030 +10031 +10032 +10033 +10034 +10035 +10036 +10037 +10038 +10040 +10041 +10042 +10043 +10044 +10045 +10046 +10047 +10048 +10049 +10050 +10051 +10052 +10053 +10054 +10055 +10056 +10057 +10058 +10059 +10060 +10061 +10062 +10063 +10064 +10066 +10067 +10068 +10070 +10071 +10072 +10073 +10074 +10075 +10076 +10077 +10078 +10079 +10080 +10081 +10082 +10083 +10084 +10085 +10086 +10087 +10088 +10089 +10090 +10091 +10092 +10093 +10094 +10095 +10097 +10098 +10099 +10100 +10101 +10102 +10103 +10104 +10105 +10106 +10107 +10108 +10109 +10110 +10111 +10112 +10113 +10116 +10117 +10118 +10119 +10120 +10121 +10122 +10123 +10124 +10125 +10126 +10127 +10128 +10129 +10130 +10132 +10133 +10134 +10135 +10136 +10137 +10138 +10139 +10141 +10142 +10143 +10144 +10145 +10146 +10147 +10148 +10149 +10150 +10151 +10152 +10153 +10154 +10155 +10156 +10157 +10158 +10159 +10160 +10161 +10162 +10163 +10164 +10165 +10166 +10167 +10168 +10169 +10170 +10171 +10173 +10174 +10175 +10177 +10179 +10180 +10181 +10182 +10183 +10184 +10185 +10188 +10189 +10190 +10191 +10192 +10194 +10195 +10196 +10197 +10198 +10199 +10200 +10201 +10202 +10203 +10204 +10205 +10206 +10207 +10208 +10209 +10211 +10212 +10213 +10214 +10215 +10216 +10217 +10218 +10219 +10220 +10221 +10222 +10223 +10224 +10225 +10226 +10227 +10230 +10231 +10232 +10233 +10234 +10235 +10236 +10237 +10238 +10239 +10241 +10243 +10244 +10245 +10246 +10247 +10249 +10250 +10251 +10252 +10255 +10256 +10257 +10258 +10261 +10265 +10266 +10267 +10268 +10269 +10270 +10271 +10272 +10273 +10274 +10275 +10276 +10277 +10278 +10279 +10280 +10281 +10282 +10283 +10284 +10285 +10287 +10288 +10289 +10291 +10292 +10293 +10294 +10295 +10297 +10298 +10299 +10301 +10302 +10303 +10305 +10307 +10308 +10309 +10310 +10311 +10312 +10313 +10314 +10315 +10317 +10318 +10319 +10320 +10321 +10323 +10324 +10325 +10326 +10327 +10328 +10329 +10330 +10331 +10332 +10333 +10334 +10335 +10336 +10337 +10338 +10339 +10340 +10341 +10342 +10343 +10344 +10345 +10346 +10347 +10348 +10349 +10350 +10352 +10353 +10354 +10355 +10356 +10357 +10358 +10360 +10361 +10362 +10363 +10364 +10366 +10367 +10370 +10371 +10372 +10375 +10376 +10377 +10378 +10380 +10382 +10383 +10384 +10385 +10386 +10387 +10389 +10390 +10391 +10392 +10393 +10394 +10395 +10396 +10397 +10398 +10399 +10400 +10401 +10402 +10403 +10404 +10405 +10406 +10407 +10408 +10409 +10411 +10413 +10414 +10415 +10416 +10417 +10418 +10419 +10420 +10421 +10423 +10424 +10425 +10426 +10427 +10428 +10429 +10430 +10431 +10432 +10433 +10434 +10436 +10437 +10438 +10439 +10441 +10442 +10443 +10444 +10445 +10446 +10447 +10448 +10449 +10450 +10451 +10452 +10453 +10457 +10458 +10459 +10460 +10463 +10464 +10467 +10468 +10469 +10470 +10471 +10472 +10473 +10474 +10475 +10476 +10477 +10479 +10480 +10481 +10482 +10483 +10485 +10486 +10487 +10488 +10490 +10491 +10492 +10493 +10494 +10495 +10496 +10497 +10498 +10499 +10500 +10501 +10502 +10503 +10504 +10505 +10506 +10507 +10508 +10509 +10510 +10511 +10512 +10513 +10514 +10515 +10516 +10517 +10518 +10519 +10520 +10521 +10522 +10523 +10524 +10528 +10529 +10530 +10531 +10532 +10534 +10538 +10539 +10540 +10541 +10542 +10544 +10545 +10546 +10547 +10548 +10549 +10550 +10551 +10552 +10553 +10555 +10556 +10557 +10559 +10560 +10561 +10562 +10563 +10564 +10565 +10566 +10567 +10569 +10571 +10572 +10573 +10574 +10575 +10576 +10578 +10580 +10581 +10582 +10583 +10584 +10585 +10586 +10587 +10588 +10589 +10590 +10591 +10592 +10593 +10594 +10595 +10596 +10597 +10598 +10599 +10600 +10601 +10602 +10603 +10604 +10605 +10606 +10607 +10608 +10609 +10610 +10611 +10612 +10613 +10614 +10615 +10616 +10618 +10621 +10622 +10623 +10624 +10625 +10626 +10627 +10628 +10629 +10630 +10631 +10633 +10635 +10637 +10638 +10640 +10643 +10644 +10645 +10646 +10647 +10648 +10649 +10650 +10651 +10652 +10653 +10654 +10655 +10656 +10657 +10658 +10659 +10660 +10663 +10664 +10665 +10666 +10667 +10670 +10671 +10672 +10673 +10674 +10676 +10679 +10680 +10681 +10682 +10683 +10684 +10685 +10686 +10687 +10688 +10689 +10690 +10691 +10692 +10693 +10694 +10695 +10696 +10698 +10699 +10700 +10701 +10702 +10703 +10704 +10705 +10706 +10707 +10708 +10709 +10710 +10711 +10712 +10713 +10714 +10715 +10716 +10717 +10718 +10719 +10720 +10721 +10722 +10724 +10727 +10728 +10729 +10730 +10732 +10733 +10734 +10735 +10736 +10737 +10738 +10740 +10743 +10744 +10745 +10746 +10747 +10748 +10749 +10751 +10752 +10753 +10754 +10756 +10757 +10758 +10759 +10760 +10761 +10762 +10763 +10764 +10765 +10766 +10767 +10769 +10770 +10771 +10772 +10773 +10774 +10775 +10776 +10777 +10778 +10780 +10781 +10783 +10784 +10785 +10786 +10787 +10788 +10789 +10790 +10792 +10794 +10795 +10796 +10797 +10798 +10799 +10800 +10801 +10803 +10804 +10805 +10806 +10807 +10808 +10809 +10811 +10812 +10813 +10814 +10815 +10816 +10817 +10819 +10820 +10821 +10822 +10823 +10825 +10826 +10827 +10828 +10829 +10831 +10835 +10836 +10838 +10839 +10840 +10841 +10842 +10845 +10846 +10847 +10848 +10849 +10850 +10851 +10852 +10853 +10854 +10855 +10855 +10857 +10859 +10860 +10861 +10862 +10864 +10865 +10866 +10867 +10868 +10869 +10870 +10872 +10873 +10874 +10875 +10876 +10877 +10878 +10881 +10882 +10883 +10884 +10885 +10886 +10887 +10888 +10889 +10890 +10891 +10892 +10893 +10894 +10895 +10897 +10898 +10899 +10900 +10901 +10903 +10904 +10905 +10906 +10907 +10908 +10909 +10910 +10911 +10912 +10913 +10914 +10915 +10917 +10918 +10919 +10920 +10921 +10922 +10923 +10924 +10925 +10926 +10927 +10928 +10929 +10930 +10932 +10934 +10936 +10939 +10940 +10942 +10943 +10944 +15001 +15002 +15003 +15004 +15005 +15007 +15008 +15010 +15011 +15012 +15013 +15014 +15015 +15016 +15017 +15018 +15019 +15020 +15021 +15022 +15023 +20001 +20002 +20004 +20006 +20007 +20008 +20009 +20011 +20012 +20013 +20014 +20015 +20016 +20017 +20018 +20019 +20020 +20021 +20022 +20023 +20024 +20025 +20026 +20027 +20028 +20029 +20030 +20031 +20033 +20034 +20035 +20036 +20037 +20038 +20039 +20040 +20041 +20042 +20043 +20045 +20046 +20047 +20049 +20050 +20051 +20054 +20055 +20056 +20057 +20058 +20059 +20061 +20062 +20064 +20065 +20066 +20067 +20068 +20071 +20072 +20073 +20075 +20076 +20077 +20078 +20081 +20083 +20084 +20085 +20086 +20087 +20088 +20089 +20090 +20091 +20092 +20093 +20094 +20095 +20096 +20097 +20098 +20099 +20100 +20101 +20102 +20103 +20104 +20105 +20106 +20107 +20108 +20109 +20110 +20111 +20112 +20113 +20115 +20116 +20117 +20118 +20119 +20120 +20121 +20123 +20124 +20125 +20127 +20128 +20129 +20130 +20131 +20132 +20133 +20134 +20135 +20136 +20137 +20138 +20139 +20140 +20141 +20142 +20143 +20144 +20145 +20146 +20149 +20151 +20152 +20153 +20154 +20155 +20156 +20158 +20159 +20160 +20161 +20162 +20163 +20164 +20165 +20166 +20167 +20169 +20170 +20171 +20172 +20173 +20174 +20175 +20176 +20177 +20178 +20179 +20180 +20181 +20182 +20183 +20185 +20187 +20189 +20191 +20192 +20193 +20194 +20195 +20197 +20198 +20201 +20202 +20203 +20205 +20206 +20207 +20208 +20209 +20211 +20212 +20213 +20214 +20215 +20216 +20219 +20220 +20221 +20223 +20224 +20225 +20226 +20227 +20228 +20229 +20230 +20231 +20232 +20233 +20235 +20236 +20237 +20238 +20239 +20240 +20241 +20242 +20243 +20246 +20247 +20248 +20249 +20252 +20253 +20254 +20255 +20256 +20257 +20258 +20259 +20260 +20262 +20263 +20264 +20265 +20266 +20267 +20268 +20269 +20270 +20272 +20273 +20274 +20275 +20276 +20278 +20279 +20280 +20281 +20283 +20284 +20285 +20286 +20287 +20288 +20289 +20290 +20291 +20292 +20293 +20294 +20295 +20296 +20297 +20298 +20299 +20300 +20301 +20302 +20303 +20304 +20305 +20306 +20307 +20308 +20309 +20310 +20311 +20312 +20313 +20314 +20315 +20316 +20317 +20318 +20319 +20320 +20321 +20322 +20323 +20326 +20328 +20330 +20331 +20332 +20333 +20334 +20335 +20336 +20337 +20338 +20339 +20340 +20341 +20342 +20343 +20344 +20345 +20346 +20347 +20348 +20349 +20350 +20352 +20353 +20354 +20355 +20356 +20357 +20360 +20361 +20362 +20363 +20364 +20366 +20367 +20368 +20369 +20370 +20371 +20372 +20373 +20375 +20376 +20377 +20378 +20379 +20380 +20381 +20382 +20383 +20384 +20385 +20386 +20388 +20389 +20390 +20392 +20393 +25001 +25003 +25004 +25005 +25007 +25009 +25010 +25011 +25012 +25013 +25014 +25015 +25016 +25017 +25018 +25019 +25020 +25021 +25022 +25023 +25024 +25027 +25028 +25029 +25032 +25034 +25035 +25038 +25039 +25040 +25041 +25042 +25043 +25044 +25046 +25047 +25048 +25049 +25050 +25051 +25052 +25053 +25054 +25055 +25056 +25057 +25058 +25059 +25060 +25061 +25062 +25064 +25065 +25067 +25068 +25070 +25071 +25072 +25073 +25074 +25076 +25077 +25078 +25079 +25080 +25081 +25084 +25086 +25087 +25088 +25089 +25090 +25091 +25092 +25093 +25094 +25095 +25097 +25098 +25099 +25100 +25101 +25102 +25103 +25104 +25105 +25106 +25107 +25108 +25109 +25111 +25112 +25114 +25115 +25116 +25117 +25118 +25119 +25120 +25121 +25122 +25123 +25124 +25125 +25126 +25127 +25128 +25129 +25130 +25131 +25132 +25134 +25135 +25136 +25137 +25138 +25139 +25140 +25141 +25142 +25143 +25145 +25146 +25147 +25148 +25149 +25150 +25152 +25153 +25154 +25155 +25157 +25158 +25159 +25160 +25161 +25162 +25163 +25164 +25165 +25166 +25167 +25168 +25169 +25170 +25171 +25172 +25174 +25175 +25177 +25178 +25179 +25180 +25181 +25182 +25183 +25185 +25186 +25187 +25188 +25189 +25190 +25191 +25192 +25193 +25194 +25195 +25196 +25197 +25198 +25199 +25200 +25201 +25203 +25204 +25206 +25207 +25209 +25212 +25213 +25214 +25215 +25216 +25217 +25218 +25220 +25221 +25222 +25223 +25224 +25225 +25227 +25228 +25229 +25231 +25232 +25233 +25234 +25235 +25236 +25237 +25238 +25239 +25240 +25242 +25243 +25244 +25245 +25246 +25248 +25249 +25250 +25251 +25252 +25253 +25254 +25256 +25257 +25259 +25260 +25261 +25262 +25264 +25266 +25267 +25268 +25269 +25270 +25271 +25272 +25273 +25274 +25275 +25277 +25278 From 7e8840c58fa2825231b1a69a642d16c80f09a1ad Mon Sep 17 00:00:00 2001 From: Lissa Pearson Date: Mon, 19 Aug 2024 17:27:20 -0500 Subject: [PATCH 2/4] Updates to imaging --- src/app.py | 60 +++++++++++++++++++++++++++++------------- src/data_processing.py | 53 ++++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 31 deletions(-) diff --git a/src/app.py b/src/app.py index 1374924..4ae6c39 100644 --- a/src/app.py +++ b/src/app.py @@ -82,6 +82,11 @@ def filter_imaging_by_date(imaging_df, start_date = None, end_date = None): filtered_imaging = imaging_df.copy() filtered_imaging['acquisition_week'] = pd.to_datetime(filtered_imaging['acquisition_week']).dt.date + # convert string dates to date object + start_date = datetime.strptime(start_date, '%Y-%m-%d').date() + end_date = datetime.strptime(end_date, '%Y-%m-%d').date() + print(isinstance(start_date, date), type(start_date)) + print(isinstance(end_date, date)) if start_date and isinstance(start_date, date): filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] >= start_date] @@ -324,11 +329,14 @@ def get_processed_data(imaging, qc): return processed_data_dictionary -def get_filtered_data_store(raw_data_store, filter_type=None, start_date: datetime = None, end_date = None): +def get_filtered_data_store(raw_data_store, start_date: datetime = None, end_date = None, filter_type=None): imaging = pd.DataFrame.from_dict(raw_data_store['imaging']) qc = pd.DataFrame.from_dict(raw_data_store['qc']) + print(len(imaging)) + print(len(qc)) if imaging.empty or qc.empty: + print('empty') completions = pd.DataFrame() imaging_overview = pd.DataFrame() indicated_received = pd.DataFrame() @@ -336,14 +344,19 @@ def get_filtered_data_store(raw_data_store, filter_type=None, start_date: dateti sites = [] else: if filter_type: + print(filter_type) if filter_type == 'id_list' and isinstance(filter_type, list): imaging = imaging[imaging['subject_id'].isin(filter_type)] else: - imaging = filter_imaging_by_date(imaging, filter_type=None, start_date = start_date, end_date = end_date) + print('dates: ', start_date, end_date) + imaging = filter_imaging_by_date(imaging, start_date = start_date, end_date = end_date) else: imaging = imaging qc = filter_qc(qc, imaging) + print(len(imaging)) + print(len(qc)) + processed_data_dictionary = get_processed_data(imaging, qc) return processed_data_dictionary @@ -417,7 +430,7 @@ def create_data_stores(source, raw_data_dictionary): data_stores = html.Div([ dcc.Store(id='session_data', data = raw_data_dictionary), #storage_type='session', dcc.Store(id='tab_text', data=tab_text), - dcc.Store(id='report_data'), + dcc.Store(id='cache_data'), dcc.Store(id='filtered_data'), # html.P('Imaging Source: ' + data_dictionary['imaging_source']), # html.P('QC Source: ' + data_dictionary['qc_source']), @@ -469,6 +482,7 @@ def create_content(source, data_date, sites): ), ]), html.Button('Re-load Report', id='btn-selections', n_clicks=0), + html.Div(id='datediv'), ]), ], width=3) ], justify='end', align='center' @@ -582,19 +596,6 @@ def serve_layout(): # Date Range / Filter type # ---------------------------------------------------------------------------- -# @app.callback( -# Output("date-picker-range", "style"), -# Input("dropdown-date-range", "value"), -# State("date-picker-range", "start_date"), -# State("date-picker-range", "end_date"), -# ) -# def update_visibility(customValue, startDate, endDate): -# visible = {'display': 'block'} -# hidden = {'display': 'none'} -# if customValue == 'custom': -# return visible -# else: -# return hidden def relative_date(nDays): today = datetime.today() @@ -634,12 +635,33 @@ def update_date_range(customValue): report_dates = 'Date Range: ' + start_date.strftime("%m/%d/%Y") + ' to ' + end_date.strftime("%m/%d/%Y") return start_date, end_date, report_dates +# @app.callback( +# Output('cache_data', 'data'), +# Output('datediv','children'), +# Input('btn-selections','clicks'), +# State('session_data', 'data'), +# State("dropdown-date-range", "value"), +# State("date-picker-range", "start_date"), +# State("date-picker-range", "end_date"), +# ) +# def filtered(clicks, raw_data, selection, startDate, endDate): +# cache_data = {} +# print(selection, startDate, endDate) +# children = html.P("I'm here!") +# return cache_data, children + +# TO DO: Switch input to Re-load report button @app.callback( Output('filtered_data', 'data'), - Input('session_data', 'data') + Input('btn-selections','n_clicks'), + State('session_data', 'data'), + State("date-picker-range", "start_date"), + State("date-picker-range", "end_date") ) -def filtered(raw_data): - filtered_data = get_filtered_data_store(raw_data) +def filtered(clicks, rawData, startDate, endDate): + filtered_data = get_filtered_data_store(rawData, startDate, endDate, 'dates') + print(startDate) + print(endDate) return filtered_data # Filter diff --git a/src/data_processing.py b/src/data_processing.py index 573e41e..2c6b40b 100644 --- a/src/data_processing.py +++ b/src/data_processing.py @@ -68,29 +68,55 @@ def clean_qc(qc_full): return qc - # ---------------------------------------------------------------------------- -# Filter by date +# Filter imaging by date # ---------------------------------------------------------------------------- -def filter_imaging_by_date(imaging_df, start_date: datetime = None, end_date = None): +def relative_date(nDays): + today = datetime.today() + relativeDate = (today - pd.Timedelta(days=nDays)).date() + return relativeDate + +def filter_imaging_by_date(imaging_df, start_date = None, end_date = None): '''Filter the imaging datatable using: start_date: select imaging records acquired on or after this date end_date: select imaging records acquired on or before this date''' filtered_imaging = imaging_df.copy() - filtered_imaging['acquisition_week']= pd.to_datetime(filtered_imaging['acquisition_week'], errors = 'coerce') - - if start_date and isinstance(start_date, datetime): + # filtered_imaging['acquisition_week']= pd.to_datetime(filtered_imaging['acquisition_week'], errors = 'coerce') + filtered_imaging['acquisition_week'] = pd.to_datetime(filtered_imaging['acquisition_week']).dt.date + + if start_date and isinstance(start_date, date): filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] >= start_date] - if end_date and isinstance(end_date, datetime): + if end_date and isinstance(end_date, date): filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] <= end_date] return filtered_imaging + +# ---------------------------------------------------------------------------- +# Filter imaging by data release +# ---------------------------------------------------------------------------- + +def filter_by_release(imaging, release_list): + ''' Filter imaging list to only include the V1 visit for subjects from specific releases. ''' + filtered_imaging = imaging.copy() + filtered_imaging = filtered_imaging[(filtered_imaging['subject_id'].isin(release_list)) & (filtered_imaging['visit']=='V1') ] + + return filtered_imaging + +# ---------------------------------------------------------------------------- +# Filter qc by filtered imaging +# ---------------------------------------------------------------------------- + def filter_qc(qc, filtered_imaging): '''Filter qx records to just those subjects / visits in the filtered imaging set''' + filtered_qc = qc.copy() + filtered_qc['ses'] = filtered_qc['ses'] .astype('category') + filt_sub = filtered_imaging[['subject_id','visit']] - filt_sub.columns = ['sub','ses'] + filt_sub.columns = ['sub','ses'] + filt_sub['ses'] = filt_sub['ses'] .astype('category') + filtered_qc = qc.merge(filt_sub, how = 'left',on = ['sub','ses']) return filtered_qc @@ -158,12 +184,15 @@ def get_indicated_received(imaging_dataframe, validation_column = 'bids', valida # Imaging Overview # ---------------------------------------------------------------------------- def roll_up(imaging): + print(imaging.columns) + print(len(imaging)) cols = ['site','visit','subject_id'] df = imaging[cols].groupby(['site','visit']).count().reset_index() df = df.pivot(index='site', columns = 'visit', values = 'subject_id') - df.loc['All Sites'] = df.sum(numeric_only=True, axis=0) - df.loc[:,'Total'] = df.sum(numeric_only=True, axis=1) - df.reset_index(inplace=True) + print(len(df)) + # df.loc['All Sites'] = df.sum(numeric_only=True, axis=0) + # df.loc[:,'Total'] = df.sum(numeric_only=True, axis=1) + # df.reset_index(inplace=True) return df # ---------------------------------------------------------------------------- @@ -180,7 +209,7 @@ def get_completions(df): icols = list(scan_dict.keys()) icols2 = list(scan_dict.values()) - df['completions_id'] = df.apply(lambda x: str(x['subject_id']) + x['visit'],axis=1) + # df['completions_id'] = df.apply(lambda x: str(x['subject_id']) + x['visit'],axis=1) completions = df[['completions_id']+icols].groupby(icols).count().reset_index().rename(columns=scan_dict).rename(columns={'completions_id':'Count'}) completions['Percent'] = round(100 * completions['Count']/(completions['Count'].sum()),1) completions = completions.sort_values(by=['Count'], ascending=False) From 3e045e9d006f40da8c6fd22bc49b49c5085a4da2 Mon Sep 17 00:00:00 2001 From: Lissa Pearson Date: Tue, 20 Aug 2024 09:57:19 -0500 Subject: [PATCH 3/4] Data Filtering in place and working. Pieces that aren't building from the new data (and need to be fixed): * Overview-Bar Chart * Heat Map Also check site selection choices / dropdown. Consider creating a new one in individual tabs. --- src/app.py | 79 ++++++++++++++++++++++-------------------- src/data_processing.py | 7 ++-- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/app.py b/src/app.py index 4ae6c39..51dc32c 100644 --- a/src/app.py +++ b/src/app.py @@ -75,23 +75,18 @@ def relative_date(nDays): relativeDate = (today - pd.Timedelta(days=nDays)).date() return relativeDate -def filter_imaging_by_date(imaging_df, start_date = None, end_date = None): +def filter_imaging_by_date(imaging_df, date_col, start_date = None, end_date = None): '''Filter the imaging datatable using: start_date: select imaging records acquired on or after this date end_date: select imaging records acquired on or before this date''' filtered_imaging = imaging_df.copy() - filtered_imaging['acquisition_week'] = pd.to_datetime(filtered_imaging['acquisition_week']).dt.date + filtered_imaging[date_col] = pd.to_datetime(filtered_imaging[date_col]).dt.date - # convert string dates to date object - start_date = datetime.strptime(start_date, '%Y-%m-%d').date() - end_date = datetime.strptime(end_date, '%Y-%m-%d').date() - print(isinstance(start_date, date), type(start_date)) - print(isinstance(end_date, date)) if start_date and isinstance(start_date, date): - filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] >= start_date] + filtered_imaging = filtered_imaging[filtered_imaging[date_col] >= start_date] if end_date and isinstance(end_date, date): - filtered_imaging = filtered_imaging[filtered_imaging['acquisition_week'] <= end_date] + filtered_imaging = filtered_imaging[filtered_imaging[date_col] <= end_date] return filtered_imaging @@ -329,11 +324,12 @@ def get_processed_data(imaging, qc): return processed_data_dictionary -def get_filtered_data_store(raw_data_store, start_date: datetime = None, end_date = None, filter_type=None): +def get_report_data_store(raw_data_store, selection = None, start_date: datetime = None, end_date = None): imaging = pd.DataFrame.from_dict(raw_data_store['imaging']) qc = pd.DataFrame.from_dict(raw_data_store['qc']) print(len(imaging)) print(len(qc)) + print(selection) if imaging.empty or qc.empty: print('empty') @@ -343,16 +339,22 @@ def get_filtered_data_store(raw_data_store, start_date: datetime = None, end_dat # stacked_bar_df = pd.DataFrame() sites = [] else: - if filter_type: - print(filter_type) - if filter_type == 'id_list' and isinstance(filter_type, list): - imaging = imaging[imaging['subject_id'].isin(filter_type)] - else: - print('dates: ', start_date, end_date) - imaging = filter_imaging_by_date(imaging, start_date = start_date, end_date = end_date) - else: + if not selection or selection =='all': imaging = imaging - qc = filter_qc(qc, imaging) + qc = qc + else: + if selection == 'release 1': + imaging = filter_by_release(imaging, release1_ids) + + elif selection == 'release 2': + imaging = filter_by_release(imaging, release2_ids) + + else: + startDate = datetime.strptime(start_date, '%Y-%m-%d').date() + endDate = datetime.strptime(end_date, '%Y-%m-%d').date() + imaging = filter_imaging_by_date(imaging, 'acquisition_week', startDate, endDate) + # Filter qc by the subject IDs in the image filter + qc = filter_qc(qc, imaging) print(len(imaging)) print(len(qc)) @@ -431,7 +433,7 @@ def create_data_stores(source, raw_data_dictionary): dcc.Store(id='session_data', data = raw_data_dictionary), #storage_type='session', dcc.Store(id='tab_text', data=tab_text), dcc.Store(id='cache_data'), - dcc.Store(id='filtered_data'), + dcc.Store(id='report_data'), # html.P('Imaging Source: ' + data_dictionary['imaging_source']), # html.P('QC Source: ' + data_dictionary['qc_source']), create_content(source, data_date, sites) @@ -652,26 +654,27 @@ def update_date_range(customValue): # TO DO: Switch input to Re-load report button @app.callback( - Output('filtered_data', 'data'), + Output('report_data', 'data'), Input('btn-selections','n_clicks'), State('session_data', 'data'), + State("dropdown-date-range", "value"), State("date-picker-range", "start_date"), State("date-picker-range", "end_date") ) -def filtered(clicks, rawData, startDate, endDate): - filtered_data = get_filtered_data_store(rawData, startDate, endDate, 'dates') +def filtered(clicks, rawData, selection, startDate, endDate): + report_data = get_report_data_store(rawData, selection, startDate, endDate) print(startDate) print(endDate) - return filtered_data + return report_data # Filter @app.callback( Output('test-row','children'), - Input('filtered_data', 'data') + Input('report_data', 'data') ) -def see_filtering(filtered_data): +def see_filtering(report_data): kids = html.Div() - # kids = html.Div(json.dumps(filtered_data['qc'])) + # kids = html.Div(json.dumps(report_data['qc'])) return kids @app.callback(Output("tab-content", "children"), @@ -817,7 +820,7 @@ def switch_tab(at, tab_text): # Toggle Stacked bar toggle_stackedbar graph_stackedbar @app.callback( Output('overview_div', 'children'), - Input('filtered_data', 'data') + Input('report_data', 'data') ) def update_overview_section(data): imaging_overview = pd.DataFrame.from_dict(data['imaging_overview']) @@ -825,7 +828,7 @@ def update_overview_section(data): @app.callback( Output('discrepancies_section', 'children'), - Input('filtered_data', 'data') + Input('report_data', 'data') ) def update_discrepancies_section(data): # Load imaging data from data store @@ -876,7 +879,7 @@ def update_discrepancies_section(data): @app.callback( Output('cuff_section', 'children'), - Input('filtered_data', 'data') + Input('report_data', 'data') ) def update_cuff_section(data): # Load imaging data from data store @@ -892,12 +895,12 @@ def update_cuff_section(data): @app.callback( Output('graph_stackedbar_div', 'children'), - Input('filtered_data', 'data'), + Input('report_data', 'data'), Input('toggle_stackedbar', 'value'), Input('toggle_visit', 'value'), Input('dropdown-bar', 'value'), ) -def update_stackedbar(filtered_data, type, visit, chart_selection): +def update_stackedbar(report_data, type, visit, chart_selection): global mcc_dict # False = Count and True = Percent # return json.dumps(mcc_dict) @@ -906,7 +909,7 @@ def update_stackedbar(filtered_data, type, visit, chart_selection): else: type = 'Count' - qc = pd.DataFrame.from_dict(filtered_data['qc']) + qc = pd.DataFrame.from_dict(report_data['qc']) count_col='sub' color_col = 'rating' @@ -935,7 +938,7 @@ def update_stackedbar(filtered_data, type, visit, chart_selection): @app.callback( Output('completions_section', 'children'), Input('dropdown-sites', 'value'), - State('filtered_data', 'data') + State('report_data', 'data') ) def update_image_report(sites, data): imaging = pd.DataFrame.from_dict(data['imaging']) @@ -953,12 +956,12 @@ def update_image_report(sites, data): @app.callback( Output('pie_charts', 'children'), Input('dropdown-sites', 'value'), - Input('filtered_data', 'data'), + Input('report_data', 'data'), State('dropdown-sites', 'options') ) -def update_pie(sites, filtered_data, options): +def update_pie(sites, report_data, options): sites_list = sites.split(",") - ratings = pd.DataFrame.from_dict(filtered_data['ratings']) + ratings = pd.DataFrame.from_dict(report_data['ratings']) site_label = [x['label'] for x in options if x['value'] == sites] pie_df = ratings[ratings['Site'].isin(sites_list)] @@ -993,7 +996,7 @@ def update_pie(sites, filtered_data, options): @app.callback( Output('heatmap', 'children'), Input('dropdown-sites', 'value'), - State('filtered_data', 'data') + State('report_data', 'data') ) def update_heatmap(sites, data): global color_mapping_list diff --git a/src/data_processing.py b/src/data_processing.py index 2c6b40b..6c80adf 100644 --- a/src/data_processing.py +++ b/src/data_processing.py @@ -189,10 +189,9 @@ def roll_up(imaging): cols = ['site','visit','subject_id'] df = imaging[cols].groupby(['site','visit']).count().reset_index() df = df.pivot(index='site', columns = 'visit', values = 'subject_id') - print(len(df)) - # df.loc['All Sites'] = df.sum(numeric_only=True, axis=0) - # df.loc[:,'Total'] = df.sum(numeric_only=True, axis=1) - # df.reset_index(inplace=True) + df.loc['All Sites'] = df.sum(numeric_only=True, axis=0) + df.loc[:,'Total'] = df.sum(numeric_only=True, axis=1) + df.reset_index(inplace=True) return df # ---------------------------------------------------------------------------- From ae0b4da9f8d6f89e8af8eb2403662623d3ecc88d Mon Sep 17 00:00:00 2001 From: Lissa Pearson Date: Tue, 20 Aug 2024 10:44:55 -0500 Subject: [PATCH 4/4] Change filter_qc to inner merger to fix barchart and heatmap. TO DO: double check that inner merge is right data manipualtion here --- src/app.py | 2 +- src/data_processing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 51dc32c..f2e80a6 100644 --- a/src/app.py +++ b/src/app.py @@ -996,7 +996,7 @@ def update_pie(sites, report_data, options): @app.callback( Output('heatmap', 'children'), Input('dropdown-sites', 'value'), - State('report_data', 'data') + Input('report_data', 'data') ) def update_heatmap(sites, data): global color_mapping_list diff --git a/src/data_processing.py b/src/data_processing.py index 6c80adf..b4298c7 100644 --- a/src/data_processing.py +++ b/src/data_processing.py @@ -117,7 +117,7 @@ def filter_qc(qc, filtered_imaging): filt_sub.columns = ['sub','ses'] filt_sub['ses'] = filt_sub['ses'] .astype('category') - filtered_qc = qc.merge(filt_sub, how = 'left',on = ['sub','ses']) + filtered_qc = qc.merge(filt_sub, how = 'inner',on = ['sub','ses']) return filtered_qc # ----------------------------------------------------------------------------