From 041b4b06a02b594321dad9adb064e3c7ea7c4b85 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Thu, 19 Dec 2024 15:35:35 +1100 Subject: [PATCH 01/12] initial version of ourdna dashboard --- web/src/Routes.tsx | 2 +- web/src/pages/report/OurDnaDashboard.tsx | 369 ++++++++++++++++++ .../pages/report/chart/MetricFromQuery.tsx | 66 ++++ web/src/pages/report/chart/PlotFromQuery.tsx | 148 +++++++ web/src/pages/report/chart/TableFromQuery.tsx | 92 ++++- 5 files changed, 675 insertions(+), 2 deletions(-) create mode 100644 web/src/pages/report/OurDnaDashboard.tsx create mode 100644 web/src/pages/report/chart/MetricFromQuery.tsx create mode 100644 web/src/pages/report/chart/PlotFromQuery.tsx diff --git a/web/src/Routes.tsx b/web/src/Routes.tsx index 9055a7f82..0c52d477f 100644 --- a/web/src/Routes.tsx +++ b/web/src/Routes.tsx @@ -18,10 +18,10 @@ import DocumentationArticle from './pages/docs/Documentation' import { FamilyPage } from './pages/family/FamilyView' import Details from './pages/insights/Details' import Summary from './pages/insights/Summary' -import OurDnaDashboard from './pages/ourdna/OurDnaDashboard' import { ParticipantPage } from './pages/participant/ParticipantViewContainer' import AnalysisRunnerSummary from './pages/project/AnalysisRunnerView/AnalysisRunnerSummary' import ProjectOverview from './pages/project/ProjectOverview' +import OurDnaDashboard from './pages/report/OurDnaDashboard' import SqlQueryUI from './pages/report/SqlQueryUI' import SampleView from './pages/sample/SampleView' import ErrorBoundary from './shared/utilities/errorBoundary' diff --git a/web/src/pages/report/OurDnaDashboard.tsx b/web/src/pages/report/OurDnaDashboard.tsx new file mode 100644 index 000000000..e77f4434f --- /dev/null +++ b/web/src/pages/report/OurDnaDashboard.tsx @@ -0,0 +1,369 @@ +import { Box, Typography } from '@mui/material' +import * as Plot from '@observablehq/plot' +import { MetricFromQueryCard } from './chart/MetricFromQuery' +import { PlotFromQueryCard } from './chart/PlotFromQuery' +import { TableFromQueryCard } from './chart/TableFromQuery' +import { useProjectDbQuery } from './data/projectDatabase' + +/* + +other charts: + +- chart of samples where we don't have consent / registration information for a participant. + - could be a few reasons, but want to capture badly entered participant ids + + + sections: + - recruitment + - processing times + + - demographics + + - data quality + +*/ +export default function OurDnaDashboard() { + const result = useProjectDbQuery( + 'ourdna', + + ` + select * from participant limit 10 + + + ` + ) + if (result?.status === 'success') { + console.log(result.data.toArray().map((r) => r.toJSON())) + } + + return ( + + + + Recruitment + + + + + + Plot.plot({ + marginLeft: 160, + inset: 10, + height: 170, + width, + marks: [ + Plot.barX(data, { + y: 'stage', + x: 'count', + tip: true, + fill: 'stage', + }), + ], + }) + } + /> + + + + Plot.plot({ + width, + height: 150, + color: { legend: true }, + marks: [ + Plot.barX( + data, + Plot.stackX({ + x: 'count', + fill: 'collection_type', + inset: 0.5, + tip: true, + }) + ), + ], + }) + } + /> + + + + + + Processing times + + + + + 0 + `} + plot={(data, { width }) => + Plot.plot({ + y: { grid: true, padding: 5, label: 'duration(hrs)' }, + x: { grid: true, padding: 5 }, + width: width, + height: 400, + marginTop: 20, + marginRight: 20, + marginBottom: 30, + + marginLeft: 40, + marks: [ + Plot.ruleY([24, 48, 72], { stroke: '#ff725c' }), + Plot.dot(data, { + x: 'collection', + y: 'duration', + stroke: null, + fill: '#4269d0', + fillOpacity: 0.5, + channels: { process_end: 'process_end' }, + tip: true, + }), + ], + }) + } + /> + + + = 24 AND duration < 30 THEN '24-30 hours' + WHEN duration >= 30 AND duration < 33 THEN '30-33 hours' + WHEN duration >= 33 AND duration < 48 THEN '33-48 hours' + WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' + ELSE '72+ hours' + END AS duration + + from durations group by 2 order by 2 + `} + plot={(data, { width }) => + Plot.plot({ + marginLeft: 100, + inset: 10, + height: 410, + color: { + scheme: 'RdYlGn', + reverse: true, + }, + width, + marks: [ + Plot.barX(data, { + y: 'duration', + x: 'count', + tip: true, + fill: 'duration', + }), + ], + }) + } + /> + + + + + + + + + + + + Demographics + + + + + + + + + + Plot.plot({ + marginLeft: 40, + inset: 10, + width: width, + height: 400, + y: { + grid: true, + label: 'Participants', + }, + x: { + type: 'time', + }, + marks: [ + Plot.areaY(data, { + x: 'week', + y: 'count', + fill: 'ancestry', + tip: true, + order: 'sum', + }), + Plot.ruleY([0]), + ], + }) + } + /> + + + + ) +} diff --git a/web/src/pages/report/chart/MetricFromQuery.tsx b/web/src/pages/report/chart/MetricFromQuery.tsx new file mode 100644 index 000000000..4758cd6f8 --- /dev/null +++ b/web/src/pages/report/chart/MetricFromQuery.tsx @@ -0,0 +1,66 @@ +import { Box, Card, CardContent, Divider, Typography } from '@mui/material' +import CircularProgress from '@mui/material/CircularProgress' +import { fromArrow } from 'arquero' +import { ArrowTable } from 'arquero/dist/types/format/types' +import { Fragment, ReactChild } from 'react' +import { useProjectDbQuery } from '../data/projectDatabase' + +type Props = { + project: string + query: string + title?: ReactChild + subtitle?: ReactChild + description?: ReactChild +} + +export function MetricFromQueryCard(props: Props) { + const { project, query } = props + const result = useProjectDbQuery(project, query) + + const data = result && result.status === 'success' ? result.data : undefined + + if (!data || !result || result.status === 'loading') return + + const table = fromArrow(data as ArrowTable) + + const columns = table.columnNames() + + return ( + + + {(props.title || props.subtitle) && ( + + {props.title && ( + + {props.title} + + )} + {props.subtitle && {props.subtitle}} + + )} + + + {columns.map((col, index) => ( + + {index !== 0 && } + + + {col} + + + {table.get(col, 0)} + + + + ))} + + + {props.description && ( + + {props.description} + + )} + + + ) +} diff --git a/web/src/pages/report/chart/PlotFromQuery.tsx b/web/src/pages/report/chart/PlotFromQuery.tsx new file mode 100644 index 000000000..5efc615a4 --- /dev/null +++ b/web/src/pages/report/chart/PlotFromQuery.tsx @@ -0,0 +1,148 @@ +import OpenInFullIcon from '@mui/icons-material/OpenInFull' +import TableChartIcon from '@mui/icons-material/TableChart' +import { + Alert, + Box, + Card, + CardActions, + CardContent, + CircularProgress, + IconButton, + Modal, + Tooltip, + Typography, +} from '@mui/material' +import * as Plot from '@observablehq/plot' +import { Table, TypeMap } from 'apache-arrow' +import { ReactChild, useEffect, useRef, useState } from 'react' +import { useMeasure } from 'react-use' +import { useProjectDbQuery } from '../data/projectDatabase' +import { TableFromQuery } from './TableFromQuery' + +type PlotOptions = { + width: number +} + +type PlotInputFunc = ( + data: Table, + options: PlotOptions +) => (HTMLElement | SVGSVGElement) & Plot.Plot + +type Props = { + project: string + query: string + title?: ReactChild + subtitle?: ReactChild + description?: ReactChild + plot: PlotInputFunc +} + +const modalStyle = { + position: 'absolute', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + maxHeight: '100vh', + overflow: 'hidden', + width: 'calc(100% - 50px)', + bgcolor: 'background.paper', + boxShadow: 24, + p: 4, +} + +export function PlotFromQueryCard(props: Props) { + const [showingTable, setShowingTable] = useState(false) + const [expanded, setExpanded] = useState(false) + + return ( + + + + setShowingTable(true)}> + + + + + setExpanded(true)}> + + + + + + + + + setExpanded(false)}> + + + + + + setShowingTable(false)}> + +
+ +
+
+
+
+ ) +} + +export function PlotFromQuery(props: Props) { + const containerRef = useRef(null) + + const { project, query, plot } = props + const result = useProjectDbQuery(project, query) + + const [measureRef, { width }] = useMeasure() + + const data = result && result.status === 'success' ? result.data : undefined + + useEffect(() => { + if (!data) return + const _plot = plot(data, { width }) + containerRef.current?.append(_plot) + return () => _plot.remove() + }, [data, width, plot]) + + return ( + + {(props.title || props.subtitle) && ( + + {props.title && ( + + {props.title} + + )} + {props.subtitle && {props.subtitle}} + + )} + +
+ {!result || (result.status === 'loading' && )} +
+ + {result && result.status === 'error' && ( + {result.errorMessage} + )} + {props.description && ( + + {props.description} + + )} + + ) +} diff --git a/web/src/pages/report/chart/TableFromQuery.tsx b/web/src/pages/report/chart/TableFromQuery.tsx index e8d432ddf..2282b9aad 100644 --- a/web/src/pages/report/chart/TableFromQuery.tsx +++ b/web/src/pages/report/chart/TableFromQuery.tsx @@ -1,4 +1,15 @@ -import { Alert, Box } from '@mui/material' +import OpenInFullIcon from '@mui/icons-material/OpenInFull' +import { + Alert, + Box, + Card, + CardActions, + CardContent, + IconButton, + Modal, + Tooltip, + Typography, +} from '@mui/material' import CircularProgress from '@mui/material/CircularProgress' import { DataGrid, @@ -9,6 +20,7 @@ import { } from '@mui/x-data-grid' import { fromArrow } from 'arquero' import { ArrowTable } from 'arquero/dist/types/format/types' +import { ReactChild, useState } from 'react' import { Link } from 'react-router-dom' import { useProjectDbQuery } from '../data/projectDatabase' @@ -18,6 +30,84 @@ type TableProps = { showToolbar?: boolean } +type TableCardProps = TableProps & { + height: number | string + title?: ReactChild + subtitle?: ReactChild + description?: ReactChild +} + +const modalStyle = { + position: 'absolute', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + maxHeight: '100vh', + overflow: 'hidden', + width: 'calc(100% - 50px)', + bgcolor: 'background.paper', + boxShadow: 24, + p: 4, +} + +export function TableFromQueryCard(props: TableCardProps) { + const [expanded, setExpanded] = useState(false) + + return ( + + + + setExpanded(true)}> + + + + + + {(props.title || props.subtitle) && ( + + {props.title && ( + + {props.title} + + )} + {props.subtitle && {props.subtitle}} + + )} + + + + + + {props.description && ( + + {props.description} + + )} + + + setExpanded(false)}> + +
+ +
+
+
+
+ ) +} + // Provide custom rendering for some known columns, this allows adding links to the table const knownColumnMap: Record> = { participant_id: { From feed57bc3318ba814f424fb85d034730f8546084 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Wed, 8 Jan 2025 16:47:59 +1100 Subject: [PATCH 02/12] add link to view/edit chart sql --- .../pages/report/chart/MetricFromQuery.tsx | 29 ++++++++++++++++++- web/src/pages/report/chart/PlotFromQuery.tsx | 9 ++++++ web/src/pages/report/chart/TableFromQuery.tsx | 9 ++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/web/src/pages/report/chart/MetricFromQuery.tsx b/web/src/pages/report/chart/MetricFromQuery.tsx index 4758cd6f8..5428ceeda 100644 --- a/web/src/pages/report/chart/MetricFromQuery.tsx +++ b/web/src/pages/report/chart/MetricFromQuery.tsx @@ -1,4 +1,15 @@ -import { Box, Card, CardContent, Divider, Typography } from '@mui/material' +import CodeIcon from '@mui/icons-material/Code' + +import { + Box, + Card, + CardActions, + CardContent, + Divider, + IconButton, + Tooltip, + Typography, +} from '@mui/material' import CircularProgress from '@mui/material/CircularProgress' import { fromArrow } from 'arquero' import { ArrowTable } from 'arquero/dist/types/format/types' @@ -27,6 +38,22 @@ export function MetricFromQueryCard(props: Props) { return ( + + + + + + + {(props.title || props.subtitle) && ( diff --git a/web/src/pages/report/chart/PlotFromQuery.tsx b/web/src/pages/report/chart/PlotFromQuery.tsx index 5efc615a4..caeab2e98 100644 --- a/web/src/pages/report/chart/PlotFromQuery.tsx +++ b/web/src/pages/report/chart/PlotFromQuery.tsx @@ -1,3 +1,4 @@ +import CodeIcon from '@mui/icons-material/Code' import OpenInFullIcon from '@mui/icons-material/OpenInFull' import TableChartIcon from '@mui/icons-material/TableChart' import { @@ -63,6 +64,14 @@ export function PlotFromQueryCard(props: Props) { right: 0, }} > + + + + + setShowingTable(true)}> diff --git a/web/src/pages/report/chart/TableFromQuery.tsx b/web/src/pages/report/chart/TableFromQuery.tsx index 2282b9aad..a43955cc9 100644 --- a/web/src/pages/report/chart/TableFromQuery.tsx +++ b/web/src/pages/report/chart/TableFromQuery.tsx @@ -1,3 +1,4 @@ +import CodeIcon from '@mui/icons-material/Code' import OpenInFullIcon from '@mui/icons-material/OpenInFull' import { Alert, @@ -62,6 +63,14 @@ export function TableFromQueryCard(props: TableCardProps) { right: 0, }} > + + + + + setExpanded(true)}> From 713c29f8a783572d38e08f29107ad64d9a98a0b4 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Thu, 9 Jan 2025 14:46:39 +1100 Subject: [PATCH 03/12] memoise table rendering --- web/src/pages/report/OurDnaDashboard.tsx | 4 ++-- web/src/pages/report/SqlQueryUI.tsx | 10 ++-------- web/src/pages/report/chart/TableFromQuery.tsx | 18 +++++++++++++++--- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/web/src/pages/report/OurDnaDashboard.tsx b/web/src/pages/report/OurDnaDashboard.tsx index e77f4434f..58f17cec3 100644 --- a/web/src/pages/report/OurDnaDashboard.tsx +++ b/web/src/pages/report/OurDnaDashboard.tsx @@ -245,8 +245,8 @@ export default function OurDnaDashboard() { select s.participant_id, s.sample_id as sample_id, - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection, - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end, + "meta_collection-time" as collection, + "meta_process-end-time" as process_end, date_diff('hour', try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S')) as "duration (hrs)" from sample s join participant p diff --git a/web/src/pages/report/SqlQueryUI.tsx b/web/src/pages/report/SqlQueryUI.tsx index b41b016f4..c24a7f57a 100644 --- a/web/src/pages/report/SqlQueryUI.tsx +++ b/web/src/pages/report/SqlQueryUI.tsx @@ -24,7 +24,7 @@ import { } from '@mui/material' import { debounce } from 'lodash' import { editor, KeyCode, KeyMod } from 'monaco-editor' -import { Fragment, memo, useCallback, useContext, useEffect, useRef, useState } from 'react' +import { Fragment, useCallback, useContext, useEffect, useRef, useState } from 'react' import { Link, useNavigate, useParams } from 'react-router-dom' import { ThemeContext } from '../../shared/components/ThemeProvider' import { TableFromQuery } from './chart/TableFromQuery' @@ -32,12 +32,6 @@ import { useProjectDbSetup } from './data/projectDatabase' const SIDEBAR_WIDTH = 250 -// Memoized version of the result table to avoid rerenders, this is very important for performance -// without this each keystroke takes forever to render -const QueryResultTable = memo(function QueryResultTable(props: { project: string; query: string }) { - return -}) - // This fn is debounced to try and limit the amount of history entries. const persistQueryToUrl = debounce(function persistQueryToUrl(query: string) { const url = new URL(window.location.href) @@ -429,7 +423,7 @@ export default function SqlQueryUi() { {projectName && (selectedTableQuery || tableQueryValue) && ( - diff --git a/web/src/pages/report/chart/TableFromQuery.tsx b/web/src/pages/report/chart/TableFromQuery.tsx index a43955cc9..90919d5cf 100644 --- a/web/src/pages/report/chart/TableFromQuery.tsx +++ b/web/src/pages/report/chart/TableFromQuery.tsx @@ -21,7 +21,7 @@ import { } from '@mui/x-data-grid' import { fromArrow } from 'arquero' import { ArrowTable } from 'arquero/dist/types/format/types' -import { ReactChild, useState } from 'react' +import { memo, ReactChild, useState } from 'react' import { Link } from 'react-router-dom' import { useProjectDbQuery } from '../data/projectDatabase' @@ -137,7 +137,19 @@ function CustomTableToolbar() { ) } -export function TableFromQuery(props: TableProps) { +// Memoized version of the result table to avoid rerenders, they can be very expensive +// if the result set is large +export const TableFromQuery = memo(function TableFromQuery(props: TableProps) { + return ( + + ) +}) + +function TableFromQueryUnmemoized(props: TableProps) { const { project, query, showToolbar } = props const result = useProjectDbQuery(project, query) @@ -149,7 +161,7 @@ export function TableFromQuery(props: TableProps) { if (!data || !result || result.status === 'loading') return - const table = fromArrow(data as ArrowTable) + const table = fromArrow(data as ArrowTable, { useDate: true }) const columns = table.columnNames().map( (colName): GridColDef => ({ From 3db74e819a8fa7dc1af894c7133c78add60e8606 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Fri, 10 Jan 2025 15:00:20 +1100 Subject: [PATCH 04/12] allow for constructing a query from multiple query parts --- web/package-lock.json | 16 ++- web/package.json | 2 + web/src/pages/report/OurDnaDashboard.tsx | 130 +++++++++++-------- web/src/pages/report/chart/PlotFromQuery.tsx | 23 +++- web/src/pages/report/data/combineQueries.ts | 49 +++++++ 5 files changed, 158 insertions(+), 62 deletions(-) create mode 100644 web/src/pages/report/data/combineQueries.ts diff --git a/web/package-lock.json b/web/package-lock.json index 24fcfe4b6..edd2ad7a2 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "metamist", - "version": "7.4.3", + "version": "7.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "metamist", - "version": "7.4.3", + "version": "7.6.0", "dependencies": { "@apollo/client": "^3.11.5", "@duckdb/duckdb-wasm": "^1.29.0", @@ -19,6 +19,7 @@ "@observablehq/plot": "^0.6.16", "arquero": "^7.2.0", "axios": "^0.24.0", + "common-tags": "^1.8.2", "d3": "^7.9.0", "d3-scale-chromatic": "^3.1.0", "graphql": "^16.6.0", @@ -48,6 +49,7 @@ "@testing-library/jest-dom": "^5.15.0", "@testing-library/react": "^11.2.7", "@testing-library/user-event": "^12.8.3", + "@types/common-tags": "^1.8.4", "@types/d3": "^7.4.3", "@types/jest": "^26.0.24", "@types/lodash": "^4.14.178", @@ -3839,6 +3841,12 @@ "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==" }, + "node_modules/@types/common-tags": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/@types/common-tags/-/common-tags-1.8.4.tgz", + "integrity": "sha512-S+1hLDJPjWNDhcGxsxEbepzaxWqURP/o+3cP4aa2w7yBXgdcmKGQtZzP8JbyfOd0m+33nh+8+kvxYE2UJtBDkg==", + "dev": true + }, "node_modules/@types/d3": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", @@ -5626,8 +5634,8 @@ }, "node_modules/common-tags": { "version": "1.8.2", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", "engines": { "node": ">=4.0.0" } diff --git a/web/package.json b/web/package.json index b9d7c5edd..d0cb19f14 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "@observablehq/plot": "^0.6.16", "arquero": "^7.2.0", "axios": "^0.24.0", + "common-tags": "^1.8.2", "d3": "^7.9.0", "d3-scale-chromatic": "^3.1.0", "graphql": "^16.6.0", @@ -71,6 +72,7 @@ "@testing-library/jest-dom": "^5.15.0", "@testing-library/react": "^11.2.7", "@testing-library/user-event": "^12.8.3", + "@types/common-tags": "^1.8.4", "@types/d3": "^7.4.3", "@types/jest": "^26.0.24", "@types/lodash": "^4.14.178", diff --git a/web/src/pages/report/OurDnaDashboard.tsx b/web/src/pages/report/OurDnaDashboard.tsx index 58f17cec3..880a04cd5 100644 --- a/web/src/pages/report/OurDnaDashboard.tsx +++ b/web/src/pages/report/OurDnaDashboard.tsx @@ -283,59 +283,83 @@ export default function OurDnaDashboard() { Plot.plot({ marginLeft: 40, diff --git a/web/src/pages/report/chart/PlotFromQuery.tsx b/web/src/pages/report/chart/PlotFromQuery.tsx index caeab2e98..943c8cf48 100644 --- a/web/src/pages/report/chart/PlotFromQuery.tsx +++ b/web/src/pages/report/chart/PlotFromQuery.tsx @@ -15,8 +15,10 @@ import { } from '@mui/material' import * as Plot from '@observablehq/plot' import { Table, TypeMap } from 'apache-arrow' +import { stripIndent } from 'common-tags' import { ReactChild, useEffect, useRef, useState } from 'react' import { useMeasure } from 'react-use' +import combineQueries from '../data/combineQueries' import { useProjectDbQuery } from '../data/projectDatabase' import { TableFromQuery } from './TableFromQuery' @@ -29,14 +31,21 @@ type PlotInputFunc = ( options: PlotOptions ) => (HTMLElement | SVGSVGElement) & Plot.Plot +type QueryProps = + | { + query: string + } + | { + queries: { name: string; query: string }[] + } + type Props = { project: string - query: string title?: ReactChild subtitle?: ReactChild description?: ReactChild plot: PlotInputFunc -} +} & QueryProps const modalStyle = { position: 'absolute', @@ -55,6 +64,8 @@ export function PlotFromQueryCard(props: Props) { const [showingTable, setShowingTable] = useState(false) const [expanded, setExpanded] = useState(false) + let query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) + return ( @@ -102,7 +113,7 @@ export function PlotFromQueryCard(props: Props) { flexDirection: 'column', }} > - +
@@ -113,7 +124,9 @@ export function PlotFromQueryCard(props: Props) { export function PlotFromQuery(props: Props) { const containerRef = useRef(null) - const { project, query, plot } = props + const { project, plot } = props + + const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) const result = useProjectDbQuery(project, query) const [measureRef, { width }] = useMeasure() diff --git a/web/src/pages/report/data/combineQueries.ts b/web/src/pages/report/data/combineQueries.ts new file mode 100644 index 000000000..ab4b7d4a4 --- /dev/null +++ b/web/src/pages/report/data/combineQueries.ts @@ -0,0 +1,49 @@ +import { stripIndent } from 'common-tags' + +type Query = { + name: string + query: string +} + +// Take a list of queries and combine them into a single query using CTEs +// This allows viewing of steps that are used to arrive at a final aggregated query +export default function combineQueries(queries: Query[]) { + const cleanQueries = queries.map((qq) => { + return { + ...qq, + // remove any trailing semicolons from the query, in case they were added accidentally + query: stripIndent(qq.query.replace(/;[\w\s]*$/g, '')), + } + }) + + const ctes = cleanQueries.slice(0, cleanQueries.length - 1).map((qq) => { + // To format the query nicely, need to indent each line by the same amount, but no way + // of knowing what that amount is, as it depends on the code formatting :/ so use a sentinel + // to represent the query, then check how indented it is and replace it with the real query + // while adding the correct indentation to each line (other than the first). + const querySentinel = `___QUERY_REPLACEMENT___` + + const queryPlaceholder = stripIndent` + ${qq.name} AS ( + ${querySentinel} + ) + ` + + const matcher = new RegExp(`^(\\s*)${querySentinel}`, 'm') + + const queryIndent = queryPlaceholder.match(matcher)?.[1] || '' + + const indentedQuery = qq.query + .split('\n') + .map((line, index) => (index === 0 ? line : queryIndent + line)) + .join('\n') + + return queryPlaceholder.replace(querySentinel, indentedQuery) + }) + + const finalQuery = cleanQueries[cleanQueries.length - 1] + + return stripIndent`WITH ${ctes.join(',\n')} +${finalQuery.query} + ` +} From e5a579c6ada38623344e9f70d31461b8159479e5 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Tue, 14 Jan 2025 12:13:50 +1100 Subject: [PATCH 05/12] query updates --- web/src/pages/report/OurDnaDashboard.tsx | 277 ++++++++++++++---- web/src/pages/report/chart/TableFromQuery.tsx | 30 +- 2 files changed, 231 insertions(+), 76 deletions(-) diff --git a/web/src/pages/report/OurDnaDashboard.tsx b/web/src/pages/report/OurDnaDashboard.tsx index 880a04cd5..3693d5705 100644 --- a/web/src/pages/report/OurDnaDashboard.tsx +++ b/web/src/pages/report/OurDnaDashboard.tsx @@ -3,7 +3,6 @@ import * as Plot from '@observablehq/plot' import { MetricFromQueryCard } from './chart/MetricFromQuery' import { PlotFromQueryCard } from './chart/PlotFromQuery' import { TableFromQueryCard } from './chart/TableFromQuery' -import { useProjectDbQuery } from './data/projectDatabase' /* @@ -22,20 +21,32 @@ other charts: - data quality */ -export default function OurDnaDashboard() { - const result = useProjectDbQuery( - 'ourdna', - ` - select * from participant limit 10 - +// Common queries: +// queries that are used in multiple charts - ` - ) - if (result?.status === 'success') { - console.log(result.data.toArray().map((r) => r.toJSON())) - } +// Query for processing times, this can be used for most visualistions +// of processing time +const PROCESS_DURATION_QUERY = ` + select + s.sample_id, + cast(s.participant_id as string) as participant_id, + coalesce(s."meta_processing-site", 'Unknown') as processing_site, + + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, + date_diff( + 'hour', + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') + ) as duration + from sample s + join participant p + on p.participant_id = s.participant_id + where s.type = 'blood' +` +export default function OurDnaDashboard() { return ( @@ -144,18 +155,18 @@ export default function OurDnaDashboard() { title="Processing times for all samples" description="This excludes any samples with > 100 hour processing times, and any that have missing or malformed collection or processing times" project="ourdna" - query={` - with durations as ( - select - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection, - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end, - date_diff('hour', try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S')) as duration, - from sample s - join participant p - on p.participant_id = s.participant_id - where s.type = 'blood' - ) select * from durations where duration < 100 and duration > 0 - `} + queries={[ + { + name: 'durations', + query: PROCESS_DURATION_QUERY, + }, + { + name: 'result', + query: ` + select * from durations where duration < 100 and duration > 0 + `, + }, + ]} plot={(data, { width }) => Plot.plot({ y: { grid: true, padding: 5, label: 'duration(hrs)' }, @@ -170,12 +181,16 @@ export default function OurDnaDashboard() { marks: [ Plot.ruleY([24, 48, 72], { stroke: '#ff725c' }), Plot.dot(data, { - x: 'collection', + x: 'collection_time', y: 'duration', stroke: null, fill: '#4269d0', fillOpacity: 0.5, - channels: { process_end: 'process_end' }, + channels: { + process_end: 'process_end_time', + sample_id: 'sample_id', + participant_id: 'participant_id', + }, tip: true, }), ], @@ -188,35 +203,29 @@ export default function OurDnaDashboard() { title="Processing time buckets" description="Count of participants with their samples processed in each bucket" project="ourdna" - query={` - with durations as ( + queries={[ + { name: 'durations', query: PROCESS_DURATION_QUERY }, + { + name: 'result', + query: ` select - s.participant_id, - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection, - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end, - date_diff('hour', try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S')) as duration, - from sample s - join participant p - on p.participant_id = s.participant_id - where s.type = 'blood' - ) select - count(distinct participant_id) as count, - CASE - WHEN duration < 24 THEN '0-24 hours' - WHEN duration >= 24 AND duration < 30 THEN '24-30 hours' - WHEN duration >= 30 AND duration < 33 THEN '30-33 hours' - WHEN duration >= 33 AND duration < 48 THEN '33-48 hours' - WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' - ELSE '72+ hours' - END AS duration - - from durations group by 2 order by 2 - `} + count(distinct participant_id) as count, + CASE + WHEN duration < 24 THEN '0-24 hours' + WHEN duration >= 24 AND duration < 36 THEN '24-36 hours' + WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' + WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' + ELSE '72+ hours' + END AS duration + from durations group by 2 order by 2 + `, + }, + ]} plot={(data, { width }) => Plot.plot({ marginLeft: 100, inset: 10, - height: 410, + height: 400, color: { scheme: 'RdYlGn', reverse: true, @@ -235,25 +244,79 @@ export default function OurDnaDashboard() { /> - - + + = 24 AND duration < 36 THEN '24-36 hours' + WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' + WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' + ELSE '72+ hours' + END AS duration + from durations group by 2,3 order by 2,3 + `, + }, + ]} + plot={(data, { width }) => + Plot.plot({ + marginLeft: 100, + marginRight: 100, + inset: 10, + height: 450, + color: { + scheme: 'RdYlGn', + reverse: true, + }, + width, + marks: [ + Plot.barX(data, { + y: 'processing_site', + fy: 'duration', + x: 'count', + tip: true, + fill: 'duration', + }), + ], + }) + } + /> + + + @@ -388,6 +451,90 @@ export default function OurDnaDashboard() { /> + + + + Plot.plot({ + width, + height: 450, + color: { legend: true }, + marks: [ + Plot.rectY( + data, + Plot.binX( + { y: 'count' }, + { x: 'age', fill: 'reported_sex', tip: true } + ) + ), + ], + }) + } + /> + + + + Plot.plot({ + marginLeft: 160, + inset: 10, + height: 450, + color: { + scheme: 'Observable10', + reverse: true, + }, + width, + marks: [ + Plot.barX(data, { + y: 'ancestry', + fill: 'ancestry', + x: 'count', + tip: true, + sort: { + y: 'x', + reverse: true, + }, + }), + ], + }) + } + /> + + + + + + Choices + + ) } diff --git a/web/src/pages/report/chart/TableFromQuery.tsx b/web/src/pages/report/chart/TableFromQuery.tsx index 90919d5cf..6b92a2c25 100644 --- a/web/src/pages/report/chart/TableFromQuery.tsx +++ b/web/src/pages/report/chart/TableFromQuery.tsx @@ -19,17 +19,27 @@ import { GridToolbarExport, GridToolbarQuickFilter, } from '@mui/x-data-grid' +import combineQueries from '../data/combineQueries' + import { fromArrow } from 'arquero' import { ArrowTable } from 'arquero/dist/types/format/types' +import { stripIndent } from 'common-tags' import { memo, ReactChild, useState } from 'react' import { Link } from 'react-router-dom' import { useProjectDbQuery } from '../data/projectDatabase' +type QueryProps = + | { + query: string + } + | { + queries: { name: string; query: string }[] + } + type TableProps = { project: string - query: string showToolbar?: boolean -} +} & QueryProps type TableCardProps = TableProps & { height: number | string @@ -54,6 +64,8 @@ const modalStyle = { export function TableFromQueryCard(props: TableCardProps) { const [expanded, setExpanded] = useState(false) + const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) + return ( @@ -140,17 +152,13 @@ function CustomTableToolbar() { // Memoized version of the result table to avoid rerenders, they can be very expensive // if the result set is large export const TableFromQuery = memo(function TableFromQuery(props: TableProps) { - return ( - - ) + return }) function TableFromQueryUnmemoized(props: TableProps) { - const { project, query, showToolbar } = props + const { project, showToolbar } = props + const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) + const result = useProjectDbQuery(project, query) const data = result && result.status === 'success' ? result.data : undefined From 9ffb70955ad840c93cd97dab49edaacc7792750c Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Tue, 28 Jan 2025 18:39:17 +1100 Subject: [PATCH 06/12] Add report layout and initial version of ourdna dashboard --- web/src/Routes.tsx | 11 +- web/src/index.css | 12 + web/src/pages/report/OurDnaDashboard.tsx | 540 ------------------ web/src/pages/report/ProjectReport.tsx | 46 ++ web/src/pages/report/SqlQueryUI.tsx | 2 +- .../pages/report/chart/MetricFromQuery.tsx | 93 --- web/src/pages/report/chart/PlotFromQuery.tsx | 170 ------ web/src/pages/report/chart/TableFromQuery.tsx | 200 ------- .../report/components/MetricFromQuery.tsx | 54 ++ .../pages/report/components/PlotFromQuery.tsx | 62 ++ web/src/pages/report/components/Report.tsx | 9 + .../pages/report/components/ReportItem.tsx | 180 ++++++ .../report/components/ReportItemActions.tsx | 80 +++ .../report/components/ReportItemLoader.tsx | 16 + web/src/pages/report/components/ReportRow.tsx | 9 + .../report/components/TableFromQuery.tsx | 92 +++ .../{combineQueries.ts => formatQuery.ts} | 9 +- web/src/pages/report/reportIndex.ts | 39 ++ .../report/reports/ourdna/Demographics.tsx | 227 ++++++++ .../report/reports/ourdna/ProcessingTimes.tsx | 194 +++++++ .../report/reports/ourdna/Recruitment.tsx | 96 ++++ 21 files changed, 1133 insertions(+), 1008 deletions(-) delete mode 100644 web/src/pages/report/OurDnaDashboard.tsx create mode 100644 web/src/pages/report/ProjectReport.tsx delete mode 100644 web/src/pages/report/chart/MetricFromQuery.tsx delete mode 100644 web/src/pages/report/chart/PlotFromQuery.tsx delete mode 100644 web/src/pages/report/chart/TableFromQuery.tsx create mode 100644 web/src/pages/report/components/MetricFromQuery.tsx create mode 100644 web/src/pages/report/components/PlotFromQuery.tsx create mode 100644 web/src/pages/report/components/Report.tsx create mode 100644 web/src/pages/report/components/ReportItem.tsx create mode 100644 web/src/pages/report/components/ReportItemActions.tsx create mode 100644 web/src/pages/report/components/ReportItemLoader.tsx create mode 100644 web/src/pages/report/components/ReportRow.tsx create mode 100644 web/src/pages/report/components/TableFromQuery.tsx rename web/src/pages/report/data/{combineQueries.ts => formatQuery.ts} (86%) create mode 100644 web/src/pages/report/reportIndex.ts create mode 100644 web/src/pages/report/reports/ourdna/Demographics.tsx create mode 100644 web/src/pages/report/reports/ourdna/ProcessingTimes.tsx create mode 100644 web/src/pages/report/reports/ourdna/Recruitment.tsx diff --git a/web/src/Routes.tsx b/web/src/Routes.tsx index 0c52d477f..4c3bcb71a 100644 --- a/web/src/Routes.tsx +++ b/web/src/Routes.tsx @@ -21,7 +21,8 @@ import Summary from './pages/insights/Summary' import { ParticipantPage } from './pages/participant/ParticipantViewContainer' import AnalysisRunnerSummary from './pages/project/AnalysisRunnerView/AnalysisRunnerSummary' import ProjectOverview from './pages/project/ProjectOverview' -import OurDnaDashboard from './pages/report/OurDnaDashboard' +import ProjectReport from './pages/report/ProjectReport' +import OurDnaDashboard from './pages/report/reports/ourdna/OurDnaDashboard' import SqlQueryUI from './pages/report/SqlQueryUI' import SampleView from './pages/sample/SampleView' import ErrorBoundary from './shared/utilities/errorBoundary' @@ -49,6 +50,14 @@ const Routes: React.FunctionComponent = () => ( } /> + + + + } + /> .ui.modals.dimmer.visible { display: flex !important; } + + +/* + hax to fix bad cascading styles. + There are a few conflicting css frameworks on metamist for strange historical reasons. + While we're working to move to mui and remove other conflicting styles at the moment it + is sometimes necessary to remove styles that cascade from bad global definitions. +*/ + +.MuiTablePagination-selectLabel, .MuiTablePagination-displayedRows { + margin-bottom: 0 !important; +} \ No newline at end of file diff --git a/web/src/pages/report/OurDnaDashboard.tsx b/web/src/pages/report/OurDnaDashboard.tsx deleted file mode 100644 index 3693d5705..000000000 --- a/web/src/pages/report/OurDnaDashboard.tsx +++ /dev/null @@ -1,540 +0,0 @@ -import { Box, Typography } from '@mui/material' -import * as Plot from '@observablehq/plot' -import { MetricFromQueryCard } from './chart/MetricFromQuery' -import { PlotFromQueryCard } from './chart/PlotFromQuery' -import { TableFromQueryCard } from './chart/TableFromQuery' - -/* - -other charts: - -- chart of samples where we don't have consent / registration information for a participant. - - could be a few reasons, but want to capture badly entered participant ids - - - sections: - - recruitment - - processing times - - - demographics - - - data quality - -*/ - -// Common queries: -// queries that are used in multiple charts - -// Query for processing times, this can be used for most visualistions -// of processing time -const PROCESS_DURATION_QUERY = ` - select - s.sample_id, - cast(s.participant_id as string) as participant_id, - coalesce(s."meta_processing-site", 'Unknown') as processing_site, - - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, - date_diff( - 'hour', - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') - ) as duration - from sample s - join participant p - on p.participant_id = s.participant_id - where s.type = 'blood' -` - -export default function OurDnaDashboard() { - return ( - - - - Recruitment - - - - - - Plot.plot({ - marginLeft: 160, - inset: 10, - height: 170, - width, - marks: [ - Plot.barX(data, { - y: 'stage', - x: 'count', - tip: true, - fill: 'stage', - }), - ], - }) - } - /> - - - - Plot.plot({ - width, - height: 150, - color: { legend: true }, - marks: [ - Plot.barX( - data, - Plot.stackX({ - x: 'count', - fill: 'collection_type', - inset: 0.5, - tip: true, - }) - ), - ], - }) - } - /> - - - - - - Processing times - - - - - 0 - `, - }, - ]} - plot={(data, { width }) => - Plot.plot({ - y: { grid: true, padding: 5, label: 'duration(hrs)' }, - x: { grid: true, padding: 5 }, - width: width, - height: 400, - marginTop: 20, - marginRight: 20, - marginBottom: 30, - - marginLeft: 40, - marks: [ - Plot.ruleY([24, 48, 72], { stroke: '#ff725c' }), - Plot.dot(data, { - x: 'collection_time', - y: 'duration', - stroke: null, - fill: '#4269d0', - fillOpacity: 0.5, - channels: { - process_end: 'process_end_time', - sample_id: 'sample_id', - participant_id: 'participant_id', - }, - tip: true, - }), - ], - }) - } - /> - - - = 24 AND duration < 36 THEN '24-36 hours' - WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' - WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' - ELSE '72+ hours' - END AS duration - from durations group by 2 order by 2 - `, - }, - ]} - plot={(data, { width }) => - Plot.plot({ - marginLeft: 100, - inset: 10, - height: 400, - color: { - scheme: 'RdYlGn', - reverse: true, - }, - width, - marks: [ - Plot.barX(data, { - y: 'duration', - x: 'count', - tip: true, - fill: 'duration', - }), - ], - }) - } - /> - - - - - = 24 AND duration < 36 THEN '24-36 hours' - WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' - WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' - ELSE '72+ hours' - END AS duration - from durations group by 2,3 order by 2,3 - `, - }, - ]} - plot={(data, { width }) => - Plot.plot({ - marginLeft: 100, - marginRight: 100, - inset: 10, - height: 450, - color: { - scheme: 'RdYlGn', - reverse: true, - }, - width, - marks: [ - Plot.barX(data, { - y: 'processing_site', - fy: 'duration', - x: 'count', - tip: true, - fill: 'duration', - }), - ], - }) - } - /> - - - - - - - - - - Demographics - - - - - - - - - - Plot.plot({ - marginLeft: 40, - inset: 10, - width: width, - height: 400, - y: { - grid: true, - label: 'Participants', - }, - x: { - type: 'time', - }, - marks: [ - Plot.areaY(data, { - x: 'week', - y: 'count', - fill: 'ancestry', - tip: true, - order: 'sum', - }), - Plot.ruleY([0]), - ], - }) - } - /> - - - - - - Plot.plot({ - width, - height: 450, - color: { legend: true }, - marks: [ - Plot.rectY( - data, - Plot.binX( - { y: 'count' }, - { x: 'age', fill: 'reported_sex', tip: true } - ) - ), - ], - }) - } - /> - - - - Plot.plot({ - marginLeft: 160, - inset: 10, - height: 450, - color: { - scheme: 'Observable10', - reverse: true, - }, - width, - marks: [ - Plot.barX(data, { - y: 'ancestry', - fill: 'ancestry', - x: 'count', - tip: true, - sort: { - y: 'x', - reverse: true, - }, - }), - ], - }) - } - /> - - - - - - Choices - - - - ) -} diff --git a/web/src/pages/report/ProjectReport.tsx b/web/src/pages/report/ProjectReport.tsx new file mode 100644 index 000000000..bf6efc0ea --- /dev/null +++ b/web/src/pages/report/ProjectReport.tsx @@ -0,0 +1,46 @@ +import { Box, Tab, Tabs, Typography } from '@mui/material' +import { Link, useParams } from 'react-router-dom' +import MuckError from '../../shared/components/MuckError' +import { reports } from './reportIndex' + +const NotFound = () => + +export default function ProjectReport() { + const { projectName, reportName, tabName } = useParams() + + if (!projectName || !reportName || !reports[projectName] || !reports[projectName][reportName]) { + return + } + + const reportTabs = reports[projectName][reportName].tabs + const urlActiveTabIndex = reportTabs.findIndex((tab) => tab.id === tabName) + const activeTabIndex = urlActiveTabIndex > -1 ? urlActiveTabIndex : 0 + const activeTab = reportTabs[activeTabIndex] + + const Report = activeTab.content + + return ( + + + + {reportTabs.map((tab) => ( + + ))} + + + + + {activeTab.title} + + + + + + + ) +} diff --git a/web/src/pages/report/SqlQueryUI.tsx b/web/src/pages/report/SqlQueryUI.tsx index c24a7f57a..32291966f 100644 --- a/web/src/pages/report/SqlQueryUI.tsx +++ b/web/src/pages/report/SqlQueryUI.tsx @@ -27,7 +27,7 @@ import { editor, KeyCode, KeyMod } from 'monaco-editor' import { Fragment, useCallback, useContext, useEffect, useRef, useState } from 'react' import { Link, useNavigate, useParams } from 'react-router-dom' import { ThemeContext } from '../../shared/components/ThemeProvider' -import { TableFromQuery } from './chart/TableFromQuery' +import { TableFromQuery } from './components/TableFromQuery' import { useProjectDbSetup } from './data/projectDatabase' const SIDEBAR_WIDTH = 250 diff --git a/web/src/pages/report/chart/MetricFromQuery.tsx b/web/src/pages/report/chart/MetricFromQuery.tsx deleted file mode 100644 index 5428ceeda..000000000 --- a/web/src/pages/report/chart/MetricFromQuery.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import CodeIcon from '@mui/icons-material/Code' - -import { - Box, - Card, - CardActions, - CardContent, - Divider, - IconButton, - Tooltip, - Typography, -} from '@mui/material' -import CircularProgress from '@mui/material/CircularProgress' -import { fromArrow } from 'arquero' -import { ArrowTable } from 'arquero/dist/types/format/types' -import { Fragment, ReactChild } from 'react' -import { useProjectDbQuery } from '../data/projectDatabase' - -type Props = { - project: string - query: string - title?: ReactChild - subtitle?: ReactChild - description?: ReactChild -} - -export function MetricFromQueryCard(props: Props) { - const { project, query } = props - const result = useProjectDbQuery(project, query) - - const data = result && result.status === 'success' ? result.data : undefined - - if (!data || !result || result.status === 'loading') return - - const table = fromArrow(data as ArrowTable) - - const columns = table.columnNames() - - return ( - - - - - - - - - - {(props.title || props.subtitle) && ( - - {props.title && ( - - {props.title} - - )} - {props.subtitle && {props.subtitle}} - - )} - - - {columns.map((col, index) => ( - - {index !== 0 && } - - - {col} - - - {table.get(col, 0)} - - - - ))} - - - {props.description && ( - - {props.description} - - )} - - - ) -} diff --git a/web/src/pages/report/chart/PlotFromQuery.tsx b/web/src/pages/report/chart/PlotFromQuery.tsx deleted file mode 100644 index 943c8cf48..000000000 --- a/web/src/pages/report/chart/PlotFromQuery.tsx +++ /dev/null @@ -1,170 +0,0 @@ -import CodeIcon from '@mui/icons-material/Code' -import OpenInFullIcon from '@mui/icons-material/OpenInFull' -import TableChartIcon from '@mui/icons-material/TableChart' -import { - Alert, - Box, - Card, - CardActions, - CardContent, - CircularProgress, - IconButton, - Modal, - Tooltip, - Typography, -} from '@mui/material' -import * as Plot from '@observablehq/plot' -import { Table, TypeMap } from 'apache-arrow' -import { stripIndent } from 'common-tags' -import { ReactChild, useEffect, useRef, useState } from 'react' -import { useMeasure } from 'react-use' -import combineQueries from '../data/combineQueries' -import { useProjectDbQuery } from '../data/projectDatabase' -import { TableFromQuery } from './TableFromQuery' - -type PlotOptions = { - width: number -} - -type PlotInputFunc = ( - data: Table, - options: PlotOptions -) => (HTMLElement | SVGSVGElement) & Plot.Plot - -type QueryProps = - | { - query: string - } - | { - queries: { name: string; query: string }[] - } - -type Props = { - project: string - title?: ReactChild - subtitle?: ReactChild - description?: ReactChild - plot: PlotInputFunc -} & QueryProps - -const modalStyle = { - position: 'absolute', - top: '50%', - left: '50%', - transform: 'translate(-50%, -50%)', - maxHeight: '100vh', - overflow: 'hidden', - width: 'calc(100% - 50px)', - bgcolor: 'background.paper', - boxShadow: 24, - p: 4, -} - -export function PlotFromQueryCard(props: Props) { - const [showingTable, setShowingTable] = useState(false) - const [expanded, setExpanded] = useState(false) - - let query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) - - return ( - - - - - - - - - setShowingTable(true)}> - - - - - setExpanded(true)}> - - - - - - - - - setExpanded(false)}> - - - - - - setShowingTable(false)}> - -
- -
-
-
-
- ) -} - -export function PlotFromQuery(props: Props) { - const containerRef = useRef(null) - - const { project, plot } = props - - const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) - const result = useProjectDbQuery(project, query) - - const [measureRef, { width }] = useMeasure() - - const data = result && result.status === 'success' ? result.data : undefined - - useEffect(() => { - if (!data) return - const _plot = plot(data, { width }) - containerRef.current?.append(_plot) - return () => _plot.remove() - }, [data, width, plot]) - - return ( - - {(props.title || props.subtitle) && ( - - {props.title && ( - - {props.title} - - )} - {props.subtitle && {props.subtitle}} - - )} - -
- {!result || (result.status === 'loading' && )} -
- - {result && result.status === 'error' && ( - {result.errorMessage} - )} - {props.description && ( - - {props.description} - - )} - - ) -} diff --git a/web/src/pages/report/chart/TableFromQuery.tsx b/web/src/pages/report/chart/TableFromQuery.tsx deleted file mode 100644 index 6b92a2c25..000000000 --- a/web/src/pages/report/chart/TableFromQuery.tsx +++ /dev/null @@ -1,200 +0,0 @@ -import CodeIcon from '@mui/icons-material/Code' -import OpenInFullIcon from '@mui/icons-material/OpenInFull' -import { - Alert, - Box, - Card, - CardActions, - CardContent, - IconButton, - Modal, - Tooltip, - Typography, -} from '@mui/material' -import CircularProgress from '@mui/material/CircularProgress' -import { - DataGrid, - GridColDef, - GridToolbarContainer, - GridToolbarExport, - GridToolbarQuickFilter, -} from '@mui/x-data-grid' -import combineQueries from '../data/combineQueries' - -import { fromArrow } from 'arquero' -import { ArrowTable } from 'arquero/dist/types/format/types' -import { stripIndent } from 'common-tags' -import { memo, ReactChild, useState } from 'react' -import { Link } from 'react-router-dom' -import { useProjectDbQuery } from '../data/projectDatabase' - -type QueryProps = - | { - query: string - } - | { - queries: { name: string; query: string }[] - } - -type TableProps = { - project: string - showToolbar?: boolean -} & QueryProps - -type TableCardProps = TableProps & { - height: number | string - title?: ReactChild - subtitle?: ReactChild - description?: ReactChild -} - -const modalStyle = { - position: 'absolute', - top: '50%', - left: '50%', - transform: 'translate(-50%, -50%)', - maxHeight: '100vh', - overflow: 'hidden', - width: 'calc(100% - 50px)', - bgcolor: 'background.paper', - boxShadow: 24, - p: 4, -} - -export function TableFromQueryCard(props: TableCardProps) { - const [expanded, setExpanded] = useState(false) - - const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) - - return ( - - - - - - - - - setExpanded(true)}> - - - - - - {(props.title || props.subtitle) && ( - - {props.title && ( - - {props.title} - - )} - {props.subtitle && {props.subtitle}} - - )} - - - - - - {props.description && ( - - {props.description} - - )} - - - setExpanded(false)}> - -
- -
-
-
-
- ) -} - -// Provide custom rendering for some known columns, this allows adding links to the table -const knownColumnMap: Record> = { - participant_id: { - renderCell: (params) => {params.value}, - }, - sample_id: { - renderCell: (params) => {params.value}, - }, -} - -function CustomTableToolbar() { - return ( - - - - - - ) -} - -// Memoized version of the result table to avoid rerenders, they can be very expensive -// if the result set is large -export const TableFromQuery = memo(function TableFromQuery(props: TableProps) { - return -}) - -function TableFromQueryUnmemoized(props: TableProps) { - const { project, showToolbar } = props - const query = 'query' in props ? stripIndent(props.query) : combineQueries(props.queries) - - const result = useProjectDbQuery(project, query) - - const data = result && result.status === 'success' ? result.data : undefined - - if (result && result.status === 'error') { - return {result.errorMessage} - } - - if (!data || !result || result.status === 'loading') return - - const table = fromArrow(data as ArrowTable, { useDate: true }) - - const columns = table.columnNames().map( - (colName): GridColDef => ({ - field: colName, - ...(colName in knownColumnMap ? knownColumnMap[colName] : {}), - }) - ) - - const rows = table.objects().map((row, index) => ({ ...row, __index: index })) - - return ( - row.__index} - density="compact" - sx={{ - fontFamily: 'monospace', - }} - /> - ) -} diff --git a/web/src/pages/report/components/MetricFromQuery.tsx b/web/src/pages/report/components/MetricFromQuery.tsx new file mode 100644 index 000000000..fd7906507 --- /dev/null +++ b/web/src/pages/report/components/MetricFromQuery.tsx @@ -0,0 +1,54 @@ +import { Alert, Box, Divider, Typography } from '@mui/material' +import { fromArrow } from 'arquero' +import { ArrowTable } from 'arquero/dist/types/format/types' +import { Fragment } from 'react' +import { formatQuery, UnformattedQuery } from '../data/formatQuery' +import { useProjectDbQuery } from '../data/projectDatabase' +import ReportItemLoader from './ReportItemLoader' + +export type MetricProps = { + project: string + query: UnformattedQuery +} + +export function MetricFromQuery(props: MetricProps) { + const { project } = props + const query = formatQuery(props.query) + const result = useProjectDbQuery(project, query) + + const data = result && result.status === 'success' ? result.data : undefined + + if (!result || result.status === 'loading') return + + if (result && result.status === 'error') { + return ( + + {result.errorMessage} + + ) + } + + const table = fromArrow(data as ArrowTable) + + const columns = table.columnNames() + + return ( + + + {columns.map((col, index) => ( + + {index !== 0 && } + + + {col} + + + {table.get(col, 0)} + + + + ))} + + + ) +} diff --git a/web/src/pages/report/components/PlotFromQuery.tsx b/web/src/pages/report/components/PlotFromQuery.tsx new file mode 100644 index 000000000..8cd4ef42f --- /dev/null +++ b/web/src/pages/report/components/PlotFromQuery.tsx @@ -0,0 +1,62 @@ +import { Alert, Box } from '@mui/material' +import * as Plot from '@observablehq/plot' +import { Table, TypeMap } from 'apache-arrow' +import { useEffect, useRef } from 'react' +import { useMeasure } from 'react-use' +import { formatQuery, UnformattedQuery } from '../data/formatQuery' +import { useProjectDbQuery } from '../data/projectDatabase' +import ReportItemLoader from './ReportItemLoader' + +type PlotInputFunc = (data: Table) => Omit + +export type PlotProps = { + project: string + query: UnformattedQuery + plot: PlotInputFunc +} + +export function PlotFromQuery(props: PlotProps) { + const containerRef = useRef(null) + + const { project, plot: getPlotOptions } = props + const query = formatQuery(props.query) + const result = useProjectDbQuery(project, query) + + const [measureRef, { width, height }] = useMeasure() + + const data = result && result.status === 'success' ? result.data : undefined + + useEffect(() => { + if (!data) return + const plotOptions = getPlotOptions(data) + console.log(width, height) + const plot = Plot.plot({ + ...plotOptions, + width, + height, + }) + containerRef.current?.append(plot) + return () => plot.remove() + }, [data, width, height, getPlotOptions]) + + return ( + +
+ {!result || result.status === 'loading' ? : null} +
+ {result && result.status === 'error' && ( + {result.errorMessage} + )} + + ) +} diff --git a/web/src/pages/report/components/Report.tsx b/web/src/pages/report/components/Report.tsx new file mode 100644 index 000000000..2a013f09b --- /dev/null +++ b/web/src/pages/report/components/Report.tsx @@ -0,0 +1,9 @@ +import { Box } from '@mui/material' + +export default function Report(props: { children: React.ReactNode }) { + return ( + + {props.children} + + ) +} diff --git a/web/src/pages/report/components/ReportItem.tsx b/web/src/pages/report/components/ReportItem.tsx new file mode 100644 index 000000000..53248cf80 --- /dev/null +++ b/web/src/pages/report/components/ReportItem.tsx @@ -0,0 +1,180 @@ +import { Box, Card, CardActions, CardContent, Typography } from '@mui/material' +import { formatQuery } from '../data/formatQuery' +import { MetricFromQuery, MetricProps } from './MetricFromQuery' +import { PlotFromQuery, PlotProps } from './PlotFromQuery' +import { + ActionViewEditSql, + ActionViewExpandedPlot, + ActionViewExpandedTable, +} from './ReportItemActions' +import { TableFromQuery, TableProps } from './TableFromQuery' + +type BaseReportItemProps = { + title?: string + subtitle?: string + description?: string + height: number + flexBasis?: number | string + flexGrow: number + flexShrink?: number + minWidth?: number | string + maxWidth?: number | string +} + +type ReportItemContentProps = { + content: React.ReactNode + actions: React.ReactNode +} + +export function ReportItem(props: BaseReportItemProps & ReportItemContentProps) { + const { flexBasis, flexGrow, flexShrink, minWidth, maxWidth } = props + return ( + + + {props.actions} + + + {(props.title || props.subtitle) && ( + + {props.title && ( + + {props.title} + + )} + {props.subtitle && {props.subtitle}} + + )} + + {props.content} + + {props.description && ( + + {props.description} + + )} + + + ) +} + +type ReportItemPlotProps = BaseReportItemProps & PlotProps + +export function ReportItemPlot(props: ReportItemPlotProps) { + const { project, query: unformattedQuery, plot, ...reportItemProps } = props + const query = formatQuery(unformattedQuery) + + return ( + } + actions={ + + + + + + } + {...reportItemProps} + /> + ) +} + +type ReportItemTableProps = BaseReportItemProps & TableProps + +export function ReportItemTable(props: ReportItemTableProps) { + const { project, query: unformattedQuery, showToolbar, ...reportItemProps } = props + const query = formatQuery(unformattedQuery) + + return ( + + + + } + actions={ + + + + + } + {...reportItemProps} + /> + ) +} + +type ReportItemMetricProps = BaseReportItemProps & MetricProps + +export function ReportItemMetric(props: ReportItemMetricProps) { + const { project, query: unformattedQuery, ...reportItemProps } = props + const query = formatQuery(unformattedQuery) + + return ( + + + + } + actions={ + + + + + } + {...reportItemProps} + /> + ) +} diff --git a/web/src/pages/report/components/ReportItemActions.tsx b/web/src/pages/report/components/ReportItemActions.tsx new file mode 100644 index 000000000..b31f9692c --- /dev/null +++ b/web/src/pages/report/components/ReportItemActions.tsx @@ -0,0 +1,80 @@ +import CodeIcon from '@mui/icons-material/Code' +import OpenInFullIcon from '@mui/icons-material/OpenInFull' +import TableChartIcon from '@mui/icons-material/TableChart' + +import { Box, IconButton, Modal, Tooltip } from '@mui/material' +import { useState } from 'react' +import { PlotFromQuery, PlotProps } from './PlotFromQuery' +import { TableFromQuery } from './TableFromQuery' + +const modalStyle = { + position: 'absolute', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + maxHeight: '100vh', + overflow: 'hidden', + width: 'calc(100% - 50px)', + bgcolor: 'background.paper', + boxShadow: 24, + p: 4, +} + +export function ActionViewExpandedTable({ query, project }: { project: string; query: string }) { + const [showingTable, setShowingTable] = useState(false) + + return ( + <> + + setShowingTable(true)}> + + + + setShowingTable(false)}> + +
+ +
+
+
+ + ) +} + +export function ActionViewEditSql({ query, project }: { project: string; query: string }) { + return ( + + + + + + ) +} + +export function ActionViewExpandedPlot(props: PlotProps) { + const [expanded, setExpanded] = useState(false) + + return ( + <> + + setExpanded(true)}> + + + + setExpanded(false)}> + + + + + + ) +} diff --git a/web/src/pages/report/components/ReportItemLoader.tsx b/web/src/pages/report/components/ReportItemLoader.tsx new file mode 100644 index 000000000..303298156 --- /dev/null +++ b/web/src/pages/report/components/ReportItemLoader.tsx @@ -0,0 +1,16 @@ +import { Box, CircularProgress } from '@mui/material' + +export default function ReportItemLoader() { + return ( + + + + ) +} diff --git a/web/src/pages/report/components/ReportRow.tsx b/web/src/pages/report/components/ReportRow.tsx new file mode 100644 index 000000000..768e133d0 --- /dev/null +++ b/web/src/pages/report/components/ReportRow.tsx @@ -0,0 +1,9 @@ +import { Box } from '@mui/material' + +export default function ReportRow(props: { children: React.ReactNode }) { + return ( + + {props.children} + + ) +} diff --git a/web/src/pages/report/components/TableFromQuery.tsx b/web/src/pages/report/components/TableFromQuery.tsx new file mode 100644 index 000000000..2a5769cb0 --- /dev/null +++ b/web/src/pages/report/components/TableFromQuery.tsx @@ -0,0 +1,92 @@ +import { Alert, Box } from '@mui/material' +import { + DataGrid, + GridColDef, + GridToolbarContainer, + GridToolbarExport, + GridToolbarQuickFilter, +} from '@mui/x-data-grid' + +import { fromArrow } from 'arquero' +import { ArrowTable } from 'arquero/dist/types/format/types' +import { memo } from 'react' +import { Link } from 'react-router-dom' +import { formatQuery, UnformattedQuery } from '../data/formatQuery' +import { useProjectDbQuery } from '../data/projectDatabase' +import ReportItemLoader from './ReportItemLoader' + +export type TableProps = { + project: string + query: UnformattedQuery + showToolbar?: boolean +} + +// Provide custom rendering for some known columns, this allows adding links to the table +const knownColumnMap: Record> = { + participant_id: { + renderCell: (params) => {params.value}, + }, + sample_id: { + renderCell: (params) => {params.value}, + }, +} + +function CustomTableToolbar() { + return ( + + + + + + ) +} + +// Memoized version of the result table to avoid rerenders, they can be very expensive +// if the result set is large +export const TableFromQuery = memo(function TableFromQuery(props: TableProps) { + return +}) + +function TableFromQueryUnmemoized(props: TableProps) { + const { project, showToolbar } = props + const query = formatQuery(props.query) + + const result = useProjectDbQuery(project, query) + + const data = result && result.status === 'success' ? result.data : undefined + + if (result && result.status === 'error') { + return {result.errorMessage} + } + + if (!data || !result || result.status === 'loading') return + + const table = fromArrow(data as ArrowTable, { useDate: true }) + + const columns = table.columnNames().map( + (colName): GridColDef => ({ + field: colName, + ...(colName in knownColumnMap ? knownColumnMap[colName] : {}), + }) + ) + + const rows = table.objects().map((row, index) => ({ ...row, __index: index })) + + return ( + row.__index} + density="compact" + sx={{ + fontFamily: 'monospace', + }} + /> + ) +} diff --git a/web/src/pages/report/data/combineQueries.ts b/web/src/pages/report/data/formatQuery.ts similarity index 86% rename from web/src/pages/report/data/combineQueries.ts rename to web/src/pages/report/data/formatQuery.ts index ab4b7d4a4..372b99467 100644 --- a/web/src/pages/report/data/combineQueries.ts +++ b/web/src/pages/report/data/formatQuery.ts @@ -1,14 +1,17 @@ import { stripIndent } from 'common-tags' -type Query = { +export type NamedQueryPart = { name: string query: string } +export type UnformattedQuery = string | NamedQueryPart[] + // Take a list of queries and combine them into a single query using CTEs // This allows viewing of steps that are used to arrive at a final aggregated query -export default function combineQueries(queries: Query[]) { - const cleanQueries = queries.map((qq) => { +export function formatQuery(query: UnformattedQuery) { + if (typeof query === 'string') return stripIndent(query) + const cleanQueries = query.map((qq) => { return { ...qq, // remove any trailing semicolons from the query, in case they were added accidentally diff --git a/web/src/pages/report/reportIndex.ts b/web/src/pages/report/reportIndex.ts new file mode 100644 index 000000000..2d51671a4 --- /dev/null +++ b/web/src/pages/report/reportIndex.ts @@ -0,0 +1,39 @@ +import OurDnaDemographics from './reports/ourdna/Demographics' +import OurDnaProcessingTimes from './reports/ourdna/ProcessingTimes' +import OurDnaRecruitment from './reports/ourdna/Recruitment' + +type ReportTab = { + title: string + id: string + content: React.ComponentType +} + +type ReportDefinition = { + title: string + tabs: ReportTab[] +} + +export const reports: Record> = { + ourdna: { + dashboard: { + title: 'OurDNA Dashboard', + tabs: [ + { + title: 'Recruitment', + id: 'recruitment', + content: OurDnaRecruitment, + }, + { + title: 'Processing Times', + id: 'processing_times', + content: OurDnaProcessingTimes, + }, + { + title: 'Demographics', + id: 'demographics', + content: OurDnaDemographics, + }, + ], + }, + }, +} diff --git a/web/src/pages/report/reports/ourdna/Demographics.tsx b/web/src/pages/report/reports/ourdna/Demographics.tsx new file mode 100644 index 000000000..68f237008 --- /dev/null +++ b/web/src/pages/report/reports/ourdna/Demographics.tsx @@ -0,0 +1,227 @@ +import * as Plot from '@observablehq/plot' +import Report from '../../components/Report' +import { ReportItemMetric, ReportItemPlot } from '../../components/ReportItem' +import ReportRow from '../../components/ReportRow' + +const ROW_HEIGHT = 450 +const PROJECT = 'ourdna' + +const PROCESS_DURATION_QUERY = ` + select + s.sample_id, + cast(s.participant_id as string) as participant_id, + coalesce(s."meta_processing-site", 'Unknown') as processing_site, + + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, + date_diff( + 'hour', + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') + ) as duration + from sample s + join participant p + on p.participant_id = s.participant_id + where s.type = 'blood' +` + +export default function Demographics() { + return ( + + + + ({ + marginLeft: 40, + inset: 10, + y: { + grid: true, + label: 'Participants', + }, + x: { + type: 'time', + }, + marks: [ + Plot.areaY(data, { + x: 'week', + y: 'count', + fill: 'ancestry', + tip: true, + order: 'sum', + }), + Plot.ruleY([0]), + ], + })} + /> + + + ({ + color: { legend: true }, + marks: [ + Plot.rectY( + data, + Plot.binX( + { y: 'count' }, + { x: 'age', fill: 'reported_sex', tip: true } + ) + ), + ], + })} + /> + ({ + marginLeft: 160, + inset: 10, + color: { + scheme: 'Observable10', + reverse: true, + }, + marks: [ + Plot.barX(data, { + y: 'ancestry', + fill: 'ancestry', + x: 'count', + tip: true, + sort: { + y: 'x', + reverse: true, + }, + }), + ], + })} + /> + + + ) +} diff --git a/web/src/pages/report/reports/ourdna/ProcessingTimes.tsx b/web/src/pages/report/reports/ourdna/ProcessingTimes.tsx new file mode 100644 index 000000000..6504d2bed --- /dev/null +++ b/web/src/pages/report/reports/ourdna/ProcessingTimes.tsx @@ -0,0 +1,194 @@ +import * as Plot from '@observablehq/plot' +import Report from '../../components/Report' +import { ReportItemPlot, ReportItemTable } from '../../components/ReportItem' +import ReportRow from '../../components/ReportRow' + +const ROW_HEIGHT = 450 +const PROJECT = 'ourdna' + +const PROCESS_DURATION_QUERY = ` + select + s.sample_id, + cast(s.participant_id as string) as participant_id, + coalesce(s."meta_processing-site", 'Unknown') as processing_site, + + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, + date_diff( + 'hour', + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), + try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') + ) as duration + from sample s + join participant p + on p.participant_id = s.participant_id + where s.type = 'blood' +` + +export default function ProcessingTimes() { + return ( + + + 0 + `, + }, + ]} + plot={(data) => ({ + y: { grid: true, padding: 5, label: 'duration(hrs)' }, + x: { grid: true, padding: 5 }, + marginTop: 20, + marginRight: 20, + marginBottom: 30, + + marginLeft: 40, + marks: [ + Plot.ruleY([24, 48, 72], { stroke: '#ff725c' }), + Plot.dot(data, { + x: 'collection_time', + y: 'duration', + stroke: null, + fill: '#4269d0', + fillOpacity: 0.5, + channels: { + process_end: 'process_end_time', + sample_id: 'sample_id', + participant_id: 'participant_id', + }, + tip: true, + }), + ], + })} + /> + + = 24 AND duration < 36 THEN '24-36 hours' + WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' + WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' + ELSE '72+ hours' + END AS duration + from durations group by 2 order by 2 + `, + }, + ]} + plot={(data) => ({ + marginLeft: 100, + inset: 10, + color: { + scheme: 'RdYlGn', + reverse: true, + }, + marks: [ + Plot.barX(data, { + y: 'duration', + x: 'count', + tip: true, + fill: 'duration', + }), + ], + })} + /> + + + = 24 AND duration < 36 THEN '24-36 hours' + WHEN duration >= 36 AND duration < 48 THEN '36-48 hours' + WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' + ELSE '72+ hours' + END AS duration + from durations group by 2,3 order by 2,3 + `, + }, + ]} + plot={(data) => ({ + marginLeft: 100, + marginRight: 100, + inset: 10, + color: { + scheme: 'RdYlGn', + reverse: true, + }, + marks: [ + Plot.barX(data, { + y: 'processing_site', + fy: 'duration', + x: 'count', + tip: true, + fill: 'duration', + }), + ], + })} + /> + + + + + ) +} diff --git a/web/src/pages/report/reports/ourdna/Recruitment.tsx b/web/src/pages/report/reports/ourdna/Recruitment.tsx new file mode 100644 index 000000000..2adffdd20 --- /dev/null +++ b/web/src/pages/report/reports/ourdna/Recruitment.tsx @@ -0,0 +1,96 @@ +import * as Plot from '@observablehq/plot' +import Report from '../../components/Report' +import { ReportItemPlot } from '../../components/ReportItem' +import ReportRow from '../../components/ReportRow' + +const ROW_HEIGHT = 400 +const PROJECT = 'ourdna' + +export default function Recruitment() { + return ( + + + ({ + marginLeft: 160, + inset: 10, + marks: [ + Plot.barX(data, { + y: 'stage', + x: 'count', + tip: true, + fill: 'stage', + }), + ], + })} + /> + + ({ + color: { legend: true }, + marks: [ + Plot.barX( + data, + Plot.stackX({ + x: 'count', + fill: 'collection_type', + inset: 0.5, + tip: true, + }) + ), + ], + })} + /> + + + ) +} From 4a295d4646fdf2fb4c2e8d32d13b1026d0ceae73 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Tue, 28 Jan 2025 18:40:49 +1100 Subject: [PATCH 07/12] remove ourdna dashboard from nav --- web/src/Routes.tsx | 3 --- web/src/shared/components/Header/NavBar.tsx | 7 +------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/web/src/Routes.tsx b/web/src/Routes.tsx index 4c3bcb71a..3aec04a6b 100644 --- a/web/src/Routes.tsx +++ b/web/src/Routes.tsx @@ -22,7 +22,6 @@ import { ParticipantPage } from './pages/participant/ParticipantViewContainer' import AnalysisRunnerSummary from './pages/project/AnalysisRunnerView/AnalysisRunnerSummary' import ProjectOverview from './pages/project/ProjectOverview' import ProjectReport from './pages/report/ProjectReport' -import OurDnaDashboard from './pages/report/reports/ourdna/OurDnaDashboard' import SqlQueryUI from './pages/report/SqlQueryUI' import SampleView from './pages/sample/SampleView' import ErrorBoundary from './shared/utilities/errorBoundary' @@ -118,8 +117,6 @@ const Routes: React.FunctionComponent = () => ( } /> - } /> - } /> } /> diff --git a/web/src/shared/components/Header/NavBar.tsx b/web/src/shared/components/Header/NavBar.tsx index c1f730220..74181b55a 100644 --- a/web/src/shared/components/Header/NavBar.tsx +++ b/web/src/shared/components/Header/NavBar.tsx @@ -12,7 +12,6 @@ import DescriptionIcon from '@mui/icons-material/Description' import DisplaySettingsIcon from '@mui/icons-material/DisplaySettings' import ExploreIcon from '@mui/icons-material/Explore' import InsightsIcon from '@mui/icons-material/Insights' -import PeopleIcon from '@mui/icons-material/People' import SummarizeIcon from '@mui/icons-material/Summarize' import TroubleshootIcon from '@mui/icons-material/Troubleshoot' import MuckTheDuck from '../MuckTheDuck' @@ -133,11 +132,7 @@ const NavBar = () => { url: '/analysis-runner', icon: , }, - { - title: 'OurDNA', - url: '/ourdna', - icon: , - }, + { title: 'Swagger', url: '/swagger', From bfeadb4002bd449befce62321650bb997011812d Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Wed, 29 Jan 2025 10:31:20 +1100 Subject: [PATCH 08/12] add remaining charts for ourdna dashboard --- .../pages/report/components/ReportItem.tsx | 2 +- web/src/pages/report/data/formatQuery.ts | 2 + web/src/pages/report/reportIndex.ts | 12 +- .../report/reports/ourdna/DataQuality.tsx | 115 ++++++++++++++++++ .../report/reports/ourdna/Demographics.tsx | 75 +++++++++--- .../report/reports/ourdna/Recruitment.tsx | 50 ++++++++ ...ocessingTimes.tsx => SampleProcessing.tsx} | 33 +++++ 7 files changed, 266 insertions(+), 23 deletions(-) create mode 100644 web/src/pages/report/reports/ourdna/DataQuality.tsx rename web/src/pages/report/reports/ourdna/{ProcessingTimes.tsx => SampleProcessing.tsx} (86%) diff --git a/web/src/pages/report/components/ReportItem.tsx b/web/src/pages/report/components/ReportItem.tsx index 53248cf80..bb8ba80a6 100644 --- a/web/src/pages/report/components/ReportItem.tsx +++ b/web/src/pages/report/components/ReportItem.tsx @@ -61,7 +61,7 @@ export function ReportItem(props: BaseReportItemProps & ReportItemContentProps) }} > {(props.title || props.subtitle) && ( - + {props.title && ( {props.title} diff --git a/web/src/pages/report/data/formatQuery.ts b/web/src/pages/report/data/formatQuery.ts index 372b99467..ee1bf495d 100644 --- a/web/src/pages/report/data/formatQuery.ts +++ b/web/src/pages/report/data/formatQuery.ts @@ -19,6 +19,8 @@ export function formatQuery(query: UnformattedQuery) { } }) + if (cleanQueries.length === 1) return cleanQueries[0].query + const ctes = cleanQueries.slice(0, cleanQueries.length - 1).map((qq) => { // To format the query nicely, need to indent each line by the same amount, but no way // of knowing what that amount is, as it depends on the code formatting :/ so use a sentinel diff --git a/web/src/pages/report/reportIndex.ts b/web/src/pages/report/reportIndex.ts index 2d51671a4..2e64b3f82 100644 --- a/web/src/pages/report/reportIndex.ts +++ b/web/src/pages/report/reportIndex.ts @@ -1,6 +1,7 @@ +import OurDnaDataQuality from './reports/ourdna/DataQuality' import OurDnaDemographics from './reports/ourdna/Demographics' -import OurDnaProcessingTimes from './reports/ourdna/ProcessingTimes' import OurDnaRecruitment from './reports/ourdna/Recruitment' +import OurDnaProcessingTimes from './reports/ourdna/SampleProcessing' type ReportTab = { title: string @@ -24,8 +25,8 @@ export const reports: Record> = { content: OurDnaRecruitment, }, { - title: 'Processing Times', - id: 'processing_times', + title: 'Sample Processing', + id: 'sample_processing', content: OurDnaProcessingTimes, }, { @@ -33,6 +34,11 @@ export const reports: Record> = { id: 'demographics', content: OurDnaDemographics, }, + { + title: 'Data Quality', + id: 'data_quality', + content: OurDnaDataQuality, + }, ], }, }, diff --git a/web/src/pages/report/reports/ourdna/DataQuality.tsx b/web/src/pages/report/reports/ourdna/DataQuality.tsx new file mode 100644 index 000000000..443c6e84b --- /dev/null +++ b/web/src/pages/report/reports/ourdna/DataQuality.tsx @@ -0,0 +1,115 @@ +import Report from '../../components/Report' +import { ReportItemMetric } from '../../components/ReportItem' +import ReportRow from '../../components/ReportRow' + +const ROW_HEIGHT = 220 +const PROJECT = 'ourdna' + +export default function DataQuality() { + return ( + + + + + + + + + + + + + ) +} diff --git a/web/src/pages/report/reports/ourdna/Demographics.tsx b/web/src/pages/report/reports/ourdna/Demographics.tsx index 68f237008..0bcc8f17b 100644 --- a/web/src/pages/report/reports/ourdna/Demographics.tsx +++ b/web/src/pages/report/reports/ourdna/Demographics.tsx @@ -6,25 +6,6 @@ import ReportRow from '../../components/ReportRow' const ROW_HEIGHT = 450 const PROJECT = 'ourdna' -const PROCESS_DURATION_QUERY = ` - select - s.sample_id, - cast(s.participant_id as string) as participant_id, - coalesce(s."meta_processing-site", 'Unknown') as processing_site, - - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, - date_diff( - 'hour', - try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S'), - try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') - ) as duration - from sample s - join participant p - on p.participant_id = s.participant_id - where s.type = 'blood' -` - export default function Demographics() { return ( @@ -168,6 +149,7 @@ export default function Demographics() { `} plot={(data) => ({ color: { legend: true }, + y: { label: 'count' }, marks: [ Plot.rectY( data, @@ -192,6 +174,8 @@ export default function Demographics() { p.participant_id, unnest(p."meta_ancestry-participant-ancestry") as ancestry from participant p + join sample s on s.participant_id = p.participant_id + where s.type = 'blood' ) select count(distinct participant_id) as count, @@ -222,6 +206,59 @@ export default function Demographics() { })} /> + + ({ + marginLeft: 100, + marginRight: 100, + inset: 10, + y: { + inset: 1, + }, + color: { + scheme: 'Observable10', + reverse: true, + }, + marks: [ + Plot.barX(data, { + y: 'processing_site', + fy: 'ancestry', + fill: 'ancestry', + x: 'count', + tip: true, + sort: { + fy: 'x', + reverse: true, + }, + }), + ], + })} + /> + ) } diff --git a/web/src/pages/report/reports/ourdna/Recruitment.tsx b/web/src/pages/report/reports/ourdna/Recruitment.tsx index 2adffdd20..ebd1e5a4a 100644 --- a/web/src/pages/report/reports/ourdna/Recruitment.tsx +++ b/web/src/pages/report/reports/ourdna/Recruitment.tsx @@ -91,6 +91,56 @@ export default function Recruitment() { })} /> + + ({ + color: { legend: true }, + marginLeft: 100, + marks: [ + Plot.barX( + data, + Plot.stackX({ + x: 'count', + y: 'processing_site', + fill: 'collection_type', + inset: 0.5, + tip: true, + }) + ), + ], + })} + /> + ) } diff --git a/web/src/pages/report/reports/ourdna/ProcessingTimes.tsx b/web/src/pages/report/reports/ourdna/SampleProcessing.tsx similarity index 86% rename from web/src/pages/report/reports/ourdna/ProcessingTimes.tsx rename to web/src/pages/report/reports/ourdna/SampleProcessing.tsx index 6504d2bed..917127548 100644 --- a/web/src/pages/report/reports/ourdna/ProcessingTimes.tsx +++ b/web/src/pages/report/reports/ourdna/SampleProcessing.tsx @@ -189,6 +189,39 @@ export default function ProcessingTimes() { ]} /> + + ({ + marginLeft: 100, + marks: [ + Plot.barX(data, { + y: 'type', + x: 'count', + fill: 'type', + tip: true, + }), + ], + })} + /> + ) } From 42bc3dac8343bb6226f41974743f48f77dbf950b Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Wed, 29 Jan 2025 10:47:11 +1100 Subject: [PATCH 09/12] add reports to project summary page --- web/src/index.css | 4 +- web/src/pages/project/ProjectSummary.tsx | 42 +++++++++++++++++-- .../report/reports/ourdna/Demographics.tsx | 2 +- .../report/reports/ourdna/Recruitment.tsx | 4 +- .../reports/ourdna/SampleProcessing.tsx | 8 ++-- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/web/src/index.css b/web/src/index.css index e30da7580..e648367a9 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -491,7 +491,7 @@ html[data-theme='dark-mode'] .ui.table { } -/* +/* hax to fix bad cascading styles. There are a few conflicting css frameworks on metamist for strange historical reasons. While we're working to move to mui and remove other conflicting styles at the moment it @@ -500,4 +500,4 @@ html[data-theme='dark-mode'] .ui.table { .MuiTablePagination-selectLabel, .MuiTablePagination-displayedRows { margin-bottom: 0 !important; -} \ No newline at end of file +} diff --git a/web/src/pages/project/ProjectSummary.tsx b/web/src/pages/project/ProjectSummary.tsx index cd4c99c89..40f67a86e 100644 --- a/web/src/pages/project/ProjectSummary.tsx +++ b/web/src/pages/project/ProjectSummary.tsx @@ -3,6 +3,7 @@ import LoadingDucks from '../../shared/components/LoadingDucks/LoadingDucks' import MuckError from '../../shared/components/MuckError' import { ProjectSummary, WebApi } from '../../sm-api' +import { Box, Button } from '@mui/material' import { Link } from 'react-router-dom' import BatchStatistics from './BatchStatistics' import MultiQCReports from './MultiQCReports' @@ -11,6 +12,8 @@ import SeqrSync from './SeqrSync' import SummaryStatistics from './SummaryStatistics' import TotalsStats from './TotalsStats' +import { reports } from '../report/reportIndex' + interface ProjectSummaryProps { projectName: string } @@ -53,10 +56,39 @@ export const ProjectSummaryView: React.FunctionComponent = ) } + const customReports = reports[projectName] + return ( - <> - SQL Query - + + + + + + + + {customReports &&

Custom Reports

}
+ + + +
+
    + {customReports && + Object.entries(customReports).map(([key, report]) => ( +
  • + + {report.title} + +
  • + ))} +
+
+
= cramSeqrStats={summary?.cram_seqr_stats ?? {}} batchSequenceStats={summary?.batch_sequencing_group_stats ?? {}} /> +
+ - +
) } diff --git a/web/src/pages/report/reports/ourdna/Demographics.tsx b/web/src/pages/report/reports/ourdna/Demographics.tsx index 0bcc8f17b..58cb959b1 100644 --- a/web/src/pages/report/reports/ourdna/Demographics.tsx +++ b/web/src/pages/report/reports/ourdna/Demographics.tsx @@ -88,7 +88,7 @@ export default function Demographics() { coalesce(r.participants, 0) as participants, strftime(w.week, '%Y-%m-%d') as week, a.ancestry, - + from weeks w join ancestries a on true left join counts_by_week r diff --git a/web/src/pages/report/reports/ourdna/Recruitment.tsx b/web/src/pages/report/reports/ourdna/Recruitment.tsx index ebd1e5a4a..04011066b 100644 --- a/web/src/pages/report/reports/ourdna/Recruitment.tsx +++ b/web/src/pages/report/reports/ourdna/Recruitment.tsx @@ -65,7 +65,7 @@ export default function Recruitment() { WHEN 'OSS' THEN 'Once Stop Shop' - + WHEN '__NULL__' THEN 'Unknown' @@ -112,7 +112,7 @@ export default function Recruitment() { WHEN 'OSS' THEN 'Once Stop Shop' - + WHEN '__NULL__' THEN 'Unknown' diff --git a/web/src/pages/report/reports/ourdna/SampleProcessing.tsx b/web/src/pages/report/reports/ourdna/SampleProcessing.tsx index 917127548..05ff785ab 100644 --- a/web/src/pages/report/reports/ourdna/SampleProcessing.tsx +++ b/web/src/pages/report/reports/ourdna/SampleProcessing.tsx @@ -11,7 +11,7 @@ const PROCESS_DURATION_QUERY = ` s.sample_id, cast(s.participant_id as string) as participant_id, coalesce(s."meta_processing-site", 'Unknown') as processing_site, - + try_strptime(nullif("meta_collection-time", ' '), '%Y-%m-%d %H:%M:%S') as collection_time, try_strptime(nullif("meta_process-end-time", ' '), '%Y-%m-%d %H:%M:%S') as process_end_time, date_diff( @@ -94,7 +94,7 @@ export default function ProcessingTimes() { WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' ELSE '72+ hours' END AS duration - from durations group by 2 order by 2 + from durations group by 2 order by 2 `, }, ]} @@ -139,7 +139,7 @@ export default function ProcessingTimes() { WHEN duration >= 48 AND duration < 72 THEN '48-72 hours' ELSE '72+ hours' END AS duration - from durations group by 2,3 order by 2,3 + from durations group by 2,3 order by 2,3 `, }, ]} @@ -205,7 +205,7 @@ export default function ProcessingTimes() { count(distinct participant_id) as count, type from sample s - group by 2 + group by 2 `, }, ]} From 18018b35e3ff1059eb0f18044aec944e19b4b5e7 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Wed, 29 Jan 2025 10:52:47 +1100 Subject: [PATCH 10/12] remove v1 ourdna dashboard frontend code --- docs/installation.md | 1 - web/src/index.css | 21 -- web/src/ourdna-logo-rgb.svg | 79 ----- web/src/ourdna-white.svg | 75 ----- web/src/pages/ourdna/OurDnaDashboard.tsx | 299 ------------------ web/src/shared/components/OurDNALogo.tsx | 15 - web/src/shared/components/ourdna/BarChart.tsx | 255 --------------- web/src/shared/components/ourdna/Colours.tsx | 17 - .../components/ourdna/OurDonutChart.tsx | 169 ---------- web/src/shared/components/ourdna/StatTile.tsx | 54 ---- .../shared/components/ourdna/TableTile.tsx | 64 ---- web/src/shared/components/ourdna/Tile.tsx | 46 --- .../ourdna/dashboardIcons/AlarmIcon.tsx | 42 --- .../ourdna/dashboardIcons/BloodSampleIcon.tsx | 42 --- .../ourdna/dashboardIcons/ClipboardIcon.tsx | 35 -- .../ourdna/dashboardIcons/ClockIcon.tsx | 42 --- .../ourdna/dashboardIcons/RocketIcon.tsx | 42 --- .../ourdna/dashboardIcons/SyringeIcon.tsx | 35 -- .../ourdna/dashboardIcons/TestTubeIcon.tsx | 42 --- .../ourdna/dashboardIcons/TruckIcon.tsx | 35 -- 20 files changed, 1410 deletions(-) delete mode 100644 web/src/ourdna-logo-rgb.svg delete mode 100644 web/src/ourdna-white.svg delete mode 100644 web/src/pages/ourdna/OurDnaDashboard.tsx delete mode 100644 web/src/shared/components/OurDNALogo.tsx delete mode 100644 web/src/shared/components/ourdna/BarChart.tsx delete mode 100644 web/src/shared/components/ourdna/Colours.tsx delete mode 100644 web/src/shared/components/ourdna/OurDonutChart.tsx delete mode 100644 web/src/shared/components/ourdna/StatTile.tsx delete mode 100644 web/src/shared/components/ourdna/TableTile.tsx delete mode 100644 web/src/shared/components/ourdna/Tile.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/AlarmIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/BloodSampleIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/ClipboardIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/ClockIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/RocketIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/SyringeIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/TestTubeIcon.tsx delete mode 100644 web/src/shared/components/ourdna/dashboardIcons/TruckIcon.tsx diff --git a/docs/installation.md b/docs/installation.md index 20608a80b..58e875151 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -97,7 +97,6 @@ export PATH="$HB_PREFIX/opt/mariadb@10.8/bin:$PATH" export SM_ENVIRONMENT=LOCAL # good default to have export SM_DEV_DB_USER=sm_api # makes it easier to copy liquibase update command -export VITE_OURDNA_PROJECT_NAME=greek-myth # points the dashboard to the project to fetch data from ``` You can also add these to your shell config file e.g `.zshrc` or `.bashrc` for persistence to new sessions. diff --git a/web/src/index.css b/web/src/index.css index e648367a9..f119aba34 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -42,18 +42,6 @@ --color-pedigree-affected: rgb(76, 76, 76); --color-pedigree-unaffected: rgb(240, 240, 240); - --ourdna-blue-transparent: rgba(113, 172, 225, 0.5); - --ourdna-red-transparent: rgba(191, 0, 59, 0.5); - --ourdna-yellow-transparent: rgba(232, 199, 29, 0.5); - --ourdna-green-transparent: rgba(159, 201, 54, 0.5); - --ourdna-charcoal-transparent: rgba(85, 85, 85, 0.5); - - --ourdna-blue: rgba(113, 172, 225); - --ourdna-red: rgba(191, 0, 59); - --ourdna-yellow: rgba(232, 199, 29); - --ourdna-green: rgba(159, 201, 54); - --ourdna-charcoal: rgba(85, 85, 85); - --color-header-row: #f9f9f9; --color-exome-row: #ededa1; --color-genome-row: #efc7aa; @@ -358,16 +346,7 @@ html[data-theme='dark-mode'] .ui.table { color: var(--color-text-primary); } -.dashboard-tile { - color: var(--color-text-primary) !important; -} -.ourdna-logo-responsive { - width: 50%; - height: auto; - max-width: 200px; - max-height: 100px; -} /* Project Insights dashboard */ .html-tooltip .MuiTooltip-tooltip { diff --git a/web/src/ourdna-logo-rgb.svg b/web/src/ourdna-logo-rgb.svg deleted file mode 100644 index f2743ea3e..000000000 --- a/web/src/ourdna-logo-rgb.svg +++ /dev/null @@ -1,79 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/web/src/ourdna-white.svg b/web/src/ourdna-white.svg deleted file mode 100644 index bb0558272..000000000 --- a/web/src/ourdna-white.svg +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/web/src/pages/ourdna/OurDnaDashboard.tsx b/web/src/pages/ourdna/OurDnaDashboard.tsx deleted file mode 100644 index d97ed1747..000000000 --- a/web/src/pages/ourdna/OurDnaDashboard.tsx +++ /dev/null @@ -1,299 +0,0 @@ -import { useQuery } from '@apollo/client' -import { Container, Grid, GridColumn, GridRow } from 'semantic-ui-react' -import { gql } from '../../__generated__' -import { PaddedPage } from '../../shared/components/Layout/PaddedPage' -import LoadingDucks from '../../shared/components/LoadingDucks/LoadingDucks' -import MuckError from '../../shared/components/MuckError' -import BarChart from '../../shared/components/ourdna/BarChart' -import { ourdnaColours } from '../../shared/components/ourdna/Colours' -import AlarmIcon from '../../shared/components/ourdna/dashboardIcons/AlarmIcon' -import BloodSampleIcon from '../../shared/components/ourdna/dashboardIcons/BloodSampleIcon' -import ClipboardIcon from '../../shared/components/ourdna/dashboardIcons/ClipboardIcon' -import ClockIcon from '../../shared/components/ourdna/dashboardIcons/ClockIcon' -import RocketIcon from '../../shared/components/ourdna/dashboardIcons/RocketIcon' -import SyringeIcon from '../../shared/components/ourdna/dashboardIcons/SyringeIcon' -import TestTubeIcon from '../../shared/components/ourdna/dashboardIcons/TestTubeIcon' -import TruckIcon from '../../shared/components/ourdna/dashboardIcons/TruckIcon' -import OurDonutChart from '../../shared/components/ourdna/OurDonutChart' -import StatTile from '../../shared/components/ourdna/StatTile' -import TableTile from '../../shared/components/ourdna/TableTile' -import Tile from '../../shared/components/ourdna/Tile' -import OurDNALogo from '../../shared/components/OurDNALogo' - -const GET_OURDNA_DASHBOARD = gql(` - query DashboardQuery($project: String!) { - project(name: $project) { - ourdnaDashboard { - collectionToProcessEndTime - collectionToProcessEndTime24h - collectionToProcessEndTimeStatistics - collectionToProcessEndTimeBucketStatistics - participantsConsentedNotCollected - participantsSignedNotConsented - processingTimesBySite - processingTimesByCollectionSite - samplesConcentrationGt1ug - samplesLostAfterCollection { - collectionLab - collectionTime - courier - courierActualDropoffTime - courierActualPickupTime - courierScheduledDropoffTime - courierScheduledPickupTime - courierTrackingNumber - processEndTime - processStartTime - receivedBy - receivedTime - sampleId - timeSinceCollection - } - totalSamplesByCollectionEventName - } - } - } -`) - -const iconStyle = { - marginRight: '10px', - width: '24px', - height: '24px', -} - -const Dashboard = () => { - const { loading, error, data } = useQuery(GET_OURDNA_DASHBOARD, { - variables: { project: import.meta.env.VITE_OURDNA_PROJECT_NAME || 'ourdna' }, - }) - - if (!data) return - if (loading) return - if (error) return <>Error! {error.message} - - const samplesLostAfterCollections = - data.project.ourdnaDashboard.samplesLostAfterCollection.reduce( - (rr: Record, sample) => { - rr[sample.sampleId] = - sample.timeSinceCollection != null ? sample.timeSinceCollection / 3600 : 0 - return rr - }, - {} - ) - - return ( - <> - - - - - - - - - - } - /> - - - - } - /> - - - - } - /> - - - - } - /> - - - - - - } - /> - - - - - - } - /> - - - - - 72h']}`, - units: '> 72 hours', - unitsColour: 'ourdna-yellow-transparent', - }, - ]} - description="Count of samples processed within each 24h bucket." - icon={ - - } - /> - - - - - - - } - /> - - - - - - } - /> - - - - } - /> - - - - - - - ) -} - -export default function DashboardPage() { - return ( - - - - ) -} diff --git a/web/src/shared/components/OurDNALogo.tsx b/web/src/shared/components/OurDNALogo.tsx deleted file mode 100644 index 9bcf1529d..000000000 --- a/web/src/shared/components/OurDNALogo.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import * as React from 'react' -import ourDNA from '../../ourdna-logo-rgb.svg' -import ourDNADark from '../../ourdna-white.svg' -import { ThemeContext } from './ThemeProvider' - -const OurDNALogo: React.FunctionComponent< - React.DetailedHTMLProps, HTMLImageElement> -> = (props) => { - const theme = React.useContext(ThemeContext) - const isDarkMode = theme.theme === 'dark-mode' - - const ourDNAlogo = isDarkMode ? ourDNADark : ourDNA - return OurDNA Logo -} -export default OurDNALogo diff --git a/web/src/shared/components/ourdna/BarChart.tsx b/web/src/shared/components/ourdna/BarChart.tsx deleted file mode 100644 index 649c2c56c..000000000 --- a/web/src/shared/components/ourdna/BarChart.tsx +++ /dev/null @@ -1,255 +0,0 @@ -import * as d3 from 'd3' -import * as React from 'react' -import { Card, Container } from 'semantic-ui-react' - -import { ourdnaColours } from './Colours' - -export const chartOptions = { - responsive: true, - plugins: { - legend: { - position: 'bottom' as const, - labels: { - padding: 20, - }, - }, - title: { - display: false, - text: 'Chart.js Bar Chart', - }, - }, -} - -interface HistogramProps { - icon: React.ReactNode - header: string - data: object -} - -const HistogramChart: React.FC = ({ icon, header, data }) => { - const svgRef = React.useRef(null) - interface TransformedData { - site: string - hour: string - count: number - } - - const [dimensions, setDimensions] = React.useState<{ width: number; height: number }>({ - width: 0, - height: 0, - }) - - // @ts-ignore - const transformedData: TransformedData[] = Object.entries(data).flatMap(([site, hours]) => - Object.entries(hours).map(([hour, count]) => ({ - site, - hour, - count, - })) - ) - - const groupedData = d3.groups(transformedData, (d) => d.hour) - - React.useEffect(() => { - // Function to handle window resize events and update dimensions - const handleResize = () => { - if (svgRef.current) { - const { width, height } = svgRef.current.getBoundingClientRect() - setDimensions({ width, height }) - } - } - - // Initial call to set dimensions - handleResize() - - // Add event listener for window resize - window.addEventListener('resize', handleResize) - - // Cleanup event listener on component unmount - return () => window.removeEventListener('resize', handleResize) - }, []) - - React.useEffect(() => { - if (dimensions.width === 0 || dimensions.height === 0) return - - const svg = d3.select(svgRef.current) - const margin = { top: 20, right: 100, bottom: 60, left: 60 } - const width = dimensions.width - margin.left - margin.right - const height = dimensions.height - margin.top - margin.bottom - - svg.selectAll('*').remove() // Clear previous content - - const g = svg.append('g').attr('transform', `translate(${margin.left},${margin.top})`) - - const x0 = d3 - .scaleBand() - .domain(transformedData.map((d) => d.hour)) - .rangeRound([0, width]) - .paddingInner(0.1) - - const y = d3 - .scaleLinear() - .domain([0, d3.max(transformedData, (d) => d.count) || 1]) - .nice() - .rangeRound([height, 0]) - - const color = d3 - .scaleOrdinal() - .domain(transformedData.map((d) => d.site)) - .range(Object.values(ourdnaColours)) - - groupedData.forEach(([hour, values]) => { - if (x0(hour) === undefined) { - return - } - - const x1 = d3 - .scaleBand() - .domain(values.map((d) => d.site)) - .rangeRound([0, x0.bandwidth()]) - .padding(0.05) - - if (Number.isNaN(x0(hour))) { - return - } - - g.append('g') - .attr('transform', `translate(${x0(hour)},0)`) - .selectAll('rect') - .data(values) - .enter() - .append('rect') - // @ts-ignore - .attr('x', (d) => { - const x = x1(d.site) - if (Number.isNaN(x)) { - return 0 - } - return x - }) - .attr('y', (d) => y(d.count)) - .attr('width', x1.bandwidth()) - .attr('height', (d) => Math.max(0, height - y(d.count))) - .attr('fill', (d) => color(d.site)) - }) - - g.append('g') - .attr('class', 'axis-label') - .attr('transform', `translate(0,${height})`) - .call(d3.axisBottom(x0)) - .selectAll('text') - .attr('dy', '1em') // Adjust the position of the x-axis labels - - g.append('g') - .attr('class', 'axis-label') - // Added a `.ticks(5)` to reduce the number of ticks on the y-axis - .call(d3.axisLeft(y).ticks(5).tickFormat(d3.format('d'))) - .selectAll('text') - .attr('dx', '-1em') // Adjust the position of the y-axis labels - - g.append('text') - .attr('class', 'axis-label') - .attr('text-anchor', 'end') - .attr('x', width / 2) - .attr('y', height + margin.bottom - 15) - .text('Hours Taken') - - g.append('text') - .attr('class', 'axis-label') - .attr('text-anchor', 'end') - .attr('transform', 'rotate(-90)') - .attr('x', -height / 2) - .attr('y', -margin.left + 15) - .text('Sample Count') - - const legend = svg - .append('g') - .attr('font-family', 'sans-serif') - .attr('font-size', 10) - .attr('text-anchor', 'start') - .attr('transform', `translate(${dimensions.width - margin.right + 10},${margin.top})`) - .selectAll('g') - .data(transformedData.map((d) => d.site).filter((v, i, a) => a.indexOf(v) === i)) - .enter() - .append('g') - .attr('transform', (d, i) => `translate(0,${i * 20})`) - - // Append X axis - g.append('g') - .attr('class', 'axis-label') - .attr('transform', `translate(0,${height})`) - .call(d3.axisBottom(x0)) - .selectAll('text') - .attr('dy', '1em') // Adjust the position of the x-axis labels - .style('fill', 'var(--color-text-primary)') // Change the color of the axis labels - - // Change the color of the tick marks - g.selectAll('.tick line').style('stroke', 'var(--color-text-primary)') // Change the color of the tick marks - // Append Y axis - g.append('g') - .attr('class', 'axis-label') - // Added a `.ticks(5)` to reduce the number of ticks on the y-axis - .call(d3.axisLeft(y).ticks(5).tickFormat(d3.format('d'))) - .selectAll('text') - .attr('dx', '-1em') // Adjust the position of the y-axis labels - .style('fill', 'var(--color-text-primary)') // Change the color of the axis labels - - // Append X axis line - g.append('g') - .attr('class', 'axis-line') - .append('line') - .attr('x1', 0) - .attr('x2', width) - .attr('y1', height) - .attr('y2', height) - .style('stroke', 'var(--color-text-primary)') // Change the color of the axis line - - // Append Y axis line - g.append('g') - .attr('class', 'axis-line') - .append('line') - .attr('x1', 0) - .attr('x2', 0) - .attr('y1', 0) - .attr('y2', height) - .style('stroke', 'var(--color-text-primary)') // Change the color of the axis line - - legend - .append('rect') - .attr('x', 0) - .attr('width', 19) - .attr('height', 19) - .attr('fill', (d) => color(d)) - - legend - .append('text') - .attr('x', 24) - .attr('y', 9.5) - .attr('dy', '0.32em') - .text((d) => d) - }, [data, dimensions, groupedData, transformedData]) - - return ( - - - - {icon} - {header} - - - - - - - - - ) -} - -export default HistogramChart diff --git a/web/src/shared/components/ourdna/Colours.tsx b/web/src/shared/components/ourdna/Colours.tsx deleted file mode 100644 index 8521a8e32..000000000 --- a/web/src/shared/components/ourdna/Colours.tsx +++ /dev/null @@ -1,17 +0,0 @@ -const ourdnaColours = { - red: 'var(--ourdna-red)', - yellow: 'var(--ourdna-yellow)', - green: 'var(--ourdna-green)', - blue: 'var(--ourdna-blue)', - charcoal: 'var(--ourdna-charcoal)', -} - -const ourdnaColoursTransparent = { - red: 'var(--ourdna-red-transparent)', - yellow: 'var(--ourdna-yellow-transparent)', - green: 'var(--ourdna-green-transparent)', - blue: 'var(--ourdna-blue-transparent)', - charcoal: 'var(--ourdna-charcoal-transparent)', -} - -export { ourdnaColours, ourdnaColoursTransparent } diff --git a/web/src/shared/components/ourdna/OurDonutChart.tsx b/web/src/shared/components/ourdna/OurDonutChart.tsx deleted file mode 100644 index fa4b7ca12..000000000 --- a/web/src/shared/components/ourdna/OurDonutChart.tsx +++ /dev/null @@ -1,169 +0,0 @@ -import * as d3 from 'd3' -import * as React from 'react' -import { Card, Container } from 'semantic-ui-react' - -import { ourdnaColours } from './Colours' - -interface DonutChartData { - [key: string]: number -} - -interface DonutChartProps { - header: string - data: DonutChartData - icon: React.ReactNode -} - -const OurDonutChart: React.FC = ({ header, data, icon }) => { - const svgRef = React.useRef(null) - - const [dimensions, setDimensions] = React.useState<{ width: number; height: number }>({ - width: 0, - height: 0, - }) - - React.useEffect(() => { - const handleResize = () => { - if (svgRef.current) { - const { width, height } = svgRef.current.getBoundingClientRect() - setDimensions({ width, height }) - } - } - - handleResize() - - window.addEventListener('resize', handleResize) - return () => window.removeEventListener('resize', handleResize) - }, []) - - React.useEffect(() => { - if (dimensions.width === 0 || dimensions.height === 0) return - - const svg = d3.select(svgRef.current) - const margin = { top: 40, right: 40, bottom: 50, left: 40 } - const width = dimensions.width - margin.left - margin.right - const height = dimensions.height - margin.top - margin.bottom - const radius = Math.min(width, height) / 2 - - svg.selectAll('*').remove() // Clear previous content - - const g = svg.append('g').attr('transform', `translate(${width / 2},${height / 2})`) - - const color = d3 - .scaleOrdinal() - .domain(Object.keys(data)) - .range(Object.values(ourdnaColours) || d3.schemeCategory10) - - // Correctly map Object.entries to objects with { key, value } format - const formattedData = Object.entries(data).map(([key, value]) => ({ key, value })) - - const pie = d3.pie<{ key: string; value: number }>().value((d) => d.value) - const data_ready = pie(formattedData) - - const arc = d3 - .arc>() - .innerRadius(radius * 0.4) - .outerRadius(radius * 0.8) - - const tooltip = d3 - .select('body') - .append('div') - .style('position', 'absolute') - .style('visibility', 'hidden') - .style('background', 'rgba(0, 0, 0, 0.7)') - .style('color', '#fff') - .style('padding', '5px 10px') - .style('border-radius', '4px') - .style('text-align', 'center') - .text('') - - g.selectAll('allSlices') - .data(data_ready) - .enter() - .append('path') - .attr('d', arc) - .attr('fill', (d) => color(d.data.key)) // Correct reference to key - .attr('stroke', 'white') - .style('stroke-width', '2px') - .style('opacity', 1) - .on('mouseover', function (event, d) { - tooltip.text(`${d.data.key}: ${d.data.value}`) // Correctly reference key and value - return tooltip.style('visibility', 'visible') - }) - .on('mousemove', function (event) { - return tooltip - .style('top', `${event.pageY - 10}px`) - .style('left', `${event.pageX + 10}px`) - }) - .on('mouseout', function () { - return tooltip.style('visibility', 'hidden') - }) - - // Remove any existing legend to avoid overlap - d3.selectAll('.legend').remove() - - // Add legends at the bottom - const legend = svg - .append('g') - .attr('class', 'legend') - .attr('transform', `translate(${0}, ${height / 2 + radius + 40})`) - - // Calculate how many items fit per row dynamically based on width - const itemsPerRow = Math.floor(width / 120) // Estimate based on width - const legendItems = legend - .selectAll('g') - .data(data_ready) - .enter() - .append('g') - .attr('transform', (d, i) => { - const row = Math.floor(i / itemsPerRow) // Wrap items based on available space - const col = i % itemsPerRow - return `translate(${col * 120}, ${row * 20})` - }) - - legendItems - .append('rect') - .attr('width', 18) - .attr('height', 18) - .attr('fill', (d) => color(d.data.key)) // Correct reference to key - - legendItems - .append('text') - .attr('x', 24) - .attr('y', 9) - .attr('dy', '0.35em') - .text((d) => d.data.key) // Correct reference to key - .style('font-size', '12px') - .style('fill', 'var(--color-text-primary)') - }, [data, dimensions]) - - return ( - - - - {icon} - {header} - - - - - - - - - ) -} - -export default OurDonutChart diff --git a/web/src/shared/components/ourdna/StatTile.tsx b/web/src/shared/components/ourdna/StatTile.tsx deleted file mode 100644 index da2557f5a..000000000 --- a/web/src/shared/components/ourdna/StatTile.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import * as React from 'react' -import { Card, Grid, Label, SemanticWIDTHS, Statistic } from 'semantic-ui-react' - -interface StatTileProps { - header: string - stats: { value: string; units: string; unitsColour: string }[] - icon: React.ReactNode - description: string -} - -const StatTile: React.FC = ({ header, stats, icon, description }) => ( - - - - {icon} - {header} - - - - {stats.map((stat, index) => ( - - - - {stat.value} - - - - - - - ))} - - - - - {description} - - -) - -export default StatTile diff --git a/web/src/shared/components/ourdna/TableTile.tsx b/web/src/shared/components/ourdna/TableTile.tsx deleted file mode 100644 index 4243986b4..000000000 --- a/web/src/shared/components/ourdna/TableTile.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import * as React from 'react' -import { Card, Table } from 'semantic-ui-react' - -interface TableTileProps { - header: string - data: Record - columns: Array - icon: React.ReactNode -} - -const TableTile: React.FC = ({ header, data, columns, icon }) => ( - - - - {icon} - {header} - - -
- - - - {columns && - columns.map((column, index) => ( - - {column} - - ))} - - - - {data && - Object.keys(data).map((key, index) => ( - - {key} - {data[key]} - - ))} - -
-
-
-
-
-) - -export default TableTile diff --git a/web/src/shared/components/ourdna/Tile.tsx b/web/src/shared/components/ourdna/Tile.tsx deleted file mode 100644 index 082989b90..000000000 --- a/web/src/shared/components/ourdna/Tile.tsx +++ /dev/null @@ -1,46 +0,0 @@ -import * as React from 'react' -import { Card, Label, Statistic } from 'semantic-ui-react' - -interface TileProps { - header: string - stat: string - units: string - unitsColour: string - description: string - icon: React.ReactNode -} - -const Tile: React.FC = ({ header, stat, units, unitsColour, description, icon }) => ( - - - - {icon} - {header} - - - - {stat} - - - - - - - - {description} - - -) - -export default Tile diff --git a/web/src/shared/components/ourdna/dashboardIcons/AlarmIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/AlarmIcon.tsx deleted file mode 100644 index 6298cae37..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/AlarmIcon.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const AlarmIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - - - - - - - - -) - -export default AlarmIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/BloodSampleIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/BloodSampleIcon.tsx deleted file mode 100644 index 92d99e2c9..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/BloodSampleIcon.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const BloodSampleIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - - - - - - - - -) - -export default BloodSampleIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/ClipboardIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/ClipboardIcon.tsx deleted file mode 100644 index c9e91a76e..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/ClipboardIcon.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const ClipboardIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - -) - -export default ClipboardIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/ClockIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/ClockIcon.tsx deleted file mode 100644 index f174b046b..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/ClockIcon.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const ClockIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - - - - - - - - -) - -export default ClockIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/RocketIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/RocketIcon.tsx deleted file mode 100644 index fa4da74bd..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/RocketIcon.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const RocketIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - - - - - - - - -) - -export default RocketIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/SyringeIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/SyringeIcon.tsx deleted file mode 100644 index 7e1f177f9..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/SyringeIcon.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const SyringeIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - -) - -export default SyringeIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/TestTubeIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/TestTubeIcon.tsx deleted file mode 100644 index 07e17c3eb..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/TestTubeIcon.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const TestTubeIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - - - - - - - - -) - -export default TestTubeIcon diff --git a/web/src/shared/components/ourdna/dashboardIcons/TruckIcon.tsx b/web/src/shared/components/ourdna/dashboardIcons/TruckIcon.tsx deleted file mode 100644 index eddf7d5f8..000000000 --- a/web/src/shared/components/ourdna/dashboardIcons/TruckIcon.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react' -import { Image } from 'semantic-ui-react' - -interface IconProps { - fill?: string - width?: string - height?: string - className?: string - style?: React.CSSProperties -} - -const TruckIcon: React.FC = ({ - fill, - width = '100%', - height = '100%', - className, - style, -}) => ( - - - - -) - -export default TruckIcon From d741d186f659cb1b30f13176ad1730430b8963f0 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Wed, 29 Jan 2025 10:56:02 +1100 Subject: [PATCH 11/12] remove ourdna dashboard v1 backend code --- api/graphql/schema.py | 38 -- db/python/layers/__init__.py | 1 - db/python/layers/ourdna/__init__.py | 0 db/python/layers/ourdna/dashboard.py | 461 --------------------- models/models/__init__.py | 1 - models/models/ourdna.py | 42 -- test/test_ourdna_dashboard.py | 593 --------------------------- web/src/index.css | 6 +- 8 files changed, 2 insertions(+), 1140 deletions(-) delete mode 100644 db/python/layers/ourdna/__init__.py delete mode 100644 db/python/layers/ourdna/dashboard.py delete mode 100644 models/models/ourdna.py delete mode 100644 test/test_ourdna_dashboard.py diff --git a/api/graphql/schema.py b/api/graphql/schema.py index 382d363f0..f8dd4adff 100644 --- a/api/graphql/schema.py +++ b/api/graphql/schema.py @@ -33,7 +33,6 @@ AssayLayer, CohortLayer, FamilyLayer, - OurDnaDashboardLayer, SampleLayer, SequencingGroupLayer, ) @@ -68,7 +67,6 @@ DiscussionInternal, ) from models.models.family import PedRowInternal -from models.models.ourdna import OurDNADashboard, OurDNALostSample from models.models.project import ( FullWriteAccessRoles, ProjectId, @@ -110,30 +108,6 @@ async def m(info: Info[GraphQLContext, 'Query']) -> list[str]: GraphQLAnalysisStatus = strawberry.enum(AnalysisStatus) # type: ignore -@strawberry.experimental.pydantic.type(model=OurDNALostSample, all_fields=True) # type: ignore -class GraphQLOurDNALostSample: - """OurDNA Lost Sample GraphQL model to be used in OurDNA Dashboard""" - - pass # pylint: disable=unnecessary-pass - - -@strawberry.experimental.pydantic.type(model=OurDNADashboard) # type: ignore -class GraphQLOurDNADashboard: - """OurDNA Dashboard model""" - - collection_to_process_end_time: strawberry.scalars.JSON - collection_to_process_end_time_statistics: strawberry.scalars.JSON - collection_to_process_end_time_bucket_statistics: strawberry.scalars.JSON - collection_to_process_end_time_24h: strawberry.scalars.JSON - processing_times_by_site: strawberry.scalars.JSON - processing_times_by_collection_site: strawberry.scalars.JSON - total_samples_by_collection_event_name: strawberry.scalars.JSON - samples_lost_after_collection: list[GraphQLOurDNALostSample] - samples_concentration_gt_1ug: strawberry.scalars.JSON - participants_consented_not_collected: list[int] - participants_signed_not_consented: list[int] - - # Create cohort GraphQL model @strawberry.type class GraphQLCohort: @@ -416,18 +390,6 @@ async def analysis_runner( analysis_runners = await alayer.query(filter_) return [GraphQLAnalysisRunner.from_internal(ar) for ar in analysis_runners] - @strawberry.field - async def ourdna_dashboard( - self, info: Info, root: 'Project' - ) -> 'GraphQLOurDNADashboard': - connection = info.context['connection'] - ourdna_layer = OurDnaDashboardLayer(connection) - if not root.id: - raise ValueError('Project must have an id') - ourdna_dashboard = await ourdna_layer.query(project_id=root.id) - # pylint: disable=no-member - return GraphQLOurDNADashboard.from_pydantic(ourdna_dashboard) - @strawberry.field() async def pedigree( self, diff --git a/db/python/layers/__init__.py b/db/python/layers/__init__.py index 56969ccaf..0c73249d6 100644 --- a/db/python/layers/__init__.py +++ b/db/python/layers/__init__.py @@ -7,7 +7,6 @@ from db.python.layers.cohort import CohortLayer from db.python.layers.comment import CommentLayer from db.python.layers.family import FamilyLayer -from db.python.layers.ourdna.dashboard import OurDnaDashboardLayer from db.python.layers.participant import ParticipantLayer from db.python.layers.project_insights import ProjectInsightsLayer from db.python.layers.sample import SampleLayer diff --git a/db/python/layers/ourdna/__init__.py b/db/python/layers/ourdna/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/db/python/layers/ourdna/dashboard.py b/db/python/layers/ourdna/dashboard.py deleted file mode 100644 index 39110845b..000000000 --- a/db/python/layers/ourdna/dashboard.py +++ /dev/null @@ -1,461 +0,0 @@ -# pylint: disable=too-many-locals -import asyncio -from collections import defaultdict -from datetime import datetime -from functools import cached_property -from math import ceil -from typing import Any - -from db.python.connect import Connection -from db.python.filters import GenericFilter -from db.python.layers.base import BaseLayer -from db.python.layers.participant import ParticipantLayer -from db.python.layers.sample import SampleLayer -from db.python.tables.sample import SampleFilter -from models.models import OurDNADashboard, OurDNALostSample, ProjectId, Sample -from models.models.participant import ParticipantInternal - - -class SampleProcessMeta: - """Helper class to encapsulate sample metadata properties and calculations.""" - - def __init__(self, sample: Sample): - self.sample = sample - self.meta = sample.meta - - def get_property(self, property_name: str) -> Any: - """Get a property from the meta field of a sample.""" - return self.meta.get(property_name) or self.meta.get( - property_name.replace('-', '_') - ) - - @staticmethod - def try_parse_datetime(d: str) -> datetime | None: - """ - Attempts to parse a datetime string in the format '%Y-%m-%d %H:%M:%S'. - - Args: - d (str): The datetime string to parse. - - Returns: - datetime | None: A datetime object if parsing is successful, otherwise None. - """ - if not d or d.strip() == '': - return None - try: - return datetime.strptime(d, '%Y-%m-%d %H:%M:%S') - except TypeError as e: - # Optionally log the error message - print(f'Datetime passed is not a str: {e}') - return None - except ValueError: - # Optionally log the error message - try: - return datetime.strptime(d, '%d/%m/%Y %H:%M:%S') - except TypeError as e: - # Optionally log the error message - print(f'Datetime passed is not a str: {e}') - return None - except ValueError as e: - print(f'Error parsing datetime: {e}') - return None - - @cached_property - def collection_time(self) -> datetime | None: - """Returns the collection time for a sample.""" - return self.try_parse_datetime(self.get_property('collection-time')) - - @cached_property - def process_start_time(self) -> datetime | None: - """Returns the process start time for a sample.""" - return self.try_parse_datetime(self.get_property('process-start-time')) - - @cached_property - def process_end_time(self) -> datetime | None: - """Returns the process end time for a sample.""" - return self.try_parse_datetime(self.get_property('process-end-time')) - - @cached_property - def processing_time(self) -> int | None: - """Get processing time for a sample.""" - if not (self.process_start_time and self.process_end_time): - return None - return int((self.process_end_time - self.process_start_time).total_seconds()) - - @cached_property - def processing_time_by_site(self) -> tuple[str | None, int | None]: - """Get processing times and site for a sample.""" - return self.get_property('processing-site'), self.processing_time - - @cached_property - def processing_time_by_collection_lab(self) -> tuple[str | None, int | None]: - """Get processing times and site for a sample.""" - return self.get_property('collection-lab'), self.processing_time - - @cached_property - def collection_to_process_end_time(self) -> int | None: - """Get the time taken from collection to process end.""" - if self.collection_time and self.process_end_time: - return int((self.process_end_time - self.collection_time).total_seconds()) - return None - - @cached_property - def collection_to_process_start_time(self) -> int | None: - """Get the time taken from collection to process start.""" - if self.collection_time and self.process_start_time: - return int((self.process_start_time - self.collection_time).total_seconds()) - return None - - @cached_property - def time_since_collection(self) -> int | None: - """Get the time since the sample was collected.""" - if self.collection_time: - return int((datetime.now() - self.collection_time).total_seconds()) - return None - - @cached_property - def get_lost_sample_properties(self) -> OurDNALostSample: - """Returns the normalised properties to report for a sample that has been lost""" - return OurDNALostSample( - sample_id=self.sample.id, - collection_time=self.get_property('collection-time'), - process_start_time=self.get_property('process-start-time'), - process_end_time=self.get_property('process-end-time'), - received_time=self.get_property('received-time'), - received_by=self.get_property('received-by'), - collection_lab=self.get_property('collection-lab'), - courier=self.get_property('courier'), - courier_tracking_number=self.get_property('courier-tracking-number'), - courier_scheduled_pickup_time=self.get_property( - 'courier-scheduled-pickup-time' - ), - courier_actual_pickup_time=self.get_property('courier-actual-pickup-time'), - courier_scheduled_dropoff_time=self.get_property( - 'courier-scheduled-dropoff-time' - ), - courier_actual_dropoff_time=self.get_property( - 'courier-actual-dropoff-time' - ), - time_since_collection=self.time_since_collection, - ) - - @cached_property - def is_lost(self) -> bool: - """Returns True if the sample is considered lost, otherwise False.""" - # if time since collection time is > 72 hours and process_start_time is None, return True else False - if self.collection_time: - return ( - datetime.now() - self.collection_time - ).total_seconds() > 72 * 60 * 60 and self.process_start_time is None - return False - - -class OurDnaDashboardLayer(BaseLayer): - """Layer for analysis logic""" - - def __init__(self, connection: Connection): - super().__init__(connection) - - self.sample_layer = SampleLayer(connection) - self.participant_layer = ParticipantLayer(connection) - - self._24_hr_threshold = 24 * 60 * 60 - self._48_hr_threshold = 48 * 60 * 60 - self._72_hr_threshold = 72 * 60 * 60 - - async def query( - self, - project_id: ProjectId, - ) -> OurDNADashboard: - """Get dashboard data""" - samples: list[Sample] = [] - participants: list[ParticipantInternal] = [] - - s, participants = await asyncio.gather( - self.sample_layer.query( - filter_=SampleFilter( - project=GenericFilter(eq=project_id), - # Get the top level samples only - sample_root_id=GenericFilter(isnull=True), - ) - ), - self.participant_layer.get_participants(project=project_id), - ) - - # Converting to external to show stats per sample (with XPG ID) via the GraphQL API - samples = [sample.to_external() for sample in s] - participants_by_id = {p.id: p for p in participants} - - grouped_participant_samples: dict[int, list] = defaultdict(list) - - # Group instances of A by their foreign key - for sample in samples: - if sample.participant_id: - grouped_participant_samples[sample.participant_id].append(sample) - - # Data to be returned - collection_to_process_end_time: dict[str, int] = ( - self.process_collection_to_process_end_times(samples=samples) - ) - collection_to_process_end_time_statistics: dict[str, float | None] = ( - self.process_collection_to_process_end_times_statistics( - collection_to_process_end_time=collection_to_process_end_time - ) - ) - collection_to_process_end_time_bucket_statistics: dict[str, int] = ( - self.process_collection_to_process_end_times_bucket_statistics( - collection_to_process_end_time=collection_to_process_end_time - ) - ) - collection_to_process_end_time_24h: dict[str, int] = ( - self.process_collection_to_process_end_times_24h(samples=samples) - ) - processing_times_by_site: dict[str, dict[int, int]] = ( - self.process_processing_times_by_site(samples=samples) - ) - processing_times_by_collection_site: dict[str, dict[int, int]] = ( - self.process_processing_times_by_collection_site(samples=samples) - ) - total_samples_by_collection_event_name: dict[str, int] = ( - self.process_total_samples_by_collection_event_name(samples=samples) - ) - samples_lost_after_collection: list[OurDNALostSample] = ( - self.process_samples_lost_after_collection(samples=samples) - ) - samples_concentration_gt_1ug: dict[str, float] = ( - self.process_samples_concentration_gt_1ug(samples=samples) - ) - participants_consented_not_collected: list[int] = ( - self.process_participants_consented_not_collected( - participants_by_id, grouped_participant_samples - ) - ) - participants_signed_not_consented: list[int] = ( - self.process_participants_signed_not_consented( - participants_by_id, grouped_participant_samples - ) - ) - - return OurDNADashboard( - collection_to_process_end_time=collection_to_process_end_time, - collection_to_process_end_time_statistics=collection_to_process_end_time_statistics, - collection_to_process_end_time_bucket_statistics=collection_to_process_end_time_bucket_statistics, - collection_to_process_end_time_24h=collection_to_process_end_time_24h, - processing_times_by_site=processing_times_by_site, - processing_times_by_collection_site=processing_times_by_collection_site, - total_samples_by_collection_event_name=total_samples_by_collection_event_name, - samples_lost_after_collection=samples_lost_after_collection, - samples_concentration_gt_1ug=samples_concentration_gt_1ug, - participants_consented_not_collected=participants_consented_not_collected, - participants_signed_not_consented=participants_signed_not_consented, - ) - - def process_collection_to_process_end_times(self, samples: list[Sample]) -> dict: - """Get the time between blood collection and sample processing""" - collection_to_process_end_time: dict[str, int] = {} - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - if processed_meta.collection_to_process_end_time is not None: - collection_to_process_end_time[sample.id] = ( - processed_meta.collection_to_process_end_time - ) - return collection_to_process_end_time - - def process_collection_to_process_end_times_statistics( - self, collection_to_process_end_time: dict[str, int] - ) -> dict[str, float | None]: - """Get the statistics for the time between blood collection and sample processing""" - collection_to_process_end_time_statistics: dict[str, float | None] = {} - - collection_to_process_end_time_statistics['average'] = ( - sum(collection_to_process_end_time.values()) - / len(collection_to_process_end_time) - if collection_to_process_end_time - else None - ) - - collection_to_process_end_time_statistics['min'] = ( - min(collection_to_process_end_time.values()) - if collection_to_process_end_time - else None - ) - - collection_to_process_end_time_statistics['max'] = ( - max(collection_to_process_end_time.values()) - if collection_to_process_end_time - else None - ) - - return collection_to_process_end_time_statistics - - def process_collection_to_process_end_times_bucket_statistics( - self, collection_to_process_end_time: dict[str, int] - ) -> dict[str, int]: - """Get the statistics for the time between blood collection and sample processing in buckets of 24 hours, 48 hours and 72 hours.""" - collection_to_process_end_time_bucket_statistics: dict[str, int] = defaultdict( - int - ) - - collection_to_process_end_time_bucket_statistics['24h'] = sum( - 1 - for time in collection_to_process_end_time.values() - if time <= self._24_hr_threshold - ) - - collection_to_process_end_time_bucket_statistics['48h'] = sum( - 1 - for time in collection_to_process_end_time.values() - if self._24_hr_threshold < time <= self._48_hr_threshold - ) - - collection_to_process_end_time_bucket_statistics['72h'] = sum( - 1 - for time in collection_to_process_end_time.values() - if self._48_hr_threshold < time <= self._72_hr_threshold - ) - - collection_to_process_end_time_bucket_statistics['>72h'] = sum( - 1 - for time in collection_to_process_end_time.values() - if time > self._72_hr_threshold - ) - - return collection_to_process_end_time_bucket_statistics - - def process_collection_to_process_end_times_24h( - self, samples: list[Sample] - ) -> dict: - """Get the time between blood collection and sample processing""" - collection_to_process_end_time_24h: dict[str, int] = {} - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - if ( - processed_meta.collection_to_process_end_time - and processed_meta.collection_to_process_end_time - > self._24_hr_threshold - ): - collection_to_process_end_time_24h[sample.id] = ( - processed_meta.collection_to_process_end_time - ) - return collection_to_process_end_time_24h - - def process_processing_times_by_site(self, samples: list[Sample]) -> dict: - """Get the processing times by site""" - processing_times_by_site: dict[str, dict[int, int]] = defaultdict( - lambda: defaultdict(int) - ) - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - processing_site, processing_time = processed_meta.processing_time_by_site - if processing_site and processing_time: - hour_bucket = ceil(processing_time / 3600) - processing_times_by_site[processing_site][hour_bucket] += 1 - - # ENABLE THIS IF WE WANT VALUES FOR ALL HOURS - # for site in processing_times_by_site: - # min_bucket = min(processing_times_by_site[site]) - # max_bucket = max(processing_times_by_site[site]) - # for i in range(min_bucket, max_bucket + 1): - # processing_times_by_site[site].setdefault(i, 0) - - return processing_times_by_site - - def process_processing_times_by_collection_site( - self, samples: list[Sample] - ) -> dict: - """Get the processing times by collection site""" - processing_times_by_collection_site: dict[str, dict[int, int]] = defaultdict( - lambda: defaultdict(int) - ) - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - processing_collection_site, processing_time = ( - processed_meta.processing_time_by_collection_lab - ) - if processing_collection_site and processing_time: - hour_bucket = ceil(processing_time / 3600) - processing_times_by_collection_site[processing_collection_site][ - hour_bucket - ] += 1 - # ENABLE THIS IF WE WANT VALUES FOR ALL HOURS - # for site in processing_times_by_collection_site: - # min_bucket = min(processing_times_by_collection_site[site]) - # max_bucket = max(processing_times_by_collection_site[site]) - # for i in range(min_bucket, max_bucket + 1): - # processing_times_by_collection_site[site].setdefault(i, 0) - - return processing_times_by_collection_site - - def process_total_samples_by_collection_event_name( - self, samples: list[Sample] - ) -> dict: - """Get total number of samples collected from each type of collection-event-name""" - total_samples_by_collection_event_name: dict[str, int] = defaultdict(int) - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - _collection_event_name = processed_meta.get_property( - 'collection-event-type' - ) - total_samples_by_collection_event_name[ - _collection_event_name or 'Unknown' - ] += 1 - return total_samples_by_collection_event_name - - def process_samples_lost_after_collection( - self, samples: list[Sample] - ) -> list[OurDNALostSample]: - """Get total number of many samples have been lost, EG: participants have been consented, blood collected, not processed (etc), Alert here (highlight after 72 hours)""" - samples_lost_after_collection: list[OurDNALostSample] = [] - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - if processed_meta.is_lost: - samples_lost_after_collection.append( - processed_meta.get_lost_sample_properties - ) - - return samples_lost_after_collection - - def process_samples_concentration_gt_1ug(self, samples: list[Sample]) -> dict: - """Get the concentration of the sample where the concentration is more than 1 ug of DNA""" - samples_concentration_gt_1ug: dict[str, float] = {} - - for sample in samples: - processed_meta = SampleProcessMeta(sample) - concentration = processed_meta.get_property('concentration') - if concentration and float(concentration) > 1: - samples_concentration_gt_1ug[sample.id] = float(concentration) - return samples_concentration_gt_1ug - - def process_participants_consented_not_collected( - self, - participants: dict[int, ParticipantInternal], - grouped_participants_samples: dict[int, list[Sample]], - ) -> list[int]: - """Get the participants who have been consented but have not had a sample collected""" - filtered_participants: list[int] = [] - for participant_id, samples in grouped_participants_samples.items(): - participant = participants[participant_id] - if participant.meta.get('consent') and any( - SampleProcessMeta(sample).collection_time is None for sample in samples - ): - filtered_participants.append(participant.id) - return filtered_participants - - def process_participants_signed_not_consented( - self, - participants: dict[int, ParticipantInternal], - grouped_participants_samples: dict[int, list[Sample]], - ) -> list[int]: - """Get the participants who have signed but have not been consented""" - filtered_participants: list[int] = [] - for participant_id in grouped_participants_samples: - participant = participants[participant_id] - if not participant.meta.get('consent'): - filtered_participants.append(participant.id) - return filtered_participants diff --git a/models/models/__init__.py b/models/models/__init__.py index b7d937684..aa5c781a8 100644 --- a/models/models/__init__.py +++ b/models/models/__init__.py @@ -29,7 +29,6 @@ FamilySimpleInternal, PedRowInternal, ) -from models.models.ourdna import OurDNADashboard, OurDNALostSample from models.models.output_file import OutputFileInternal from models.models.participant import ( NestedParticipant, diff --git a/models/models/ourdna.py b/models/models/ourdna.py deleted file mode 100644 index 9aff10164..000000000 --- a/models/models/ourdna.py +++ /dev/null @@ -1,42 +0,0 @@ -from collections import defaultdict - -from pydantic import BaseModel - - -class OurDNALostSample(BaseModel): - """Model for OurDNA Lost Sample""" - - sample_id: str - time_since_collection: int | None - collection_time: str - process_start_time: str | None - process_end_time: str | None - received_time: str | None - received_by: str | None - collection_lab: str - courier: str | None - courier_tracking_number: str | None - courier_scheduled_pickup_time: str | None - courier_actual_pickup_time: str | None - courier_scheduled_dropoff_time: str | None - courier_actual_dropoff_time: str | None - - -class OurDNADashboard(BaseModel): - """Model for OurDNA Dashboard""" - - collection_to_process_end_time: dict[str, int] = {} - collection_to_process_end_time_statistics: dict[str, float | None] = {} - collection_to_process_end_time_bucket_statistics: dict[str, int] = {} - collection_to_process_end_time_24h: dict[str, int] = {} - processing_times_by_site: dict[str, dict[int, int]] = defaultdict( - lambda: defaultdict(int) - ) - processing_times_by_collection_site: dict[str, dict[int, int]] = defaultdict( - lambda: defaultdict(int) - ) - total_samples_by_collection_event_name: dict[str, int] = defaultdict(int) - samples_lost_after_collection: list[OurDNALostSample] = [] - samples_concentration_gt_1ug: dict[str, float] = {} - participants_consented_not_collected: list[int] = [] - participants_signed_not_consented: list[int] = [] diff --git a/test/test_ourdna_dashboard.py b/test/test_ourdna_dashboard.py deleted file mode 100644 index db8618260..000000000 --- a/test/test_ourdna_dashboard.py +++ /dev/null @@ -1,593 +0,0 @@ -from collections import defaultdict -from datetime import datetime -from math import ceil -from test.testbase import DbIsolatedTest, run_as_sync - -from db.python.enum_tables.sample_type import SampleTypeTable -from db.python.layers.ourdna.dashboard import OurDnaDashboardLayer -from db.python.layers.participant import ParticipantLayer -from db.python.layers.sample import SampleLayer -from models.models import ( - PRIMARY_EXTERNAL_ORG, - OurDNADashboard, - ParticipantUpsertInternal, - SampleUpsert, - SampleUpsertInternal, -) - - -def str_to_datetime(timestamp_str): - """Convert string timestamp to datetime""" - return datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S') - - -def convert_to_dict(d): - """Converts a defaultdict into a dict recursively""" - if isinstance(d, defaultdict): - d = {k: convert_to_dict(v) for k, v in d.items()} - return d - - -class OurDNADashboardTest(DbIsolatedTest): - """Test sample class""" - - @run_as_sync - async def setUp(self) -> None: - # don't need to await because it's tagged @run_as_sync - super().setUp() - self.odd = OurDnaDashboardLayer(self.connection) - self.sl = SampleLayer(self.connection) - self.pl = ParticipantLayer(self.connection) - - self.stt = SampleTypeTable(self.connection) - await self.stt.insert('ebld') - - participants = await self.pl.upsert_participants( - [ - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX01'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 1}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test01'}, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - 'process-start-time': '2022-07-06 16:28:00', - 'process-end-time': '2022-07-06 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'collection-event-type': 'walk-in', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.45, - }, - type='ebld', - active=True, - nested_samples=[ - # add a random sample here to test it's not collected - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test01-01'}, - type='blood', - meta={ - # something wild - 'collection-time': '1999-01-01 12:34:56', - }, - ) - ], - ) - ], - ), - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX02'}, - reported_sex=1, - karyotype='XY', - meta={'field': 2}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test02'}, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'BBV', - # 'process-start-time': '2022-07-06 16:28:00', - # 'process-end-time': '2022-07-06 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'collection-event-type': 'OSS', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 0.98, - }, - type='ebld', - active=True, - ) - ], - ), - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX03'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 3}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test03'}, - meta={ - # 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - # 'process-start-time': '2022-07-03 16:28:00', - # 'process-end-time': '2022-07-03 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.66, - }, - type='ebld', - active=True, - ) - ], - ), - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX04'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 3}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test04'}, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - 'process-start-time': '2022-07-03 16:28:00', - 'process-end-time': '2022-07-04 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.66, - }, - type='ebld', - active=True, - ) - ], - ), - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX05'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 3}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test05'}, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - 'process-start-time': '2022-07-03 16:28:00', - 'process-end-time': '2022-07-05 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.66, - }, - type='ebld', - active=True, - ) - ], - ), - ParticipantUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX06'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 3}, - samples=[ - SampleUpsertInternal( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test06'}, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - 'process-start-time': '2022-07-03 16:28:00', - 'process-end-time': '2022-07-06 11:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.66, - }, - type='ebld', - active=True, - ) - ], - ), - ], - ) - - self.participants_external_objects = [ - participant.to_external() for participant in participants - ] - - self.sample_external_objects: list[SampleUpsert] = [] - self.sample_internal_objects: list[SampleUpsertInternal] = [] - - self.sample_internal_objects.extend( - sample for participant in participants for sample in participant.samples - ) - self.sample_external_objects.extend( - sample.to_external() for sample in self.sample_internal_objects - ) - - @run_as_sync - async def test_get_dashboard(self): - """Test get_dashboard""" - dashboard = await self.odd.query(project_id=self.project_id) - - # Check that the dashboard is not empty and is a dict - self.assertTrue(dashboard) - self.assertIsInstance(dashboard, OurDNADashboard) - - @run_as_sync - async def test_collection_to_process_end_time(self): - """I want to know how long it took between blood collection and sample processing""" - dashboard = await self.odd.query(project_id=self.project_id) - collection_to_process_end_time = dashboard.collection_to_process_end_time - - # Check that collection_to_process_end_time is not empty and is a dict - self.assertTrue(collection_to_process_end_time) - self.assertIsInstance(collection_to_process_end_time, dict) - - samples_filtered: list[str] = [] - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - collection_time = sample.meta.get('collection-time') - process_end_time = sample.meta.get('process-end-time') - # Skip samples that don't have collection_time or process_end_time - if not collection_time or not process_end_time: - continue - time_difference = str_to_datetime(process_end_time) - str_to_datetime( - collection_time - ) - if time_difference.total_seconds(): - # Check that the time difference matches - assert isinstance(sample.id, str) - - samples_filtered.append(sample.id) - - self.assertEqual( - time_difference.total_seconds(), - collection_to_process_end_time[sample.id], - ) - - # Check the number of samples in the cohort - self.assertCountEqual(collection_to_process_end_time.keys(), samples_filtered) - - for _sample_id, time_diff in collection_to_process_end_time.items(): - self.assertGreater(time_diff, 0) - self.assertIsInstance(time_diff, int) - - @run_as_sync - async def test_collection_to_process_end_time_24h(self): - """I want to know which samples took more than 24 hours between blood collection and sample processing completion""" - dashboard = await self.odd.query(project_id=self.project_id) - collection_to_process_end_time_24h = ( - dashboard.collection_to_process_end_time_24h - ) - - # Check that collection_to_process_end_time is not empty and is a dict - self.assertTrue(collection_to_process_end_time_24h) - self.assertIsInstance(collection_to_process_end_time_24h, dict) - - samples_filtered: list[str] = [] - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - collection_time = sample.meta.get('collection-time') - process_end_time = sample.meta.get('process-end-time') - # Skip samples that don't have collection_time or process_end_time - if not collection_time or not process_end_time: - continue - time_difference = str_to_datetime(process_end_time) - str_to_datetime( - collection_time - ) - if time_difference.total_seconds() > 24 * 3600: - # Check that the time difference matches - assert isinstance(sample.id, str) - - samples_filtered.append(sample.id) - - self.assertEqual( - time_difference.total_seconds(), - collection_to_process_end_time_24h[sample.id], - ) - - # check that there are a correct number of matching results - self.assertCountEqual( - collection_to_process_end_time_24h.keys(), samples_filtered - ) - - @run_as_sync - async def test_processing_times_per_site(self): - """I want to know what the sample processing times were for samples at each designated site""" - dashboard = await self.odd.query(project_id=self.project_id) - processing_times_by_site = dashboard.processing_times_by_site - - # Check that processing_times_per_site is not empty and is a dict - self.assertTrue(processing_times_by_site) - self.assertIsInstance(processing_times_by_site, dict) - - sample_tally: dict[str, dict[int, int]] = defaultdict(lambda: defaultdict(int)) - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - processing_site = sample.meta.get('processing-site', 'Unknown') - process_start_time = sample.meta.get('process-start-time') - process_end_time = sample.meta.get('process-end-time') - # Skip samples that don't have process_start_time or process_end_time - if not process_start_time or not process_end_time: - continue - time_difference = str_to_datetime(process_end_time) - str_to_datetime( - process_start_time - ) - current_bucket = ceil(time_difference.total_seconds() / 3600) - sample_tally[processing_site][current_bucket] += 1 - - # ENABLE THIS IF WE WANT VALUES FOR ALL HOURS - # for site in sample_tally: - # min_bucket = min(sample_tally[site]) - # max_bucket = max(sample_tally[site]) - # for i in range(min_bucket, max_bucket + 1): - # sample_tally[site].setdefault(i, 0) - - # Checks that we have identical dicts (by extension, keys and their values) - self.assertDictEqual(processing_times_by_site, convert_to_dict(sample_tally)) - - @run_as_sync - async def test_processing_times_per_collection_site(self): - """I want to know how long it took to process samples collected at each site""" - dashboard = await self.odd.query(project_id=self.project_id) - processing_times_by_collection_site = ( - dashboard.processing_times_by_collection_site - ) - - # Check that processing_times_per_site is not empty and is a dict - self.assertTrue(processing_times_by_collection_site) - self.assertIsInstance(processing_times_by_collection_site, dict) - - sample_tally: dict[str, dict[int, int]] = defaultdict(lambda: defaultdict(int)) - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - collection_site = sample.meta.get('collection-lab', 'Unknown') - process_start_time = sample.meta.get('process-start-time') - process_end_time = sample.meta.get('process-end-time') - # Skip samples that don't have process_start_time or process_end_time - if not process_start_time or not process_end_time: - continue - time_difference = str_to_datetime(process_end_time) - str_to_datetime( - process_start_time - ) - current_bucket = ceil(time_difference.total_seconds() / 3600) - sample_tally[collection_site][current_bucket] += 1 - - # ENABLE THIS IF WE WANT VALUES FOR ALL HOURS - # for site in sample_tally: - # min_bucket = min(sample_tally[site]) - # max_bucket = max(sample_tally[site]) - # for i in range(min_bucket, max_bucket + 1): - # sample_tally[site].setdefault(i, 0) - - # Checks that we have identical dicts (by extension, keys and their values) - self.assertDictEqual(processing_times_by_collection_site, sample_tally) - - @run_as_sync - async def test_total_samples_by_collection_event_name(self): - """I want to know how many samples were collected from walk-ins vs during events or scheduled activities""" - dashboard = await self.odd.query(project_id=self.project_id) - total_samples_by_collection_event_name = ( - dashboard.total_samples_by_collection_event_name - ) - - # Check that total_samples_by_collection_event_name is not empty and is a dict - self.assertTrue(total_samples_by_collection_event_name) - self.assertIsInstance(total_samples_by_collection_event_name, dict) - - sample_tally: dict[str, int] = defaultdict() - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - event_name = sample.meta.get('collection-event-type', 'Unknown') - if event_name in sample_tally: - sample_tally[event_name] += 1 - else: - sample_tally[event_name] = 1 - - # Check that the tally and the total_samples_by_collection_event_name are the same, by extension, keys and their values - self.assertDictEqual(total_samples_by_collection_event_name, sample_tally) - - @run_as_sync - async def test_samples_lost_after_collection(self): - """I need to know how many samples have been lost, EG: participants have been consented, blood collected, not processed""" - dashboard = await self.odd.query(project_id=self.project_id) - samples_lost_after_collection = dashboard.samples_lost_after_collection - - # Check that samples_lost_after_collection is not empty and is a dict - self.assertTrue(samples_lost_after_collection) - self.assertIsInstance(samples_lost_after_collection, list) - - # Check that the number of samples in the list is correct - samples_filtered: list[str] = [] - sample_ids_lost_after_collection = [ - sample.sample_id for sample in samples_lost_after_collection - ] - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - collection_time = sample.meta.get('collection-time') - process_start_time = sample.meta.get('process-start-time') - # Skip samples that don't have collection_time or process_end_time - if not collection_time: - continue - time_difference = datetime.now() - str_to_datetime(collection_time) - if time_difference.total_seconds() > 72 * 3600 and not process_start_time: - # Check that the time difference matches - assert isinstance(sample.id, str) - - samples_filtered.append(sample.id) - - for sample_data in samples_lost_after_collection: - if sample_data.sample_id == sample.id: - self.assertEqual( - int(time_difference.total_seconds()), - sample_data.time_since_collection, - ) - - # check that there are a correct number of matching results - self.assertCountEqual(sample_ids_lost_after_collection, samples_filtered) - - # TODO: Add assertions VB - - @run_as_sync - async def test_samples_more_than_1ug_dna(self): - """I want to generate a list of samples containing more than 1 ug of DNA to prioritise them for long-read sequencing applications""" - dashboard = await self.odd.query(project_id=self.project_id) - samples_more_than_1ug_dna = dashboard.samples_concentration_gt_1ug - - # Check that samples_concentratiom_gt_1ug is not empty and is a dict - self.assertTrue(samples_more_than_1ug_dna) - self.assertIsInstance(samples_more_than_1ug_dna, dict) - - # Check that the number of samples in the list is correct - samples_filtered: list[str] = [] - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - if sample.meta['concentration'] and sample.meta['concentration'] > 1: - assert isinstance(sample.id, str) - - samples_filtered.append(sample.id) - - self.assertCountEqual(samples_more_than_1ug_dna, samples_filtered) - - @run_as_sync - async def test_participants_consented_not_collected(self): - """I want to know how many people who have consented and NOT given blood""" - dashboard = await self.odd.query(project_id=self.project_id) - # print(dashboard) - participants_consented_not_collected = ( - dashboard.participants_consented_not_collected - ) - - # Check that participants_consented_not_collected is not empty and is a dict - self.assertTrue(participants_consented_not_collected) - self.assertIsInstance(participants_consented_not_collected, list) - - # Check that the number of participants in the list is correct - participants_filtered: list[int] = [] - for participant in self.participants_external_objects: - assert isinstance(participant.meta, dict) - samples_for_participant = [ - sample - for sample in self.sample_external_objects - if sample.participant_id == participant.id - and isinstance(sample.meta, dict) - ] - if participant.meta.get('consent') and any( - isinstance(sample.meta, dict) - and sample.meta.get('collection-time') is None - for sample in samples_for_participant - ): - assert isinstance(participant.id, int) - participants_filtered.append(participant.id) - - self.assertCountEqual( - participants_consented_not_collected, participants_filtered - ) - - @run_as_sync - async def test_participants_signed_not_consented(self): - """I want to know how many people have signed up but not consented""" - dashboard = await self.odd.query(project_id=self.project_id) - # print(dashboard) - participants_signed_not_consented = dashboard.participants_signed_not_consented - - # Check that participants_signed_not_consented is not empty and is a dict - self.assertTrue(participants_signed_not_consented) - self.assertIsInstance(participants_signed_not_consented, list) - - # Check that the number of participants in the list is correct - participants_filtered: list[int] = [] - for participant in self.participants_external_objects: - assert isinstance(participant.meta, dict) - if not participant.meta.get('consent'): - assert isinstance(participant.id, int) - participants_filtered.append(participant.id) - - self.assertCountEqual(participants_signed_not_consented, participants_filtered) - - @run_as_sync - async def test_collection_to_process_end_time_bucket_statistics(self): - """Bucket the number of samples processed within 24, 48 and 72 hours""" - dashboard = await self.odd.query(project_id=self.project_id) - collection_to_process_end_time_bucket_statistics = ( - dashboard.collection_to_process_end_time_bucket_statistics - ) - - # Check that collection_to_process_end_time_bucket_statistics is not empty and is a dict - self.assertTrue(collection_to_process_end_time_bucket_statistics) - self.assertIsInstance(collection_to_process_end_time_bucket_statistics, dict) - - # Check that the number of samples in the list is correct - samples_filtered: dict[str, int] = { - '24h': 0, - '48h': 0, - '72h': 0, - '>72h': 0, - } - for sample in self.sample_external_objects: - assert isinstance(sample.meta, dict) - collection_time = sample.meta.get('collection-time') - process_end_time = sample.meta.get('process-end-time') - # Skip samples that don't have collection_time or process_end_time - if not collection_time or not process_end_time: - continue - time_difference = str_to_datetime(process_end_time) - str_to_datetime( - collection_time - ) - current_bucket = ceil(time_difference.total_seconds() / 3600) - if current_bucket <= 24: - samples_filtered['24h'] += 1 - elif current_bucket <= 48: - samples_filtered['48h'] += 1 - elif current_bucket <= 72: - samples_filtered['72h'] += 1 - else: - samples_filtered['>72h'] += 1 - - self.assertDictEqual( - collection_to_process_end_time_bucket_statistics, samples_filtered - ) diff --git a/web/src/index.css b/web/src/index.css index f119aba34..de65423d2 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -346,8 +346,6 @@ html[data-theme='dark-mode'] .ui.table { color: var(--color-text-primary); } - - /* Project Insights dashboard */ .html-tooltip .MuiTooltip-tooltip { background-color: #f5f5f9; @@ -469,7 +467,6 @@ html[data-theme='dark-mode'] .ui.table { display: flex !important; } - /* hax to fix bad cascading styles. There are a few conflicting css frameworks on metamist for strange historical reasons. @@ -477,6 +474,7 @@ html[data-theme='dark-mode'] .ui.table { is sometimes necessary to remove styles that cascade from bad global definitions. */ -.MuiTablePagination-selectLabel, .MuiTablePagination-displayedRows { +.MuiTablePagination-selectLabel, +.MuiTablePagination-displayedRows { margin-bottom: 0 !important; } From 405c74ee8e5881cdb8210deb7fc5a100d3780a14 Mon Sep 17 00:00:00 2001 From: Dan Coates Date: Thu, 30 Jan 2025 17:12:52 +1100 Subject: [PATCH 12/12] update script to generate data for ourdna dashbaord --- test/data/generate_ourdna_data.py | 305 ++++++++++++++++++++-------- web/src/pages/report/SqlQueryUI.tsx | 1 + 2 files changed, 216 insertions(+), 90 deletions(-) diff --git a/test/data/generate_ourdna_data.py b/test/data/generate_ourdna_data.py index eb12856f3..e2f3ee77e 100644 --- a/test/data/generate_ourdna_data.py +++ b/test/data/generate_ourdna_data.py @@ -1,118 +1,242 @@ #!/usr/bin/env python3 """ -This is a simple script to generate 3 participants & its samples in the ourdna project +This is a simple script to generate some participants and samples for testing ourdna Local Backend API needs to run prior executing this script -NOTE: This is WIP and will be updated with more features -If you want to regenerate the data you would need to -delete records from table sample and participant first """ import argparse import asyncio +import datetime +import random +import uuid +from typing import Sequence, Union from metamist.apis import ParticipantApi from metamist.models import ParticipantUpsert, SampleUpsert PRIMARY_EXTERNAL_ORG = '' -PARTICIPANTS = [ - ParticipantUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX01'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 1}, - samples=[ + +ANCESTRIES = [ + 'Vietnamese', + 'Filipino', + 'Australian', + 'Spanish', + 'Acehnese', + 'Afghan', + 'African American', + 'American', + 'Amhara', + 'British', + 'Chinese', + 'English', + 'German', + 'Greek', + 'Indian', + 'Irish', + 'Italian', + 'Japanese', + 'Malay', + 'Norwegian', + 'Scottish', + 'Venezuelan', +] + +BIRTHPLACES = [ + 'Philippines', + 'Vietnam', + 'Cambodia', + 'Australia', + "I don't know", + "I'd prefer not to say", + 'Thailand', +] + +LANGUAGES = [ + 'Vietnamese', + 'Filipino', + 'Tagalog', + 'Cebuano', + 'English', + 'Bisaya', + 'Ilonggo (Hiligaynon)', + 'Cantonese', + 'Other Southern Asian Languages', + 'Spanish', + 'Ilokano', + 'Bikol', + 'American Languages', + 'IIokano', + 'Hawaiian English', + 'Armenian', + 'Khmer', + 'Acehnese', + 'Other Southeast Asian Languages', + 'Urdu', + 'French', + 'Japanese', + 'Thai', + 'Italian', + 'Croatian', + 'Chin Haka', + 'Arabic', +] + + +event_type = ['OSS', 'Walk in'] +processing_site = ['bbv', 'Westmead'] + + +def random_date_range(): + "Generate a random date range" + # Throw in the occasional invalid date to simulate the current state of the data + # this should be removed once the data is cleaned up + if random.randint(0, 10) == 0: + return 'N/A', 'N/A' + start_date = datetime.datetime.now() - datetime.timedelta( + days=random.randint(1, 365) + ) + end_date = start_date + datetime.timedelta(hours=random.randint(1, 150)) + return start_date.strftime('%Y-%m-%d %H:%M:%S'), end_date.strftime( + '%Y-%m-%d %H:%M:%S' + ) + + +def random_choice( + choices: Sequence[Union[str, bool, int]], weight_by_index: bool = False +): + "Pick a random choice from a list of choices" + weighted_choices = list(choices) + if weight_by_index: + for i, choice in enumerate(choices): + weighted_choices.extend([choice] * (len(choices) - i)) + return weighted_choices[random.randint(0, len(weighted_choices) - 1)] + + +def random_list( + choices: Sequence[Union[str, bool, int]], + weight_by_index: bool = False, + min_len: int = 1, + max_len: int = 5, +): + "Generate a random list of choices" + result: list[Union[str, bool, int]] = [] + desired_len = random.randint(min_len, max_len) + if desired_len > len(choices): + raise ValueError( + f'Desired length {desired_len} is greater than the number of choices {len(choices)}' + ) + while len(result) < desired_len: + choice = random_choice(choices, weight_by_index) + if choice not in result: + result.append(choice) + + return result + + +def create_samples(): + """Create a sample with nested samples""" + start_date, end_date = random_date_range() + + meta = { + 'collection-time': start_date, + 'process-end-time': end_date, + 'collection-event-type': random_choice(event_type), + 'processing-site': random_choice(processing_site), + } + + sample = SampleUpsert( + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + type='blood', + active=True, + nested_samples=[ SampleUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test01'}, - type='blood', + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + type='guthrie-card', active=True, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - 'process-start-time': '2022-07-06 16:28:00', - 'process-end-time': '2022-07-06 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'collection-event-name': 'walk-in', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.45, - }, - ) - ], - ), - ParticipantUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX02'}, - reported_sex=1, - karyotype='XY', - meta={'field': 2}, - samples=[ + meta=meta, + ), SampleUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test02'}, - type='blood', + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + type='plasma', active=True, - meta={ - 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'BBV', - 'process-start-time': '2022-07-06 16:28:00', - 'process-end-time': '2022-07-06 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'collection-event-name': 'EventA', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 0.98, - }, - ) - ], - ), - ParticipantUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'EX03'}, - reported_sex=2, - karyotype='XX', - meta={'consent': True, 'field': 3}, - samples=[ + meta=meta, + ), + SampleUpsert( + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + type='buffy-coat', + active=True, + meta=meta, + ), SampleUpsert( - external_ids={PRIMARY_EXTERNAL_ORG: 'Test03'}, - type='blood', + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + type='pbmc', active=True, - meta={ - # 'collection-time': '2022-07-03 13:28:00', - 'processing-site': 'Garvan', - # 'process-start-time': '2022-07-03 16:28:00', - # 'process-end-time': '2022-07-03 19:28:00', - 'received-time': '2022-07-03 14:28:00', - 'received-by': 'YP', - 'collection-lab': 'XYZ LAB', - 'courier': 'ABC COURIERS', - 'courier-tracking-number': 'ABCDEF12562', - 'courier-scheduled-pickup-time': '2022-07-03 13:28:00', - 'courier-actual-pickup-time': '2022-07-03 13:28:00', - 'courier-scheduled-dropoff-time': '2022-07-03 13:28:00', - 'courier-actual-dropoff-time': '2022-07-03 13:28:00', - 'concentration': 1.66, - }, - ) + meta=meta, + ), ], - ), -] + meta=meta, + ) + + return sample + +def create_participant(): + """Create a participant with nested samples""" + participant = ParticipantUpsert( + external_ids={PRIMARY_EXTERNAL_ORG: str(uuid.uuid4())}, + reported_sex=random_choice([1, 2]), + meta={ + 'ancestry-participant-ancestry': random_list( + ANCESTRIES, weight_by_index=True, min_len=1, max_len=2 + ), + 'ancestry-mother-ancestry': random_list( + ANCESTRIES, weight_by_index=True, min_len=1, max_len=2 + ), + 'ancestry-father-ancestry': random_list( + ANCESTRIES, weight_by_index=True, min_len=1, max_len=2 + ), + 'ancestry-mother-birthplace': random_list( + BIRTHPLACES, weight_by_index=True, min_len=1, max_len=2 + ), + 'ancestry-father-birthplace': random_list( + BIRTHPLACES, weight_by_index=True, min_len=1, max_len=2 + ), + 'ancestry-language-other-than-english': random_list( + LANGUAGES, weight_by_index=True, min_len=1, max_len=2 + ), + 'birth-year': random.randint(1900, 2010), + 'blood-consent': random_choice(['yes', 'no']), + 'informed-consent': random_choice(['yes', 'no']), + 'choice-receive-genetic-info': random_choice(['yes', 'no']), + 'choice-family-receive-genetic-info': random_choice(['yes', 'no']), + 'choice-recontact': random_choice(['yes', 'no']), + 'choice-general-updates': random_choice(['yes', 'no']), + 'choice-use-of-cells-in-future-research-consent': random_choice( + ['yes', 'no'] + ), + 'choice-use-of-cells-in-future-research-understanding': random_list( + [ + 'grown_indefinitely', + 'used_by_approved_researchers', + ], + min_len=1, + max_len=2, + ), + }, + samples=[create_samples()], + ) + + return participant -async def main(project='ourdna'): + +async def main(project='ourdna', num_participants=10): """Doing the generation for you""" participant_api = ParticipantApi() - participants_rec = participant_api.upsert_participants(project, PARTICIPANTS) + + participants = [create_participant() for _ in range(num_participants)] + participants_rec = participant_api.upsert_participants(project, participants) print('inserted participants:', participants_rec) @@ -121,5 +245,6 @@ async def main(project='ourdna'): description='Script for generating data in the ourdna test project' ) parser.add_argument('--project', type=str, default='ourdna') + parser.add_argument('--num-participants', type=str, default=10) args = vars(parser.parse_args()) asyncio.new_event_loop().run_until_complete(main(**args)) diff --git a/web/src/pages/report/SqlQueryUI.tsx b/web/src/pages/report/SqlQueryUI.tsx index 32291966f..778438b45 100644 --- a/web/src/pages/report/SqlQueryUI.tsx +++ b/web/src/pages/report/SqlQueryUI.tsx @@ -425,6 +425,7 @@ export default function SqlQueryUi() { {projectName && (selectedTableQuery || tableQueryValue) && ( )}