Billing cost by category (#629)

* Added simple Total Cost By Batch Page.

* Fixed autoselect day format.

* Fixing day format for autoselect (missing leading 0)

* Added first draft of billing page to show detail SKU per selected cost category over selected time periods (day, week, month or invoice month)

* Small fix for BillingCostByBatch page, disable search if searchBy is empty or < 6 chars.

* New: Billing API GET namespaces, added namespace to allowed fields for total cost.

* Implemented HorizontalStackedBarChart, updated Billing By Invoice Month page to enable toggle between chart and table view.

api/routes/billing.py

@@ -253,6 +253,24 @@ async def get_invoice_months(
     return records
 
 
+@router.get(
+    '/namespaces',
+    response_model=list[str],
+    operation_id='getNamespaces',
+)
+@alru_cache(ttl=BILLING_CACHE_RESPONSE_TTL)
+async def get_namespaces(
+    author: str = get_author,
+) -> list[str]:
+    """
+    Get list of all namespaces in database
+    Results are sorted DESC
+    """
+    billing_layer = initialise_billing_layer(author)
+    records = await billing_layer.get_namespaces()
+    return records
+
+
 @router.post(
     '/query', response_model=list[BillingRowRecord], operation_id='queryBilling'
 )
@@ -469,6 +487,19 @@ async def get_total_cost(
             "order_by": {"cost": true}
         }
 
+    18. Get weekly total cost by sku for selected cost_category, order by day ASC:
+
+        {
+            "fields": ["sku"],
+            "start_date": "2022-11-01",
+            "end_date": "2023-12-07",
+            "filters": {
+                "cost_category": "Cloud Storage"
+            },
+            "order_by": {"day": false},
+            "time_periods": "week"
+        }
+
     """
     billing_layer = initialise_billing_layer(author)
     records = await billing_layer.get_total_cost(query)

db/python/layers/billing_db.py

@@ -1,7 +1,7 @@
 import re
 from collections import Counter, defaultdict
 from datetime import datetime
-from typing import Any
+from typing import Any, Tuple
 
 from google.cloud import bigquery
 
@@ -20,6 +20,7 @@
     BillingColumn,
     BillingCostBudgetRecord,
     BillingRowRecord,
+    BillingTimePeriods,
     BillingTotalCostQueryModel,
     BillingTotalCostRecord,
 )
@@ -32,6 +33,29 @@ def abbrev_cost_category(cost_category: str) -> str:
     return 'S' if cost_category == 'Cloud Storage' else 'C'
 
 
+def prepare_time_periods(query: BillingTotalCostQueryModel) -> Tuple[str, str, str]:
+    """Prepare Time periods grouping and parsing formulas"""
+    day_parse_formula = ''
+    day_field = ''
+    day_grp = 'day, '
+
+    # Based on specified time period, add the corresponding column
+    if query.time_periods == BillingTimePeriods.DAY:
+        day_field = 'day, '
+        day_parse_formula = 'day, '
+    elif query.time_periods == BillingTimePeriods.WEEK:
+        day_field = 'FORMAT_DATE("%Y%W", day) as day, '
+        day_parse_formula = 'PARSE_DATE("%Y%W", day) as day, '
+    elif query.time_periods == BillingTimePeriods.MONTH:
+        day_field = 'FORMAT_DATE("%Y%m", day) as day, '
+        day_parse_formula = 'PARSE_DATE("%Y%m", day) as day, '
+    elif query.time_periods == BillingTimePeriods.INVOICE_MONTH:
+        day_field = 'invoice_month as day, '
+        day_parse_formula = 'PARSE_DATE("%Y%m", day) as day, '
+
+    return day_field, day_grp, day_parse_formula
+
+
 class BillingDb(BqDbBase):
     """Db layer for billing related routes"""
 
@@ -356,6 +380,18 @@ async def get_total_cost(
 
         fields_selected = ','.join(columns)
 
+        # prepare grouping by time periods
+        day_parse_formula = ''
+        day_field = ''
+        day_grp = ''
+        if query.time_periods:
+            # remove existing day column, if added to fields
+            # this is to prevent duplicating various time periods in one query
+            if BillingColumn.DAY in query.fields:
+                columns.remove(BillingColumn.DAY)
+
+            day_field, day_grp, day_parse_formula = prepare_time_periods(query)
+
         # construct filters
         and_filters = []
         query_parameters = []
@@ -415,11 +451,14 @@ async def get_total_cost(
         order_by_str = f'ORDER BY {",".join(order_by_cols)}' if order_by_cols else ''
 
         _query = f"""
-        SELECT {fields_selected}, SUM(cost) as cost
-        FROM `{view_to_use}`
-        {filter_str}
-        GROUP BY {fields_selected}
-        {order_by_str}
+        WITH t AS (
+            SELECT {day_field}{fields_selected}, SUM(cost) as cost
+            FROM `{view_to_use}`
+            {filter_str}
+            GROUP BY {day_grp}{fields_selected}
+            {order_by_str}
+        )
+        SELECT {day_parse_formula}{fields_selected}, cost FROM t
         """
 
         # append LIMIT and OFFSET if present
@@ -808,10 +847,11 @@ async def get_running_cost(
             BillingColumn.COMPUTE_CATEGORY,
             BillingColumn.WDL_TASK_NAME,
             BillingColumn.CROMWELL_SUB_WORKFLOW_NAME,
+            BillingColumn.NAMESPACE,
         ):
             raise ValueError(
                 'Invalid field only topic, dataset, gcp-project, compute_category, '
-                'wdl_task_name & cromwell_sub_workflow_name are allowed'
+                'wdl_task_name, cromwell_sub_workflow_name & namespace are allowed'
             )
 
         (

db/python/layers/billing_layer.py

@@ -123,6 +123,15 @@ async def get_invoice_months(
         billing_db = BillingDb(self.connection)
         return await billing_db.get_invoice_months()
 
+    async def get_namespaces(
+        self,
+    ) -> list[str] | None:
+        """
+        Get All namespaces values in database
+        """
+        billing_db = BillingDb(self.connection)
+        return await billing_db.get_extended_values('namespace')
+
     async def query(
         self,
         _filter: BillingFilter,

models/models/__init__.py

@@ -8,11 +8,15 @@
     ProportionalDateTemporalMethod,
     SequencingGroupSizeModel,
 )
-from models.models.assay import (
-    Assay,
-    AssayInternal,
-    AssayUpsert,
-    AssayUpsertInternal,
+from models.models.assay import Assay, AssayInternal, AssayUpsert, AssayUpsertInternal
+from models.models.billing import (
+    BillingColumn,
+    BillingCostBudgetRecord,
+    BillingCostDetailsRecord,
+    BillingRowRecord,
+    BillingTimePeriods,
+    BillingTotalCostQueryModel,
+    BillingTotalCostRecord,
 )
 from models.models.family import (
     Family,
@@ -62,11 +66,3 @@
     ProjectSummaryInternal,
     WebProject,
 )
-from models.models.billing import (
-    BillingRowRecord,
-    BillingTotalCostRecord,
-    BillingTotalCostQueryModel,
-    BillingColumn,
-    BillingCostBudgetRecord,
-    BillingCostDetailsRecord,
-)

models/models/billing.py

@@ -142,6 +142,7 @@ class BillingColumn(str, Enum):
     CROMWELL_WORKFLOW_ID = 'cromwell_workflow_id'
     GOOG_PIPELINES_WORKER = 'goog_pipelines_worker'
     WDL_TASK_NAME = 'wdl_task_name'
+    NAMESPACE = 'namespace'
 
     @classmethod
     def extended_cols(cls) -> list[str]:
@@ -158,6 +159,7 @@ def extended_cols(cls) -> list[str]:
             'cromwell_workflow_id',
             'goog_pipelines_worker',
             'wdl_task_name',
+            'namespace',
         ]
 
     @staticmethod
@@ -169,6 +171,16 @@ def generate_all_title(record) -> str:
         return f'All {record.title()}s'
 
 
+class BillingTimePeriods(str, Enum):
+    """List of billing grouping time periods"""
+
+    # grouping time periods
+    DAY = 'day'
+    WEEK = 'week'
+    MONTH = 'month'
+    INVOICE_MONTH = 'invoice_month'
+
+
 class BillingTotalCostQueryModel(SMBase):
     """
     Used to query for billing total cost
@@ -192,6 +204,9 @@ class BillingTotalCostQueryModel(SMBase):
     limit: int | None = None
     offset: int | None = None
 
+    # default to day, can be day, week, month, invoice_month
+    time_periods: BillingTimePeriods | None = None
+
     def __hash__(self):
         """Create hash for this object to use in caching"""
         return hash(self.json())
@@ -205,6 +220,7 @@ class BillingTotalCostRecord(SMBase):
     gcp_project: str | None
     cost_category: str | None
     sku: str | None
+    invoice_month: str | None
     ar_guid: str | None
     # extended columns
     dataset: str | None
@@ -217,6 +233,7 @@ class BillingTotalCostRecord(SMBase):
     cromwell_workflow_id: str | None
     goog_pipelines_worker: str | None
     wdl_task_name: str | None
+    namespace: str | None
 
     cost: float
     currency: str | None
@@ -230,6 +247,7 @@ def from_json(record):
             gcp_project=record.get('gcp_project'),
             cost_category=record.get('cost_category'),
             sku=record.get('sku'),
+            invoice_month=record.get('invoice_month'),
             ar_guid=record.get('ar_guid'),
             dataset=record.get('dataset'),
             batch_id=record.get('batch_id'),
@@ -241,6 +259,7 @@ def from_json(record):
             cromwell_workflow_id=record.get('cromwell_workflow_id'),
             goog_pipelines_worker=record.get('goog_pipelines_worker'),
             wdl_task_name=record.get('wdl_task_name'),
+            namespace=record.get('namespace'),
             cost=record.get('cost'),
             currency=record.get('currency'),
         )

web/src/Routes.tsx

@@ -8,6 +8,7 @@ import {
     BillingCostByTime,
     BillingCostByBatch,
     BillingInvoiceMonthCost,
+    BillingCostByCategory,
 } from './pages/billing'
 import DocumentationArticle from './pages/docs/Documentation'
 import SampleView from './pages/sample/SampleView'
@@ -66,6 +67,14 @@ const Routes: React.FunctionComponent = () => (
                 </ErrorBoundary>
             }
         />
+        <Route
+            path="/billing/costByCategory"
+            element={
+                <ErrorBoundary>
+                    <BillingCostByCategory />
+                </ErrorBoundary>
+            }
+        />
         <Route
             path="/billing/seqrPropMap"
             element={

web/src/index.css

@@ -241,3 +241,9 @@ html[data-theme='dark-mode'] .ui.table {
 .ui.toggle.checkbox input:checked ~ label {
     color: var(--color-text-primary) !important;
 }
+
+/* charts */
+.tooltip {
+    background-color: var(--color-bg-card);
+    color: var(--color-text-primary);
+}

web/src/pages/billing/BillingCostByBatch.tsx

@@ -49,10 +49,15 @@ const BillingCostByBatch: React.FunctionComponent = () => {
     const [error, setError] = React.useState<string | undefined>()
 
     const [start, setStart] = React.useState<string>(
-        searchParams.get('start') ?? `${now.getFullYear()}-${now.getMonth() + 1}-01`
+        searchParams.get('start') ??
+            `${now.getFullYear()}-${now.getMonth().toString().padStart(2, '0')}-01`
     )
     const [end, setEnd] = React.useState<string>(
-        searchParams.get('end') ?? `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`
+        searchParams.get('end') ??
+            `${now.getFullYear()}-${(now.getMonth() + 1).toString().padStart(2, '0')}-${now
+                .getDate()
+                .toString()
+                .padStart(2, '0')}`
     )
 
     const [data, setData] = React.useState<BillingTotalCostRecord[]>([])
@@ -67,13 +72,10 @@ const BillingCostByBatch: React.FunctionComponent = () => {
 
     const updateNav = (searchBy: string | undefined) => {
         let url = `${location.pathname}`
-        if (searchBy) url += '?'
-
-        const params: string[] = []
-        if (searchBy) params.push(`searchBy=${searchBy}`)
-
-        url += params.join('&')
-        navigate(url)
+        if (searchBy) {
+            url += `?searchBy=${searchBy}`
+            navigate(url)
+        }
     }
 
     const getData = (query: BillingTotalCostQueryModel) => {
@@ -89,6 +91,11 @@ const BillingCostByBatch: React.FunctionComponent = () => {
     }
 
     const handleSearch = () => {
+        if (searchTxt === undefined || searchTxt.length < 6) {
+            // Seaarch text is not large enough
+            setIsLoading(false)
+            return
+        }
         updateNav(searchTxt)
         getData({
             fields: [