feat: ci-metrics data quality + dashboard improvements

ci3/ci-metrics/billing/billing-dashboard.html

@@ -58,6 +58,7 @@
     <a href="/cost-overview">cost overview</a>
     <a href="/namespace-billing" class="active">namespace billing</a>
     <a href="/ci-insights">ci insights</a>
+    <a href="/test-timings">test timings</a>
   </div>
   <h2 style="margin:8px 0;color:#ccc;">namespace billing</h2>
 

ci3/ci-metrics/db.py

@@ -7,7 +7,8 @@
 import sqlite3
 import threading
 
-_DB_PATH = os.path.join(os.getenv('LOGS_DISK_PATH', '/logs-disk'), 'metrics.db')
+_DB_PATH = os.getenv('METRICS_DB_PATH',
+                     os.path.join(os.getenv('LOGS_DISK_PATH', '/logs-disk'), 'metrics.db'))
 _local = threading.local()
 
 SCHEMA = """
@@ -64,6 +65,18 @@
 CREATE INDEX IF NOT EXISTS idx_ci_runs_ts ON ci_runs(timestamp_ms);
 CREATE INDEX IF NOT EXISTS idx_ci_runs_name ON ci_runs(name);
 CREATE INDEX IF NOT EXISTS idx_ci_runs_dashboard ON ci_runs(dashboard);
+
+CREATE TABLE IF NOT EXISTS test_daily_stats (
+    date          TEXT NOT NULL,
+    test_cmd      TEXT NOT NULL,
+    dashboard     TEXT NOT NULL DEFAULT '',
+    passed        INTEGER NOT NULL DEFAULT 0,
+    failed        INTEGER NOT NULL DEFAULT 0,
+    flaked        INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY (date, test_cmd, dashboard)
+);
+CREATE INDEX IF NOT EXISTS idx_tds_date ON test_daily_stats(date);
+CREATE INDEX IF NOT EXISTS idx_tds_dashboard ON test_daily_stats(dashboard);
 """
 
 

ci3/ci-metrics/ec2_pricing.py

@@ -16,12 +16,20 @@
 # ---- Hardcoded fallback rates (us-east-2, USD/hr) ----
 
 _HARDCODED_RATES = {
-    ('m6a.48xlarge', True):  8.31,   # spot
-    ('m6a.48xlarge', False): 16.56,  # on-demand
-    ('m6a.32xlarge', True):  5.54,
-    ('m6a.32xlarge', False): 11.04,
+    ('m6a.xlarge',   True):  0.07,   # spot
+    ('m6a.xlarge',   False): 0.1728, # on-demand
+    ('m6a.4xlarge',  True):  0.28,
+    ('m6a.4xlarge',  False): 0.6912,
+    ('m6a.8xlarge',  True):  0.55,
+    ('m6a.8xlarge',  False): 1.3824,
     ('m6a.16xlarge', True):  2.77,
     ('m6a.16xlarge', False): 5.52,
+    ('m6a.24xlarge', True):  1.66,
+    ('m6a.24xlarge', False): 4.1472,
+    ('m6a.32xlarge', True):  5.54,
+    ('m6a.32xlarge', False): 11.04,
+    ('m6a.48xlarge', True):  8.31,
+    ('m6a.48xlarge', False): 16.56,
     ('m7a.48xlarge', True):  8.31,
     ('m7a.48xlarge', False): 16.56,
     ('m7a.16xlarge', True):  2.77,
@@ -145,8 +153,19 @@ def _fetch_all_spot(instance_types: list[str]) -> dict[str, float]:
 # ---- Cache refresh ----
 
 def _get_known_instance_types() -> list[str]:
-    """Return the set of instance types we need pricing for."""
-    return sorted({itype for itype, _ in _HARDCODED_RATES})
+    """Return the set of instance types we need pricing for (hardcoded + from DB)."""
+    types = {itype for itype, _ in _HARDCODED_RATES}
+    try:
+        import db
+        conn = db.get_db()
+        rows = conn.execute(
+            "SELECT DISTINCT instance_type FROM ci_runs "
+            "WHERE instance_type IS NOT NULL AND instance_type != '' AND instance_type != 'unknown'"
+        ).fetchall()
+        types.update(r['instance_type'] for r in rows)
+    except Exception:
+        pass
+    return sorted(types)
 
 
 def _refresh_cache():