Feat/simplified query

.changeset/simplified-query.md

@@ -0,0 +1,28 @@
+---
+"@kopai/core": minor
+"@kopai/sdk": minor
+"@kopai/api": minor
+"@kopai/clickhouse-datasource": minor
+"@kopai/sqlite-datasource": minor
+"@kopai/ui": minor
+"@kopai/ui-core": minor
+"@kopai/cli": patch
+---
+
+Add KopaiQuery — a unified, type-safe query surface across traces, logs, and metrics, in both raw and aggregate modes.
+
+KopaiQuery gives every signal type one query model with one set of semantics, compiled to backend-specific SQL and exposed end to end: a builder and client methods in the SDK, HTTP routes in the API, datasource implementations for ClickHouse and SQLite, and a dashboard `DataSource` variant in the UI.
+
+**`@kopai/core`** — new `kopaiQuery` (query model + zod schemas) and `kopaiQueryCompiler` (compiler + `KopaiQueryValidationError`) modules, exported from the package root; `metricsBaseSchema` is now exported too. The `ReadTelemetryDatasource` interface gains a `ReadQueryDatasource` member (7 `query*` methods).
+
+**`@kopai/sdk`** — new `kq` query builder and `KopaiClient` methods (`query()`, `queryTracesRaw/Aggregate`, `queryLogsRaw/Aggregate`, `queryMetricsRaw/Aggregate`), plus `KopaiQueryResponse`, `KopaiQueryBuildError`, and `KopaiQueryBuildIssue` exports. A new Node-only subpath export `@kopai/sdk/node` (`clientFromConfig`, `loadConfig`, `resolveConnection`, `DEFAULT_URL`, `CONFIG_FILENAME`) reads `.kopairc` and builds a configured client for code-mode scripts; the package root stays platform-neutral (browser-safe). `KopaiError.message` now includes the RFC 7807 `detail` text (composed as `"<title>: <detail>"`, falling back to title-only then `HTTP <status>`), so a server-side validation failure logs the actionable explanation — e.g. `"Invalid query: Percentile measures (P50-P999) are not yet supported on the sqlite backend."` — instead of just the generic title. The `detail`, `code`, `status`, and `type` fields are unchanged.
+
+**`@kopai/api`** — new `POST /signals/query/{traces,logs,metrics}/{raw,aggregate}` routes; the error handler maps `KopaiQueryValidationError` to a 400.
+
+**`@kopai/clickhouse-datasource` / `@kopai/sqlite-datasource`** — implement the new query methods. `ZeroThreshold` is excluded from the KopaiQuery surface so both backends behave identically: the ClickHouse OTel-collector schema has no `ZeroThreshold` column on the exponential-histogram table (it is coerced to `undefined` on read) while SQLite stores it, so a raw `ExponentialHistogram` query previously returned a different shape per backend, and filtering/grouping/aggregating on the field would have generated SQL against a non-existent ClickHouse column. `ZeroThreshold` is removed from the `MetricColumn` enum (moved to `METRIC_EXCLUDED`) and is no longer projected by the SQLite raw `ExponentialHistogram` query. The field remains in the underlying storage schemas and the legacy `getMetrics` read paths; only the unified KopaiQuery surface excludes it.
+
+**`@kopai/ui` / `@kopai/ui-core`** — the dashboard `DataSource` union gains a `query` variant (KopaiQuery), wired through `useKopaiData` and the renderer, and the observability catalog's `acceptsDataFrom` lists now include `"query"` for the log/trace/metric renderers — letting dashboard components source data from KopaiQuery. The metric renderers (`MetricTimeSeries`, `MetricHistogram`, `MetricStat`, `MetricTable`) now surface an explicit error when a `query` dataSource returns rows they can't draw — most commonly an aggregate-mode result, or a query for a different signal — instead of silently falling back to an empty panel and hiding the misconfiguration. Empty result sets and not-yet-loaded responses still render normally (no error); a shared `narrowQueryRows` helper distinguishes a genuine shape mismatch from an empty/absent response. As part of this work `@kopai/ui` now re-exports DOM-free symbols from `@kopai/ui-core` instead of shipping its own copies (public API unchanged — additive only; new code should prefer importing non-DOM symbols from `@kopai/ui-core` directly), and `CatalogueComponentProps` is added to the `@kopai/ui-core` public barrel so `@kopai/ui`'s dashboard primitives can use it.
+
+**`@kopai/cli`** — `.kopairc` reading and connection resolution now come from `@kopai/sdk/node` (single source of truth) instead of a private copy. No change to CLI behavior or flags.
+
+**Compatibility:** widening `ReadTelemetryDatasource` is purely additive for callers of the interface and for existing HTTP/SDK clients. Anyone _implementing_ `ReadTelemetryDatasource` outside this repo must add the new `query*` methods. All consumers within this repo are updated.

examples/ui-react-app/src/custom-observability-catalog.tsx

@@ -28,7 +28,6 @@ import {
   type OtelLogsRow,
   type OtelMetricsRow,
   type OtelTracesRow,
-  type SearchResult,
   type TraceSummaryRow,
 } from "@kopai/sdk";
 import {
@@ -86,6 +85,62 @@ function NoSource({ name }: { name: string }) {
   );
 }
 
+/** Shown when a (polymorphic `query`) response doesn't match the shape a
+ * signal-specific renderer expects — better than rendering garbage off a
+ * blind cast. */
+function UnsupportedShape({ name }: { name: string }) {
+  return (
+    <div style={{ color: "#999", fontSize: 12, padding: 8 }}>
+      {name}: unsupported response shape.
+    </div>
+  );
+}
+
+// The `query` method is polymorphic: a renderer wired to it can receive any of
+// the six KopaiQuery result shapes. `narrowRows` validates that a response's
+// `data` is an array of the expected row type (every element, not just a
+// sample) and returns null otherwise, so each renderer can fall back instead
+// of casting blindly. Same pattern as TraceDetail's isTraceRows below.
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return typeof v === "object" && v !== null;
+}
+
+function narrowRows<T>(
+  response: unknown,
+  guard: (v: unknown) => v is T
+): T[] | null {
+  if (!isRecord(response)) return null;
+  const data = response.data;
+  if (!Array.isArray(data)) return null;
+  // Accumulate through the guard so each row is narrowed to T — no `as T[]`.
+  const rows: T[] = [];
+  for (const row of data) {
+    if (!guard(row)) return null;
+    rows.push(row);
+  }
+  return rows;
+}
+
+function hasMetricRowShape(v: unknown): v is OtelMetricsRow {
+  if (!isRecord(v)) return false;
+  // Only metric data points carry TimeUnix (trace/log rows use Timestamp).
+  return typeof v.TimeUnix === "string";
+}
+
+function hasLogRowShape(v: unknown): v is OtelLogsRow {
+  if (!isRecord(v)) return false;
+  // Logs use Timestamp (not TimeUnix), but so do spans — a log-only structural
+  // key (Body/SeverityText/…) is what separates them, since a trace-correlated
+  // log also has SpanId/TraceId. Production code should validate the full schema.
+  return (
+    typeof v.Timestamp === "string" &&
+    typeof v.TimeUnix !== "string" &&
+    ("Body" in v || "SeverityText" in v || "SeverityNumber" in v) &&
+    !("SpanName" in v) &&
+    !("Duration" in v)
+  );
+}
+
 // =============================================================================
 // 3. Primitive renderers
 // =============================================================================
@@ -294,22 +349,32 @@ function MetricStat(props: MetricStatProps) {
   if (!props.hasData) return <NoSource name="MetricStat" />;
   const { label } = props.element.props;
 
+  let body: ReactNode;
+  if (isAggregatedMetricStat(props)) {
+    body = (
+      <span style={{ fontSize: 24, fontWeight: 600 }}>
+        {props.response?.data[0]?.value ?? "—"}
+      </span>
+    );
+  } else {
+    // searchMetricsPage or a `query` returning metric rows.
+    const rows = narrowRows(props.response, hasMetricRowShape);
+    body =
+      rows === null ? (
+        <UnsupportedShape name="MetricStat" />
+      ) : (
+        <span style={{ fontSize: 14 }}>
+          {rows.length} rows
+          {rows[0]?.MetricName ? ` · ${rows[0].MetricName}` : ""}
+        </span>
+      );
+  }
+
   return (
     <RequestState loading={props.loading} error={props.error}>
       <div style={{ display: "flex", flexDirection: "column", gap: 4 }}>
         {label && <span style={{ color: "#666", fontSize: 12 }}>{label}</span>}
-        {isAggregatedMetricStat(props) ? (
-          <span style={{ fontSize: 24, fontWeight: 600 }}>
-            {props.response?.data[0]?.value ?? "—"}
-          </span>
-        ) : (
-          <span style={{ fontSize: 14 }}>
-            {props.response?.data.length ?? 0} rows
-            {props.response?.data[0]?.MetricName
-              ? ` · ${props.response.data[0].MetricName}`
-              : ""}
-          </span>
-        )}
+        {body}
       </div>
     </RequestState>
   );
@@ -318,33 +383,43 @@ function MetricStat(props: MetricStatProps) {
 // ---------- MetricTimeSeries ------------------------------------------------
 function MetricTimeSeries(props: RendererProps<"MetricTimeSeries">) {
   if (!props.hasData) return <NoSource name="MetricTimeSeries" />;
-  const rows: OtelMetricsRow[] = props.response?.data ?? [];
+  const rows = narrowRows(props.response, hasMetricRowShape);
   return (
     <RequestState loading={props.loading} error={props.error}>
-      <div style={{ fontSize: 13, color: "#666" }}>
-        {rows.length} points
-        {rows[0]?.MetricName ? ` · ${rows[0].MetricName}` : ""}{" "}
-        <em>(render a line chart here)</em>
-      </div>
+      {rows === null ? (
+        <UnsupportedShape name="MetricTimeSeries" />
+      ) : (
+        <div style={{ fontSize: 13, color: "#666" }}>
+          {rows.length} points
+          {rows[0]?.MetricName ? ` · ${rows[0].MetricName}` : ""}{" "}
+          <em>(render a line chart here)</em>
+        </div>
+      )}
     </RequestState>
   );
 }
 
 // ---------- MetricHistogram -------------------------------------------------
 function MetricHistogram(props: RendererProps<"MetricHistogram">) {
   if (!props.hasData) return <NoSource name="MetricHistogram" />;
-  const rows: OtelMetricsRow[] = props.response?.data ?? [];
-  const first = rows[0];
+  const rows = narrowRows(props.response, hasMetricRowShape);
+  const first = rows?.[0];
   return (
     <RequestState loading={props.loading} error={props.error}>
-      <div style={{ fontSize: 13, color: "#666" }}>
-        {rows.length} data points
-        {first?.MetricName ? ` · ${first.MetricName}` : ""}
-        {first && first.MetricType === "Histogram" && first.Count !== undefined
-          ? ` · count=${first.Count}`
-          : ""}{" "}
-        <em>(render a histogram here)</em>
-      </div>
+      {rows === null ? (
+        <UnsupportedShape name="MetricHistogram" />
+      ) : (
+        <div style={{ fontSize: 13, color: "#666" }}>
+          {rows.length} data points
+          {first?.MetricName ? ` · ${first.MetricName}` : ""}
+          {first &&
+          first.MetricType === "Histogram" &&
+          first.Count !== undefined
+            ? ` · count=${first.Count}`
+            : ""}{" "}
+          <em>(render a histogram here)</em>
+        </div>
+      )}
     </RequestState>
   );
 }
@@ -353,7 +428,14 @@ function MetricHistogram(props: RendererProps<"MetricHistogram">) {
 function MetricTable(props: RendererProps<"MetricTable">) {
   if (!props.hasData) return <NoSource name="MetricTable" />;
   const maxRows = props.element.props.maxRows ?? 5;
-  const rows: OtelMetricsRow[] = props.response?.data ?? [];
+  const rows = narrowRows(props.response, hasMetricRowShape);
+  if (rows === null) {
+    return (
+      <RequestState loading={props.loading} error={props.error}>
+        <UnsupportedShape name="MetricTable" />
+      </RequestState>
+    );
+  }
   const shown = rows.slice(0, maxRows);
   return (
     <RequestState loading={props.loading} error={props.error}>
@@ -421,7 +503,14 @@ function MetricDiscovery(props: RendererProps<"MetricDiscovery">) {
 // ---------- LogTimeline -----------------------------------------------------
 function LogTimeline(props: RendererProps<"LogTimeline">) {
   if (!props.hasData) return <NoSource name="LogTimeline" />;
-  const rows: OtelLogsRow[] = props.response?.data ?? [];
+  const rows = narrowRows(props.response, hasLogRowShape);
+  if (rows === null) {
+    return (
+      <RequestState loading={props.loading} error={props.error}>
+        <UnsupportedShape name="LogTimeline" />
+      </RequestState>
+    );
+  }
   return (
     <RequestState loading={props.loading} error={props.error}>
       {rows.length === 0 ? (
@@ -453,25 +542,61 @@ function LogTimeline(props: RendererProps<"LogTimeline">) {
   );
 }
 
-// ---------- TraceDetail (two accepted methods) ------------------------------
+// ---------- TraceDetail (three accepted methods) ----------------------------
 type TraceDetailProps = RendererProps<"TraceDetail">;
-function isTraceSummaries(
+
+function hasTraceRowShape(v: unknown): v is OtelTracesRow {
+  if (!isRecord(v)) return false;
+  // A trace-correlated log also has SpanId/TraceId, so require a span-only
+  // structural key to avoid mis-accepting log rows here.
+  return (
+    typeof v.SpanId === "string" &&
+    typeof v.TraceId === "string" &&
+    typeof v.TimeUnix !== "string" &&
+    ("SpanName" in v || "SpanKind" in v || "Duration" in v) &&
+    !("Body" in v) &&
+    !("SeverityText" in v)
+  );
+}
+
+function hasTraceSummaryShape(v: unknown): v is TraceSummaryRow {
+  if (!isRecord(v)) return false;
+  return typeof v.traceId === "string" && typeof v.durationNs === "string";
+}
+
+// Trace summaries come only from `searchTraceSummariesPage` (not from a
+// `query`). Gate on the method, then validate the row shape.
+function traceSummaryRows(
   props: TraceDetailProps & { hasData: true }
-): props is TraceDetailProps & {
-  hasData: true;
-  response: SearchResult<TraceSummaryRow> | null;
-} {
-  return props.element.dataSource?.method === "searchTraceSummariesPage";
+): TraceSummaryRow[] | null {
+  if (props.element.dataSource?.method !== "searchTraceSummariesPage")
+    return null;
+  return narrowRows(props.response, hasTraceSummaryShape);
+}
+
+// `query` can return any shape (raw traces/logs/metrics or aggregate rows), so
+// only narrow to OtelTracesRow[] when the response actually looks like trace
+// rows — otherwise fall through to the unsupported-shape fallback rather than
+// casting blindly. (Same narrow-or-fallback pattern the other renderers use.)
+function traceRows(
+  props: TraceDetailProps & { hasData: true }
+): OtelTracesRow[] | null {
+  const method = props.element.dataSource?.method;
+  if (method !== "searchTracesPage" && method !== "query") return null;
+  return narrowRows(props.response, hasTraceRowShape);
 }
 
 function TraceDetail(props: TraceDetailProps) {
   if (!props.hasData) return <NoSource name="TraceDetail" />;
 
+  const summaries = traceSummaryRows(props);
+  const traces = summaries === null ? traceRows(props) : null;
+
   return (
     <RequestState loading={props.loading} error={props.error}>
-      {isTraceSummaries(props) ? (
+      {summaries !== null ? (
         <ul style={{ listStyle: "none", margin: 0, padding: 0, fontSize: 12 }}>
-          {(props.response?.data ?? []).slice(0, 10).map((t) => (
+          {summaries.slice(0, 10).map((t) => (
             <li
               key={t.traceId}
               style={{ padding: "4px 0", borderBottom: "1px solid #eee" }}
@@ -487,21 +612,21 @@ function TraceDetail(props: TraceDetailProps) {
             </li>
           ))}
         </ul>
-      ) : (
+      ) : traces !== null ? (
         <ul style={{ listStyle: "none", margin: 0, padding: 0, fontSize: 12 }}>
-          {((props.response?.data ?? []) as OtelTracesRow[])
-            .slice(0, 10)
-            .map((s) => (
-              <li
-                key={s.SpanId}
-                style={{ padding: "4px 0", borderBottom: "1px solid #eee" }}
-              >
-                <code style={{ color: "#666" }}>{s.SpanId}</code>{" "}
-                {s.SpanName ?? "—"} · {s.ServiceName ?? "—"} ·{" "}
-                {s.Duration ?? "—"}ns
-              </li>
-            ))}
+          {traces.slice(0, 10).map((s) => (
+            <li
+              key={s.SpanId}
+              style={{ padding: "4px 0", borderBottom: "1px solid #eee" }}
+            >
+              <code style={{ color: "#666" }}>{s.SpanId}</code>{" "}
+              {s.SpanName ?? "—"} · {s.ServiceName ?? "—"} · {s.Duration ?? "—"}
+              ns
+            </li>
+          ))}
         </ul>
+      ) : (
+        <UnsupportedShape name="TraceDetail" />
       )}
     </RequestState>
   );