Switch out old APIs

api/src/main/scala/ai/chronon/api/Constants.scala

@@ -77,12 +77,7 @@ object Constants {
   val extensionsToIgnore: Array[String] = Array(".class", ".csv", ".java", ".scala", ".py", ".DS_Store")
   val foldersToIgnore: Array[String] = Array(".git")
 
-  // import base64
-  // text_bytes = "chronon".encode('utf-8')
-  // base64_str = base64.b64encode(text_bytes)
-  // int.from_bytes(base64.b64decode(base64_str), "big")
-  //
-  // output: 27980863399423854
-
-  val magicNullDouble: java.lang.Double = -27980863399423854.0
+  // A negative integer within the safe range for both long and double in JavaScript, Java, Scala, Python
+  val magicNullLong: java.lang.Long = -1234567890L
+  val magicNullDouble: java.lang.Double = -1234567890.0
 }

api/src/test/scala/ai/chronon/api/test/TileSeriesSerializationTest.scala

@@ -4,10 +4,13 @@ import ai.chronon.api.Constants
 import ai.chronon.api.ScalaJavaConversions.JListOps
 import ai.chronon.api.ThriftJsonCodec
 import ai.chronon.observability.TileDriftSeries
+import ai.chronon.observability.TileSummarySeries
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
 import java.lang.{Double => JDouble}
+import java.lang.{Long => JLong}
+import scala.jdk.CollectionConverters.asScalaBufferConverter
 
 class TileSeriesSerializationTest extends AnyFlatSpec with Matchers {
 
@@ -27,8 +30,49 @@ class TileSeriesSerializationTest extends AnyFlatSpec with Matchers {
 
     val jsonStr = ThriftJsonCodec.toJsonStr(tileDriftSeries)
 
-    jsonStr should be ("""{"percentileDriftSeries":[0.1,-2.7980863399423856E16,-2.7980863399423856E16,-2.7980863399423856E16,0.5]}""")
+    jsonStr should be (s"""{"percentileDriftSeries":[0.1,${Constants.magicNullDouble},${Constants.magicNullDouble},${Constants.magicNullDouble},0.5]}""")
   }
 
+  it should "deserialize double values correctly" in {
+    val json = s"""{"percentileDriftSeries":[0.1,${Constants.magicNullDouble},${Constants.magicNullDouble},${Constants.magicNullDouble},0.5]}"""
+
+    val series = ThriftJsonCodec.fromJsonStr[TileDriftSeries](json, true, classOf[TileDriftSeries])(manifest[TileDriftSeries])
+
+    val drifts = series.getPercentileDriftSeries.asScala.toList
+    drifts.size should be (5)
+    drifts(0) should be (0.1)
+    drifts(1) should be (Constants.magicNullDouble)
+    drifts(2) should be (Constants.magicNullDouble)
+    drifts(3) should be (Constants.magicNullDouble)
+    drifts(4) should be (0.5)
+  }
+
+  "TileSummarySeries" should "serialize with nulls and special long values" in {
+    val tileSummarySeries = new TileSummarySeries()
+
+    val counts: Seq[JLong] = Seq(100L, null, Long.MaxValue, Constants.magicNullLong, 500L)
+      .map(v => if (v == null) Constants.magicNullLong else v.asInstanceOf[JLong])
+
+    val countsList: java.util.List[JLong] = counts.toJava
+    tileSummarySeries.setCount(countsList)
+
+    val jsonStr = ThriftJsonCodec.toJsonStr(tileSummarySeries)
+
+    jsonStr should be (s"""{"count":[100,${Constants.magicNullLong},9223372036854775807,${Constants.magicNullLong},500]}""")
+  }
+
+  it should "deserialize long values correctly" in {
+    val json = s"""{"count":[100,${Constants.magicNullLong},9223372036854775807,${Constants.magicNullLong},500]}"""
+
+    val series = ThriftJsonCodec.fromJsonStr[TileSummarySeries](json, true, classOf[TileSummarySeries])(manifest[TileSummarySeries])
+
+    val counts = series.getCount.asScala.toList
+    counts.size should be (5)
+    counts(0) should be (100L)
+    counts(1) should be (Constants.magicNullLong)
+    counts(2) should be (Long.MaxValue)
+    counts(3) should be (Constants.magicNullLong)
+    counts(4) should be (500L)
+  }
 
 }

api/thrift/api.thrift

@@ -432,4 +432,4 @@ struct Model {
     3: optional TDataType outputSchema
     4: optional Source source
     5: optional map<string, string> modelParams
-}
+}

api/thrift/hub.thrift

@@ -115,3 +115,39 @@ struct Submission {
     20: optional i64 finishedTs
     21: optional DateRange dateRange
 }
+
+enum ConfType{
+   STAGING_QUERY = 1
+   GROUP_BY = 2
+   JOIN = 3
+   MODEL = 4
+}
+
+struct ConfRequest {
+   1: optional string confName
+   2: optional ConfType confType
+
+   // one of either branch or version are set - otherwise we will pull conf for main branch
+   3: optional string branch 
+   4: optional string version
+}
+
+/**
+  * lists all confs of the specified type
+  */
+struct ConfListRequest {
+  1: optional ConfType confType
+
+  // if not specified we will pull conf list for main branch
+  2: optional string branch
+}
+
+/**
+  * Response for listing configurations of a specific type
+  */
+struct ConfListResponse {
+  1: optional list<api.Join> joins
+  2: optional list<api.GroupBy> groupBys
+  3: optional list<api.Model> models
+  4: optional list<api.StagingQuery> stagingQueries
+}

api/thrift/observability.thrift

@@ -139,4 +139,24 @@ struct DriftSpec {
 
     // default drift metric to use
     6: optional DriftMetric driftMetric = DriftMetric.JENSEN_SHANNON
-}
+}
+
+struct JoinDriftRequest {
+    1: required string name
+    2: required i64 startTs
+    3: required i64 endTs
+    6: optional string offset // Format: "24h" or "7d"
+    7: optional DriftMetric algorithm
+    8: optional string columnName
+}
+
+struct JoinDriftResponse {
+    1: required list<TileDriftSeries> driftSeries
+}
+
+struct JoinSummaryRequest {
+    1: required string name
+    2: required i64 startTs
+    3: required i64 endTs
+    8: required string columnName
+}

frontend/src/lib/api/api.test.ts

@@ -30,9 +30,9 @@ describe('API module', () => {
 				text: () => Promise.resolve(JSON.stringify(mockResponse))
 			});
 
-			const result = await api.getModels();
+			const result = await api.getModelList();
 
-			expect(mockFetch).toHaveBeenCalledWith(`/api/v1/models`, {
+			expect(mockFetch).toHaveBeenCalledWith(`/api/v1/conf/list?confType=MODEL`, {
 				method: 'GET',
 				headers: {
 					'Content-Type': 'application/json'
@@ -47,7 +47,7 @@ describe('API module', () => {
 				text: () => Promise.resolve('')
 			});
 
-			const result = await api.getModels();
+			const result = await api.getModelList();
 
 			expect(result).toEqual({});
 		});
@@ -58,7 +58,7 @@ describe('API module', () => {
 				status: 404
 			});
 
-			await api.getModels();
+			await api.getModelList();
 
 			expect(error).toHaveBeenCalledWith(404);
 		});

frontend/src/lib/api/api.ts

@@ -1,10 +1,17 @@
 import { error } from '@sveltejs/kit';
+import type { FeatureResponse, JoinTimeSeriesResponse } from '$lib/types/Model/Model';
 import type {
-	FeatureResponse,
-	JoinsResponse,
-	JoinTimeSeriesResponse,
-	ModelsResponse
-} from '$lib/types/Model/Model';
+	Join,
+	GroupBy,
+	Model,
+	StagingQuery,
+	IJoinDriftRequestArgs,
+	IJoinDriftResponseArgs,
+	ITileSummarySeries,
+	IJoinSummaryRequestArgs
+} from '$lib/types/codegen';
+import { ConfType, DriftMetric } from '$lib/types/codegen';
+import type { ConfListResponse } from '$lib/types/codegen/ConfListResponse';
 
 export type ApiOptions = {
 	base?: string;
@@ -30,25 +37,35 @@ export class Api {
 		this.#accessToken = opts.accessToken;
 	}
 
-	// TODO: eventually move this to a model-specific file/decide on a good project structure for organizing api calls
-	async getModels() {
-		return this.#send<ModelsResponse>('models');
-	}
-
-	async getJoins(offset: number = 0, limit: number = 10) {
+	async getConf(name: string, type: ConfType) {
 		const params = new URLSearchParams({
-			offset: offset.toString(),
-			limit: limit.toString()
+			confName: name,
+			confType: ConfType[type]
 		});
-		return this.#send<JoinsResponse>(`joins?${params.toString()}`);
+		return this.#send<Join | GroupBy | Model>(`conf?${params.toString()}`);
+	}
+
+	async getJoin(name: string): Promise<Join> {
+		return this.getConf(name, ConfType.JOIN) as Promise<Join>;
 	}
 
-	async search(term: string, limit: number = 20) {
+	async getGroupBy(name: string): Promise<GroupBy> {
+		return this.getConf(name, ConfType.GROUP_BY) as Promise<GroupBy>;
+	}
+
+	async getModel(name: string): Promise<Model> {
+		return this.getConf(name, ConfType.MODEL) as Promise<Model>;
+	}
+
+	async getStagingQuery(name: string): Promise<StagingQuery> {
+		return this.getConf(name, ConfType.STAGING_QUERY) as Promise<StagingQuery>;
+	}
+
+	async search(term: string) {
 		const params = new URLSearchParams({
-			term,
-			limit: limit.toString()
+			confName: term
 		});
-		return this.#send<ModelsResponse>(`search?${params.toString()}`);
+		return this.#send<ConfListResponse>(`search?${params.toString()}`);
 	}
 
 	async getJoinTimeseries({
@@ -115,6 +132,74 @@ export class Api {
 		);
 	}
 
+	async getConfList(type: ConfType): Promise<ConfListResponse> {
+		const params = new URLSearchParams({
+			confType: ConfType[type]
+		});
+		return this.#send<ConfListResponse>(`conf/list?${params.toString()}`);
+	}
+
+	async getJoinList(): Promise<ConfListResponse> {
+		return this.getConfList(ConfType.JOIN);
+	}
+
+	async getGroupByList(): Promise<ConfListResponse> {
+		return this.getConfList(ConfType.GROUP_BY);
+	}
+
+	async getModelList(): Promise<ConfListResponse> {
+		return this.getConfList(ConfType.MODEL);
+	}
+
+	async getStagingQueryList(): Promise<ConfListResponse> {
+		return this.getConfList(ConfType.STAGING_QUERY);
+	}
+
+	async getJoinDrift({
+		name,
+		startTs,
+		endTs,
+		offset = '10h',
+		algorithm = DriftMetric.PSI
+	}: IJoinDriftRequestArgs) {
+		const params = new URLSearchParams({
+			startTs: startTs.toString(),
+			endTs: endTs.toString(),
+			offset,
+			algorithm: DriftMetric[algorithm]
+		});
+		return this.#send<IJoinDriftResponseArgs>(`join/${name}/drift?${params.toString()}`);
+	}
+
+	async getColumnDrift({
+		name,
+		columnName,
+		startTs,
+		endTs,
+		offset = '10h',
+		algorithm = DriftMetric.PSI
+	}: IJoinDriftRequestArgs) {
+		const params = new URLSearchParams({
+			startTs: startTs.toString(),
+			endTs: endTs.toString(),
+			offset,
+			algorithm: DriftMetric[algorithm]
+		});
+		return this.#send<IJoinDriftResponseArgs>(
+			`join/${name}/column/${columnName}/drift?${params.toString()}`
+		);
+	}
+
+	async getColumnSummary({ name, columnName, startTs, endTs }: IJoinSummaryRequestArgs) {
+		const params = new URLSearchParams({
+			startTs: startTs.toString(),
+			endTs: endTs.toString()
+		});
+		return this.#send<ITileSummarySeries>(
+			`join/${name}/column/${columnName}/summary?${params.toString()}`
+		);
+	}
+
 	async #send<Data = unknown>(resource: string, options?: ApiRequestOptions) {
 		let url = `${this.#base}/${resource}`;
 

frontend/src/lib/components/LogicalNodeTable.svelte

@@ -0,0 +1,64 @@
+<script lang="ts">
+	import type { IJoin, IGroupBy, IModel, IStagingQuery } from '$lib/types/codegen';
+	import {
+		Table,
+		TableBody,
+		TableCell,
+		TableHead,
+		TableHeader,
+		TableRow
+	} from '$lib/components/ui/table';
+	import Separator from '$lib/components/ui/separator/separator.svelte';
+	import PageHeader from '$lib/components/PageHeader.svelte';
+	import ActionButtons from '$lib/components/ActionButtons.svelte';
+
+	const {
+		title,
+		items,
+		basePath
+	}: { title: string; items: (IJoin | IGroupBy | IModel | IStagingQuery)[]; basePath: string } =
+		$props();
+</script>
+
+<PageHeader {title} />
+
+<div class="w-full">
+	<ActionButtons class="mb-4" />
+</div>
+
+<Separator fullWidthExtend={true} />
+
+<Table>
+	<TableHeader>
+		<TableRow>
+			<TableHead>{title}</TableHead>
+		</TableRow>
+	</TableHeader>
+	<TableBody>
+		{#if items.length === 0}
+			<TableRow>
+				<TableCell>
+					No {title.toLowerCase()} found.
+				</TableCell>
+			</TableRow>
+		{:else}
+			{#each items as item}
+				<TableRow>
+					<TableCell>
+						<!-- todo: enable all items once we have data for them -->
+						<a
+							href={`${basePath}/${encodeURIComponent(item.metaData?.name ?? '')}`}
+							class="hover:underline {item.metaData?.name !== 'risk.user_transactions.txn_join'
+								? 'pointer-events-none opacity-50'
+								: ''}"
+						>
+							{item.metaData?.name}
+						</a>
+					</TableCell>
+				</TableRow>
+			{/each}
+		{/if}
+	</TableBody>
+</Table>
+
+<Separator fullWidthExtend={true} />