diff --git a/frontend/src/lib/api/api.ts b/frontend/src/lib/api/api.ts index 61c9172524..234a8b9e8f 100644 --- a/frontend/src/lib/api/api.ts +++ b/frontend/src/lib/api/api.ts @@ -16,7 +16,7 @@ import type { } from '$lib/types/codegen'; import { ConfType, DriftMetric, Status } from '$lib/types/codegen'; import type { ConfListResponse } from '$lib/types/codegen/ConfListResponse'; -import { joinToLineage } from './utils'; +import { confToLineage } from './utils'; export type ApiOptions = { base?: string; @@ -160,7 +160,38 @@ export class Api { // TODO: Remove this once we have the API endpoint return this.getJoin(name!).then((join) => { - return joinToLineage(join); + return confToLineage(join); + }); + } + + async getGroupByLineage({ + name + // type, + // branch, + // direction + }: ILineageRequestArgs): Promise { + // TODO: Remove this once we have the API endpoint + return this.getGroupBy(name!).then((groupBy) => { + return confToLineage(groupBy); + }); + } + + async getModelLineage({ + name + // type, + // branch, + // direction + }: ILineageRequestArgs): Promise { + // TODO: Remove this once we have the API endpoint + return this.getModel(name!).then((model) => { + return confToLineage(model); + }); + } + + async getStagingQueryLineage({ name }: ILineageRequestArgs): Promise { + // TODO: Remove this once we have the API endpoint + return this.getStagingQuery(name!).then((stagingQuery) => { + return confToLineage(stagingQuery); }); } diff --git a/frontend/src/lib/api/utils.ts b/frontend/src/lib/api/utils.ts index 17212951f4..299590537d 100644 --- a/frontend/src/lib/api/utils.ts +++ b/frontend/src/lib/api/utils.ts @@ -2,36 +2,83 @@ import { InternMap } from 'd3'; import { LogicalType, type IJoin, + type IJoinPart, type ILineageResponse, type ILogicalNode, type INodeKey, type ISource, type NodeGraph } from '../types/codegen'; +import { getLogicalNodeType, type CombinedLogicalNode } from '../types/LogicalNode'; /** Convert Join to LineageResponse by walking joinParts */ -export function joinToLineage(join: IJoin, excludeLeft = false): ILineageResponse { +export function confToLineage(conf: CombinedLogicalNode, excludeLeft = false): ILineageResponse { // Use `InternMap` insteaad of `Map` to support object keys (instances will be different once serialized/fetched from API) - https://d3js.org/d3-array/intern // @ts-expect-error: Bad typing const connections: NodeGraph['connections'] = new InternMap([], JSON.stringify); // @ts-expect-error: Bad typing const infoMap: NodeGraph['infoMap'] = new InternMap([], JSON.stringify); - const joinNodeKey: INodeKey = { - name: join.metaData?.name, - logicalType: LogicalType.JOIN + const logicalType = getLogicalNodeType(conf); + + const confNodeKey: INodeKey = { + name: + 'metaData' in conf && conf.metaData + ? conf.metaData.name + : 'table' in conf + ? conf.table + : 'Unknown', + logicalType }; - infoMap.set(joinNodeKey, { - conf: join as ILogicalNode + infoMap.set(confNodeKey, { + conf: conf as ILogicalNode }); - const joinParents: INodeKey[] = []; - connections.set(joinNodeKey, { parents: joinParents }); + const confParents: INodeKey[] = []; + connections.set(confNodeKey, { parents: confParents }); + + /* + * Join + */ + if ('left' in conf && conf.left && !excludeLeft) { + processSource(conf.left, infoMap, connections, confParents); + } + + if ('joinParts' in conf && conf.joinParts) { + processJoinParts(conf.joinParts, infoMap, connections, confParents); + } + + /* + * GroupBy + */ + if ('sources' in conf && conf.sources) { + for (const source of conf.sources ?? []) { + processSource(source, infoMap, connections, confParents); + } + } - if (join.left && !excludeLeft) { - processSource(join.left, infoMap, connections, joinParents); + /* + * Model + */ + if ('source' in conf && conf.source) { + processSource(conf.source, infoMap, connections, confParents); } - for (const jp of join.joinParts ?? []) { + return { + nodeGraph: { + connections, + infoMap + }, + mainNode: confNodeKey + }; +} + +function processJoinParts( + joinParts: IJoinPart[], + infoMap: NonNullable, + connections: NonNullable, + parents: INodeKey[] +) { + for (const jp of joinParts ?? []) { if (jp.groupBy) { const groupByNodeKey: INodeKey = { name: jp.groupBy.metaData?.name, @@ -40,7 +87,7 @@ export function joinToLineage(join: IJoin, excludeLeft = false): ILineageRespons infoMap.set(groupByNodeKey, { conf: jp.groupBy as ILogicalNode }); - joinParents.push(groupByNodeKey); + parents.push(groupByNodeKey); const groupByParents: INodeKey[] = []; connections.set(groupByNodeKey, { parents: groupByParents }); @@ -50,14 +97,6 @@ export function joinToLineage(join: IJoin, excludeLeft = false): ILineageRespons } } } - - return { - nodeGraph: { - connections, - infoMap - }, - mainNode: joinNodeKey - }; } function processSource( @@ -99,7 +138,7 @@ function processSource( parents.push(joinNodeKey); // Transfer connections and infoMap from joinSource join to root join graph - const joinSourceLineage = joinToLineage(source.joinSource.join as IJoin); + const joinSourceLineage = confToLineage(source.joinSource.join as IJoin); for (const [key, nodeConnections] of joinSourceLineage.nodeGraph?.connections ?? []) { connections.set(key === joinSourceLineage.mainNode ? joinNodeKey : key, nodeConnections); diff --git a/frontend/src/lib/components/ActionButtons.svelte b/frontend/src/lib/components/ActionButtons.svelte index 7c3bc25226..b984b3a5f6 100644 --- a/frontend/src/lib/components/ActionButtons.svelte +++ b/frontend/src/lib/components/ActionButtons.svelte @@ -5,14 +5,10 @@ import { cn } from '$lib/utils'; import IconArrowsUpDown from '~icons/heroicons/arrows-up-down-16-solid'; - import IconPlus from '~icons/heroicons/plus-16-solid'; - import IconSquare3Stack3d from '~icons/heroicons/square-3-stack-3d-16-solid'; - import IconXMark from '~icons/heroicons/x-mark-16-solid'; import { getSortParamKey, type SortContext, getSortParamsConfig } from '$lib/util/sort'; let { - showCluster = false, class: className, showSort = false, context = 'drift' @@ -23,8 +19,6 @@ context?: SortContext; } = $props(); - let activeCluster = showCluster ? 'GroupBys' : null; - const sortKey = $derived(getSortParamKey(context)); const params = $derived( queryParameters(getSortParamsConfig(context), { pushHistory: false, showDefaults: false }) @@ -35,40 +29,11 @@ } -
- - {#if activeCluster} -
- - - -
- {/if} - - -
- {#if showSort} - - {/if} - - {#if showCluster} - - {/if} -
+ {/if}
diff --git a/frontend/src/lib/components/ChartControls.svelte b/frontend/src/lib/components/ChartControls.svelte index 21027fe555..1bbf30dbd7 100644 --- a/frontend/src/lib/components/ChartControls.svelte +++ b/frontend/src/lib/components/ChartControls.svelte @@ -3,9 +3,8 @@ import DriftMetricToggle from '$lib/components/DriftMetricToggle.svelte'; import DateRangeSelector from '$lib/components/DateRangeSelector.svelte'; import ActionButtons from '$lib/components/ActionButtons.svelte'; - import * as Alert from '$lib/components/ui/alert/index.js'; - import { formatDate } from '$lib/util/format'; import type { SortContext } from '$lib/util/sort'; + import { fromAbsolute, getLocalTimeZone } from '@internationalized/date'; let { isZoomed = false, @@ -29,30 +28,24 @@
- {#if isUsingFallbackDates} -
- - - No data for that date range. Showing data between {formatDate(dateRange.startTimestamp)} and - {formatDate(dateRange.endTimestamp)} - -
- {/if} -
{#if isZoomed} {/if} - + {#if context === 'drift'} {/if} -
- {#if showActionButtons} -
+ {#if showActionButtons} -
- {/if} + {/if} +
diff --git a/frontend/src/lib/components/DateRangeSelector.svelte b/frontend/src/lib/components/DateRangeSelector.svelte index 1eb151f4fe..30167672a3 100644 --- a/frontend/src/lib/components/DateRangeSelector.svelte +++ b/frontend/src/lib/components/DateRangeSelector.svelte @@ -19,6 +19,10 @@ import { cn } from '$lib/utils'; import { RangeCalendar } from '$lib/components/ui/range-calendar/index'; import { getDateRangeParamsConfig } from '$lib/util/date-ranges'; + import IconInformationCircle from '~icons/heroicons/information-circle'; + import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'; + + const { fallbackDateRange }: { fallbackDateRange?: DateRange } = $props(); const params = queryParameters(getDateRangeParamsConfig(), { pushHistory: false, @@ -87,66 +91,99 @@ } -
- - - - - - {#each getNonCustomDateRanges() as range} - - {/each} - - - - - - - - - - - -
+ + + + {#each getNonCustomDateRanges() as range} + + {/each} + + + + + + + + + + + + + + + {#if fallbackDateRange} + + {#if calendarDateRange?.start && calendarDateRange?.end} + No data for + {df.format(calendarDateRange.start.toDate(getLocalTimeZone()))} - {df.format( + calendarDateRange.end.toDate(getLocalTimeZone()) + )}. Showing fallback data + {/if} + + {/if} + diff --git a/frontend/src/lib/types/LogicalNode.ts b/frontend/src/lib/types/LogicalNode.ts index 99b784f9b9..3aed688529 100644 --- a/frontend/src/lib/types/LogicalNode.ts +++ b/frontend/src/lib/types/LogicalNode.ts @@ -13,27 +13,32 @@ export const logicalNodeConfig = { [LogicalType.GROUP_BY]: { label: 'GroupBys', icon: getEntity(EntityTypes.GROUPBYS).icon, - color: '50 80% 50%' + color: '50 80% 50%', + url: '/groupbys' }, [LogicalType.JOIN]: { label: 'Joins', icon: getEntity(EntityTypes.JOINS).icon, - color: '100 80% 50%' + color: '100 80% 50%', + url: '/joins' }, [LogicalType.STAGING_QUERY]: { label: 'Staging Queries', icon: getEntity(EntityTypes.STAGINGQUERIES).icon, - color: '150 80% 50%' + color: '150 80% 50%', + url: '/stagingqueries' }, [LogicalType.MODEL]: { label: 'Models', icon: getEntity(EntityTypes.MODELS).icon, - color: '200 80% 50%' + color: '200 80% 50%', + url: '/models' }, [LogicalType.TABULAR_DATA]: { label: 'Tabular Data', icon: IconTableCells, - color: '220 80% 50%' + color: '220 80% 50%', + url: null // Not available } }; diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index a2da2124e5..d31edd4de7 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -25,15 +25,15 @@
-
-
+
+
{@render children()}
diff --git a/frontend/src/routes/+layout.ts b/frontend/src/routes/+layout.ts index e4b3dda3c8..a3d15781a7 100644 --- a/frontend/src/routes/+layout.ts +++ b/frontend/src/routes/+layout.ts @@ -1 +1 @@ -// export const ssr = false; +export const ssr = false; diff --git a/frontend/src/routes/[conf]/+page.server.ts b/frontend/src/routes/[conf]/+page.server.ts new file mode 100644 index 0000000000..70b1dd8942 --- /dev/null +++ b/frontend/src/routes/[conf]/+page.server.ts @@ -0,0 +1,44 @@ +import { Api } from '$lib/api/api'; +import { ConfType } from '$lib/types/codegen'; +import type { IConfListResponse } from '$lib/types/codegen/ConfListResponse'; +import { entityConfig } from '$lib/types/Entity/Entity'; + +const ConfResponseMap: Record = { + [ConfType.MODEL]: 'models', + [ConfType.STAGING_QUERY]: 'stagingQueries', + [ConfType.GROUP_BY]: 'groupBys', + [ConfType.JOIN]: 'joins' +}; + +export async function load({ fetch, url, params }) { + const path = url.pathname; + const entityMatch = entityConfig.find((entity) => + params.conf.startsWith(entity.path.substring(1)) + ); + + if (!entityMatch) { + return { + items: [], + basePath: path, + title: '' + }; + } + + try { + const api = new Api({ fetch }); + const response = await api.getConfList(entityMatch.type); + if (!response) throw new Error(`Failed to fetch ${entityMatch.label.toLowerCase()}`); + + const responseKey = ConfResponseMap[entityMatch.type]; + const items = response[responseKey] ?? []; + + return { + items: items, + basePath: path, + title: entityMatch.label + }; + } catch (error) { + console.error(`Failed to load ${entityMatch.label.toLowerCase()}:`, error); + throw error; + } +} diff --git a/frontend/src/routes/joins/+page.svelte b/frontend/src/routes/[conf]/+page.svelte similarity index 57% rename from frontend/src/routes/joins/+page.svelte rename to frontend/src/routes/[conf]/+page.svelte index 68ed74d44f..01f051f933 100644 --- a/frontend/src/routes/joins/+page.svelte +++ b/frontend/src/routes/[conf]/+page.svelte @@ -4,9 +4,9 @@ const { data } = $props(); - // TODO: remove this once we have data for all joins - const sortedItems = sort(data.items, (d) => - d.metaData?.name === 'risk.user_transactions.txn_join' ? -1 : 1 + // TODO: remove this once we have observability data for all joins + const sortedItems = $derived( + sort(data.items, (d) => (d.metaData?.name === 'risk.user_transactions.txn_join' ? -1 : 1)) ); diff --git a/frontend/src/routes/joins/[slug]/+layout.svelte b/frontend/src/routes/[conf]/[name]/+layout.svelte similarity index 55% rename from frontend/src/routes/joins/[slug]/+layout.svelte rename to frontend/src/routes/[conf]/[name]/+layout.svelte index 8bf3005589..aba7f86c2f 100644 --- a/frontend/src/routes/joins/[slug]/+layout.svelte +++ b/frontend/src/routes/[conf]/[name]/+layout.svelte @@ -13,22 +13,26 @@ const pageName = $derived(page.url.pathname.split('/').at(-1)); - + - - - + + + - Overview - + Overview + + - Job tracking - - - Observability + Job tracking + + + + {#if page.params.conf === 'joins'} + + + Observability + + {/if} diff --git a/frontend/src/routes/[conf]/[name]/+layout.ts b/frontend/src/routes/[conf]/[name]/+layout.ts new file mode 100644 index 0000000000..01a09d8a08 --- /dev/null +++ b/frontend/src/routes/[conf]/[name]/+layout.ts @@ -0,0 +1,43 @@ +import type { PageServerLoad } from './$types'; +import { Api } from '$lib/api/api'; +import { ConfType } from '$src/lib/types/codegen'; +import { getEntity, type EntityId } from '$src/lib/types/Entity/Entity'; + +function getConfApi(api: Api, confType: ConfType, confName: string) { + switch (confType) { + case ConfType.JOIN: + return { + conf: api.getJoin(confName), + lineage: api.getJoinLineage({ name: confName }) + }; + case ConfType.GROUP_BY: + return { + conf: api.getGroupBy(confName), + lineage: api.getGroupByLineage({ name: confName }) + }; + case ConfType.MODEL: + return { + conf: api.getModel(confName), + lineage: api.getModelLineage({ name: confName }) + }; + case ConfType.STAGING_QUERY: + return { + conf: api.getStagingQuery(confName), + lineage: api.getStagingQueryLineage({ name: confName }) + }; + } +} + +export const load: PageServerLoad = async ({ params, fetch }) => { + const api = new Api({ fetch }); + + const confType = getEntity(params.conf as EntityId).type; + const confApi = getConfApi(api, confType, params.name); + + const [conf, lineage] = await Promise.all([confApi?.conf, confApi?.lineage]); + + return { + conf, + lineage + }; +}; diff --git a/frontend/src/routes/joins/[slug]/+page.server.ts b/frontend/src/routes/[conf]/[name]/+page.server.ts similarity index 100% rename from frontend/src/routes/joins/[slug]/+page.server.ts rename to frontend/src/routes/[conf]/[name]/+page.server.ts diff --git a/frontend/src/routes/[conf]/[name]/+page.svelte b/frontend/src/routes/[conf]/[name]/+page.svelte new file mode 100644 index 0000000000..23ba388e94 --- /dev/null +++ b/frontend/src/routes/[conf]/[name]/+page.svelte @@ -0,0 +1 @@ + diff --git a/frontend/src/routes/joins/[slug]/job-tracking/+page.svelte b/frontend/src/routes/[conf]/[name]/job-tracking/+page.svelte similarity index 100% rename from frontend/src/routes/joins/[slug]/job-tracking/+page.svelte rename to frontend/src/routes/[conf]/[name]/job-tracking/+page.svelte diff --git a/frontend/src/routes/[conf]/[name]/job-tracking/+page.ts b/frontend/src/routes/[conf]/[name]/job-tracking/+page.ts new file mode 100644 index 0000000000..5f87fc4d6e --- /dev/null +++ b/frontend/src/routes/[conf]/[name]/job-tracking/+page.ts @@ -0,0 +1,40 @@ +import type { PageLoad } from './$types'; +import { buildJobTrackerTree } from '$lib/job/tree-builder/tree-builder'; +import type { IJobTrackerResponseArgs, INodeKeyArgs } from '$lib/types/codegen'; +import { Api } from '$lib/api/api'; + +export const load: PageLoad = async ({ parent, fetch }) => { + const { lineage } = await parent(); + const api = new Api({ fetch }); + + // Get job tracker data for all nodes in lineage + const nodes = new Set(); + + // Add main nodes + lineage.nodeGraph?.connections?.forEach((_, node) => { + nodes.add(node); + }); + + // Add parent nodes + lineage.nodeGraph?.connections?.forEach((connection) => { + connection.parents?.forEach((parent) => { + nodes.add(parent); + }); + }); + + // todo think about where this is best called - might be worth lazily lower levels + const jobTrackerDataMap = new Map(); + await Promise.all( + Array.from(nodes).map(async (node) => { + const data = await api.getJobTrackerData(node); + jobTrackerDataMap.set(node.name ?? '', data); + }) + ); + + const jobTrackerData = buildJobTrackerTree(lineage, jobTrackerDataMap); + + return { + jobTree: jobTrackerData.jobTree, + dates: jobTrackerData.dates + }; +}; diff --git a/frontend/src/routes/[conf]/[name]/observability/+layout.ts b/frontend/src/routes/[conf]/[name]/observability/+layout.ts new file mode 100644 index 0000000000..1771b64110 --- /dev/null +++ b/frontend/src/routes/[conf]/[name]/observability/+layout.ts @@ -0,0 +1,58 @@ +import { Api } from '$src/lib/api/api'; +import { parseDateRangeParams } from '$lib/util/date-ranges'; +import { getDriftMetricFromParams } from '$src/lib/util/drift-metric'; +import { type IJoinDriftResponse } from '$lib/types/codegen'; + +const FALLBACK_START_TS = 1672531200000; // 2023-01-01 +const FALLBACK_END_TS = 1677628800000; // 2023-03-01 + +export async function load({ url, params }) { + const requestedDateRange = parseDateRangeParams(url.searchParams); + const driftMetric = getDriftMetricFromParams(url.searchParams); + + const confName = params.name; + + let dateRange = { + ...requestedDateRange, + isUsingFallback: false + }; + + const api = new Api({ fetch }); + + let joinDrift: IJoinDriftResponse | undefined = undefined; + try { + // Try with requested date range first + joinDrift = await api.getJoinDrift({ + name: confName, + startTs: dateRange.startTimestamp, + endTs: dateRange.endTimestamp, + algorithm: driftMetric + }); + + // If empty data is results, use fallback date range + const useFallback = joinDrift.driftSeries.every((ds) => Object.keys(ds).length <= 1); // Only `key` returned on results + if (useFallback) { + dateRange = { + ...dateRange, + startTimestamp: FALLBACK_START_TS, + endTimestamp: FALLBACK_END_TS, + isUsingFallback: true + }; + + joinDrift = await api.getJoinDrift({ + name: confName, + startTs: dateRange.startTimestamp, + endTs: dateRange.endTimestamp, + algorithm: driftMetric + }); + } + } catch { + // Drift series data not available for join + } + + return { + joinDrift, + driftMetric, + dateRange + }; +} diff --git a/frontend/src/routes/joins/[slug]/observability/+page.server.ts b/frontend/src/routes/[conf]/[name]/observability/+page.server.ts similarity index 100% rename from frontend/src/routes/joins/[slug]/observability/+page.server.ts rename to frontend/src/routes/[conf]/[name]/observability/+page.server.ts diff --git a/frontend/src/routes/joins/[slug]/observability/ObservabilityNavTabs.svelte b/frontend/src/routes/[conf]/[name]/observability/ObservabilityNavTabs.svelte similarity index 100% rename from frontend/src/routes/joins/[slug]/observability/ObservabilityNavTabs.svelte rename to frontend/src/routes/[conf]/[name]/observability/ObservabilityNavTabs.svelte diff --git a/frontend/src/routes/joins/[slug]/observability/distributions/+page.svelte b/frontend/src/routes/[conf]/[name]/observability/distributions/+page.svelte similarity index 93% rename from frontend/src/routes/joins/[slug]/observability/distributions/+page.svelte rename to frontend/src/routes/[conf]/[name]/observability/distributions/+page.svelte index 29c7ce1bd6..a24b8ba71d 100644 --- a/frontend/src/routes/joins/[slug]/observability/distributions/+page.svelte +++ b/frontend/src/routes/[conf]/[name]/observability/distributions/+page.svelte @@ -7,9 +7,8 @@ import { getSortParamsConfig, getSortParamKey } from '$lib/util/sort'; import type { ITileSummarySeries } from '$src/lib/types/codegen'; import ChartControls from '$src/lib/components/ChartControls.svelte'; - import ObservabilityNavTabs from '$routes/joins/[slug]/observability/ObservabilityNavTabs.svelte'; + import ObservabilityNavTabs from '$routes/[conf]/[name]/observability/ObservabilityNavTabs.svelte'; import { Separator } from '$src/lib/components/ui/separator'; - import ModelTable from '../ModelTable.svelte'; import PercentileLineChart from '$src/lib/components/charts/PercentileLineChart.svelte'; const { data } = $props(); @@ -43,10 +42,6 @@ } -{#if data.model} - -{/if} -
{ - const { join, joinDrift, dateRange } = await parent(); + const { conf, joinDrift, dateRange } = await parent(); const api = new Api({ fetch }); - const joinName = join?.metaData?.name?.replace('/', '.') ?? 'Unknown'; + const confName = conf?.metaData?.name?.replace('/', '.') ?? 'Unknown'; const columnNames = joinDrift?.driftSeries.map((ds) => ds.key?.column ?? 'Unknown') ?? []; // Fetch percentile data for each column @@ -12,7 +12,7 @@ export const load = async ({ parent, fetch }) => { const distributionsPromise = Promise.allSettled( columnNames.map((columnName) => api.getColumnSummary({ - name: joinName, + name: confName, columnName: columnName, startTs: dateRange.startTimestamp, endTs: dateRange.endTimestamp diff --git a/frontend/src/routes/joins/[slug]/observability/drift/+page.svelte b/frontend/src/routes/[conf]/[name]/observability/drift/+page.svelte similarity index 97% rename from frontend/src/routes/joins/[slug]/observability/drift/+page.svelte rename to frontend/src/routes/[conf]/[name]/observability/drift/+page.svelte index 58ed383ea3..77c2496c9e 100644 --- a/frontend/src/routes/joins/[slug]/observability/drift/+page.svelte +++ b/frontend/src/routes/[conf]/[name]/observability/drift/+page.svelte @@ -14,8 +14,6 @@ import { formatDate } from '$lib/util/format'; import { DRIFT_METRIC_SCALES } from '$lib/util/drift-metric'; import ChartControls from '$lib/components/ChartControls.svelte'; - import type { JoinData } from '$routes/joins/[slug]/services/joins.service'; - import ModelTable from '$routes/joins/[slug]/observability/ModelTable.svelte'; import Separator from '$lib/components/ui/separator/separator.svelte'; import ObservabilityNavTabs from '../ObservabilityNavTabs.svelte'; import FeaturesLineChart from '$lib/components/charts/FeaturesLineChart.svelte'; @@ -35,7 +33,7 @@ const api = new Api(); - const { data }: { data: JoinData } = $props(); + const { data } = $props(); const sortContext = 'drift'; const sortKey = getSortParamKey(sortContext); @@ -44,6 +42,7 @@ showDefaults: false }); const sortDirection = $derived(params[sortKey]); + // TODO: Determine why sortDirection is not updating if client-side routed to page (page refresh or HMR fixes it though) const baselineOffset: Duration = { days: 7 }; @@ -90,7 +89,7 @@ if (seriesPoint) { try { - const joinName = data.join?.metaData?.name?.replace('/', '.') ?? 'Unknown'; + const joinName = data.conf?.metaData?.name?.replace('/', '.') ?? 'Unknown'; const columnName = seriesPoint.series.key.toString(); // TODO: Add loading and error states @@ -120,10 +119,6 @@ } -{#if data.model} - -{/if} -
['data']; type CustomNode = Node & { id: string; key: INodeKey; value: INodeInfo }; const { data } = $props(); - const { connections, infoMap } = data.lineage.nodeGraph ?? {}; + const { connections, infoMap } = $derived(data.lineage.nodeGraph ?? {}); const nodes: DagreData['nodes'] = $derived( Array.from(infoMap?.entries() ?? []).map(([key, value]) => ({ @@ -57,7 +60,7 @@ }) ?? [] ); } - const edges = getEdges(data.lineage.mainNode!); + const edges = $derived(getEdges(data.lineage.mainNode!)); const chartData = $derived({ nodes, edges }); @@ -111,14 +114,14 @@ {#snippet collapsibleContent()} - + {/snippet} -
-
+
+
{/each} @@ -315,7 +318,7 @@ classes={{ label: cn(tooltipProps.item.classes?.label, 'self-start') }} valueAlign="left" > - {#each Object.entries(data.value.conf.query.selects) as [key, _]} + {#each Object.entries(data.value.conf.query?.selects ?? {}) as [key, _]}
{key}
{/each} @@ -340,21 +343,36 @@ {@const config = getLogicalNodeConfig(selectedNode)} {@const Icon = config?.icon} -
-
- -
-
-
- {namespace} +
+
+
+
-
- {nameParts.join('.')} +
+
+ {namespace} +
+
+ {nameParts.join('.')} +
+ + {#if config.url && selectedNode.id !== data.lineage.mainNode?.name} +
+ +
+ {/if}
diff --git a/frontend/src/routes/joins/[slug]/overview/ConfProperties.svelte b/frontend/src/routes/[conf]/[name]/overview/ConfProperties.svelte similarity index 76% rename from frontend/src/routes/joins/[slug]/overview/ConfProperties.svelte rename to frontend/src/routes/[conf]/[name]/overview/ConfProperties.svelte index b7af589e2c..55da5e9975 100644 --- a/frontend/src/routes/joins/[slug]/overview/ConfProperties.svelte +++ b/frontend/src/routes/[conf]/[name]/overview/ConfProperties.svelte @@ -19,7 +19,9 @@ type ISource, type IMetaData, type IJoinSource, - Operation + Operation, + ModelType, + DataKind } from '$src/lib/types/codegen'; import { keys } from '@layerstack/utils'; import Self from './ConfProperties.svelte'; @@ -75,19 +77,23 @@
{#if conf?.metaData}
-
{metaDataLabel}
+ {#if metaDataLabel} +
{metaDataLabel}
+ {/if}
{#each METADATA_PROPERTIES as prop} - - - {prop} - - - {conf.metaData[prop]} - - + {#if prop in conf.metaData} + + + {prop} + + + {conf.metaData[prop]} + + + {/if} {/each}
@@ -95,22 +101,52 @@
{/if} - {#if conf?.left && includeUpstream} + {#if 'modelType' in conf}
-
- Left ({conf.left.entities ? 'entity' : conf.left.events ? 'event' : 'join'}) -
+
Model
+ + modelType + - + {ModelType[conf.modelType ?? 0]} + + + + + + modelParams + + + {#each Object.entries(conf.modelParams ?? {}) as [key, value]} +
+ {key}: + {value} +
+ {/each} + + + + + + outputSchema + + +
+ name: + {conf.outputSchema?.name} +
+
+ kind: + {DataKind[conf.outputSchema?.kind ?? 0]} +
+
+ params: + {conf.outputSchema?.params?.map((p) => `${p.name}: ${p.dataType}`).join(', ')} +
@@ -119,6 +155,35 @@ {/if} + {#if (conf?.left || conf?.source) && includeUpstream} + {@const source = conf.left ?? conf.source} + {#if source} +
+
+ {conf.left ? 'Left' : 'Source'} ({source.entities + ? 'entity' + : source.events + ? 'event' + : 'join'}) +
+
+
+ + + + + + + +
+
+
+ {/if} + {/if} + {#if conf?.rowIds}
Row IDs
diff --git a/frontend/src/routes/groupbys/+page.server.ts b/frontend/src/routes/groupbys/+page.server.ts deleted file mode 100644 index ddc6c06a47..0000000000 --- a/frontend/src/routes/groupbys/+page.server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { PageServerLoad } from './$types'; -import { loadConfList } from '$lib/server/conf-loader'; - -export const load: PageServerLoad = loadConfList; diff --git a/frontend/src/routes/groupbys/+page.svelte b/frontend/src/routes/groupbys/+page.svelte deleted file mode 100644 index f10bb22943..0000000000 --- a/frontend/src/routes/groupbys/+page.svelte +++ /dev/null @@ -1,7 +0,0 @@ - - - diff --git a/frontend/src/routes/joins/+page.server.ts b/frontend/src/routes/joins/+page.server.ts deleted file mode 100644 index ddc6c06a47..0000000000 --- a/frontend/src/routes/joins/+page.server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { PageServerLoad } from './$types'; -import { loadConfList } from '$lib/server/conf-loader'; - -export const load: PageServerLoad = loadConfList; diff --git a/frontend/src/routes/joins/[slug]/+layout.ts b/frontend/src/routes/joins/[slug]/+layout.ts deleted file mode 100644 index a33815259d..0000000000 --- a/frontend/src/routes/joins/[slug]/+layout.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { PageServerLoad } from './$types'; -import { Api } from '$lib/api/api'; -import { parseDateRangeParams } from '$lib/util/date-ranges'; -import { getDriftMetricFromParams } from '$src/lib/util/drift-metric'; -import { getJoinData } from '$routes/joins/[slug]/services/joins.service'; - -export const load: PageServerLoad = async ({ params, url, fetch }) => { - const api = new Api({ fetch }); - const requestedDateRange = parseDateRangeParams(url.searchParams); - const joinName = params.slug; - const driftMetric = getDriftMetricFromParams(url.searchParams); - - const [joinData] = await Promise.all([ - getJoinData(api, joinName, requestedDateRange, driftMetric) - ]); - - return { - ...joinData - }; -}; diff --git a/frontend/src/routes/joins/[slug]/+page.svelte b/frontend/src/routes/joins/[slug]/+page.svelte deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/src/routes/joins/[slug]/job-tracking/+page.ts b/frontend/src/routes/joins/[slug]/job-tracking/+page.ts deleted file mode 100644 index 00e3796bb9..0000000000 --- a/frontend/src/routes/joins/[slug]/job-tracking/+page.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { PageLoad } from './$types'; -import { buildJobTrackerTree } from '$lib/job/tree-builder/tree-builder'; - -export const load: PageLoad = async ({ parent }) => { - const { lineage, jobTrackerDataMap } = await parent(); - - const jobTrackerData = buildJobTrackerTree(lineage, jobTrackerDataMap); - - return { - jobTree: jobTrackerData.jobTree, - dates: jobTrackerData.dates - }; -}; diff --git a/frontend/src/routes/joins/[slug]/observability/ModelTable.svelte b/frontend/src/routes/joins/[slug]/observability/ModelTable.svelte deleted file mode 100644 index 1b67fecdf2..0000000000 --- a/frontend/src/routes/joins/[slug]/observability/ModelTable.svelte +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - diff --git a/frontend/src/routes/joins/[slug]/services/joins.service.ts b/frontend/src/routes/joins/[slug]/services/joins.service.ts deleted file mode 100644 index e8eecab153..0000000000 --- a/frontend/src/routes/joins/[slug]/services/joins.service.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { Api } from '$lib/api/api'; -import { - DriftMetric, - type IJoin, - type IJoinDriftResponse, - type ILineageResponse, - type IModel, - type IJobTrackerResponseArgs, - type INodeKeyArgs -} from '$lib/types/codegen'; - -const FALLBACK_START_TS = 1672531200000; // 2023-01-01 -const FALLBACK_END_TS = 1677628800000; // 2023-03-01 - -export type JoinData = { - join: IJoin; - lineage: ILineageResponse; - model?: IModel; - joinDrift?: IJoinDriftResponse; - driftMetric: DriftMetric; - dateRange: { - startTimestamp: number; - endTimestamp: number; - dateRangeValue: string; - isUsingFallback: boolean; - }; - jobTrackerDataMap: Map; -}; - -export async function getJoinData( - api: Api, - joinName: string, - requestedDateRange: { startTimestamp: number; endTimestamp: number; dateRangeValue: string }, - driftMetric: DriftMetric -): Promise { - const [join, lineage, model] = await Promise.all([ - api.getJoin(joinName), - api.getJoinLineage({ name: joinName }), - api - .getModelList() - .then((models) => - models.models?.find((m) => m.source?.joinSource?.join?.metaData?.name === joinName) - ) - ]); - - // Get job tracker data for all nodes in lineage - const nodes = new Set(); - - // Add main nodes - lineage.nodeGraph?.connections?.forEach((_, node) => { - nodes.add(node); - }); - - // Add parent nodes - lineage.nodeGraph?.connections?.forEach((connection) => { - connection.parents?.forEach((parent) => { - nodes.add(parent); - }); - }); - - // todo think about where this is best called - might be worth lazily lower levels - const jobTrackerDataMap = new Map(); - await Promise.all( - Array.from(nodes).map(async (node) => { - const data = await api.getJobTrackerData(node); - jobTrackerDataMap.set(node.name ?? '', data); - }) - ); - - let dateRange = { - ...requestedDateRange, - isUsingFallback: false - }; - - let joinDrift: IJoinDriftResponse | undefined = undefined; - try { - // Try with requested date range first - joinDrift = await api.getJoinDrift({ - name: joinName, - startTs: dateRange.startTimestamp, - endTs: dateRange.endTimestamp, - algorithm: driftMetric - }); - - // If empty data is results, use fallback date range - const useFallback = joinDrift.driftSeries.every((ds) => Object.keys(ds).length <= 1); // Only `key` returned on results - if (useFallback) { - dateRange = { - ...dateRange, - startTimestamp: FALLBACK_START_TS, - endTimestamp: FALLBACK_END_TS, - isUsingFallback: true - }; - - joinDrift = await api.getJoinDrift({ - name: joinName, - startTs: dateRange.startTimestamp, - endTs: dateRange.endTimestamp, - algorithm: driftMetric - }); - } - } catch { - // Drift series data not available for join - } - - return { - join, - lineage, - model, - joinDrift, - driftMetric, - dateRange, - jobTrackerDataMap - }; -} diff --git a/frontend/src/routes/models/+page.server.ts b/frontend/src/routes/models/+page.server.ts deleted file mode 100644 index ddc6c06a47..0000000000 --- a/frontend/src/routes/models/+page.server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { PageServerLoad } from './$types'; -import { loadConfList } from '$lib/server/conf-loader'; - -export const load: PageServerLoad = loadConfList; diff --git a/frontend/src/routes/models/+page.svelte b/frontend/src/routes/models/+page.svelte deleted file mode 100644 index f10bb22943..0000000000 --- a/frontend/src/routes/models/+page.svelte +++ /dev/null @@ -1,7 +0,0 @@ - - - diff --git a/frontend/src/routes/stagingqueries/+page.server.ts b/frontend/src/routes/stagingqueries/+page.server.ts deleted file mode 100644 index ddc6c06a47..0000000000 --- a/frontend/src/routes/stagingqueries/+page.server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { PageServerLoad } from './$types'; -import { loadConfList } from '$lib/server/conf-loader'; - -export const load: PageServerLoad = loadConfList; diff --git a/frontend/src/routes/stagingqueries/+page.svelte b/frontend/src/routes/stagingqueries/+page.svelte deleted file mode 100644 index f10bb22943..0000000000 --- a/frontend/src/routes/stagingqueries/+page.svelte +++ /dev/null @@ -1,7 +0,0 @@ - - -