diff --git a/docs/docs/vis/spatial-canvas-status.mdx b/docs/docs/vis/spatial-canvas-status.mdx index e6dacc4e..15d01110 100644 --- a/docs/docs/vis/spatial-canvas-status.mdx +++ b/docs/docs/vis/spatial-canvas-status.mdx @@ -19,6 +19,12 @@ sidebar_position: 1 - **View state** conversion between SpatialCanvas and Viv is still **2D-oriented**; full 3D orbit state is not fully round-tripped. - **`useLayerData`** prefers explicit `LayerConfig.channels` when arrays are non-empty; further **override flags** may still be useful for edge cases. +## Upstream Viv Follow-ups + +- **Labels currently carry local Viv workarounds:** SpatialCanvas labels use local single-scale and multiscale wrappers instead of Viv `ImageLayer` / `MultiscaleImageLayer` because the current Viv prop schemas reject `null` callback defaults such as `onHover`, `onClick`, and `onViewportLoad`. +- **Viv interpolation prop typing also needs cleanup:** current Viv layer defaults still type `interpolation` incorrectly on some paths, which triggers deck.gl prop validation when labels request `'nearest'` / `'linear'`. +- **Shader path should converge back toward upstream Viv:** labels already use GLSL 300 ES + UBO-style uniforms locally; once the Viv shader/toolchain work is published we should revisit these wrappers and delete any compatibility code that is no longer needed. + ## Related code - [`packages/vis/src/SpatialCanvas/index.tsx`](https://github.com/Taylor-CCB-Group/SpatialData.js/blob/main/packages/vis/src/SpatialCanvas/index.tsx) — `SpatialCanvas` UI and store wiring diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5783371d..ea52dd51 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,3 +9,4 @@ export * from './types.js'; export * from './store/index.js'; export * from './models/index.js'; export * from './spatialViewFit.js'; +export * from './tooltip.js'; diff --git a/packages/core/src/tooltip.ts b/packages/core/src/tooltip.ts new file mode 100644 index 00000000..c06400ab --- /dev/null +++ b/packages/core/src/tooltip.ts @@ -0,0 +1,203 @@ +import type { LabelsElement, ShapesElement } from './models'; +import type { SpatialData } from './store'; +import type { TableColumnData } from './types'; + +export type SpatialFeatureTooltipItem = { + label: string; + value: string; +}; + +export type SpatialFeatureTooltipData = { + title?: string; + items: SpatialFeatureTooltipItem[]; +}; + +interface BaseTooltipMetadata { + tooltipSignature?: string; + tooltipFields?: string[]; + tooltipColumns?: Array; +} + +export interface ShapesTooltipMetadata extends BaseTooltipMetadata { + featureIds?: string[]; + tooltipRowIndices?: Int32Array; +} + +export interface LabelsTooltipMetadata extends BaseTooltipMetadata { + tooltipRowIndexByFeatureId?: Map; +} + +interface AssociatedTableTooltipData extends BaseTooltipMetadata { + rowIds?: string[]; + rowIndexByFeatureId?: Map; +} + +export function getTooltipSignature(tooltipFields?: string[]): string { + return (tooltipFields ?? []).join('\u0001'); +} + +export function normalizeTooltipValue( + value: TableColumnData | undefined, + rowIndex: number, +): string { + if (!value) return ''; + const row = value[rowIndex]; + if (row === null || row === undefined) return ''; + return String(row); +} + +export function resolveTooltipItems( + tooltipFields: string[] | undefined, + tooltipColumns: Array | undefined, + rowIndex: number, +): SpatialFeatureTooltipItem[] { + if (!tooltipFields || !tooltipColumns || rowIndex < 0) { + return []; + } + return tooltipFields + .map((field, fieldIndex) => ({ + label: field, + value: normalizeTooltipValue(tooltipColumns[fieldIndex], rowIndex), + })) + .filter((item) => item.value !== ''); +} + +function tableRegionMatches(kind: 'shapes' | 'labels', regionValue: string, key: string) { + return regionValue === key || regionValue === `${kind}/${key}`; +} + +async function loadAssociatedTableTooltipData({ + spatialData, + kind, + key, + tooltipFields, +}: { + spatialData: SpatialData | undefined; + kind: 'shapes' | 'labels'; + key: string; + tooltipFields: string[]; +}): Promise { + if (tooltipFields.length === 0) { + return { + tooltipSignature: '', + tooltipFields: [], + tooltipColumns: undefined, + rowIds: undefined, + rowIndexByFeatureId: undefined, + }; + } + + if (!spatialData) { + return { + tooltipSignature: undefined, + tooltipFields, + tooltipColumns: undefined, + rowIds: undefined, + rowIndexByFeatureId: undefined, + }; + } + + const associated = spatialData.getAssociatedTable(kind, key); + if (!associated) { + return { + tooltipSignature: undefined, + tooltipFields, + tooltipColumns: undefined, + rowIds: undefined, + rowIndexByFeatureId: undefined, + }; + } + + const tooltipSignature = getTooltipSignature(tooltipFields); + const [, table] = associated; + const { regionKey } = table.getTableKeys(); + const requestedColumns = Array.from(new Set([regionKey, ...tooltipFields])); + const rowIds = await table.loadObsIndex(); + const columns = await table.loadObsColumns(requestedColumns); + const regionColumn = columns[0]; + const tooltipColumns = columns.slice(1); + const filteredRowIds: string[] = []; + const rowIndexByFeatureId = new Map(); + + for (let rowIndex = 0; rowIndex < rowIds.length; rowIndex++) { + const regionValue = normalizeTooltipValue(regionColumn, rowIndex); + if (regionValue && !tableRegionMatches(kind, regionValue, key)) { + continue; + } + const rowId = String(rowIds[rowIndex]); + filteredRowIds.push(rowId); + rowIndexByFeatureId.set(rowId, rowIndex); + } + + return { + tooltipSignature, + tooltipFields, + tooltipColumns, + rowIds: filteredRowIds, + rowIndexByFeatureId, + }; +} + +export async function loadShapesTooltipMetadata( + spatialData: SpatialData | undefined, + element: ShapesElement, + tooltipFields: string[], +): Promise { + const featureIdsRaw = await element.loadFeatureIds(); + const featureIds = featureIdsRaw + ? Array.from(featureIdsRaw, (value: unknown) => String(value)) + : undefined; + + const associated = await loadAssociatedTableTooltipData({ + spatialData, + kind: 'shapes', + key: element.key, + tooltipFields, + }); + + let tooltipRowIndices: Int32Array | undefined; + if (featureIds && associated.rowIds && associated.rowIndexByFeatureId) { + const isDirectlyAligned = + associated.rowIds.length === featureIds.length && + associated.rowIds.every((rowId, index) => rowId === featureIds[index]); + + if (!isDirectlyAligned) { + tooltipRowIndices = new Int32Array(featureIds.length); + tooltipRowIndices.fill(-1); + for (const [featureIndex, featureId] of featureIds.entries()) { + const matchedRowIndex = associated.rowIndexByFeatureId.get(featureId); + if (matchedRowIndex !== undefined) { + tooltipRowIndices[featureIndex] = matchedRowIndex; + } + } + } + } + + return { + featureIds, + tooltipSignature: associated.tooltipSignature, + tooltipFields: associated.tooltipFields, + tooltipColumns: associated.tooltipColumns, + tooltipRowIndices, + }; +} + +export async function loadLabelsTooltipMetadata( + spatialData: SpatialData | undefined, + element: LabelsElement, + tooltipFields: string[], +): Promise { + const associated = await loadAssociatedTableTooltipData({ + spatialData, + kind: 'labels', + key: element.key, + tooltipFields, + }); + + return { + tooltipSignature: associated.tooltipSignature, + tooltipFields: associated.tooltipFields, + tooltipColumns: associated.tooltipColumns, + tooltipRowIndexByFeatureId: associated.rowIndexByFeatureId, + }; +} diff --git a/packages/layers/package.json b/packages/layers/package.json index f7ae3968..7e9c011a 100644 --- a/packages/layers/package.json +++ b/packages/layers/package.json @@ -21,6 +21,8 @@ "deck.gl": "catalog:" }, "dependencies": { + "@deck.gl/core": "catalog:", + "@hms-dbmi/viv": "catalog:", "@math.gl/core": "catalog:", "zod": "catalog:" }, diff --git a/packages/layers/src/LabelsBitmaskTileLayer.ts b/packages/layers/src/LabelsBitmaskTileLayer.ts new file mode 100644 index 00000000..957036ad --- /dev/null +++ b/packages/layers/src/LabelsBitmaskTileLayer.ts @@ -0,0 +1,338 @@ +import type { GetPickingInfoParams, PickingInfo } from '@deck.gl/core'; +import { project32, picking } from '@deck.gl/core'; +import { Matrix4 } from '@math.gl/core'; +import { XRLayer } from '@hms-dbmi/viv'; +import { fs, labelsBitmaskUniforms, vs } from './labelsBitmaskLayerShaders'; + +const MAX_LABEL_CHANNELS = 7; + +function padWithDefault(arr: readonly T[] | undefined, defaultValue: T, targetLength: number): T[] { + const next = arr ? [...arr] : []; + while (next.length < targetLength) { + next.push(defaultValue); + } + return next; +} + +function getNormalizedColor(color?: readonly number[]): [number, number, number] { + if (!color || color.length < 3) { + return [0, 0, 0]; + } + return [color[0] / 255, color[1] / 255, color[2] / 255]; +} + +function getLabelCoordinate( + coordinate: number[] | undefined, + modelMatrix: unknown, +): [number, number, number] | null { + if (!coordinate || coordinate.length < 2) { + return null; + } + const point: [number, number, number] = [ + coordinate[0] ?? 0, + coordinate[1] ?? 0, + coordinate[2] ?? 0, + ]; + if (!modelMatrix) { + return point; + } + try { + return new Matrix4(modelMatrix as any).invert().transformAsPoint(point) as [number, number, number]; + } catch { + return point; + } +} + +function getTopmostLabelAtPixel( + props: any, + pixelX: number, + pixelY: number, +): { channelIndex: number; labelId: number; selection?: unknown } | null { + const { + channelData, + channelsVisible, + selections, + channelColors, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + } = props; + const channelArrays = channelData?.data; + const width = channelData?.width; + const height = channelData?.height; + if (!channelArrays?.length || !width || !height) { + return null; + } + + const actualChannelCount = Math.max( + 1, + Math.min( + MAX_LABEL_CHANNELS, + channelArrays.length ?? + selections?.length ?? + channelColors?.length ?? + channelsVisible?.length ?? + channelOpacities?.length ?? + channelOutlineOpacities?.length ?? + channelsFilled?.length ?? + channelStrokeWidths?.length ?? + 1 + ) + ); + const pixelIndex = pixelY * width + pixelX; + for (let channelIndex = actualChannelCount - 1; channelIndex >= 0; channelIndex--) { + if (!(channelsVisible?.[channelIndex] ?? true)) { + continue; + } + const labelValue = Number(channelArrays[channelIndex]?.[pixelIndex] ?? 0); + if (Number.isFinite(labelValue) && labelValue > 0) { + return { + channelIndex, + labelId: labelValue, + selection: selections?.[channelIndex], + }; + } + } + return null; +} + +type LabelsBitmaskTileLayerProps = { + channelColors?: Array<[number, number, number]>; + channelsFilled?: boolean[]; + channelOpacities?: number[]; + channelOutlineOpacities?: number[]; + channelsVisible?: boolean[]; + channelStrokeWidths?: number[]; + maxZoom?: number; + opacity?: number; + zoom?: number; +}; + +const UntypedXRLayer = XRLayer as any; + +export class LabelsBitmaskTileLayer extends UntypedXRLayer { + static layerName = 'LabelsBitmaskTileLayer'; + + static defaultProps = { + channelColors: { type: 'array', value: [[255, 255, 255]], compare: true }, + channelsFilled: { type: 'array', value: [true], compare: true }, + channelOpacities: { type: 'array', value: [0.18], compare: true }, + channelOutlineOpacities: { type: 'array', value: [0.95], compare: true }, + channelsVisible: { type: 'array', value: [true], compare: true }, + channelStrokeWidths: { type: 'array', value: [1.5], compare: true }, + }; + + constructor(...args: any[]) { + super(...args); + } + + getShaders(): any { + return { + fs, + vs, + modules: [project32, picking, labelsBitmaskUniforms], + }; + } + + getPickingInfo(params: GetPickingInfoParams): PickingInfo { + const info = super.getPickingInfo(params); + const localCoordinate = getLabelCoordinate( + info.coordinate as number[] | undefined, + this.props.modelMatrix, + ); + const { bounds, channelData } = this.props; + const width = channelData?.width; + const height = channelData?.height; + + if ( + !localCoordinate || + !bounds || + !width || + !height || + !Number.isFinite(bounds[0]) || + !Number.isFinite(bounds[1]) || + !Number.isFinite(bounds[2]) || + !Number.isFinite(bounds[3]) + ) { + return info; + } + + const xDenominator = bounds[2] - bounds[0]; + const yDenominator = bounds[1] - bounds[3]; + if (xDenominator === 0 || yDenominator === 0) { + return info; + } + + const u = (localCoordinate[0] - bounds[0]) / xDenominator; + const v = (localCoordinate[1] - bounds[3]) / yDenominator; + if (!(u >= 0 && u <= 1 && v >= 0 && v <= 1)) { + return info; + } + + const pixelX = Math.max(0, Math.min(width - 1, Math.floor(u * width))); + const pixelY = Math.max(0, Math.min(height - 1, Math.floor(v * height))); + const pickedLabel = getTopmostLabelAtPixel(this.props, pixelX, pixelY); + if (!pickedLabel) { + info.object = null; + return info; + } + + info.object = { + ...pickedLabel, + pixel: [pixelX, pixelY] as const, + }; + info.index = pickedLabel.channelIndex; + return info; + } + + dataToTexture(data: unknown, width: number, height: number) { + return this.context.device.createTexture({ + width, + height, + dimension: '2d', + data: new Float32Array(data as ArrayLike), + mipmaps: false, + sampler: { + minFilter: 'nearest', + magFilter: 'nearest', + addressModeU: 'clamp-to-edge', + addressModeV: 'clamp-to-edge', + }, + format: 'r32float', + }); + } + + // Override the parent implementation to support up to seven label channels. + loadChannelTextures(channelData: { data?: unknown[]; width: number; height: number }) { + const textures: Record = { + channel0: null, + channel1: null, + channel2: null, + channel3: null, + channel4: null, + channel5: null, + channel6: null, + }; + + if (this.state.textures) { + Object.values( + this.state.textures as Record void } | null> + ).forEach((tex) => { + tex?.delete?.(); + }); + } + + if (channelData?.data?.length) { + channelData.data.forEach((data, index) => { + if (index >= MAX_LABEL_CHANNELS) { + return; + } + textures[`channel${index}`] = this.dataToTexture(data, channelData.width, channelData.height); + }); + for (const key of Object.keys(textures)) { + if (!textures.channel0) { + throw new Error('Bad labels texture state.'); + } + if (!textures[key]) { + textures[key] = textures.channel0; + } + } + (this as any)._setNewTexturesFromLoadThisFrame?.(textures); + this.setState({ textures }); + return; + } + + (this as any)._setNewTexturesFromLoadThisFrame?.(null); + } + + draw(_opts: { uniforms?: Record }) { + const { model, textures } = this.state as { + model?: { + shaderInputs?: { setProps: (props: Record) => void }; + setBindings: (bindings: Record) => void; + draw: (renderPass?: unknown) => void; + }; + textures?: Record; + }; + if (!model || !textures) { + return; + } + + const { + channelColors, + channelsFilled, + channelOpacities, + channelOutlineOpacities, + channelsVisible, + channelStrokeWidths, + maxZoom, + opacity = 1, + zoom, + channelData, + selections, + } = this.props; + + const actualChannelCount = Math.max( + 1, + Math.min( + MAX_LABEL_CHANNELS, + channelData?.data?.length ?? + selections?.length ?? + channelColors?.length ?? + channelsVisible?.length ?? + channelOpacities?.length ?? + channelOutlineOpacities?.length ?? + channelsFilled?.length ?? + channelStrokeWidths?.length ?? + 1 + ) + ); + + const normalizedColors = Array.from({ length: MAX_LABEL_CHANNELS }, (_, index) => + getNormalizedColor( + index < actualChannelCount ? channelColors?.[index] ?? [255, 255, 255] : [255, 255, 255] + ) + ); + + const zoomDelta = + typeof zoom === 'number' && typeof maxZoom === 'number' ? maxZoom - zoom : 0; + const scaleFactor = 1 / (2 ** zoomDelta); + + const filled = Array.from( + { length: MAX_LABEL_CHANNELS }, + (_, index) => (index < actualChannelCount ? channelsFilled?.[index] ?? true : false) + ); + const opacities = Array.from( + { length: MAX_LABEL_CHANNELS }, + (_, index) => (index < actualChannelCount ? channelOpacities?.[index] ?? 0.18 : 0) + ); + const outlineOpacities = Array.from( + { length: MAX_LABEL_CHANNELS }, + (_, index) => (index < actualChannelCount ? channelOutlineOpacities?.[index] ?? 0.95 : 0) + ); + const visible = Array.from( + { length: MAX_LABEL_CHANNELS }, + (_, index) => (index < actualChannelCount ? channelsVisible?.[index] ?? true : false) + ); + const strokeWidths = Array.from( + { length: MAX_LABEL_CHANNELS }, + (_, index) => (index < actualChannelCount ? channelStrokeWidths?.[index] ?? 1.5 : 1) + ); + + const labelsBitmask = Object.fromEntries([ + ...normalizedColors.map((color, index) => [`color${index}`, [...color, 1] as const]), + ...filled.map((value, index) => [`channelFilled${index}`, value ? 1 : 0]), + ...opacities.map((value, index) => [`channelOpacity${index}`, value]), + ...outlineOpacities.map((value, index) => [`channelOutlineOpacity${index}`, value]), + ...visible.map((value, index) => [`channelVisible${index}`, value ? 1 : 0]), + ...strokeWidths.map((value, index) => [`channelStrokeWidth${index}`, value]), + ['scaleFactor', scaleFactor], + ['labelOpacity', opacity], + ]); + + model.shaderInputs?.setProps({ labelsBitmask }); + model.setBindings(textures); + model.draw((this.context as { renderPass?: unknown }).renderPass); + } +} diff --git a/packages/layers/src/LabelsLayer.ts b/packages/layers/src/LabelsLayer.ts new file mode 100644 index 00000000..325e9149 --- /dev/null +++ b/packages/layers/src/LabelsLayer.ts @@ -0,0 +1,330 @@ +import type { Matrix4 } from '@math.gl/core'; +import { getImageSize } from '@hms-dbmi/viv'; +import { CompositeLayer, TileLayer, type Layer, type LayersList } from 'deck.gl'; +import { LabelsBitmaskTileLayer } from './LabelsBitmaskTileLayer'; + +export const MAX_LABEL_CHANNELS = 7 as const; + +export type LabelsSelection = Partial<{ z: number; c: number; t: number }>; + +export interface LabelsLayerProps { + id: string; + loader: unknown; + selections: LabelsSelection[]; + visible?: boolean; + opacity?: number; + modelMatrix?: Matrix4; + channelColors?: Array<[number, number, number]>; + channelsVisible?: boolean[]; + channelOpacities?: number[]; + channelOutlineOpacities?: number[]; + channelsFilled?: boolean[]; + channelStrokeWidths?: number[]; + onClick?: (info: unknown) => void; + onHover?: (info: unknown) => void; +} + +const VIV_SIGNAL_ABORTED = '__vivSignalAborted'; +const UntypedTileLayer = TileLayer as any; + +function isMultiscaleLoader(loader: unknown): loader is unknown[] { + return Array.isArray(loader) && loader.length > 1; +} + +function getBaseLoader(loader: any) { + return Array.isArray(loader) ? loader[0] : loader; +} + +class SingleScaleLabelsLayer extends CompositeLayer { + static layerName = 'SingleScaleLabelsLayer'; + + finalizeState(context: any): void { + (this.state.abortController as AbortController | undefined)?.abort(); + super.finalizeState(context); + } + + updateState({ props, oldProps }: { props: any; oldProps: any }): void { + const loaderChanged = props.loader !== oldProps.loader; + const selectionsChanged = props.selections !== oldProps.selections; + + if (!loaderChanged && !selectionsChanged) { + return; + } + + (this.state.abortController as AbortController | undefined)?.abort(); + const abortController = new AbortController(); + this.setState({ abortController }); + + const { signal } = abortController; + const getRaster = (selection: any) => props.loader.getRaster({ selection, signal }); + const dataPromises = (props.selections ?? []).map(getRaster); + + Promise.all(dataPromises) + .then((rasters) => { + if (signal.aborted) { + return; + } + const raster = { + data: rasters.map((rasterData) => rasterData.data), + width: rasters[0]?.width, + height: rasters[0]?.height, + }; + if (typeof props.onViewportLoad === 'function') { + props.onViewportLoad(raster); + } + this.setState(raster); + }) + .catch((error) => { + if (signal.aborted || error?.name === 'AbortError') { + return; + } + throw error; + }); + } + + renderLayers(): Layer | null { + const { + id, + onClick, + onHover, + channelColors, + channelsVisible, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + selections, + } = this.props; + const { width, height, data } = this.state as { + width?: number; + height?: number; + data?: unknown[]; + }; + + if (!(width && height) || !data) { + return null; + } + + const bounds = [0, height, width, 0] as const; + + return new LabelsBitmaskTileLayer( + this.getSubLayerProps({ + id: 'single-scale-labels-bitmask', + pickable: true, + ...(typeof onClick === 'function' ? { onClick } : {}), + ...(typeof onHover === 'function' ? { onHover } : {}), + }), + { + channelData: { data, height, width }, + channelColors, + channelsVisible, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + selections, + bounds, + id: `image-sub-layer-${bounds}-${id}`, + interpolation: 'nearest', + maxZoom: 0, + minZoom: 0, + zoom: 0, + } + ) as unknown as Layer; + } +} + +// Temporary Viv workaround: current Viv ImageLayer/MultiscaleImageLayer prop schemas +// reject null callback defaults and mis-type interpolation, which breaks labels paths +// that otherwise mirror Viv. Replace these local wrappers once the upstream fixes land. +class MultiscaleLabelsTileLayer extends UntypedTileLayer { + static layerName = 'MultiscaleLabelsTileLayer'; + + constructor(...args: any[]) { + super(...args); + } + + _updateTileset(): void { + if (!this.props.viewportId) { + super._updateTileset(); + return; + } + if ( + this.context.viewport.id === this.props.viewportId || + !this.state.tileset?._viewport + ) { + super._updateTileset(); + } + } +} + +function renderSubBitmaskLayers(props: any) { + const { + bbox: { + left, top, right, bottom, + }, + index: { x, y, z }, + zoom, + } = props.tile; + const { + data, id, loader, maxZoom, minZoom, zoomOffset, + } = props; + + if (!data) { + return null; + } + + const base = getBaseLoader(loader); + if (!base?.shape) { + return null; + } + const [height, width] = base.shape.slice(-2); + const bounds = [ + left, + data.height < base.tileSize ? height : bottom, + data.width < base.tileSize ? width : right, + top, + ]; + + return new LabelsBitmaskTileLayer(props, { + channelData: data, + bounds, + id: `sub-layer-${bounds}-${id}`, + tileId: { x, y, z }, + zoom, + minZoom, + maxZoom, + zoomOffset, + }) as unknown as Layer; +} + +export class LabelsLayer extends CompositeLayer { + static layerName = 'LabelsLayer'; + + static defaultProps = { + opacity: 1, + visible: true, + selections: [{}], + channelColors: [[255, 255, 255]], + channelsVisible: [true], + channelOpacities: [0.18], + channelOutlineOpacities: [0.95], + channelsFilled: [true], + channelStrokeWidths: [1.5], + } satisfies Partial; + + renderLayers(): Layer | null | LayersList { + const { + loader, + selections, + visible = true, + opacity = 1, + modelMatrix, + channelColors = [[255, 255, 255]], + channelsVisible = [true], + channelOpacities = [0.18], + channelOutlineOpacities = [0.95], + channelsFilled = [true], + channelStrokeWidths = [1.5], + onClick, + onHover, + } = this.props; + + if (!visible || !loader) { + return null; + } + + const nextLoader = Array.isArray(loader) && loader.length === 1 ? loader[0] : loader; + const commonProps = { + loader: nextLoader, + selections, + modelMatrix, + opacity, + channelsVisible, + channelColors, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + } as const; + const interactionProps = { + ...(typeof onClick === 'function' ? { onClick } : {}), + ...(typeof onHover === 'function' ? { onHover } : {}), + }; + + if (isMultiscaleLoader(loader)) { + const baseLoader = loader[0] as any; + const { height, width } = getImageSize(baseLoader); + const tileSize = baseLoader.tileSize; + const zoomOffset = Math.round( + Math.log2(modelMatrix ? modelMatrix.getScale()[0] : 1) + ); + const getTileData = async ({ + index: { x, y, z }, + signal, + }: { + index: { x: number; y: number; z: number }; + signal: AbortSignal; + }) => { + if (!selections || selections.length === 0) { + return null; + } + + const resolution = Math.max(0, Math.min(loader.length - 1, Math.round(-z))); + const getTile = (selection: LabelsSelection) => + (loader[resolution] as any).getTile({ x, y, selection, signal }); + + try { + const tiles = await Promise.all(selections.map(getTile)); + return { + data: tiles.map((tile) => tile.data), + width: tiles[0]?.width, + height: tiles[0]?.height, + }; + } catch (error) { + if ( + error === VIV_SIGNAL_ABORTED || + signal.aborted || + (error as { name?: string } | null)?.name === 'AbortError' + ) { + return null; + } + throw error; + } + }; + + return new MultiscaleLabelsTileLayer( + this.getSubLayerProps({ + id: 'labels', + pickable: true, + visible, + }), + { + ...commonProps, + ...interactionProps, + getTileData, + tileSize, + extent: [0, 0, width, height], + zoomOffset, + minZoom: Math.round(-(loader.length - 1)), + maxZoom: 0, + refinementStrategy: opacity === 1 ? 'best-available' : 'no-overlap', + onTileError: baseLoader.onTileError, + renderSubLayers: renderSubBitmaskLayers, + } as any + ) as unknown as Layer; + } + + return new SingleScaleLabelsLayer( + this.getSubLayerProps({ + id: 'labels', + pickable: true, + visible, + }), + { + ...commonProps, + ...interactionProps, + } as any + ) as unknown as Layer; + } +} diff --git a/packages/layers/src/index.ts b/packages/layers/src/index.ts index b135ebb3..2744c657 100644 --- a/packages/layers/src/index.ts +++ b/packages/layers/src/index.ts @@ -1,5 +1,7 @@ export { SpatialLayer } from './SpatialLayer'; export type { SpatialLayerProps } from './spatialLayerProps'; +export { LabelsLayer, MAX_LABEL_CHANNELS } from './LabelsLayer'; +export type { LabelsLayerProps, LabelsSelection } from './LabelsLayer'; export { spatialLayerPropsSchema, spatialSublayerSchema, diff --git a/packages/layers/src/labelsBitmaskLayerShaders.ts b/packages/layers/src/labelsBitmaskLayerShaders.ts new file mode 100644 index 00000000..eadb7629 --- /dev/null +++ b/packages/layers/src/labelsBitmaskLayerShaders.ts @@ -0,0 +1,191 @@ +const CHANNEL_INDICES = [0, 1, 2, 3, 4, 5, 6] as const; + +const labelsUniformBlock = `\ +uniform labelsBitmaskUniforms { +${CHANNEL_INDICES.map((index) => ` vec4 color${index};`).join('\n')} +${CHANNEL_INDICES.map((index) => ` float channelOpacity${index};`).join('\n')} +${CHANNEL_INDICES.map((index) => ` float channelOutlineOpacity${index};`).join('\n')} +${CHANNEL_INDICES.map((index) => ` float channelStrokeWidth${index};`).join('\n')} +${CHANNEL_INDICES.map((index) => ` float channelVisible${index};`).join('\n')} +${CHANNEL_INDICES.map((index) => ` float channelFilled${index};`).join('\n')} + float scaleFactor; + float labelOpacity; +} labelsBitmask; +`; + +export const labelsBitmaskUniforms = { + name: 'labelsBitmask', + fs: labelsUniformBlock, + uniformTypes: Object.fromEntries([ + ...CHANNEL_INDICES.map((index) => [`color${index}`, 'vec4']), + ...CHANNEL_INDICES.map((index) => [`channelOpacity${index}`, 'f32']), + ...CHANNEL_INDICES.map((index) => [`channelOutlineOpacity${index}`, 'f32']), + ...CHANNEL_INDICES.map((index) => [`channelStrokeWidth${index}`, 'f32']), + ...CHANNEL_INDICES.map((index) => [`channelVisible${index}`, 'f32']), + ...CHANNEL_INDICES.map((index) => [`channelFilled${index}`, 'f32']), + ['scaleFactor', 'f32'], + ['labelOpacity', 'f32'], + ]), +} as const; + +export const vs = `#version 300 es +#define SHADER_NAME labels-bitmask-layer-vertex-shader + +in vec2 texCoords; +in vec3 positions; +in vec3 instancePickingColors; + +out vec2 vTexCoord; + +void main(void) { + geometry.worldPosition = positions; + geometry.uv = texCoords; + geometry.pickingColor = instancePickingColors; + gl_Position = project_position_to_clipspace(positions, vec3(0.0), vec3(0.0), geometry.position); + DECKGL_FILTER_GL_POSITION(gl_Position, geometry); + vTexCoord = texCoords; + vec4 color = vec4(0.0); + DECKGL_FILTER_COLOR(color, geometry); +} +`; + +export const fs = `#version 300 es +#define SHADER_NAME labels-bitmask-layer-fragment-shader +precision highp float; + +uniform sampler2D channel0; +uniform sampler2D channel1; +uniform sampler2D channel2; +uniform sampler2D channel3; +uniform sampler2D channel4; +uniform sampler2D channel5; +uniform sampler2D channel6; + +in vec2 vTexCoord; + +out vec4 fragColor; + +float labelMatch(float sampledLabel, float referenceLabel) { + return 1.0 - step(0.5, abs(sampledLabel - referenceLabel)); +} + +float getCoverage(sampler2D dataTex, vec2 coord, float sampledData) { + vec2 coordDx = dFdx(coord) * 0.5; + vec2 coordDy = dFdy(coord) * 0.5; + return 0.25 * ( + labelMatch(texture(dataTex, coord + coordDx + coordDy).r, sampledData) + + labelMatch(texture(dataTex, coord + coordDx - coordDy).r, sampledData) + + labelMatch(texture(dataTex, coord - coordDx + coordDy).r, sampledData) + + labelMatch(texture(dataTex, coord - coordDx - coordDy).r, sampledData) + ); +} + +float getEdgeAtRadius(sampler2D dataTex, vec2 coord, float sampledData, float radius) { + vec2 texel = 1.0 / vec2(textureSize(dataTex, 0)); + vec2 offsetX = vec2(texel.x * radius, 0.0); + vec2 offsetY = vec2(0.0, texel.y * radius); + float diff = 0.0; + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord + offsetX).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord - offsetX).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord + offsetY).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord - offsetY).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord + offsetX + offsetY).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord + offsetX - offsetY).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord - offsetX + offsetY).r, sampledData)); + diff = max(diff, 1.0 - labelMatch(texture(dataTex, coord - offsetX - offsetY).r, sampledData)); + return diff; +} + +float getEdgeMask( + sampler2D dataTex, + vec2 coord, + float sampledData, + float strokeWidth, + float coverage +) { + if (strokeWidth <= 0.0) { + return 0.0; + } + + float scaledStrokeWidth = max(1.0, strokeWidth * max(labelsBitmask.scaleFactor, 1.0)); + float lowerRadius = max(1.0, floor(scaledStrokeWidth)); + float upperRadius = max(1.0, ceil(scaledStrokeWidth)); + float lowerEdge = getEdgeAtRadius(dataTex, coord, sampledData, lowerRadius); + float upperEdge = getEdgeAtRadius(dataTex, coord, sampledData, upperRadius); + float edge = mix(lowerEdge, upperEdge, fract(scaledStrokeWidth)); + return edge * coverage; +} + +vec4 sampleAndGetData(sampler2D dataTex, vec2 coord, float isFilled, float strokeWidth, float isOn) { + float sampledData = texture(dataTex, coord).r; + if (isOn < 0.5 || sampledData <= 0.0) { + return vec4(0.0, sampledData, 0.0, 0.0); + } + + float coverage = getCoverage(dataTex, coord, sampledData); + float isEdge = getEdgeMask(dataTex, coord, sampledData, strokeWidth, coverage); + + return vec4(1.0, sampledData, coverage, isEdge); +} + +vec4 dataToColor( + vec4 sampledDataAndCoverage, + vec4 channelColor, + float fillOpacity, + float outlineOpacity, + float isFilled +) { + float hasData = sampledDataAndCoverage.x; + float fillCoverage = sampledDataAndCoverage.z; + float isEdge = sampledDataAndCoverage.w; + float fillAlpha = hasData * fillOpacity * fillCoverage * step(0.5, isFilled); + float edgeAlpha = hasData * outlineOpacity * isEdge; + vec4 fillColor = vec4(channelColor.rgb, fillAlpha); + vec4 edgeColor = vec4(mix(channelColor.rgb, vec3(1.0), 0.4), edgeAlpha); + float outAlpha = edgeColor.a + fillColor.a * (1.0 - edgeColor.a); + vec3 outRgb = outAlpha > 0.0 + ? ( + edgeColor.rgb * edgeColor.a + + fillColor.rgb * fillColor.a * (1.0 - edgeColor.a) + ) / outAlpha + : vec3(0.0); + return vec4(outRgb, outAlpha); +} + +void main() { + vec4 dat0 = sampleAndGetData(channel0, vTexCoord, labelsBitmask.channelFilled0, labelsBitmask.channelStrokeWidth0, labelsBitmask.channelVisible0); + vec4 dat1 = sampleAndGetData(channel1, vTexCoord, labelsBitmask.channelFilled1, labelsBitmask.channelStrokeWidth1, labelsBitmask.channelVisible1); + vec4 dat2 = sampleAndGetData(channel2, vTexCoord, labelsBitmask.channelFilled2, labelsBitmask.channelStrokeWidth2, labelsBitmask.channelVisible2); + vec4 dat3 = sampleAndGetData(channel3, vTexCoord, labelsBitmask.channelFilled3, labelsBitmask.channelStrokeWidth3, labelsBitmask.channelVisible3); + vec4 dat4 = sampleAndGetData(channel4, vTexCoord, labelsBitmask.channelFilled4, labelsBitmask.channelStrokeWidth4, labelsBitmask.channelVisible4); + vec4 dat5 = sampleAndGetData(channel5, vTexCoord, labelsBitmask.channelFilled5, labelsBitmask.channelStrokeWidth5, labelsBitmask.channelVisible5); + vec4 dat6 = sampleAndGetData(channel6, vTexCoord, labelsBitmask.channelFilled6, labelsBitmask.channelStrokeWidth6, labelsBitmask.channelVisible6); + + if ( + dat0.x == 0.0 && dat1.x == 0.0 && dat2.x == 0.0 && dat3.x == 0.0 && + dat4.x == 0.0 && dat5.x == 0.0 && dat6.x == 0.0 + ) { + discard; + } + + vec4 val0 = dataToColor(dat0, labelsBitmask.color0, labelsBitmask.channelOpacity0, labelsBitmask.channelOutlineOpacity0, labelsBitmask.channelFilled0); + vec4 val1 = dataToColor(dat1, labelsBitmask.color1, labelsBitmask.channelOpacity1, labelsBitmask.channelOutlineOpacity1, labelsBitmask.channelFilled1); + vec4 val2 = dataToColor(dat2, labelsBitmask.color2, labelsBitmask.channelOpacity2, labelsBitmask.channelOutlineOpacity2, labelsBitmask.channelFilled2); + vec4 val3 = dataToColor(dat3, labelsBitmask.color3, labelsBitmask.channelOpacity3, labelsBitmask.channelOutlineOpacity3, labelsBitmask.channelFilled3); + vec4 val4 = dataToColor(dat4, labelsBitmask.color4, labelsBitmask.channelOpacity4, labelsBitmask.channelOutlineOpacity4, labelsBitmask.channelFilled4); + vec4 val5 = dataToColor(dat5, labelsBitmask.color5, labelsBitmask.channelOpacity5, labelsBitmask.channelOutlineOpacity5, labelsBitmask.channelFilled5); + vec4 val6 = dataToColor(dat6, labelsBitmask.color6, labelsBitmask.channelOpacity6, labelsBitmask.channelOutlineOpacity6, labelsBitmask.channelFilled6); + + fragColor = val0; + fragColor = (val1 == fragColor || val1 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val1, val1.a).rgb, max(fragColor.a, val1.a)); + fragColor = (val2 == fragColor || val2 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val2, val2.a).rgb, max(fragColor.a, val2.a)); + fragColor = (val3 == fragColor || val3 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val3, val3.a).rgb, max(fragColor.a, val3.a)); + fragColor = (val4 == fragColor || val4 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val4, val4.a).rgb, max(fragColor.a, val4.a)); + fragColor = (val5 == fragColor || val5 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val5, val5.a).rgb, max(fragColor.a, val5.a)); + fragColor = (val6 == fragColor || val6 == vec4(0.0)) ? fragColor : vec4(mix(fragColor, val6, val6.a).rgb, max(fragColor.a, val6.a)); + fragColor.a = fragColor.a * labelsBitmask.labelOpacity; + + geometry.uv = vTexCoord; + DECKGL_FILTER_COLOR(fragColor, geometry); +} +`; diff --git a/packages/layers/src/spatialLayerProps.ts b/packages/layers/src/spatialLayerProps.ts index 845bdbbf..f5b21dfe 100644 --- a/packages/layers/src/spatialLayerProps.ts +++ b/packages/layers/src/spatialLayerProps.ts @@ -23,10 +23,18 @@ export const spatialShapesSublayerSchema = sublayerBase.extend({ tooltipFields: z.array(z.string()).optional(), }); +export const spatialLabelsSublayerSchema = sublayerBase.extend({ + kind: z.literal('labels'), + /** OME-Zarr labels URL understood by the labels bitmask layer. */ + url: z.string().optional(), + tooltipFields: z.array(z.string()).optional(), +}); + export const spatialSublayerSchema = z.discriminatedUnion('kind', [ spatialImageSublayerSchema, spatialScatterSublayerSchema, spatialShapesSublayerSchema, + spatialLabelsSublayerSchema, ]); export type SpatialSublayer = z.infer; diff --git a/packages/layers/vite.config.ts b/packages/layers/vite.config.ts index 429d1aac..2cdfa994 100644 --- a/packages/layers/vite.config.ts +++ b/packages/layers/vite.config.ts @@ -26,7 +26,7 @@ export default defineConfig({ formats: ['es'], }, rollupOptions: { - external: ['deck.gl', 'zod'], + external: ['@deck.gl/core', '@hms-dbmi/viv', '@math.gl/core', 'deck.gl', 'zod'], }, }, test: { diff --git a/packages/vis/src/SpatialCanvas/LabelsChannelPanel.tsx b/packages/vis/src/SpatialCanvas/LabelsChannelPanel.tsx new file mode 100644 index 00000000..7c496057 --- /dev/null +++ b/packages/vis/src/SpatialCanvas/LabelsChannelPanel.tsx @@ -0,0 +1,368 @@ +import type { CSSProperties } from 'react'; +import { clampVivSelectionsToAxes } from '@spatialdata/avivatorish'; +import type { LabelsLayerConfig } from './types'; +import type { LabelsLoaderData } from './useLayerData'; + +const MAX_CHANNELS = 7; + +const inputStyle: CSSProperties = { + width: '100%', + boxSizing: 'border-box', + backgroundColor: '#2a2a2a', + color: '#eee', + border: '1px solid #444', + borderRadius: 4, + padding: '4px 6px', + fontSize: '12px', +}; + +const labelStyle: CSSProperties = { + color: '#888', + fontSize: '11px', + display: 'block', + marginBottom: 2, +}; + +type AxisSizes = Partial>; +type SelectionRow = Partial>; +type LabelsChannelsConfig = NonNullable; + +type MergedLabelsDisplay = { + channelCount: number; + channelIds: string[]; + colors: [number, number, number][]; + channelsVisible: boolean[]; + channelOpacities: number[]; + channelOutlineOpacities: number[]; + channelsFilled: boolean[]; + channelStrokeWidths: number[]; + selections: SelectionRow[]; +}; + +function emptySelectionRow(axisSizes: AxisSizes | undefined): SelectionRow { + if (axisSizes === undefined) { + return { z: 0, c: 0, t: 0 }; + } + const next: SelectionRow = {}; + for (const dim of ['z', 'c', 't'] as const) { + if (axisSizes[dim] !== undefined) { + next[dim] = 0; + } + } + return next; +} + +function mergeSelectionRow( + override: SelectionRow | undefined, + fallback: SelectionRow, + axisSizes: AxisSizes | undefined, +): SelectionRow { + const merged: SelectionRow = { ...fallback, ...override }; + if (axisSizes === undefined) { + return { + z: merged.z ?? 0, + c: merged.c ?? 0, + t: merged.t ?? 0, + }; + } + if (Object.keys(axisSizes).length === 0) { + return {}; + } + return clampVivSelectionsToAxes([merged], axisSizes)[0] ?? {}; +} + +function pad(arr: T[], len: number, fill: T): T[] { + const next = arr.slice(0, len); + while (next.length < len) { + next.push(fill); + } + return next; +} + +function mergeForDisplay( + config: LabelsLayerConfig, + defaults: LabelsLoaderData | undefined, + layerId: string, +): MergedLabelsDisplay { + const axisSizes = defaults?.selectionAxisSizes; + const ch = config.channels; + const baseColors = defaults?.colors?.length + ? defaults.colors + : [[255, 255, 255] as [number, number, number]]; + const baseVisible = defaults?.channelsVisible?.length ? defaults.channelsVisible : [true]; + const baseFillOpacities = defaults?.channelOpacities?.length ? defaults.channelOpacities : [0.18]; + const baseOutlineOpacities = defaults?.channelOutlineOpacities?.length + ? defaults.channelOutlineOpacities + : [0.95]; + const baseFilled = defaults?.channelsFilled?.length ? defaults.channelsFilled : [true]; + const baseStrokeWidths = defaults?.channelStrokeWidths?.length + ? defaults.channelStrokeWidths + : [1.5]; + const baseSelections = + defaults?.selections?.length && defaults.selections.length > 0 + ? defaults.selections.map((selection) => ({ ...selection })) + : [emptySelectionRow(axisSizes)]; + + const colors = ch?.colors && ch.colors.length > 0 ? [...ch.colors] : [...baseColors]; + const channelsVisible = + ch?.channelsVisible && ch.channelsVisible.length > 0 + ? [...ch.channelsVisible] + : [...baseVisible]; + const channelOpacities = + ch?.channelOpacities && ch.channelOpacities.length > 0 + ? [...ch.channelOpacities] + : [...baseFillOpacities]; + const channelOutlineOpacities = + ch?.channelOutlineOpacities && ch.channelOutlineOpacities.length > 0 + ? [...ch.channelOutlineOpacities] + : [...baseOutlineOpacities]; + const channelsFilled = + ch?.channelsFilled && ch.channelsFilled.length > 0 + ? [...ch.channelsFilled] + : [...baseFilled]; + const channelStrokeWidths = + ch?.channelStrokeWidths && ch.channelStrokeWidths.length > 0 + ? [...ch.channelStrokeWidths] + : [...baseStrokeWidths]; + + const selections = + ch?.selections && ch.selections.length > 0 + ? ch.selections.map((selection, index) => + mergeSelectionRow( + selection, + baseSelections[index] ?? baseSelections[0] ?? emptySelectionRow(axisSizes), + axisSizes, + ), + ) + : baseSelections.map((selection) => mergeSelectionRow(undefined, selection, axisSizes)); + + const channelCount = Math.min( + MAX_CHANNELS, + Math.max( + colors.length, + channelsVisible.length, + channelOpacities.length, + channelOutlineOpacities.length, + channelsFilled.length, + channelStrokeWidths.length, + selections.length, + 1, + ), + ); + + const fillSelection = emptySelectionRow(axisSizes); + const channelIds = Array.from( + { length: channelCount }, + (_, index) => ch?.channelIds?.[index] ?? `${layerId}:labels:${index}`, + ); + + return { + channelCount, + channelIds, + colors: pad(colors, channelCount, [255, 255, 255] as [number, number, number]), + channelsVisible: pad(channelsVisible, channelCount, true), + channelOpacities: pad(channelOpacities, channelCount, 0.18), + channelOutlineOpacities: pad(channelOutlineOpacities, channelCount, 0.95), + channelsFilled: pad(channelsFilled, channelCount, true), + channelStrokeWidths: pad(channelStrokeWidths, channelCount, 1.5), + selections: pad( + selections, + channelCount, + mergeSelectionRow(undefined, fillSelection, axisSizes), + ), + }; +} + +export interface LabelsChannelPanelProps { + layerId: string; + config: LabelsLayerConfig; + defaults?: LabelsLoaderData; + updateLayer: (id: string, updates: Partial) => void; +} + +export function LabelsChannelPanel({ + layerId, + config, + defaults, + updateLayer, +}: LabelsChannelPanelProps) { + const m = mergeForDisplay(config, defaults, layerId); + const axisSizes = defaults?.selectionAxisSizes; + + const setChannels = (next: Partial) => { + updateLayer(layerId, { channels: { ...config.channels, ...next } }); + }; + + const axisActive = (dim: 'z' | 'c' | 't') => + axisSizes === undefined ? true : axisSizes[dim] !== undefined; + const showAxisGrid = + axisSizes === undefined || Object.keys(axisSizes).some((k) => axisSizes[k as keyof AxisSizes] !== undefined); + + return ( +
+
+ Labels channels (max {MAX_CHANNELS}) +
+ {Array.from({ length: m.channelCount }, (_, i) => ( +
+ Channel {i + 1} +
+ + +
+
+ RGB +
+ {( + [ + { band: 'r' as const, j: 0 }, + { band: 'g' as const, j: 1 }, + { band: 'b' as const, j: 2 }, + ] as const + ).map(({ band, j }) => ( + { + const value = Number(e.target.value); + const colors = m.colors.map((color) => [...color] as [number, number, number]); + if (!colors[i]) colors[i] = [255, 255, 255]; + colors[i][j] = Number.isFinite(value) + ? Math.min(255, Math.max(0, value)) + : 0; + setChannels({ colors }); + }} + /> + ))} +
+
+
+ Fill opacity ({(m.channelOpacities[i] ?? 0).toFixed(2)}) + { + const channelOpacities = [...m.channelOpacities]; + channelOpacities[i] = Number(e.target.value); + setChannels({ channelOpacities }); + }} + /> +
+
+ + Outline opacity ({(m.channelOutlineOpacities[i] ?? 0).toFixed(2)}) + + { + const channelOutlineOpacities = [...m.channelOutlineOpacities]; + channelOutlineOpacities[i] = Number(e.target.value); + setChannels({ channelOutlineOpacities }); + }} + /> +
+
+ Outline width + { + const nextValue = Number(e.target.value); + const channelStrokeWidths = [...m.channelStrokeWidths]; + channelStrokeWidths[i] = Number.isFinite(nextValue) + ? Math.max(0, nextValue) + : 1.5; + setChannels({ channelStrokeWidths }); + }} + /> +
+ {showAxisGrid ? ( +
+ {(['z', 'c', 't'] as const).map((dim) => { + const active = axisActive(dim); + const size = axisSizes?.[dim]; + const maxIdx = size !== undefined ? Math.max(0, size - 1) : undefined; + return ( +
+ {dim} + {active ? ( + { + const nextValue = Number(e.target.value); + const selections = m.selections.map((selection) => ({ ...selection })); + const nextSelection = { ...(selections[i] ?? {}) }; + nextSelection[dim] = Number.isFinite(nextValue) ? Math.max(0, nextValue) : 0; + selections[i] = mergeSelectionRow( + nextSelection, + emptySelectionRow(axisSizes), + axisSizes, + ); + setChannels({ selections }); + }} + /> + ) : ( +
n/a
+ )} +
+ ); + })} +
+ ) : null} +
+ ))} +
+ ); +} diff --git a/packages/vis/src/SpatialCanvas/SpatialFeatureTooltip.tsx b/packages/vis/src/SpatialCanvas/SpatialFeatureTooltip.tsx index 4fb82d45..2bc02421 100644 --- a/packages/vis/src/SpatialCanvas/SpatialFeatureTooltip.tsx +++ b/packages/vis/src/SpatialCanvas/SpatialFeatureTooltip.tsx @@ -1,17 +1,14 @@ import type { CSSProperties } from 'react'; +import type { + SpatialFeatureTooltipData, + SpatialFeatureTooltipItem, +} from '@spatialdata/core'; -export type SpatialFeatureTooltipItem = { - label: string; - value: string; -}; - -/** Serializable payload for picked spatial feature hover tooltips (library-owned contract). */ -export type SpatialFeatureTooltipData = { - title?: string; - items: SpatialFeatureTooltipItem[]; -}; +export type { + SpatialFeatureTooltipData, + SpatialFeatureTooltipItem, +} from '@spatialdata/core'; -/** Props for optional `renderTooltip` on SpatialCanvas (client = viewport coordinates). */ export type SpatialCanvasTooltipRenderProps = { clientX: number; clientY: number; diff --git a/packages/vis/src/SpatialCanvas/SpatialViewer.tsx b/packages/vis/src/SpatialCanvas/SpatialViewer.tsx index eecf1c93..b4a92d88 100644 --- a/packages/vis/src/SpatialCanvas/SpatialViewer.tsx +++ b/packages/vis/src/SpatialCanvas/SpatialViewer.tsx @@ -162,5 +162,3 @@ function SpatialViewerSimple({ /> ); } - - diff --git a/packages/vis/src/SpatialCanvas/TooltipFieldsPanel.tsx b/packages/vis/src/SpatialCanvas/TooltipFieldsPanel.tsx new file mode 100644 index 00000000..a91517a3 --- /dev/null +++ b/packages/vis/src/SpatialCanvas/TooltipFieldsPanel.tsx @@ -0,0 +1,81 @@ +import type { CSSProperties } from 'react'; + +const helperTextStyle: CSSProperties = { + color: '#888', + fontSize: '11px', + marginBottom: 8, +}; + +export interface TooltipFieldsPanelProps { + tableName?: string; + availableFields: string[]; + selectedFields?: string[]; + onChange: (nextFields: string[]) => void; + noAssociatedTableMessage: string; + helperText?: string; + noFieldsMessage?: string; +} + +export function TooltipFieldsPanel({ + tableName, + availableFields, + selectedFields = [], + onChange, + noAssociatedTableMessage, + helperText, + noFieldsMessage = 'No eligible obs columns found on the associated table', +}: TooltipFieldsPanelProps) { + return ( +
+
+
+ Tooltip fields +
+ {tableName ? ( + <> +
Table: {tableName}
+ {helperText ?
{helperText}
: null} + {availableFields.length > 0 ? ( +
+ {availableFields.map((field) => { + const checked = selectedFields.includes(field); + return ( + + ); + })} +
+ ) : ( +
{noFieldsMessage}
+ )} + + ) : ( +
{noAssociatedTableMessage}
+ )} +
+
+ ); +} diff --git a/packages/vis/src/SpatialCanvas/VivSpatialViewer.tsx b/packages/vis/src/SpatialCanvas/VivSpatialViewer.tsx index 6697bbc2..a55f38bb 100644 --- a/packages/vis/src/SpatialCanvas/VivSpatialViewer.tsx +++ b/packages/vis/src/SpatialCanvas/VivSpatialViewer.tsx @@ -178,14 +178,19 @@ class VivSpatialViewer extends React.PureComponent layerProps.loader); + // If we have a loader, use Viv's default initial view state - if (this.props.vivLayerProps.length > 0 && this.props.vivLayerProps[0].loader) { + if ( + firstLayerWithLoader && + firstLayerWithLoader.loader !== null && + typeof firstLayerWithLoader.loader === 'object' + ) { try { - const loader = this.props.vivLayerProps[0].loader as any; - const defaultState = getDefaultInitialViewState(loader, { + const defaultState = getDefaultInitialViewState(firstLayerWithLoader.loader, { width: this.props.width, height: this.props.height, - }); + }, 0, false, firstLayerWithLoader.modelMatrix); return { ...defaultState, id: this.viewId, @@ -265,48 +270,62 @@ class VivSpatialViewer extends React.PureComponent = { - loader: firstLayerProps.loader, - colors: firstLayerProps.colors, - contrastLimits: firstLayerProps.contrastLimits, - channelsVisible: firstLayerProps.channelsVisible, - selections: firstLayerProps.selections, - onHover, - }; + const vivLayers: Layer[] = []; + let scaleBarAdded = false; + + for (const imageLayerProps of vivLayerProps) { + const layerProps: Record = { + loader: imageLayerProps.loader, + colors: imageLayerProps.colors, + contrastLimits: imageLayerProps.contrastLimits, + channelsVisible: imageLayerProps.channelsVisible, + selections: imageLayerProps.selections, + onHover, + }; + + // Let Viv/deck merge these with layer defaultProps (including `extensions`). + // Do not patch `layer.props` afterward with `{ ...layer.props, opacity }` — that spread + // drops non-enumerable props like `extensions` and breaks MultiscaleImageLayer._update. + if (imageLayerProps.opacity !== undefined) { + layerProps.opacity = imageLayerProps.opacity; + } + if (imageLayerProps.visible !== undefined) { + layerProps.visible = imageLayerProps.visible; + } + if (imageLayerProps.modelMatrix) { + layerProps.modelMatrix = imageLayerProps.modelMatrix; + } - // Let Viv/deck merge these with layer defaultProps (including `extensions`). - // Do not patch `layer.props` afterward with `{ ...layer.props, opacity }` — that spread - // drops non-enumerable props like `extensions` and breaks MultiscaleImageLayer._update. - if (firstLayerProps.opacity !== undefined) { - layerProps.opacity = firstLayerProps.opacity; - } - if (firstLayerProps.visible !== undefined) { - layerProps.visible = firstLayerProps.visible; - } - if (firstLayerProps.modelMatrix) { - layerProps.modelMatrix = firstLayerProps.modelMatrix; + const vivLayersResult = this.detailView.getLayers({ + viewStates, + props: layerProps, + }); + + // getLayers returns an array of arrays (one per view) + // For a single view, take the first element (like MDVivViewer does at line 385) + const layersForImage = + Array.isArray(vivLayersResult) && vivLayersResult.length > 0 + ? (Array.isArray(vivLayersResult[0]) ? vivLayersResult[0] : vivLayersResult) as Layer[] + : []; + + for (const layer of layersForImage) { + if (layer instanceof ScaleBarLayer) { + if (!scaleBarAdded) { + vivLayers.push(layer); + scaleBarAdded = true; + } + continue; + } + + // Viv uses generic source-based ids here, so overlays need a stable per-image suffix. + vivLayers.push(layer.clone({ id: `${layer.id}-${imageLayerProps.id}` })); + } } - const vivLayersResult = this.detailView.getLayers({ - viewStates, - props: layerProps, - }); - - // getLayers returns an array of arrays (one per view) - // For a single view, take the first element (like MDVivViewer does at line 385) - const vivLayers = Array.isArray(vivLayersResult) && vivLayersResult.length > 0 - ? (Array.isArray(vivLayersResult[0]) ? vivLayersResult[0] : vivLayersResult) as Layer[] - : []; - // Compose with extra layers - following MDV pattern exactly // MDV does: [otherLayers (images), ...deckProps.layers (shapes), scaleBar] return composeLayers(vivLayers, extraLayersWithVivId, deckPropsLayersWithVivId); @@ -345,4 +364,3 @@ class VivSpatialViewer extends React.PureComponent )} - {selectedConfig.type === 'image' && selectedLayerLoadState.image && ( + {(selectedConfig.type === 'image' || selectedConfig.type === 'labels') && + selectedLayerLoadState.image && (
- Image: {selectedLayerLoadState.image} + {selectedConfig.type === 'labels' ? 'Labels' : 'Image'}:{' '} + {selectedLayerLoadState.image} {!hasRenderableLayerData(selectedConfig.id) && selectedLayerLoadState.image === 'loading' ? ' (blocking)' : ''}
)} - {selectedConfig.type === 'shapes' && selectedLayerLoadState.tooltip && ( + {(selectedConfig.type === 'shapes' || selectedConfig.type === 'labels') && + selectedLayerLoadState.tooltip && (
Tooltip metadata: {selectedLayerLoadState.tooltip}
)} @@ -714,66 +721,33 @@ function SpatialCanvasInner({ tooltipContainer, renderTooltip }: SpatialCanvasIn updateLayer={actions.updateLayer} /> )} - {selectedConfig.type === 'shapes' && ( -
-
-
- Tooltip fields -
- {associatedTable ? ( - <> -
- Table: {associatedTable.key} -
- {availableTooltipFields.length > 0 ? ( -
- {availableTooltipFields.map((field) => { - const checked = - selectedConfig.tooltipFields?.includes(field) ?? false; - return ( - - ); - })} -
- ) : ( -
- No eligible obs columns found on the associated table -
- )} - - ) : ( -
- No associated table found for this shapes layer -
- )} -
-
+ {selectedConfig.type === 'labels' && ( + + )} + {(selectedConfig.type === 'shapes' || selectedConfig.type === 'labels') && ( + { + actions.updateLayer(selectedConfig.id, { tooltipFields }); + }} + helperText={ + selectedConfig.type === 'labels' + ? 'Picked label ids are always shown; selected fields are appended from the associated table.' + : undefined + } + noAssociatedTableMessage={ + selectedConfig.type === 'labels' + ? 'No associated table found for this labels layer. Hover will show the picked label id only.' + : 'No associated table found for this shapes layer' + } + /> )} )} diff --git a/packages/vis/src/SpatialCanvas/renderers/imageRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/imageRenderer.ts index 0c34be02..e06994a7 100644 --- a/packages/vis/src/SpatialCanvas/renderers/imageRenderer.ts +++ b/packages/vis/src/SpatialCanvas/renderers/imageRenderer.ts @@ -7,7 +7,7 @@ import { loadOmeZarrMultiscalesData } from '@spatialdata/avivatorish'; import type { Matrix4 } from '@math.gl/core'; -import type { ImageElement } from '@spatialdata/core'; +import type { ImageElement, LabelsElement } from '@spatialdata/core'; import type { Layer } from 'deck.gl'; export interface ImageLayerRenderConfig { @@ -89,7 +89,7 @@ export function extractChannelConfig(config: { * SpatialData only supports OME-Zarr format, so we use loadOmeZarr. */ export async function createImageLoader( - element: ImageElement, + element: ImageElement | LabelsElement, fetchMultiscales: (url: string) => Promise = loadOmeZarrMultiscalesData, ): Promise { try { @@ -99,4 +99,3 @@ export async function createImageLoader( throw error; } } - diff --git a/packages/vis/src/SpatialCanvas/renderers/index.ts b/packages/vis/src/SpatialCanvas/renderers/index.ts index e4e72567..04c25ce3 100644 --- a/packages/vis/src/SpatialCanvas/renderers/index.ts +++ b/packages/vis/src/SpatialCanvas/renderers/index.ts @@ -8,5 +8,4 @@ export { renderImageLayer, type ImageLayerRenderConfig } from './imageRenderer'; export { renderShapesLayer, type ShapesLayerRenderConfig } from './shapesRenderer'; export { renderPointsLayer, type PointsLayerRenderConfig } from './pointsRenderer'; -// export { renderLabelsLayer, type LabelsLayerRenderConfig } from './labelsRenderer'; - +export { renderLabelsLayer, type LabelsLayerRenderConfig } from './labelsRenderer'; diff --git a/packages/vis/src/SpatialCanvas/renderers/labelsRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/labelsRenderer.ts new file mode 100644 index 00000000..bed4fc65 --- /dev/null +++ b/packages/vis/src/SpatialCanvas/renderers/labelsRenderer.ts @@ -0,0 +1,62 @@ +/** + * Labels layer renderer using a custom deck.gl bitmask layer. + * + * SpatialData labels are stored as raster segmentations, so we render them via + * a custom MultiscaleImageLayer wrapper that colors non-zero label ids per + * channel instead of showing the raw integer values as grayscale. + */ + +import { LabelsLayer } from '@spatialdata/layers'; +import type { Matrix4 } from '@math.gl/core'; +import type { Layer } from 'deck.gl'; + +export interface LabelsLayerRenderConfig { + id: string; + loader: unknown; + modelMatrix: Matrix4; + opacity: number; + visible: boolean; + channelColors: [number, number, number][]; + channelsVisible: boolean[]; + channelOpacities: number[]; + channelOutlineOpacities: number[]; + channelsFilled: boolean[]; + channelStrokeWidths: number[]; + selections: Partial<{ z: number; c: number; t: number }>[]; +} + +export function renderLabelsLayer(config: LabelsLayerRenderConfig): Layer | null { + const { + id, + loader, + modelMatrix, + opacity, + visible, + channelColors, + channelsVisible, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + selections, + } = config; + + if (!visible || !loader) { + return null; + } + + return new LabelsLayer({ + id, + loader, + modelMatrix, + opacity, + visible, + channelColors, + channelsVisible, + channelOpacities, + channelOutlineOpacities, + channelsFilled, + channelStrokeWidths, + selections, + }); +} diff --git a/packages/vis/src/SpatialCanvas/renderers/shapesRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/shapesRenderer.ts index 91f22a90..b6d0980e 100644 --- a/packages/vis/src/SpatialCanvas/renderers/shapesRenderer.ts +++ b/packages/vis/src/SpatialCanvas/renderers/shapesRenderer.ts @@ -6,13 +6,10 @@ import { PolygonLayer } from 'deck.gl'; import type { Matrix4 } from '@math.gl/core'; -import type { ShapesElement } from '@spatialdata/core'; +import type { ShapesElement, SpatialFeatureTooltipData } from '@spatialdata/core'; import type { Layer } from 'deck.gl'; -export interface ShapeTooltipDatum { - title?: string; - items: Array<{ label: string; value: string }>; -} +export type ShapeTooltipDatum = SpatialFeatureTooltipData; export interface ShapesLayerRenderConfig { /** The shapes element to render */ diff --git a/packages/vis/src/SpatialCanvas/types.ts b/packages/vis/src/SpatialCanvas/types.ts index 78053084..f1dae2dd 100644 --- a/packages/vis/src/SpatialCanvas/types.ts +++ b/packages/vis/src/SpatialCanvas/types.ts @@ -82,10 +82,17 @@ export interface PointsLayerConfig extends BaseLayerConfig { export interface LabelsLayerConfig extends BaseLayerConfig { type: 'labels'; - // Labels-specific settings (colormap, etc.) - // should also be able to associate with picked feature identity - // (for example ObjectID-style raster values), so we'll need some kind of - // buffer lookup for color/filter/etc + tooltipFields?: string[]; + channels?: { + channelIds?: string[]; + colors?: [number, number, number][]; + channelsVisible?: boolean[]; + channelOpacities?: number[]; + channelOutlineOpacities?: number[]; + channelsFilled?: boolean[]; + channelStrokeWidths?: number[]; + selections?: Partial<{ z: number; c: number; t: number }>[]; + }; } export type LayerConfig = ImageLayerConfig | ShapesLayerConfig | PointsLayerConfig | LabelsLayerConfig; diff --git a/packages/vis/src/SpatialCanvas/useLayerData.ts b/packages/vis/src/SpatialCanvas/useLayerData.ts index 82d2f301..67635695 100644 --- a/packages/vis/src/SpatialCanvas/useLayerData.ts +++ b/packages/vis/src/SpatialCanvas/useLayerData.ts @@ -20,14 +20,21 @@ import { import { type AxisAlignedBounds, type ImageElement, + type LabelsElement, type PointsElement, type ShapesElement, + type LabelsTooltipMetadata, + type ShapesTooltipMetadata, + type SpatialFeatureTooltipData, type SpatialData, - type TableColumnData, boundsFromImagePixelExtents, boundsFromPoints, boundsFromPolygons, + getTooltipSignature, getPhysicalSizeScalingMatrixFromMeta, + loadLabelsTooltipMetadata, + loadShapesTooltipMetadata, + resolveTooltipItems, unionBoundsList, } from '@spatialdata/core'; import type { Layer } from 'deck.gl'; @@ -40,14 +47,10 @@ import { import { createImageLoader } from './renderers/imageRenderer'; import { type PointData, - type PointsLayerRenderConfig, renderPointsLayer, } from './renderers/pointsRenderer'; -import { - type ShapeTooltipDatum, - loadShapesData, - renderShapesLayer, -} from './renderers/shapesRenderer'; +import { renderLabelsLayer } from './renderers/labelsRenderer'; +import { loadShapesData, renderShapesLayer } from './renderers/shapesRenderer'; import type { AvailableElement, ElementsByType, LayerConfig } from './types'; export interface ImageLoaderData { @@ -60,24 +63,15 @@ export interface ImageLoaderData { selectionAxisSizes?: Partial>; } -interface LoadedShapesData { +interface LoadedShapesData extends ShapesTooltipMetadata { polygons: Array>>; - featureIds?: string[]; - tooltipSignature?: string; - tooltipFields?: string[]; - tooltipColumns?: Array; - /** - * Optional row-index lookup aligned to picked feature order. - * When omitted, picked feature index and tooltip row index are assumed to be identical. - * A value of -1 indicates no matching tooltip row for that feature. - */ - tooltipRowIndices?: Int32Array; } interface LoadedData { shapes: Map; points: Map; images: Map; // Viv loaders with computed channel data + labels: Map; } type ResourceLoadStatus = 'idle' | 'loading' | 'ready' | 'error'; @@ -89,6 +83,7 @@ export interface LayerLoadState { } export interface ImageLayerConfig { + id: string; loader: unknown; // Viv PixelSource colors: [number, number, number][]; contrastLimits: [number, number][]; @@ -99,6 +94,18 @@ export interface ImageLayerConfig { visible?: boolean; // Whether layer is visible } +export interface LabelsLoaderData extends LabelsTooltipMetadata { + loader: unknown; + colors: [number, number, number][]; + channelsVisible: boolean[]; + channelOpacities: number[]; + channelOutlineOpacities: number[]; + channelsFilled: boolean[]; + channelStrokeWidths: number[]; + selections: Array>; + selectionAxisSizes?: Partial>; +} + interface UseLayerDataResult { /** Get deck.gl layers ready for rendering (shapes, points, etc.) */ getLayers: () => Layer[]; @@ -106,12 +113,17 @@ interface UseLayerDataResult { getVivLayerProps: () => ImageLayerConfig[]; /** Raw loaded image pipeline data (defaults) for the properties UI */ getImageLayerLoadedData: (layerId: string) => ImageLoaderData | undefined; + /** Raw loaded labels pipeline data (defaults) for the properties UI */ + getLabelsLayerLoadedData: (layerId: string) => LabelsLoaderData | undefined; /** Current load state for a given layer. */ getLayerLoadState: (layerId?: string) => LayerLoadState | undefined; /** Whether a layer already has enough data to render. */ hasRenderableLayerData: (layerId: string) => boolean; /** Resolve a feature tooltip lazily from the picked row index. */ - getFeatureTooltip: (layerId: string, objectIndex: number) => ShapeTooltipDatum | undefined; + getFeatureTooltip: ( + layerId: string, + pickInfo: Pick<{ index?: number; object?: unknown }, 'index' | 'object'> + ) => SpatialFeatureTooltipData | undefined; /** Whether any layers are currently loading */ isLoading: boolean; /** Whether any visible layer is still waiting on its first renderable resource. */ @@ -124,130 +136,8 @@ interface UseLayerDataResult { getWorldBoundsForVisibleLayers: () => AxisAlignedBounds | null; } -function getTooltipSignature(config: LayerConfig | undefined): string { - if (!config || config.type !== 'shapes') { - return ''; - } - return (config.tooltipFields ?? []).join('\u0001'); -} - -function normalizeTooltipValue(value: TableColumnData | undefined, rowIndex: number): string { - if (!value) return ''; - const row = value[rowIndex]; - if (row === null || row === undefined) return ''; - return String(row); -} - -function tableRegionMatches(regionValue: string, shapeKey: string) { - return regionValue === shapeKey || regionValue === `shapes/${shapeKey}`; -} - -async function loadShapeTooltipData( - spatialData: SpatialData | undefined, - element: ShapesElement, - tooltipFields: string[] -): Promise< - Pick< - LoadedShapesData, - 'featureIds' | 'tooltipSignature' | 'tooltipFields' | 'tooltipColumns' | 'tooltipRowIndices' - > -> { - const featureIdsRaw = await element.loadFeatureIds(); - const featureIds = featureIdsRaw - ? Array.from(featureIdsRaw, (value: unknown) => String(value)) - : undefined; - - if (tooltipFields.length === 0) { - return { - featureIds, - tooltipSignature: '', - tooltipFields: [], - tooltipColumns: undefined, - tooltipRowIndices: undefined, - }; - } - - if (!featureIds) { - return { - featureIds, - tooltipSignature: undefined, - tooltipFields, - tooltipColumns: undefined, - tooltipRowIndices: undefined, - }; - } - - if (!spatialData) { - return { - featureIds, - tooltipSignature: undefined, - tooltipFields, - tooltipColumns: undefined, - tooltipRowIndices: undefined, - }; - } - - const associated = spatialData.getAssociatedTable('shapes', element.key); - if (!associated) { - return { - featureIds, - tooltipSignature: undefined, - tooltipFields, - tooltipColumns: undefined, - tooltipRowIndices: undefined, - }; - } - - const tooltipSignature = tooltipFields.join('\u0001'); - const [, table] = associated; - const { regionKey } = table.getTableKeys(); - const requestedColumns = Array.from(new Set([regionKey, ...tooltipFields])); - const rowIds = await table.loadObsIndex(); - const columns = await table.loadObsColumns(requestedColumns); - const regionColumn = columns[0]; - const tooltipColumns = columns.slice(1); - // I don't like the look of this... creating 0-length arrays then pushing data... - const filteredRowIds: string[] = []; - const filteredRowIndices: number[] = []; - - for (let rowIndex = 0; rowIndex < rowIds.length; rowIndex++) { - const rowId = rowIds[rowIndex]; - const regionValue = normalizeTooltipValue(regionColumn, rowIndex); - if (regionValue && !tableRegionMatches(regionValue, element.key)) { - continue; - } - filteredRowIds.push(String(rowId)); - filteredRowIndices.push(rowIndex); - } - let tooltipRowIndices: Int32Array | undefined; - // this looks costly? - const isDirectlyAligned = - filteredRowIds.length === featureIds.length && - filteredRowIds.every((rowId, index) => rowId === featureIds[index]); - - if (!isDirectlyAligned) { - const rowIndexByFeatureId = new Map(); - for (const [index, rowId] of filteredRowIds.entries()) { - rowIndexByFeatureId.set(rowId, filteredRowIndices[index]); - } - - tooltipRowIndices = new Int32Array(featureIds.length); - tooltipRowIndices.fill(-1); - for (const [featureIndex, featureId] of featureIds.entries()) { - const matchedRowIndex = rowIndexByFeatureId.get(featureId); - if (matchedRowIndex !== undefined) { - tooltipRowIndices[featureIndex] = matchedRowIndex; - } - } - } - - return { - featureIds, - tooltipSignature, - tooltipFields, - tooltipColumns, - tooltipRowIndices, - }; +function getLayerTooltipSignature(config: LayerConfig | undefined): string { + return config && 'tooltipFields' in config ? getTooltipSignature(config.tooltipFields) : ''; } async function loadShapesLayerData( @@ -280,6 +170,7 @@ export function useLayerData( shapes: new Map(), points: new Map(), images: new Map(), + labels: new Map(), }); const layersRef = useRef(layers); @@ -329,6 +220,7 @@ export function useLayerData( loadTooltip: boolean; loadImage: boolean; loadPoints: boolean; + loadLabels: boolean; }> = []; for (const layerId of layerOrder) { @@ -342,7 +234,7 @@ export function useLayerData( const loaded = loadedDataRef.current; if (config.type === 'shapes') { const loadedShapes = loaded.shapes.get(elem.key); - const tooltipSignature = getTooltipSignature(config); + const tooltipSignature = getLayerTooltipSignature(config); const loadGeometry = !loadedShapes; const loadTooltip = !loadedShapes || loadedShapes.tooltipSignature !== tooltipSignature; if (loadGeometry || loadTooltip) { @@ -353,6 +245,23 @@ export function useLayerData( loadTooltip, loadImage: false, loadPoints: false, + loadLabels: false, + }); + } + } else if (config.type === 'labels') { + const loadedLabels = loaded.labels.get(elem.key); + const tooltipSignature = getLayerTooltipSignature(config); + const loadLabels = !loadedLabels; + const loadTooltip = !loadedLabels || loadedLabels.tooltipSignature !== tooltipSignature; + if (loadLabels || loadTooltip) { + toLoad.push({ + layerId, + element: elem, + loadGeometry: false, + loadTooltip, + loadImage: false, + loadPoints: false, + loadLabels, }); } } else if (config.type === 'points' && !loaded.points.has(elem.key)) { @@ -363,6 +272,7 @@ export function useLayerData( loadTooltip: false, loadImage: false, loadPoints: true, + loadLabels: false, }); } else if (config.type === 'image' && !loaded.images.has(elem.key)) { toLoad.push({ @@ -372,6 +282,7 @@ export function useLayerData( loadTooltip: false, loadImage: true, loadPoints: false, + loadLabels: false, }); } } @@ -381,7 +292,15 @@ export function useLayerData( // Load in parallel await Promise.all( toLoad.map( - async ({ layerId, element, loadGeometry, loadTooltip, loadImage, loadPoints }) => { + async ({ + layerId, + element, + loadGeometry, + loadTooltip, + loadImage, + loadPoints, + loadLabels, + }) => { if (element.type === 'shapes') { const existing = loadedDataRef.current.shapes.get(element.key); if (loadGeometry) { @@ -409,16 +328,16 @@ export function useLayerData( ? layersRef.current[layerId] : undefined; const tooltipFields = shapeLayerConfig?.tooltipFields ?? []; - const requestedSignature = getTooltipSignature(shapeLayerConfig); + const requestedSignature = getLayerTooltipSignature(shapeLayerConfig); if (tooltipFields.length > 0) { setLayerResourceStatus(layerId, 'tooltip', 'loading'); const current = loadedDataRef.current.shapes.get(element.key); - const tooltipData = await loadShapeTooltipData( + const tooltipData = await loadShapesTooltipMetadata( spatialData, element.element as ShapesElement, tooltipFields ); - const latestDesired = getTooltipSignature( + const latestDesired = getLayerTooltipSignature( layersRef.current[layerId]?.type === 'shapes' ? layersRef.current[layerId] : undefined @@ -502,7 +421,7 @@ export function useLayerData( const Channels = metadata.channels; const isRgb = guessRgb({ Pixels: { Channels: Channels.map((c: any) => ({ Name: c.label })) }, - } as any); + }); if (isRgb) { if (isInterleaved(loaderObj.shape)) { @@ -524,8 +443,8 @@ export function useLayerData( } else { // Compute stats for non-RGB images const stats = await getMultiSelectionStats({ - loader: loader as any, - selections: selections as any, + loader, + selections, use3d: false, }); imageData.contrastLimits = stats.contrastLimits; @@ -594,6 +513,128 @@ export function useLayerData( setLayerResourceStatus(layerId, 'image', 'error'); console.error(`Failed to load image for ${layerId}:`, error); } + } else if (element.type === 'labels') { + const existing = loadedDataRef.current.labels.get(element.key); + if (loadLabels) { + try { + setLayerResourceStatus(layerId, 'image', 'loading'); + const loader = await createImageLoader( + element.element as LabelsElement, + getOmeZarrMultiscalesData + ); + const loaderToCheck = Array.isArray(loader) ? loader[0] : loader; + const labelsData: LabelsLoaderData = { + loader, + colors: [[255, 255, 255]], + channelsVisible: [true], + channelOpacities: [0.18], + channelOutlineOpacities: [0.95], + channelsFilled: [true], + channelStrokeWidths: [1.5], + selections: [{}], + }; + + if ( + loaderToCheck && + typeof loaderToCheck === 'object' && + 'labels' in loaderToCheck && + 'shape' in loaderToCheck + ) { + const loaderObj = loaderToCheck as VivLoaderMetadata; + const axisSizes = getVivSelectionAxisSizes(loaderObj.labels, loaderObj.shape); + const selections = clampVivSelectionsToAxes( + buildDefaultSelection({ + labels: loaderObj.labels, + shape: loaderObj.shape, + }), + axisSizes + ).slice(0, 7); + const channelCount = Math.max(selections.length, 1); + const metadataChannels = (element.element as LabelsElement).attrs.omero?.channels; + + const colors = Array.from( + { length: channelCount }, + (_, index): [number, number, number] => { + const rgb = tryParseOmeroHexColor(metadataChannels?.[index]?.color); + const palette = COLOR_PALLETE[index % COLOR_PALLETE.length]; + return rgb ?? [palette[0], palette[1], palette[2]]; + } + ); + labelsData.selectionAxisSizes = axisSizes; + labelsData.selections = selections.length > 0 ? selections : [{}]; + labelsData.colors = colors; + labelsData.channelsVisible = colors.map( + (_, index) => metadataChannels?.[index]?.active ?? true + ); + labelsData.channelOpacities = colors.map(() => 0.18); + labelsData.channelOutlineOpacities = colors.map(() => 0.95); + labelsData.channelsFilled = colors.map(() => true); + labelsData.channelStrokeWidths = colors.map(() => 1.5); + } + + loadedDataRef.current.labels.set(element.key, { + ...existing, + ...labelsData, + }); + setLayerResourceStatus(layerId, 'image', 'ready'); + } catch (error) { + setLayerResourceStatus(layerId, 'image', 'error'); + console.error(`Failed to load labels for ${layerId}:`, error); + return; + } + } else { + setLayerResourceStatus(layerId, 'image', existing ? 'ready' : 'idle'); + } + + if (loadTooltip) { + try { + const labelsLayerConfig = + layersRef.current[layerId]?.type === 'labels' + ? layersRef.current[layerId] + : undefined; + const tooltipFields = labelsLayerConfig?.tooltipFields ?? []; + const requestedSignature = getLayerTooltipSignature(labelsLayerConfig); + if (tooltipFields.length > 0) { + setLayerResourceStatus(layerId, 'tooltip', 'loading'); + const current = loadedDataRef.current.labels.get(element.key); + const tooltipData = await loadLabelsTooltipMetadata( + spatialData, + element.element as LabelsElement, + tooltipFields + ); + const latestDesired = getLayerTooltipSignature( + layersRef.current[layerId]?.type === 'labels' + ? layersRef.current[layerId] + : undefined + ); + if (latestDesired !== requestedSignature) { + return; + } + loadedDataRef.current.labels.set(element.key, { + ...current, + ...tooltipData, + } as LabelsLoaderData); + setLayerResourceStatus( + layerId, + 'tooltip', + tooltipData.tooltipSignature === undefined ? 'idle' : 'ready' + ); + } else { + const current = loadedDataRef.current.labels.get(element.key); + loadedDataRef.current.labels.set(element.key, { + ...current, + tooltipSignature: '', + tooltipFields: [], + tooltipColumns: undefined, + tooltipRowIndexByFeatureId: undefined, + } as LabelsLoaderData); + setLayerResourceStatus(layerId, 'tooltip', 'idle'); + } + } catch (error) { + setLayerResourceStatus(layerId, 'tooltip', 'error'); + console.error(`Failed to load labels tooltip for ${layerId}:`, error); + } + } } } ) @@ -611,6 +652,8 @@ export function useLayerData( loaded.points.delete(key); } else if (type === 'image') { loaded.images.delete(key); + } else if (type === 'labels') { + loaded.labels.delete(key); } // The useEffect will pick up the missing data and reload }, []); @@ -627,6 +670,9 @@ export function useLayerData( if (elem.type === 'image') { return loadedDataRef.current.images.has(elem.key); } + if (elem.type === 'labels') { + return loadedDataRef.current.labels.has(elem.key); + } return false; }, []); @@ -656,6 +702,16 @@ export function useLayerData( const physical = getPhysicalSizeScalingMatrixFromMeta(source); return boundsFromImagePixelExtents(width, height, elem.transform, physical); } + if (elem.type === 'labels') { + const labelsData = loaded.labels.get(elem.key); + if (!labelsData?.loader) return null; + const source = + Array.isArray(labelsData.loader) ? labelsData.loader[0] : labelsData.loader; + if (!source || typeof source !== 'object') return null; + const { width, height } = getImageSize(source as never); + const physical = getPhysicalSizeScalingMatrixFromMeta(source); + return boundsFromImagePixelExtents(width, height, elem.transform, physical); + } return null; } catch (err) { console.warn(`[useLayerData] getWorldBoundsForLayer failed for ${layerId}`, err); @@ -718,6 +774,49 @@ export function useLayerData( }); if (layer) deckLayers.push(layer); } + } else if (config.type === 'labels') { + const labelsData = loaded.labels.get(elem.key); + if (labelsData) { + const ch = config.channels; + const rawSelections = + ch?.selections && ch.selections.length > 0 ? ch.selections : labelsData.selections; + const selections = + labelsData.selectionAxisSizes !== undefined + ? clampVivSelectionsToAxes(rawSelections, labelsData.selectionAxisSizes) + : rawSelections; + + const layer = renderLabelsLayer({ + id: layerId, + loader: labelsData.loader, + modelMatrix: elem.transform, + opacity: config.opacity, + visible: config.visible, + channelColors: + ch?.colors && ch.colors.length > 0 ? ch.colors : labelsData.colors, + channelsVisible: + ch?.channelsVisible && ch.channelsVisible.length > 0 + ? ch.channelsVisible + : labelsData.channelsVisible, + channelOpacities: + ch?.channelOpacities && ch.channelOpacities.length > 0 + ? ch.channelOpacities + : labelsData.channelOpacities, + channelOutlineOpacities: + ch?.channelOutlineOpacities && ch.channelOutlineOpacities.length > 0 + ? ch.channelOutlineOpacities + : labelsData.channelOutlineOpacities, + channelsFilled: + ch?.channelsFilled && ch.channelsFilled.length > 0 + ? ch.channelsFilled + : labelsData.channelsFilled, + channelStrokeWidths: + ch?.channelStrokeWidths && ch.channelStrokeWidths.length > 0 + ? ch.channelStrokeWidths + : labelsData.channelStrokeWidths, + selections, + }); + if (layer) deckLayers.push(layer); + } } // Image layers are handled separately via getVivLayerProps() } @@ -731,6 +830,12 @@ export function useLayerData( return loadedDataRef.current.images.get(elem.key); }, []); + const getLabelsLayerLoadedData = useCallback((layerId: string): LabelsLoaderData | undefined => { + const elem = elementMap.current.get(layerId); + if (!elem || elem.type !== 'labels') return undefined; + return loadedDataRef.current.labels.get(elem.key); + }, []); + const getLayerLoadState = useCallback( (layerId?: string): LayerLoadState | undefined => { if (layerId === undefined) return undefined; @@ -740,9 +845,58 @@ export function useLayerData( ); const getFeatureTooltip = useCallback( - (layerId: string, objectIndex: number): ShapeTooltipDatum | undefined => { + ( + layerId: string, + pickInfo: Pick<{ index?: number; object?: unknown }, 'index' | 'object'> + ): SpatialFeatureTooltipData | undefined => { const elem = elementMap.current.get(layerId); - if (!elem || elem.type !== 'shapes') { + if (!elem) { + return undefined; + } + + if (elem.type === 'labels') { + const pickedObject = pickInfo.object as + | { labelId?: number | string; channelIndex?: number } + | undefined; + const rawLabelId = pickedObject?.labelId; + const labelId = rawLabelId === undefined || rawLabelId === null ? '' : String(rawLabelId); + if (!labelId) { + return undefined; + } + + const loadedLabelData = loadedDataRef.current.labels.get(elem.key); + const config = layersRef.current[layerId]; + const title = labelId; + const items: Array<{ label: string; value: string }> = [ + { label: 'id', value: labelId }, + ]; + + if ( + config?.type === 'labels' && + (config.tooltipFields?.length ?? 0) > 0 && + loadedLabelData?.tooltipFields && + loadedLabelData.tooltipColumns && + loadedLabelData.tooltipRowIndexByFeatureId && + getLayerTooltipSignature(config) === (loadedLabelData.tooltipSignature ?? '') + ) { + const rowIndex = loadedLabelData.tooltipRowIndexByFeatureId.get(labelId); + if (rowIndex !== undefined && rowIndex >= 0) { + const tooltipItems = resolveTooltipItems( + loadedLabelData.tooltipFields, + loadedLabelData.tooltipColumns, + rowIndex, + ); + items.push(...tooltipItems); + } + } + + return { + title, + items, + }; + } + + if (elem.type !== 'shapes') { return undefined; } @@ -756,7 +910,12 @@ export function useLayerData( } const config = layersRef.current[layerId]; - if (getTooltipSignature(config) !== (loadedShapeData.tooltipSignature ?? '')) { + if (getLayerTooltipSignature(config) !== (loadedShapeData.tooltipSignature ?? '')) { + return undefined; + } + + const objectIndex = pickInfo.index; + if (typeof objectIndex !== 'number' || objectIndex < 0) { return undefined; } @@ -772,12 +931,11 @@ export function useLayerData( return undefined; } - const items = loadedShapeData.tooltipFields - .map((field, fieldIndex) => ({ - label: field, - value: normalizeTooltipValue(loadedShapeData.tooltipColumns?.[fieldIndex], rowIndex), - })) - .filter((item) => item.value !== ''); + const items = resolveTooltipItems( + loadedShapeData.tooltipFields, + loadedShapeData.tooltipColumns, + rowIndex, + ); if (items.length === 0) { return undefined; @@ -825,6 +983,7 @@ export function useLayerData( : rawSelections; vivProps.push({ + id: config.id, loader: imageData.loader, colors, contrastLimits, @@ -857,6 +1016,9 @@ export function useLayerData( if (config.type === 'image') { return state.image === 'loading' && !hasRenderableLayerData(layerId); } + if (config.type === 'labels') { + return state.image === 'loading' && !hasRenderableLayerData(layerId); + } if (config.type === 'shapes' || config.type === 'points') { return state.geometry === 'loading' && !hasRenderableLayerData(layerId); } @@ -869,6 +1031,7 @@ export function useLayerData( getLayers, getVivLayerProps, getImageLayerLoadedData, + getLabelsLayerLoadedData, getLayerLoadState, hasRenderableLayerData, getFeatureTooltip, diff --git a/packages/vis/src/viv-typing-fixes.d.ts b/packages/vis/src/viv-typing-fixes.d.ts new file mode 100644 index 00000000..aa8f7079 --- /dev/null +++ b/packages/vis/src/viv-typing-fixes.d.ts @@ -0,0 +1,11 @@ +import type { Matrix4 } from '@math.gl/core'; + +declare module '@hms-dbmi/viv' { + export function getDefaultInitialViewState( + loader: object, + viewSize: { width: number; height: number }, + zoomBackOff?: number, + use3d?: boolean, + modelMatrix?: Matrix4 + ): { target: [number, number, number]; zoom: number }; +} diff --git a/packages/vis/tsconfig.json b/packages/vis/tsconfig.json index 52e5dca1..a4d25b32 100644 --- a/packages/vis/tsconfig.json +++ b/packages/vis/tsconfig.json @@ -14,8 +14,12 @@ "lib": ["ES2020", "DOM", "DOM.Iterable"], "baseUrl": ".", "paths": { + "@spatialdata/avivatorish": ["../avivatorish/src/index.ts"], + "@spatialdata/avivatorish/*": ["../avivatorish/src/*"], "@spatialdata/core": ["../core/src/index.ts"], "@spatialdata/core/*": ["../core/src/*"], + "@spatialdata/layers": ["../layers/src/index.ts"], + "@spatialdata/layers/*": ["../layers/src/*"], "@spatialdata/react": ["../react/src/index.ts"], "@spatialdata/react/*": ["../react/src/*"], "@spatialdata/zarrextra": ["../zarrextra/src/index.ts"], @@ -26,4 +30,3 @@ "include": ["src", "vite.config.ts"], "exclude": ["dist", "node_modules"] } - diff --git a/packages/vis/vite.config.ts b/packages/vis/vite.config.ts index 09324aa7..8f40aa63 100644 --- a/packages/vis/vite.config.ts +++ b/packages/vis/vite.config.ts @@ -1,10 +1,10 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { mergeConfig } from 'vite'; -import { defineViteConfig } from '../../vite.config.base'; +import { createWorkspaceSourceAliases, defineViteConfig } from '../../vite.config.base'; const pkgRoot = fileURLToPath(new URL('.', import.meta.url)); -const coreSrcIndex = path.resolve(pkgRoot, '../core/src/index.ts'); +const workspaceRoot = path.resolve(pkgRoot, '../..'); const baseConfig = defineViteConfig({ pkgRoot, @@ -12,20 +12,10 @@ const baseConfig = defineViteConfig({ external: ['@spatialdata/core', '@spatialdata/react'], }); -const testResolve = - process.env.VITEST !== undefined - ? { - resolve: { - alias: { - // Vitest: resolve workspace `core` from source so exports match TS without a prior build. - '@spatialdata/core': coreSrcIndex, - }, - }, - } - : {}; - export default mergeConfig(baseConfig, { - ...testResolve, + resolve: { + alias: createWorkspaceSourceAliases(workspaceRoot), + }, test: { globals: true, environment: 'jsdom', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f0bc7f23..1434aa3f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -242,6 +242,12 @@ importers: packages/layers: dependencies: + '@deck.gl/core': + specifier: 'catalog:' + version: 9.1.15 + '@hms-dbmi/viv': + specifier: 'catalog:' + version: 0.19.0(fd776ac1e5acea3be9c5cb4e7aa1f916) '@math.gl/core': specifier: 'catalog:' version: 4.1.0 @@ -249,9 +255,6 @@ importers: specifier: 'catalog:' version: 4.1.13 devDependencies: - '@deck.gl/core': - specifier: 'catalog:' - version: 9.1.15 '@types/node': specifier: 'catalog:' version: 22.18.8