From 50cd0b47c9aea7fbdb9b608ad08c50138888b57d Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 27 Dec 2024 10:11:58 -0500 Subject: [PATCH 1/8] fix looker classifications --- app/packages/looker/src/lookers/abstract.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 08991adba9..7d564ce354 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -528,6 +528,11 @@ export abstract class AbstractLooker< for (const overlay of this.pluckedOverlays ?? []) { let overlayData: LabelMask = null; + console.log(overlay); + if (!overlay.label) { + continue; + } + if ("mask" in overlay.label) { overlayData = overlay.label.mask as LabelMask; } else if ("map" in overlay.label) { @@ -743,7 +748,7 @@ export abstract class AbstractLooker< protected cleanOverlays() { for (const overlay of this.sampleOverlays ?? []) { - overlay.cleanup(); + overlay.cleanup?.(); } } From dad866d6667487771323216d108efa399ca31c14 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 27 Dec 2024 10:18:48 -0500 Subject: [PATCH 2/8] cleanup --- app/packages/looker/src/lookers/abstract.ts | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 7d564ce354..3ee1349abd 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -24,6 +24,9 @@ import { Events } from "../elements/base"; import { COMMON_SHORTCUTS, LookerElement } from "../elements/common"; import { ClassificationsOverlay, loadOverlays } from "../overlays"; import { CONTAINS, LabelMask, Overlay } from "../overlays/base"; +import DetectionOverlay from "../overlays/detection"; +import HeatmapOverlay from "../overlays/heatmap"; +import SegmentationOverlay from "../overlays/segmentation"; import processOverlays from "../processOverlays"; import { BaseState, @@ -528,15 +531,13 @@ export abstract class AbstractLooker< for (const overlay of this.pluckedOverlays ?? []) { let overlayData: LabelMask = null; - console.log(overlay); - if (!overlay.label) { - continue; - } - - if ("mask" in overlay.label) { - overlayData = overlay.label.mask as LabelMask; - } else if ("map" in overlay.label) { - overlayData = overlay.label.map as LabelMask; + if ( + overlay instanceof DetectionOverlay || + overlay instanceof SegmentationOverlay + ) { + overlayData = overlay.label.mask; + } else if (overlay instanceof HeatmapOverlay) { + overlayData = overlay.label.map; } const buffer = overlayData?.data?.buffer; @@ -551,9 +552,9 @@ export abstract class AbstractLooker< if (buffer.detached) { // most likely sample is already being processed, skip update return; - } else { - arrayBuffers.push(buffer); } + + arrayBuffers.push(buffer); } else if (buffer.byteLength) { // hope we don't run into this edge case (old browser) // sometimes detached buffers have bytelength > 0 From 1b03ebd6c9e005623bbd66d33cfeb1c7c6d90696 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 27 Dec 2024 10:37:56 -0500 Subject: [PATCH 3/8] update tsconfig --- app/packages/looker/tsconfig.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/packages/looker/tsconfig.json b/app/packages/looker/tsconfig.json index 96b76d5936..f86cf5aba7 100644 --- a/app/packages/looker/tsconfig.json +++ b/app/packages/looker/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "target": "ES2023", + "target": "ES2024", "module": "ES2022", - "lib": ["ES2023", "DOM", "DOM.Iterable"], + "lib": ["ES2024", "DOM", "DOM.Iterable"], "moduleResolution": "Node", "sourceMap": true, "resolveJsonModule": true, From cc499d5687b0be0c492532bfa4b6aaa814f5e2f3 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Mon, 30 Dec 2024 10:01:26 -0500 Subject: [PATCH 4/8] enhance filterView --- app/packages/state/src/utils.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/packages/state/src/utils.ts b/app/packages/state/src/utils.ts index 400567814c..ec81307826 100644 --- a/app/packages/state/src/utils.ts +++ b/app/packages/state/src/utils.ts @@ -57,7 +57,7 @@ export const stringifyObj = (obj) => { ); }; -export const filterView = (stages) => +export const filterView = (stages: State.Stage[]) => JSON.stringify( stages.map(({ kwargs, _cls }) => ({ kwargs: kwargs.filter((ka) => !ka[0].startsWith("_")), @@ -65,7 +65,18 @@ export const filterView = (stages) => })) ); -export const viewsAreEqual = (viewOne, viewTwo) => { +export const viewsAreEqual = ( + viewOne?: string | State.Stage[], + viewTwo?: string | State.Stage[] +) => { + if (viewOne === viewTwo) { + return true; + } + + if (!Array.isArray(viewOne) || !Array.isArray(viewTwo)) { + return false; + } + return filterView(viewOne) === filterView(viewTwo); }; From 2744d5560653deede1f11fa29686b9be6bc812ce Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Mon, 30 Dec 2024 10:51:38 -0500 Subject: [PATCH 5/8] add viewsAreEqual tests --- app/packages/state/src/utils.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/packages/state/src/utils.test.ts b/app/packages/state/src/utils.test.ts index 94e7b0cd8c..14979674b9 100644 --- a/app/packages/state/src/utils.test.ts +++ b/app/packages/state/src/utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { convertTargets } from "./utils"; +import { convertTargets, viewsAreEqual } from "./utils"; describe("convertTargets", () => { it("upper cases rgb hex targets", () => { @@ -8,3 +8,13 @@ describe("convertTargets", () => { ).toStrictEqual({ "#FFFFFF": { label: "white", intTarget: 1 } }); }); }); + +describe("filterView", () => { + it("handles saved view string names and undefined values", () => { + expect(viewsAreEqual("one", "one")).toBe(true); + expect(viewsAreEqual("one", "two")).toBe(false); + expect(viewsAreEqual("one", undefined)).toBe(false); + expect(viewsAreEqual("one", [])).toBe(false); + expect(viewsAreEqual([], [])).toBe(true); + }); +}); From 0d75567c67684db4747a6478a209feec9d847488 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 3 Jan 2025 09:04:40 -0500 Subject: [PATCH 6/8] use sample overlays --- app/packages/looker/src/lookers/abstract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index 3ee1349abd..cb96771208 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -528,7 +528,7 @@ export abstract class AbstractLooker< // we'll transfer that to the worker instead of copying it const arrayBuffers: ArrayBuffer[] = []; - for (const overlay of this.pluckedOverlays ?? []) { + for (const overlay of this.sampleOverlays ?? []) { let overlayData: LabelMask = null; if ( From 2b7297833afb8768b574f59d0411ef76cbf16001 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 3 Jan 2025 09:18:06 -0500 Subject: [PATCH 7/8] empty state overlay array buffer tests --- app/packages/looker/src/lookers/abstract.ts | 48 ++--------------- app/packages/looker/src/lookers/utils.test.ts | 46 +++++++++++++++- app/packages/looker/src/lookers/utils.ts | 52 ++++++++++++++++++- 3 files changed, 99 insertions(+), 47 deletions(-) diff --git a/app/packages/looker/src/lookers/abstract.ts b/app/packages/looker/src/lookers/abstract.ts index cb96771208..2a36514b58 100644 --- a/app/packages/looker/src/lookers/abstract.ts +++ b/app/packages/looker/src/lookers/abstract.ts @@ -23,10 +23,7 @@ import { import { Events } from "../elements/base"; import { COMMON_SHORTCUTS, LookerElement } from "../elements/common"; import { ClassificationsOverlay, loadOverlays } from "../overlays"; -import { CONTAINS, LabelMask, Overlay } from "../overlays/base"; -import DetectionOverlay from "../overlays/detection"; -import HeatmapOverlay from "../overlays/heatmap"; -import SegmentationOverlay from "../overlays/segmentation"; +import { CONTAINS, Overlay } from "../overlays/base"; import processOverlays from "../processOverlays"; import { BaseState, @@ -47,6 +44,7 @@ import { } from "../util"; import { ProcessSample } from "../worker"; import { LookerUtils } from "./shared"; +import { retrieveArrayBuffers } from "./utils"; const LABEL_LISTS_PATH = new Set(withPath(LABELS_PATH, LABEL_LISTS)); const LABEL_LIST_KEY = Object.fromEntries( @@ -524,47 +522,7 @@ export abstract class AbstractLooker< abstract updateOptions(options: Partial): void; updateSample(sample: Sample) { - // collect any mask targets array buffer that overlays might have - // we'll transfer that to the worker instead of copying it - const arrayBuffers: ArrayBuffer[] = []; - - for (const overlay of this.sampleOverlays ?? []) { - let overlayData: LabelMask = null; - - if ( - overlay instanceof DetectionOverlay || - overlay instanceof SegmentationOverlay - ) { - overlayData = overlay.label.mask; - } else if (overlay instanceof HeatmapOverlay) { - overlayData = overlay.label.map; - } - - const buffer = overlayData?.data?.buffer; - - if (!buffer) { - continue; - } - - // check for detached buffer (happens if user is switching colors too fast) - // note: ArrayBuffer.prototype.detached is a new browser API - if (typeof buffer.detached !== "undefined") { - if (buffer.detached) { - // most likely sample is already being processed, skip update - return; - } - - arrayBuffers.push(buffer); - } else if (buffer.byteLength) { - // hope we don't run into this edge case (old browser) - // sometimes detached buffers have bytelength > 0 - // if we run into this case, we'll just attempt to transfer the buffer - // might get a DataCloneError if user is switching colors too fast - arrayBuffers.push(buffer); - } - } - - this.loadSample(sample, arrayBuffers.flat()); + this.loadSample(sample, retrieveArrayBuffers(this.sampleOverlays)); } getSample(): Promise { diff --git a/app/packages/looker/src/lookers/utils.test.ts b/app/packages/looker/src/lookers/utils.test.ts index 6c0d307e6e..711e1ca949 100644 --- a/app/packages/looker/src/lookers/utils.test.ts +++ b/app/packages/looker/src/lookers/utils.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest"; +import { ClassificationsOverlay } from "../overlays"; +import DetectionOverlay from "../overlays/detection"; +import HeatmapOverlay from "../overlays/heatmap"; +import KeypointOverlay from "../overlays/keypoint"; +import PolylineOverlay from "../overlays/polyline"; +import SegmentationOverlay from "../overlays/segmentation"; import type { Buffers } from "../state"; -import { hasFrame } from "./utils"; +import { hasFrame, retrieveArrayBuffers } from "./utils"; describe("looker utilities", () => { it("determines frame availability given a buffer list", () => { @@ -16,4 +22,42 @@ describe("looker utilities", () => { expect(hasFrame(BUFFERS, frameNumber)).toBe(false); } }); + + it("retrieves array buffers without errors", () => { + expect( + retrieveArrayBuffers([new ClassificationsOverlay([])]) + ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([new DetectionOverlay("ground_truth", {})]) + ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([ + new HeatmapOverlay("ground_truth", { id: "", tags: [] }), + ]) + ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([new KeypointOverlay("ground_truth", {})]) + ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([ + new PolylineOverlay("ground_truth", { + id: "", + closed: false, + filled: false, + points: [], + tags: [], + }), + ]) + ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([ + new SegmentationOverlay("ground_truth", { id: "", tags: [] }), + ]) + ).toStrictEqual([]); + }); }); diff --git a/app/packages/looker/src/lookers/utils.ts b/app/packages/looker/src/lookers/utils.ts index ea645401f0..4d56ee520f 100644 --- a/app/packages/looker/src/lookers/utils.ts +++ b/app/packages/looker/src/lookers/utils.ts @@ -1,7 +1,57 @@ -import type { Buffers } from "../state"; +import type { LabelMask, Overlay } from "../overlays/base"; +import DetectionOverlay from "../overlays/detection"; +import HeatmapOverlay from "../overlays/heatmap"; +import SegmentationOverlay from "../overlays/segmentation"; +import type { BaseState, Buffers } from "../state"; export const hasFrame = (buffers: Buffers, frameNumber: number) => { return buffers.some( ([start, end]) => start <= frameNumber && frameNumber <= end ); }; + +export const retrieveArrayBuffers = ( + overlays?: Overlay[] +) => { + // collect any mask targets array buffer that overlays might have + // we'll transfer that to the worker instead of copying it + const arrayBuffers: ArrayBuffer[] = []; + + for (const overlay of overlays ?? []) { + let overlayData: LabelMask = null; + + if ( + overlay instanceof DetectionOverlay || + overlay instanceof SegmentationOverlay + ) { + overlayData = overlay.label.mask; + } else if (overlay instanceof HeatmapOverlay) { + overlayData = overlay.label.map; + } + + const buffer = overlayData?.data?.buffer; + + if (!buffer) { + continue; + } + + // check for detached buffer (happens if user is switching colors too fast) + // note: ArrayBuffer.prototype.detached is a new browser API + if (typeof buffer.detached !== "undefined") { + if (buffer.detached) { + // most likely sample is already being processed, skip update + return []; + } + + arrayBuffers.push(buffer); + } else if (buffer.byteLength) { + // hope we don't run into this edge case (old browser) + // sometimes detached buffers have bytelength > 0 + // if we run into this case, we'll just attempt to transfer the buffer + // might get a DataCloneError if user is switching colors too fast + arrayBuffers.push(buffer); + } + } + + return arrayBuffers; +}; From 0076aa6083ced39ff71a31f73419fc19e83b36ee Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Fri, 3 Jan 2025 09:22:53 -0500 Subject: [PATCH 8/8] temporal detections --- app/packages/looker/src/lookers/utils.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/packages/looker/src/lookers/utils.test.ts b/app/packages/looker/src/lookers/utils.test.ts index 711e1ca949..ed2b9a1100 100644 --- a/app/packages/looker/src/lookers/utils.test.ts +++ b/app/packages/looker/src/lookers/utils.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { ClassificationsOverlay } from "../overlays"; +import { TemporalDetectionOverlay } from "../overlays/classifications"; import DetectionOverlay from "../overlays/detection"; import HeatmapOverlay from "../overlays/heatmap"; import KeypointOverlay from "../overlays/keypoint"; @@ -59,5 +60,9 @@ describe("looker utilities", () => { new SegmentationOverlay("ground_truth", { id: "", tags: [] }), ]) ).toStrictEqual([]); + + expect( + retrieveArrayBuffers([new TemporalDetectionOverlay([])]) + ).toStrictEqual([]); }); });