RFC: optional local AI agent chat sidebar (pi-based) with data query, playback and layout tools

packages/suite-base/jest.config.json

@@ -2,7 +2,7 @@
   "//": "Note: we use babel-jest rather than ts-jest for performance reasons.",
   "testMatch": ["<rootDir>/src/**/*.test.ts(x)?"],
   "transform": {
-    "\\.jsx?$": ["babel-jest", { "rootMode": "upward" }],
+    "\\.m?jsx?$": ["babel-jest", { "rootMode": "upward" }],
     "\\.tsx?$": "<rootDir>/src/test/transformers/typescriptTransformerWithRawImports.js",
     "\\.ne$": "<rootDir>/src/test/transformers/neTransformer.js",
     "\\.(bin|template|wasm)$": "<rootDir>/src/test/transformers/rawTransformer.js"
@@ -23,7 +23,10 @@
     "\\.css$": "<rootDir>/src/test/mocks/MockCss.ts",
     "react-monaco-editor": "<rootDir>/src/test/stubs/MonacoEditor.tsx",
     "\\.(glb|md|png)$": "<rootDir>/src/test/mocks/fileMock.ts",
-    "@lichtblick/suite-base/(.*)": "<rootDir>/src/$1"
+    "@lichtblick/suite-base/(.*)": "<rootDir>/src/$1",
+    "^@earendil-works/pi-agent-core$": "<rootDir>/../../node_modules/@earendil-works/pi-agent-core/dist/index.js",
+    "^@earendil-works/pi-ai$": "<rootDir>/../../node_modules/@earendil-works/pi-ai/dist/index.js",
+    "^@earendil-works/pi-ai/(.*)$": "<rootDir>/../../node_modules/@earendil-works/pi-ai/dist/$1.js"
   },
   "//": "Native find is slow because it does not exclude files: https://github.com/facebook/jest/pull/11264#issuecomment-825377579",
   "haste": { "forceNodeFilesystemAPI": true }

packages/suite-base/package.json

@@ -191,5 +191,10 @@
     "vm-browserify": "1.1.2",
     "webpack": "5.108.3",
     "zustand": "4.5.7"
+  },
+  "dependencies": {
+    "@earendil-works/pi-agent-core": "0.83.0",
+    "@earendil-works/pi-ai": "0.83.0",
+    "remark-gfm": "4.0.1"
   }
 }

packages/suite-base/src/AppSetting.ts

@@ -7,6 +7,18 @@
 
 export enum AppSetting {
   // General
+  // Agent (local pi runtime). The desktop secure credential store additionally protects the
+  // provider API keys under agent.llmApiKey / agent.profile.*.llmApiKey (never AppConfiguration).
+  AGENT_ENABLED = "agent.enabled",
+  AGENT_LLM_PROVIDER = "agent.llmProvider",
+  AGENT_ANTHROPIC_BASE_URL = "agent.anthropic.baseUrl",
+  AGENT_ANTHROPIC_MODEL = "agent.anthropic.model",
+  AGENT_OPENAI_BASE_URL = "agent.openai.baseUrl",
+  AGENT_OPENAI_MODEL = "agent.openai.model",
+  /** JSON-serialized agent long-term memory. Not a secret; see services/agent/memory. */
+  AGENT_MEMORY = "agent.memory",
+  /** JSON-serialized user edits to the agent's instructions and skills; see services/agent/prompts. */
+  AGENT_PROMPT_CUSTOMIZATION = "agent.promptCustomization",
   COLOR_SCHEME = "colorScheme",
   TIMEZONE = "timezone",
   TIME_FORMAT = "time.format",

packages/suite-base/src/Workspace.agent.test.tsx

@@ -0,0 +1,215 @@
+/** @jest-environment jsdom */
+
+// SPDX-FileCopyrightText: Copyright (C) 2023-2026 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)<lichtblick@bmwgroup.com>
+// SPDX-License-Identifier: MPL-2.0
+
+import { act, renderHook } from "@testing-library/react";
+import { StrictMode, useCallback, useLayoutEffect, useMemo, useRef } from "react";
+
+import { AgentConfiguration } from "@lichtblick/suite-base/services/agent/agentSettings";
+import { useLocalAgentClient } from "@lichtblick/suite-base/services/agent/localAgentClient";
+import { PiAgentOrchestrator } from "@lichtblick/suite-base/services/agent/pi/PiAgentOrchestrator";
+import type { AgentDataQueryContext } from "@lichtblick/suite-base/services/agent/tools/toolRuntime";
+
+const validConfiguration: AgentConfiguration = {
+  apiKey: "test-api-key",
+  baseUrl: "",
+  desktop: false,
+  model: "claude-test",
+  provider: "anthropic",
+};
+
+beforeAll(() => {
+  Object.defineProperty(globalThis, "fetch", {
+    configurable: true,
+    value: jest.fn(),
+    writable: true,
+  });
+});
+
+describe("local Agent client lifecycle", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("uses the latest catalog without rebuilding the client", () => {
+    const { result, rerender } = renderHook(
+      ({ catalogVersion }: { catalogVersion: number }) => {
+        const catalogRef = useRef(catalogVersion);
+        useLayoutEffect(() => {
+          catalogRef.current = catalogVersion;
+        });
+        const getCatalog = useCallback(
+          () => ({
+            datatypes: new Map(),
+            topics: [{ name: `/catalog/${catalogRef.current}`, schemaName: "test" }],
+          }),
+          [],
+        );
+        const client = useLocalAgentClient(validConfiguration, {
+          enabled: true,
+          getCatalog,
+        });
+        return { client, getCatalog };
+      },
+      { initialProps: { catalogVersion: 1 } },
+    );
+    const initialClient = result.current.client;
+    expect(initialClient).toBeInstanceOf(PiAgentOrchestrator);
+
+    rerender({ catalogVersion: 2 });
+
+    expect(result.current.client).toBe(initialClient);
+    expect(result.current.getCatalog().topics[0]).toEqual({
+      name: "/catalog/2",
+      schemaName: "test",
+    });
+  });
+
+  it("disposes replaced, disabled, and unmounted orchestrators", async () => {
+    const dispose = jest.spyOn(PiAgentOrchestrator.prototype, "dispose");
+    const { result, rerender, unmount } = renderHook(
+      ({ enabled, model }: { enabled: boolean; model: string }) =>
+        useLocalAgentClient(
+          { ...validConfiguration, model },
+          {
+            enabled,
+            getCatalog: () => ({ datatypes: new Map(), topics: [] }),
+          },
+        ),
+      { initialProps: { enabled: true, model: "model-1" } },
+    );
+    const firstClient = result.current;
+
+    rerender({ enabled: true, model: "model-2" });
+    expect(result.current).not.toBe(firstClient);
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(1);
+    expect(dispose.mock.instances[0]).toBe(firstClient);
+
+    const secondClient = result.current;
+    rerender({ enabled: false, model: "model-2" });
+    expect(result.current).toBeUndefined();
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(2);
+    expect(dispose.mock.instances[1]).toBe(secondClient);
+
+    rerender({ enabled: true, model: "model-3" });
+    const thirdClient = result.current;
+    unmount();
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(3);
+    expect(dispose.mock.instances[2]).toBe(thirdClient);
+  });
+
+  it("replaces the orchestrator when the selected profile changes", async () => {
+    const dispose = jest.spyOn(PiAgentOrchestrator.prototype, "dispose");
+    const { result, rerender } = renderHook(
+      ({ profileId }: { profileId: string }) =>
+        useLocalAgentClient(validConfiguration, {
+          enabled: true,
+          getCatalog: () => ({ datatypes: new Map(), topics: [] }),
+          profileId,
+        }),
+      { initialProps: { profileId: "profile-1" } },
+    );
+    const firstClient = result.current;
+
+    rerender({ profileId: "profile-2" });
+
+    expect(result.current).not.toBe(firstClient);
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(1);
+    expect(dispose.mock.instances[0]).toBe(firstClient);
+  });
+
+  it("disposes both committed StrictMode instances without leaking either one", async () => {
+    const dispose = jest.spyOn(PiAgentOrchestrator.prototype, "dispose");
+    const wrapper = ({ children }: React.PropsWithChildren) => <StrictMode>{children}</StrictMode>;
+    const { unmount } = renderHook(
+      () =>
+        useLocalAgentClient(validConfiguration, {
+          enabled: true,
+          getCatalog: () => ({ datatypes: new Map(), topics: [] }),
+        }),
+      { wrapper },
+    );
+
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(1);
+
+    unmount();
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(dispose).toHaveBeenCalledTimes(2);
+  });
+
+  it("keeps the client stable while the data source loads (catalog + player changes)", async () => {
+    // Simulates the Workspace wiring during a data-source load: the catalog content changes
+    // (open_data_source → catalog-ready) and the player state changes, but every dependency of
+    // useLocalAgentClient is either stable (dataQuery via useMemo, getters via useLatest*)
+    // or independent of the data source (agent configuration, profile). The client must neither
+    // be rebuilt nor released — that is the window that previously wiped the conversation.
+    type Topic = { name: string; schemaName: string };
+    const { result, rerender } = renderHook<PiAgentOrchestrator | undefined, { topics: Topic[] }>(
+      ({ topics }: { topics: Topic[] }) => {
+        const topicsRef = useRef(topics);
+        useLayoutEffect(() => {
+          topicsRef.current = topics;
+        });
+        const getCatalog = useCallback(
+          () => ({ datatypes: new Map(), topics: topicsRef.current }),
+          [],
+        );
+        const dataQuery = useMemo(() => ({ getContext: () => ({}) as AgentDataQueryContext }), []);
+        return useLocalAgentClient(validConfiguration, {
+          enabled: true,
+          getCatalog,
+          dataQuery,
+        });
+      },
+      { initialProps: { topics: [] } },
+    );
+    await act(async () => {
+      await Promise.resolve();
+    });
+    expect(result.current).toBeDefined();
+    const firstClient = result.current;
+
+    // Data source loads: empty catalog → topics appear → more topics arrive.
+    rerender({ topics: [{ name: "/imu", schemaName: "sensor_msgs/Imu" }] });
+    expect(result.current).toBe(firstClient);
+    rerender({
+      topics: [
+        { name: "/imu", schemaName: "sensor_msgs/Imu" },
+        { name: "/gps", schemaName: "sensor_msgs/NavSatFix" },
+      ],
+    });
+    expect(result.current).toBe(firstClient);
+  });
+
+  it("does not construct a client for invalid settings", () => {
+    const { result } = renderHook(() =>
+      useLocalAgentClient(
+        { ...validConfiguration, apiKey: "" },
+        {
+          enabled: true,
+          getCatalog: () => ({ datatypes: new Map(), topics: [] }),
+        },
+      ),
+    );
+
+    expect(result.current).toBeUndefined();
+  });
+});

packages/suite-base/src/Workspace.test.tsx

@@ -6,6 +6,7 @@
 import "@testing-library/jest-dom";
 import { render, waitFor } from "@testing-library/react";
 
+import { AppSetting } from "@lichtblick/suite-base/AppSetting";
 import {
   useMessagePipeline,
   useMessagePipelineGetter,
@@ -60,6 +61,9 @@ jest.mock("@lichtblick/suite-base/api/mcapBundle/McapBundleAPI", () => ({
 }));
 
 // ── components (rendered as null — Sidebars is the exception below) ────────────
+jest.mock("@lichtblick/suite-base/components/AgentWorkspaceIntegration", () => ({
+  AgentWorkspaceIntegration: ({ children }: React.PropsWithChildren) => <>{children}</>,
+}));
 jest.mock("@lichtblick/suite-base/components/Sidebars", () => ({
   __esModule: true,
   default: jest.fn(() => undefined),
@@ -708,3 +712,86 @@ describe("Workspace - fetchLayoutFromUrl", () => {
     });
   });
 });
+
+describe("Workspace - Agent sidebar wiring", () => {
+  const installMocks = () => {
+    (useMessagePipeline as jest.Mock).mockImplementation(
+      (selector: (ctx: typeof mockPipelineContext) => unknown) => selector(mockPipelineContext),
+    );
+    (useMessagePipelineGetter as jest.Mock).mockReturnValue(() => mockPipelineContext);
+    (useWorkspaceStore as jest.Mock).mockImplementation(
+      (selector: (store: typeof mockWorkspaceStore) => unknown) => selector(mockWorkspaceStore),
+    );
+    (useWorkspaceActions as jest.Mock).mockReturnValue(mockWorkspaceActions);
+    (usePlayerSelection as jest.Mock).mockReturnValue({
+      availableSources: [],
+      selectSource: jest.fn(),
+    });
+    (useAlertCount as jest.Mock).mockReturnValue({
+      playerAlerts: [],
+      sessionAlerts: [],
+      alertCount: 0,
+    });
+    (useHandleFiles as jest.Mock).mockReturnValue({ handleFiles: jest.fn() });
+    (useCurrentUser as jest.Mock).mockReturnValue({ currentUser: undefined, signIn: undefined });
+    (useCurrentUserType as jest.Mock).mockReturnValue("unauthenticated");
+    (useEvents as jest.Mock).mockImplementation(
+      (selector: (store: { eventsSupported: boolean; selectEvent: jest.Mock }) => unknown) =>
+        selector({ eventsSupported: false, selectEvent: jest.fn() }),
+    );
+    (useAppContext as jest.Mock).mockReturnValue({
+      PerformanceSidebarComponent: undefined,
+      sidebarItems: [],
+      layoutBrowser: undefined,
+      workspaceStoreCreator: undefined,
+    });
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockWorkspaceStore.sidebars.right.item = undefined;
+    mockWorkspaceStore.sidebars.right.open = false;
+    installMocks();
+  });
+
+  it("registers the agent-chat right sidebar item only while the agent is enabled", () => {
+    (useAppConfigurationValue as jest.Mock).mockImplementation((key: string) => [
+      key === AppSetting.AGENT_ENABLED,
+    ]);
+
+    const root = render(<Workspace />);
+    let rightItems = MockedSidebars.mock.lastCall?.[0]?.rightItems as Map<string, SidebarItem>;
+    expect(rightItems.get("agent-chat")).toBeDefined();
+    expect(rightItems.get("agent-chat")?.title).toBe("workspace:agentChat");
+
+    // Disabling the agent removes the item from the right sidebar.
+    (useAppConfigurationValue as jest.Mock).mockImplementation(() => [false]);
+    root.rerender(<Workspace />);
+    rightItems = MockedSidebars.mock.lastCall?.[0]?.rightItems as Map<string, SidebarItem>;
+    expect(rightItems.get("agent-chat")).toBeUndefined();
+  });
+
+  it("normalizes the right sidebar selection to variables when the agent is disabled while agent-chat is open", () => {
+    mockWorkspaceStore.sidebars.right.item = "agent-chat" as never;
+    mockWorkspaceStore.sidebars.right.open = true;
+    (useAppConfigurationValue as jest.Mock).mockImplementation(() => [false]);
+
+    render(<Workspace />);
+
+    expect(mockWorkspaceActions.sidebarActions.right.selectItem).toHaveBeenCalledWith("variables");
+    // The sidebar stays open: only the stale item is normalized.
+    expect(mockWorkspaceActions.sidebarActions.right.setOpen).not.toHaveBeenCalled();
+  });
+
+  it("preserves the closed state when normalizing a stale agent-chat selection", () => {
+    mockWorkspaceStore.sidebars.right.item = "agent-chat" as never;
+    mockWorkspaceStore.sidebars.right.open = false;
+    (useAppConfigurationValue as jest.Mock).mockImplementation(() => [false]);
+
+    render(<Workspace />);
+
+    expect(mockWorkspaceActions.sidebarActions.right.selectItem).toHaveBeenCalledWith("variables");
+    // Selecting a new item would open the sidebar; the previous closed state is restored.
+    expect(mockWorkspaceActions.sidebarActions.right.setOpen).toHaveBeenCalledWith(false);
+  });
+});