feat(server): read Bitbucket and GitCafe credentials from a secret provider

apps/server/src/secrets/settingsSecretValue.ts

@@ -0,0 +1,31 @@
+import {
+  isSecretReference,
+  type ServerSettings,
+  type SettingsSecretValue,
+} from "@t3tools/contracts";
+import type * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as Effect from "effect/Effect";
+
+import { collectSourceControlHostSecretReferences } from "../sourceControl/sourceControlHostSecrets.ts";
+
+/** An unreadable secret comes back as `""`, so a credential that is missing behaves as unset. */
+export const resolveSettingsSecret = Effect.fn("SettingsSecrets.resolve")(function* (
+  resolver: SecretResolver.SecretResolver["Service"],
+  value: SettingsSecretValue,
+) {
+  if (!isSecretReference(value)) return value;
+  const [secret] = yield* resolver.read([value]);
+  return secret ?? "";
+});
+
+/** Settings that cannot be read clear every cached secret, so a refresh never keeps a stale one. */
+export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")(function* <E>(
+  resolver: SecretResolver.SecretResolver["Service"],
+  settings: Effect.Effect<Pick<ServerSettings, "sourceControlHosts">, E>,
+) {
+  const references = yield* settings.pipe(
+    Effect.map((current) => collectSourceControlHostSecretReferences(current.sourceControlHosts)),
+    Effect.orElseSucceed(() => undefined),
+  );
+  yield* resolver.invalidate(references);
+});

apps/server/src/server.ts

@@ -64,6 +64,7 @@ import * as SourceControlBuiltInDrivers from "./sourceControl/builtInDrivers.ts"
 import * as SourceControlSignIn from "./sourceControl/SourceControlSignIn.ts";
 import { BUILT_IN_SOURCE_CONTROL_DRIVER_LIST } from "./sourceControl/sourceControlDefinitions.ts";
 import * as TextGeneration from "./textGeneration/TextGeneration.ts";
+import * as SecretResolver from "./secrets/SecretResolver.ts";
 import * as ProviderInstanceRegistryHydration from "./provider/ProviderInstanceRegistryHydration.ts";
 import * as ProjectSettingsOverrideSweep from "./provider/ProjectSettingsOverrideSweep.ts";
 import * as TerminalManager from "./terminal/Manager.ts";
@@ -628,6 +629,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll(
   Layer.provideMerge(layerOrchestrationApplication),
   Layer.provideMerge(RuntimeLayer.layerEventInfrastructure),
   Layer.provideMerge(Layer.merge(ProjectStore.layer, ThreadSearch.layer)),
+  Layer.provideMerge(SecretResolver.layer),
   Layer.provideMerge(layerServerSettings),
   Layer.provideMerge(layerSourceControlProviderRegistry),
   Layer.provideMerge(layerGit),

apps/server/src/serverSettings.test.ts

@@ -1533,6 +1533,25 @@ it.layer(NodeServices.layer)("server settings", (it) => {
         assert.isTrue(Option.isNone(yield* stored("bitbucket", "accessToken", null)));
         assert.isTrue(Option.isNone(yield* stored("github", "tokens", "github.com")));
         assert.isTrue(Option.isSome(yield* stored("github", "tokens", "ghe.acme.test")));
+
+        const reference = {
+          provider: SecretProviderId.make("personal"),
+          reference: "op://Private/Bitbucket/token",
+        };
+        yield* serverSettings.updateSettings({
+          sourceControlHosts: { bitbucket: { accessToken: "bb-access" } },
+        });
+        assert.isTrue(Option.isSome(yield* stored("bitbucket", "accessToken", null)));
+        const referenced = yield* serverSettings.updateSettings({
+          sourceControlHosts: { bitbucket: { accessToken: reference } },
+        });
+        assert.deepEqual(referenced.sourceControlHosts.bitbucket?.accessToken, reference);
+        assert.isTrue(Option.isNone(yield* stored("bitbucket", "accessToken", null)));
+        assert.deepEqual(
+          ServerSettingsModule.redactServerSettingsForClient(referenced).sourceControlHosts
+            .bitbucket?.accessToken,
+          reference,
+        );
       }).pipe(Effect.provide(layerServerSettingsWithSecrets())),
   );
 

apps/server/src/settingsSecrets.ts

@@ -6,6 +6,7 @@ import {
   type ServerSettings,
   SimlockDeviceHostConfig,
   UsageLimitSourceConfig,
+  isSecretReference,
   secretSettingsFields,
 } from "@t3tools/contracts";
 import { acpRegistryClient } from "@t3tools/provider-acp-registry/client";
@@ -60,7 +61,7 @@ export function mapSettingsSecrets(
   ) => {
     const next = { ...fields };
     for (const field of secretFields) {
-      if (!(field in next)) continue;
+      if (!(field in next) || isSecretReference(next[field])) continue;
       next[field] = writeSecretFieldValues(
         next[field],
         readSecretFieldValues(next[field]).map(([serverHost, value]) => {

apps/server/src/sourceControl/ServerSourceControlHost.ts

@@ -1,11 +1,15 @@
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
 import * as SourceControlHost from "@t3tools/source-control-core/server/SourceControlHost";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 
+import * as SecretResolverLayer from "../secrets/SecretResolver.ts";
+import { resolveSettingsSecret } from "../secrets/settingsSecretValue.ts";
 import * as ServerSettings from "../serverSettings.ts";
 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
 import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
 import * as VcsProcess from "../vcs/VcsProcess.ts";
+import { resolveSourceControlHostSecrets } from "./sourceControlHostSecrets.ts";
 
 export const layer = Layer.effect(
   SourceControlHost.SourceControlHost,
@@ -14,8 +18,26 @@ export const layer = Layer.effect(
     const process = yield* VcsProcess.VcsProcess;
     const git = yield* GitVcsDriver.GitVcsDriver;
     const vcsRegistry = yield* VcsDriverRegistry.VcsDriverRegistry;
+    const secrets = yield* SecretResolver.SecretResolver;
     return SourceControlHost.SourceControlHost.of({
-      settings: { get: serverSettings.getSettings, update: serverSettings.updateSettings },
+      settings: {
+        get: serverSettings.getSettings,
+        getWithSecrets: (kind) =>
+          serverSettings.getSettings.pipe(
+            Effect.flatMap((settings) =>
+              resolveSourceControlHostSecrets(settings.sourceControlHosts, kind, (source) =>
+                resolveSettingsSecret(secrets, source),
+              ).pipe(
+                Effect.map((sourceControlHosts) =>
+                  sourceControlHosts === settings.sourceControlHosts
+                    ? settings
+                    : { ...settings, sourceControlHosts },
+                ),
+              ),
+            ),
+          ),
+        update: serverSettings.updateSettings,
+      },
       process: { run: process.run },
       git: {
         execute: git.execute,
@@ -35,4 +57,4 @@ export const layer = Layer.effect(
       },
     });
   }),
-);
+).pipe(Layer.provide(SecretResolverLayer.layer));

apps/server/src/sourceControl/sourceControlHostSecrets.test.ts

@@ -0,0 +1,134 @@
+import { assert, it } from "@effect/vitest";
+import { SecretProviderId } from "@t3tools/contracts";
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+
+import {
+  invalidateSettingsSecrets,
+  resolveSettingsSecret,
+} from "../secrets/settingsSecretValue.ts";
+import {
+  collectSourceControlHostSecretReferences,
+  resolveSourceControlHostSecrets,
+} from "./sourceControlHostSecrets.ts";
+
+const source = (reference: string) => ({
+  provider: SecretProviderId.make("homelab"),
+  reference,
+});
+
+const secretsByReference = new Map([["bitbucket#token", "bb_from_provider"]]);
+
+const layerResolver = Layer.succeed(SecretResolver.SecretResolver, {
+  read: (references) =>
+    Effect.succeed(references.map((reference) => secretsByReference.get(reference.reference))),
+  invalidate: () => Effect.void,
+});
+
+const fakeResolver = SecretResolver.SecretResolver.use((resolver) => Effect.succeed(resolver));
+
+const hosts = {
+  bitbucket: { email: "me@example.test", accessToken: source("bitbucket#token") },
+  gitcafe: { token: source("missing#token") },
+  github: { tokens: { "github.com": "ghp_literal" } },
+};
+
+const countingResolver = Effect.gen(function* () {
+  const resolver = yield* fakeResolver;
+  const reads: Array<string> = [];
+  const resolve = (value: Parameters<typeof resolveSettingsSecret>[1]) =>
+    Effect.sync(() => {
+      if (typeof value !== "string") reads.push(value.reference);
+    }).pipe(Effect.andThen(resolveSettingsSecret(resolver, value)));
+  return { reads, resolve };
+});
+
+it.effect(
+  "replaces only the requested kind's credential references with the secrets they name",
+  () =>
+    Effect.gen(function* () {
+      const { reads, resolve } = yield* countingResolver;
+      const resolved = yield* resolveSourceControlHostSecrets(hosts, "bitbucket", resolve);
+      assert.deepEqual(resolved, {
+        ...hosts,
+        bitbucket: { email: "me@example.test", accessToken: "bb_from_provider" },
+      });
+      assert.deepEqual(reads, ["bitbucket#token"]);
+    }).pipe(Effect.provide(layerResolver)),
+);
+
+it.effect("treats a reference the provider cannot read as an unset credential", () =>
+  Effect.gen(function* () {
+    const { reads, resolve } = yield* countingResolver;
+    const resolved = yield* resolveSourceControlHostSecrets(hosts, "gitcafe", resolve);
+    assert.deepEqual(resolved.gitcafe, { token: "" });
+    assert.deepEqual(reads, ["missing#token"]);
+  }).pipe(Effect.provide(layerResolver)),
+);
+
+it.effect("leaves the resolver unused for a kind that holds no reference", () =>
+  Effect.gen(function* () {
+    const { reads, resolve } = yield* countingResolver;
+    const resolved = yield* resolveSourceControlHostSecrets(hosts, "github", resolve);
+    assert.strictEqual(resolved, hosts);
+    assert.deepEqual(reads, []);
+  }).pipe(Effect.provide(layerResolver)),
+);
+
+it("collects the references held in credential fields and skips literal values", () => {
+  assert.deepEqual(collectSourceControlHostSecretReferences(hosts), [
+    source("missing#token"),
+    source("bitbucket#token"),
+  ]);
+  assert.deepEqual(
+    collectSourceControlHostSecretReferences({
+      bitbucket: { accessToken: "literal" },
+      github: { tokens: { "github.com": "ghp_literal" } },
+    }),
+    [],
+  );
+});
+
+it.effect("reads a collected reference again once it is invalidated", () =>
+  Effect.gen(function* () {
+    let current = "stale";
+    const cache = new Map<string, string>();
+    const resolver = SecretResolver.SecretResolver.of({
+      read: (references) =>
+        Effect.sync(() =>
+          references.map((reference) => {
+            const hit = cache.get(reference.reference);
+            if (hit !== undefined) return hit;
+            cache.set(reference.reference, current);
+            return current;
+          }),
+        ),
+      invalidate: (references) =>
+        Effect.sync(() => {
+          if (references === undefined) cache.clear();
+          else for (const reference of references) cache.delete(reference.reference);
+        }),
+    });
+    const read = () =>
+      resolveSourceControlHostSecrets(hosts, "bitbucket", (reference) =>
+        resolveSettingsSecret(resolver, reference),
+      );
+    assert.deepEqual((yield* read()).bitbucket, {
+      email: "me@example.test",
+      accessToken: "stale",
+    });
+    current = "rotated";
+    yield* invalidateSettingsSecrets(resolver, Effect.succeed({ sourceControlHosts: hosts }));
+    assert.deepEqual((yield* read()).bitbucket, {
+      email: "me@example.test",
+      accessToken: "rotated",
+    });
+    current = "rotated again";
+    yield* invalidateSettingsSecrets(resolver, Effect.fail("settings unreadable"));
+    assert.deepEqual((yield* read()).bitbucket, {
+      email: "me@example.test",
+      accessToken: "rotated again",
+    });
+  }),
+);

apps/server/src/sourceControl/sourceControlHostSecrets.ts

@@ -1,4 +1,5 @@
-import { secretSettingsFields } from "@t3tools/contracts";
+import { isSecretReference, secretSettingsFields, type SecretReference } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
 
 import { sourceControlDefinitions } from "./sourceControlDefinitions.ts";
 
@@ -30,6 +31,7 @@ export function readSecretFieldValues(
   value: unknown,
 ): ReadonlyArray<readonly [serverHost: string | null, value: string]> {
   if (typeof value === "string") return [[null, value]];
+  if (isSecretReference(value)) return [];
   if (value === null || typeof value !== "object" || Array.isArray(value)) return [];
   return Object.entries(value).flatMap(([serverHost, entry]) =>
     typeof entry === "string" ? [[serverHost.trim().toLowerCase(), entry] as const] : [],
@@ -41,5 +43,40 @@ export function writeSecretFieldValues(
   values: ReadonlyArray<readonly [serverHost: string | null, value: string]>,
 ): unknown {
   if (typeof original === "string") return values[0]?.[1] ?? "";
+  if (isSecretReference(original)) return original;
   return Object.fromEntries(values.map(([serverHost, value]) => [serverHost ?? "", value]));
 }
+
+type SourceControlHostsSettings = Readonly<Record<string, Readonly<Record<string, unknown>>>>;
+
+/** Every secret provider reference held in any host kind's credential fields. */
+export function collectSourceControlHostSecretReferences(
+  hosts: SourceControlHostsSettings,
+): ReadonlyArray<SecretReference> {
+  return [...SOURCE_CONTROL_HOST_SECRET_FIELDS].flatMap(([kind, fields]) =>
+    fields.flatMap((field) => {
+      const value = hosts[kind]?.[field];
+      return isSecretReference(value) ? [value] : [];
+    }),
+  );
+}
+
+/** Replaces each secret provider reference in one host kind's credential fields with the secret it names. */
+export const resolveSourceControlHostSecrets = Effect.fn("SourceControlHost.resolveSecrets")(
+  function* <R>(
+    hosts: SourceControlHostsSettings,
+    kind: string,
+    resolve: (source: SecretReference) => Effect.Effect<string, never, R>,
+  ) {
+    const fields = hosts[kind] ?? {};
+    const pending = (SOURCE_CONTROL_HOST_SECRET_FIELDS.get(kind) ?? []).flatMap((field) => {
+      const value = fields[field];
+      return isSecretReference(value) ? [{ field, source: value }] : [];
+    });
+    if (pending.length === 0) return hosts;
+    const resolved = yield* Effect.forEach(pending, ({ field, source }) =>
+      resolve(source).pipe(Effect.map((value) => [field, value] as const)),
+    );
+    return { ...hosts, [kind]: { ...fields, ...Object.fromEntries(resolved) } };
+  },
+);

apps/server/src/ws.test.ts

@@ -1,3 +1,4 @@
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { assert, it } from "@effect/vitest";
 import {
@@ -528,6 +529,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll(
   Layer.mock(ReviewService.ReviewService)({}),
   Layer.mock(ScheduledTasks.ScheduledTaskService)({}),
   Layer.mock(SecretRequests.SecretRequests)({}),
+  Layer.mock(SecretResolver.SecretResolver)({}),
   Layer.mock(GuestShareService.GuestShareService)({}),
   Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({}),
   Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({}),

apps/server/src/ws.ts

@@ -180,7 +180,9 @@ import { makeProviderInstallation } from "./provider/providerInstallation.ts";
 import * as ServerSelfUpdate from "./cloud/selfUpdate.ts";
 import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts";
 import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
 import * as ServerSettings from "./serverSettings.ts";
+import { invalidateSettingsSecrets } from "./secrets/settingsSecretValue.ts";
 import * as DatabaseStorage from "./persistence/DatabaseStorage.ts";
 import * as StorageCleanup from "./storageCleanup.ts";
 import * as TerminalManager from "./terminal/Manager.ts";
@@ -1335,6 +1337,7 @@ export const layerWsRpc = (
       const storageCleanup = yield* StorageCleanup.StorageCleanup;
       const databaseStorage = yield* DatabaseStorage.DatabaseStorage;
       const serverSettings = yield* ServerSettings.ServerSettingsService;
+      const secretResolver = yield* SecretResolver.SecretResolver;
       const peerLinks = yield* PeerLinks.PeerLinks;
       const peerLinkRequests = yield* PeerLinkRequests.PeerLinkRequests;
       const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
@@ -2309,6 +2312,9 @@ export const layerWsRpc = (
             if (input.instanceId === undefined) {
               yield* usageLimitSources.refresh;
             }
+            if (input.reloadSecrets === true && input.instanceId === undefined) {
+              yield* invalidateSettingsSecrets(secretResolver, serverSettings.getSettings);
+            }
             let providers = yield* input.cwd !== undefined && input.instanceId !== undefined
               ? providerRegistry.refreshWorkspaceSnapshot({
                   instanceId: input.instanceId,

apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx

@@ -1930,13 +1930,13 @@ it("reloads a workspace Word file after a preview remount", async () => {
 
 it("clears an Office metadata error when the same URL becomes previewable", async () => {
   const src = "data:application/octet-stream,invalid-office-file";
-  let resolveResponse!: (response: Response) => void;
-  const response = new Promise<Response>((resolve) => {
-    resolveResponse = resolve;
+  let resolveBody!: (body: string) => void;
+  const body = new Promise<string>((resolve) => {
+    resolveBody = resolve;
   });
   using _fetch = stubGlobal(
     "fetch",
-    vi.fn(() => response),
+    vi.fn(() => body.then((text) => new Response(text))),
   );
   await using screen = await render(
     <OfficeFilePreview src={src} name="report.docx" kind="docx" sizeBytes={51 * 1024 * 1024} />,
@@ -1949,7 +1949,7 @@ it("clears an Office metadata error when the same URL becomes previewable", asyn
     expect(screen.container.querySelector('[aria-label="Loading file"]')).not.toBeNull(),
   );
   expect(screen.container.textContent).not.toContain("under 50 MB");
-  resolveResponse(new Response("invalid-office-file"));
+  resolveBody("invalid-office-file");
   await vi.waitFor(() => expect(screen.container.textContent).toContain("not a readable ZIP"), {
     timeout: 10_000,
   });