diff --git a/.changeset/social-graph-atlas-0419.md b/.changeset/social-graph-atlas-0419.md new file mode 100644 index 000000000..7b86ff5ef --- /dev/null +++ b/.changeset/social-graph-atlas-0419.md @@ -0,0 +1,22 @@ +--- +'@xnetjs/sqlite': minor +'@xnetjs/react': minor +--- + +Imported social content is now actually in the full-text index, and a saved +lens can be projected onto a canvas with its relationships. + +`extractSearchableContent` read `content`, `description`, `body`, `name` and +`note` — but not `searchText`, the property imported social records +denormalize their full text into precisely so it can be searched. Every +imported post, comment and video transcript was therefore absent from +`nodes_fts` while the pipeline reported it as indexed: search returned a clean +empty result rather than an error. `searchText` is now indexed, with +`textPreview` as the fallback for records that carry no full text. **Existing +databases need `rebuildFTS()` to pick up already-imported rows** — new writes +are indexed correctly from here on. + +`SavedViewVisualCanvasProjectionRequest` gains an `edges` array (filtered to +relationships whose endpoints both survived the projection cap), and the new +`SavedViewCanvasProjectionEdge` type is exported. This is what lets a consumer +lay a saved view out as a graph rather than an unconnected grid of cards. diff --git a/apps/electron/src/renderer/components/CanvasView.tsx b/apps/electron/src/renderer/components/CanvasView.tsx index c373c4b83..6ef62077a 100644 --- a/apps/electron/src/renderer/components/CanvasView.tsx +++ b/apps/electron/src/renderer/components/CanvasView.tsx @@ -11,6 +11,7 @@ import type { Rect, ShapeType } from '@xnetjs/canvas' +import type { SocialCanvasProjectionPlan } from '@xnetjs/social/projection' import { Canvas, getCanvasObjectsMap, @@ -132,6 +133,7 @@ export type CanvasViewHandle = { createMindMap: () => boolean createPlanningTemplate: (templateId: CanvasPlanningTemplateId) => boolean createQueryFrameFromSavedView: (input: SavedViewCanvasQueryFrameInput) => boolean + applySocialCanvasProjection: (plan: SocialCanvasProjectionPlan) => boolean refreshSelectedQueryFrame: () => boolean createExternalReference: (url?: string) => boolean createMediaFile: () => boolean @@ -273,7 +275,8 @@ export const CanvasView = forwardRef(function selectedQueryFrameNode, selectedQueryFrameDefinition, createQueryFrameFromSavedView, - refreshSelectedQueryFrame + refreshSelectedQueryFrame, + applySocialCanvasProjection } = useCanvasQueryFrames({ doc, sceneRevision, @@ -282,6 +285,13 @@ export const CanvasView = forwardRef(function onUndoBoundary: recordSceneUndoBoundary }) + // The imperative handle reports success as a boolean; the hook returns the + // richer result for callers that want the placed/omitted counts. + const applySocialCanvasProjectionHandle = useCallback( + (plan: SocialCanvasProjectionPlan): boolean => Boolean(applySocialCanvasProjection(plan)), + [applySocialCanvasProjection] + ) + const selectedDatabaseSourceId = selectedCanvasObject?.displayType === 'database' ? (selectedCanvasObject.sourceId ?? '') : '' const undoLadder = useCanvasUndoLadder({ @@ -784,6 +794,7 @@ export const CanvasView = forwardRef(function createMindMap, createPlanningTemplate, createQueryFrameFromSavedView, + applySocialCanvasProjection: applySocialCanvasProjectionHandle, refreshSelectedQueryFrame, createExternalReference, createMediaFile, @@ -804,6 +815,7 @@ export const CanvasView = forwardRef(function createMindMap, createPlanningTemplate, createQueryFrameFromSavedView, + applySocialCanvasProjectionHandle, createMediaFile, createShape, connectSelection, diff --git a/apps/electron/src/renderer/components/DataWorkspaceView.tsx b/apps/electron/src/renderer/components/DataWorkspaceView.tsx index 59de23f66..27a3727ab 100644 --- a/apps/electron/src/renderer/components/DataWorkspaceView.tsx +++ b/apps/electron/src/renderer/components/DataWorkspaceView.tsx @@ -5,9 +5,14 @@ * and inserting saved lenses onto the canvas as frames. */ import { upsertSocialImportJobProgress } from '@xnetjs/social/import/core' -import { useDataWorkspace, DataWorkspaceBody, type SavedViewCanvasFrameInput } from '@xnetjs/views' +import { + useDataWorkspace, + useSocialFeedEnrichment, + DataWorkspaceBody, + type SavedViewCanvasFrameInput +} from '@xnetjs/views' import { Database, Import, Loader2, X } from 'lucide-react' -import React, { useEffect } from 'react' +import React, { useEffect, useMemo } from 'react' export type { SavedViewCanvasFrameInput } @@ -25,6 +30,8 @@ export function DataWorkspaceView({ onInsertSavedLensAsCanvasFrame }) const { seeding, handleSeedWorkspace, refreshSocialImportJobs } = workspace + const feedEnrichment = useSocialFeedEnrichment() + const savedViewRunnerProps = useMemo(() => ({ feedEnrichment }), [feedEnrichment]) // Bridge main-process commit jobs into the renderer's import-job store so // progress started from the main process shows up in the shared panel. @@ -84,7 +91,7 @@ export function DataWorkspaceView({

- + diff --git a/apps/electron/src/renderer/shell/use-document-shell.ts b/apps/electron/src/renderer/shell/use-document-shell.ts index aa7cff299..983ad54ef 100644 --- a/apps/electron/src/renderer/shell/use-document-shell.ts +++ b/apps/electron/src/renderer/shell/use-document-shell.ts @@ -385,15 +385,18 @@ export function useDocumentShell(): DocumentShell { const handleInsertSavedLensAsCanvasFrame = useCallback( (view: SavedViewCanvasFrameInput) => { - const inserted = - canvasViewRef.current?.createQueryFrameFromSavedView({ - viewId: view.id, - title: view.title ?? 'Saved lens', - descriptorJson: view.descriptor ?? null - }) ?? false + // A projection places the cards and the connections between them; a + // plain lens becomes a live query frame instead (0419). + const inserted = view.projection + ? (canvasViewRef.current?.applySocialCanvasProjection(view.projection) ?? false) + : (canvasViewRef.current?.createQueryFrameFromSavedView({ + viewId: view.id, + title: view.title ?? 'Saved lens', + descriptorJson: view.descriptor ?? null + }) ?? false) if (!inserted) { - console.error('Failed to insert saved lens as a canvas query frame', view.id) + console.error('Failed to place saved lens on the canvas', view.id) return } diff --git a/apps/web/src/components/CanvasView.tsx b/apps/web/src/components/CanvasView.tsx index ee6ae3aab..7a3520d3f 100644 --- a/apps/web/src/components/CanvasView.tsx +++ b/apps/web/src/components/CanvasView.tsx @@ -41,6 +41,7 @@ import { isPeekableCanvasDisplayType, shouldActivateDatabasePreviewSurface, shouldActivateInlinePageSurface, + takePendingCanvasLens, useCanvasCommands, useCanvasQueryFrames, useCanvasSourceReferences, @@ -279,7 +280,9 @@ export function CanvasView({ docId }: CanvasViewProps): JSX.Element { queryFrameTargets, manualQueryFrameRefreshRequests, selectedQueryFrameDefinition, - refreshSelectedQueryFrame + refreshSelectedQueryFrame, + createQueryFrameFromSavedView, + applySocialCanvasProjection } = useCanvasQueryFrames({ doc, sceneRevision, @@ -288,6 +291,34 @@ export function CanvasView({ docId }: CanvasViewProps): JSX.Element { onUndoBoundary: recordSceneUndoBoundary }) + // A lens sent here from the Data Workspace (0419). The workspace parks the + // request and navigates; this claims it once the doc is live. Claiming is + // one-shot, so revisiting the canvas does not re-insert the frame. + useEffect(() => { + if (!doc) return + + const pending = takePendingCanvasLens(docId) + if (!pending) return + + // A projection places the cards and their connections; without one the + // request is for a live lens frame instead. + if (pending.projection) { + if (!applySocialCanvasProjection(pending.projection)) { + console.error('Failed to project saved lens onto the canvas', pending.viewId) + } + return + } + + const inserted = createQueryFrameFromSavedView({ + viewId: pending.viewId, + title: pending.title, + descriptorJson: pending.descriptorJson + }) + if (!inserted) { + console.error('Failed to insert saved lens as a canvas query frame', pending.viewId) + } + }, [applySocialCanvasProjection, createQueryFrameFromSavedView, doc, docId]) + // Peek (0277 E4): modal preview of the selected card's source without // leaving the board; inline editing activates on zoomed-in selection. const { peekState, peekedObject, openPeek, closePeekSurface } = useCanvasPeek({ diff --git a/apps/web/src/components/DataWorkspaceView.tsx b/apps/web/src/components/DataWorkspaceView.tsx index 642f00f69..a7aa2dae7 100644 --- a/apps/web/src/components/DataWorkspaceView.tsx +++ b/apps/web/src/components/DataWorkspaceView.tsx @@ -3,11 +3,19 @@ * (@xnetjs/views, exploration 0276). Web-specific concerns: OPFS store-backed * seeding, social feed enrichment, and the moderation render gate. */ +import { useNavigate } from '@tanstack/react-router' +import { useIdentity } from '@xnetjs/react' import { useNodeStore } from '@xnetjs/react/internal' -import { useDataWorkspace, DataWorkspaceBody } from '@xnetjs/views' +import { + useDataWorkspace, + DataWorkspaceBody, + stashPendingCanvasLens, + useSocialFeedEnrichment, + type SavedViewCanvasFrameInput +} from '@xnetjs/views' import { Database, Import, Loader2 } from 'lucide-react' -import { useMemo, type ReactNode } from 'react' -import { useSocialFeedEnrichment } from '../hooks/useSocialFeedEnrichment' +import { useCallback, useMemo, type ReactNode } from 'react' +import { deskIdFor } from '../lib/desk' import { ModeratedMedia } from './ModeratedMedia' /** @@ -22,10 +30,41 @@ const gateVisualItem = (nodeId: string, content: ReactNode): ReactNode => ( export function DataWorkspaceView(): JSX.Element { const { store, isReady: storeReady } = useNodeStore() const feedEnrichment = useSocialFeedEnrichment() + const navigate = useNavigate() + const { identity } = useIdentity() + const did = identity?.did + + /** + * Send a saved lens to the Desk as a live query frame. + * + * The Desk is the right target because its id is derived from the identity + * and visiting it provisions it — so this needs no "which canvas?" prompt + * and no separate create. The request is parked, then claimed by the canvas + * once it mounts. + */ + const handleInsertSavedLensAsCanvasFrame = useCallback( + (view: SavedViewCanvasFrameInput) => { + if (!did) return + + const canvasId = deskIdFor(did) + stashPendingCanvasLens({ + canvasId, + viewId: view.id, + title: view.title ?? 'Saved lens', + descriptorJson: view.descriptor ?? null, + ...(view.projection ? { projection: view.projection } : {}) + }) + void navigate({ to: '/canvas/$canvasId', params: { canvasId } }) + }, + [did, navigate] + ) const workspace = useDataWorkspace({ seedReady: Boolean(store && storeReady), - getExistingNode: (id) => (store ? Promise.resolve(store.get(id)) : Promise.resolve(undefined)) + getExistingNode: (id) => (store ? Promise.resolve(store.get(id)) : Promise.resolve(undefined)), + // Without an identity there is no Desk to target, so the canvas actions + // stay hidden rather than failing when pressed. + ...(did ? { onInsertSavedLensAsCanvasFrame: handleInsertSavedLensAsCanvasFrame } : {}) }) const { seeding, seedReady, handleSeedWorkspace } = workspace diff --git a/apps/web/src/components/SavedViewTab.tsx b/apps/web/src/components/SavedViewTab.tsx index 0cd4aa2e0..3ea8a2c97 100644 --- a/apps/web/src/components/SavedViewTab.tsx +++ b/apps/web/src/components/SavedViewTab.tsx @@ -5,8 +5,8 @@ import { validateSavedViewDescriptor, type SavedViewDescriptor } from '@xnetjs/data' import { SavedViewSchema } from '@xnetjs/data' import { SavedViewRunner, useQuery } from '@xnetjs/react' +import { useSocialFeedEnrichment } from '@xnetjs/views' import { useMemo } from 'react' -import { useSocialFeedEnrichment } from '../hooks/useSocialFeedEnrichment' import { WORKBENCH_SAVED_VIEW_REGISTRY } from '../lib/saved-view-registry' import { usePublishTitle } from '../workbench/route-title' diff --git a/apps/web/src/lib/social-import-worker-client.ts b/apps/web/src/lib/social-import-worker-client.ts index b41a340d0..810e190e8 100644 --- a/apps/web/src/lib/social-import-worker-client.ts +++ b/apps/web/src/lib/social-import-worker-client.ts @@ -38,6 +38,7 @@ export type BrowserSocialImportStageInput = { manifest: ArchiveManifest buckets: string[] includeSensitive: boolean + fetchTranscripts?: boolean importedAt?: string } @@ -76,6 +77,7 @@ type MainThreadStagedResult = { manifest: ArchiveManifest buckets: string[] includeSensitive: boolean + fetchTranscripts: boolean importedAt: string result: SocialImportNodeDraftStreamResult streams: Map @@ -316,6 +318,7 @@ async function stageOnMainThread( readTextEntry, buckets: input.buckets, includeSensitive: input.includeSensitive, + fetchTranscripts: input.fetchTranscripts === true, importedAt, includeSourceRecords: true, onComplete: (result) => { @@ -331,6 +334,7 @@ async function stageOnMainThread( manifest: input.manifest, buckets: input.buckets, includeSensitive: input.includeSensitive, + fetchTranscripts: input.fetchTranscripts === true, importedAt, result, streams: new Map() @@ -376,6 +380,7 @@ export async function stageBrowserSocialArchive( manifest: input.manifest, buckets: input.buckets, includeSensitive: input.includeSensitive, + fetchTranscripts: input.fetchTranscripts === true, importedAt: input.importedAt }, isStageResponse @@ -496,6 +501,7 @@ async function getMainThreadStageDraftStream( readTextEntry, buckets: stagedResult.buckets, includeSensitive: stagedResult.includeSensitive, + fetchTranscripts: stagedResult.fetchTranscripts, importedAt: stagedResult.importedAt, includeSourceRecords }), diff --git a/apps/web/src/lib/social-import-worker-protocol.ts b/apps/web/src/lib/social-import-worker-protocol.ts index bfaf2be76..17d3b37b0 100644 --- a/apps/web/src/lib/social-import-worker-protocol.ts +++ b/apps/web/src/lib/social-import-worker-protocol.ts @@ -46,6 +46,8 @@ export type SocialImportWorkerStageRequest = { manifest: ArchiveManifest buckets: string[] includeSensitive: boolean + /** Fetch video transcripts for this run (0419). */ + fetchTranscripts?: boolean importedAt?: string } diff --git a/apps/web/src/routes/social-import.tsx b/apps/web/src/routes/social-import.tsx index d88dd87c8..d4f34c5aa 100644 --- a/apps/web/src/routes/social-import.tsx +++ b/apps/web/src/routes/social-import.tsx @@ -75,6 +75,9 @@ function SocialImportPage(): React.ReactElement { const [selectedBuckets, setSelectedBuckets] = useState([]) const [includeSensitive, setIncludeSensitive] = useState(false) const [includeSourceRecords, setIncludeSourceRecords] = useState(false) + // Transcript fetching is per-run and off by default (0419): the decision is + // about this archive, not a standing preference. + const [fetchTranscripts, setFetchTranscripts] = useState(false) const [stageResult, setStageResult] = useState(null) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) @@ -169,7 +172,8 @@ function SocialImportPage(): React.ReactElement { file: archive.file, manifest: archive.manifest, buckets: selectedBuckets, - includeSensitive + includeSensitive, + fetchTranscripts }) setStageResult(result) setStatus('staged') @@ -177,7 +181,7 @@ function SocialImportPage(): React.ReactElement { setStatus('picked') setError(toErrorMessage(err)) } - }, [archive, includeSensitive, selectedBuckets]) + }, [archive, fetchTranscripts, includeSensitive, selectedBuckets]) const handleCommit = useCallback(async () => { if (!archive || !stageResult || !store || !storeReady) return @@ -678,6 +682,26 @@ function SocialImportPage(): React.ReactElement { +
diff --git a/apps/web/src/workers/social-import.worker.ts b/apps/web/src/workers/social-import.worker.ts index acf5bc586..fb9b5d52b 100644 --- a/apps/web/src/workers/social-import.worker.ts +++ b/apps/web/src/workers/social-import.worker.ts @@ -33,6 +33,7 @@ type WorkerStagedResult = { manifest: Extract['manifest'] buckets: string[] includeSensitive: boolean + fetchTranscripts: boolean importedAt: string streams: Map } @@ -114,6 +115,7 @@ async function handleStage( readTextEntry, buckets: request.buckets, includeSensitive: request.includeSensitive, + fetchTranscripts: request.fetchTranscripts === true, importedAt, includeSourceRecords: true, onComplete: (result) => { @@ -130,6 +132,7 @@ async function handleStage( manifest: request.manifest, buckets: request.buckets, includeSensitive: request.includeSensitive, + fetchTranscripts: request.fetchTranscripts === true, importedAt, streams: new Map() }) diff --git a/docs/explorations/0419_[-]_SOCIAL_GRAPH_ATLAS.md b/docs/explorations/0419_[-]_SOCIAL_GRAPH_ATLAS.md new file mode 100644 index 000000000..fc4deb86c --- /dev/null +++ b/docs/explorations/0419_[-]_SOCIAL_GRAPH_ATLAS.md @@ -0,0 +1,499 @@ +--- +title: The Social Graph Atlas — your imported social life as a navigable, agent-readable place +status: partial # mirrors the [_]/[-]/[x] filename checkbox +last_updated: 2026-08-01 +tags: [social, canvas, views, enrichment, ai, retrieval] +--- + +# The Social Graph Atlas — your imported social life as a navigable, agent-readable place + +> [!TIP] +> **TL;DR** — The vision ("my YouTube/Instagram/TikTok graph on an infinite +> canvas, flip between database/calendar/feed renderings, thumbnails and +> embeds everywhere, and my AI agent can read it all") is **~80% built**. +> `packages/social` already imports all three platforms into a canonical node +> spine; `SavedViewRunner` already flips between six presentation modes +> including `canvas` and `graph`; enrichment and embeds already exist for +> YouTube and Instagram. What's missing is **wiring, not architecture**: the +> canvas projection is built but never called from the app, enrichment is +> web-only and skips TikTok, there are **no video transcripts anywhere**, and +> agent retrieval treats social nodes as ordinary text instead of a +> first-class context source. Recommendation: four surgical work packages +> (canvas wiring, enrichment completion, a transcript enrichment stage, an +> agent-facing `xnet_social_context` path), no new subsystem. + +## Problem Statement + +The user's ask, distilled: + +1. **See it** — imported social data (YouTube videos, playlists, watch + history; Instagram likes and saves; TikTok likes, favorites, collections) + rendered as a graph on the infinite canvas, and re-renderable as a + database, calendar, gallery, or feed with one gesture. +2. **Recognize it** — every item carries a thumbnail, title, description; a + click gives a live embed of the actual video or post. +3. **Feed it to the agent** — transcripts, descriptions, and saved articles + become retrievable context, so an AI agent talking to the user can draw on + what they've watched, liked, and bookmarked. + +The question is what it actually takes to pull this together on xNet +primitives — and the honest answer requires an inventory of what already +ships, because a lot of this was built across explorations 0152, 0153, 0158, +0170 and 0295. + +## Executive Summary + +| Pillar of the vision | Status | What exists / what's missing | +| --------------------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------- | +| Import YouTube/IG/TikTok archives | ✅ Shipped | `packages/social` adapters, staging pipeline, resumable jobs, deterministic IDs | +| Canonical graph schema | ✅ Shipped | 13 `social/*` schemas: Actor, Content, Interaction, Collection, CollectionItem… | +| Flip-through renderings | ✅ Shipped | `SavedViewRunner` presentation modes: `table \| cards \| timeline \| canvas \| graph \| feed` | +| Thumbnails + metadata enrichment | 🚧 Partial | YouTube + Instagram via hub `/unfurl`; **web-only**, no TikTok, no Electron | +| Live embeds | ✅ Shipped | `EMBED_PROVIDERS` + iframe policy + canvas card renderers for youtube/instagram/tiktok | +| Canvas projection of social graph | 🚧 Built, unwired | `createSocialCanvasProjectionPlan()` exists + tested, **called from no app surface** | +| Calendar of watch/like history | 🚧 Partial | Interactions carry timestamps; calendar view exists; no seeded social calendar view | +| TikTok feed views | ❌ Missing | Feed seeds cover only YouTube ×2 and Instagram ×2 | +| Video transcripts | ❌ Missing | Nothing fetches or stores them; Takeout doesn't ship them; oEmbed gives title only | +| Agent access to social context | 🚧 Partial | `xnet_search`/FTS work over `searchText`, but no transcript corpus and no social-aware retrieval profile | + +The gap analysis says: **no new subsystem is needed**. Every missing piece is +a completion of a seam that already exists, plus one genuinely new capability +(transcript enrichment) that slots into the enrichment pipeline built in 0170. + +--- + +## Current State In The Repository + +### 1. The import spine (`packages/social`) — done + +Adapters in `packages/social/src/importers/` parse the platform export +archives directly: + +- `youtube.ts` — Google Takeout: subscriptions, playlists (`playlists.csv` + + per-playlist video CSVs), comments, watch history, search history. +- `instagram.ts` — Meta ZIP: liked posts, saved posts/collections, reels, + comments, followers/following, messages. +- `tiktok.ts` — favorites, favorite collections, likes, posts, hashtags, + shares, searches, comments, relationships, DMs. +- Plus `x.ts`, `reddit.ts`, and AI-chat importers (`openai.ts`, `claude.ts`, + `grok.ts` — `claude.ts` already emits `contentKind: 'transcript'`). + +Everything lands on the canonical spine from exploration 0152 +(`packages/social/src/schemas/`): `SocialActor`, `SocialContent`, +`SocialInteraction`, `SocialConversation`, `SocialMessage`, +`SocialCollection`, `SocialCollectionItem`, `SocialSourceRecord`, +`SocialEnrichment` — platform semantics as facets (`platform`, +`contentKind`, `interactionKind`, `collectionKind`), not per-platform +tables. IDs are deterministic (`packages/social/src/import/ids.ts`), so +re-import is idempotent. + +```mermaid +erDiagram + SocialActor ||--o{ SocialContent : authors + SocialActor ||--o{ SocialInteraction : performs + SocialInteraction }o--|| SocialContent : targets + SocialCollection ||--o{ SocialCollectionItem : contains + SocialCollectionItem }o--|| SocialContent : references + SocialContent ||--o| SocialEnrichment : "enriched by" + SocialContent ||--o| MediaAsset : "thumbnail blob" + SocialSourceRecord ||--o{ SocialContent : "evidence for" +``` + +A YouTube playlist is a `SocialCollection` (`collectionKind: 'playlist'`) +with `SocialCollectionItem`s pointing at `SocialContent` +(`contentKind: 'video'`). A TikTok bookmark is a `SocialInteraction` +(`interactionKind: 'bookmark'`). An Instagram save is a `collection` +(`saved`) plus a `save` interaction. The graph the user wants to see **is +already the storage shape**. + +### 2. "Flip through the renderings" — done, and better than the ask + +The presentation switcher from exploration 0158 is exactly the requested +gesture. `packages/react/src/components/SavedViewRunner.tsx` defines +`SavedViewPresentationMode = 'table' | 'cards' | 'timeline' | 'canvas' | +'graph' | 'feed'`, with the mode persisted per saved view in +`SavedViewPresentationHint` (`packages/data/src/store/query-ast.ts`). +`savedViewVisualPreview.ts` normalizes any social node into a +`SavedViewVisualPreviewModel` (title, thumbnailUrl, embedUrl, provider, +timestamp, metrics…), and `SavedViewVisualFeed.tsx` renders the virtualized +thumbnail grid. Default social views are seeded by +`packages/social/src/views/defaults.ts` (People / Content / Interactions / +Collections…), feed views by `packages/social/src/feeds/defaults.ts`, and +graph lenses by `packages/social/src/lenses/graph-lenses.ts` + +`atlas.ts` (`createDefaultSocialGraphAtlas` — the name of this exploration +honors it). + +### 3. Thumbnails, previews, embeds — partial + +- **Provider parsing and embeds**: `packages/data/src/external-references.ts` + (`EMBED_PROVIDERS` covers YouTube incl. Shorts, Instagram p/reel/tv, + TikTok, X, Vimeo, Spotify…), iframe policy in + `packages/data/src/external-reference-embed-policy.ts`, canvas card + renderers in `packages/editor/src/components/canvasExternalReferenceCardRenderers.ts`. +- **Enrichment** (exploration 0170): `SocialEnrichmentSchema` with + deterministic IDs, hub-proxied metadata via + `packages/hub/src/routes/unfurl.ts` (SSRF-guarded, image allowlist already + includes `i.ytimg.com`, `cdninstagram.com`, `tiktokcdn`), client queue in + `apps/web/src/hooks/social-feed-enrichment.ts`. +- **Thumbnails as blobs**: `packages/data/src/blob/thumbnail.ts` generates + ≤320px thumbnails stored as *separate* tiny blobs so they sync ahead of + full media — which respects the >1MB blob-sync constraint from 0385. + +> [!WARNING] +> Enrichment is **web-only** today (it lives in `apps/web/src/hooks/`), and +> feed seeds cover only YouTube and Instagram. TikTok — whose oEmbed +> endpoint actually *sends CORS headers* (the one free pass 0170 found) — +> has importers producing bookmarks and collections that render as opaque +> IDs. Electron users get no enrichment at all. + +### 4. The canvas — built on both ends, bridge unwired + +`packages/social/src/projection/canvas.ts` exports +`createSocialCanvasProjectionPlan()`, producing `external-reference` node +drafts (`sourceCardRole: 'social-projection'`) and edge drafts, capped at 75 +nodes / 200 edges. It is exercised only by +`packages/social/src/__tests__/views-lenses-projection.test.ts` — **no app +surface calls it**. The shipped insert path +(`SavedViewVisualCanvasProjectionRequest` → +`onInsertSavedLensAsCanvasFrame` in +`packages/views/src/data-workspace/DataWorkspaceCore.tsx`) is wired in +Electron only. Meanwhile the canvas side is fully ready: source-backed +objects, `frameSourceRegistry`, live saved-view query frames +(`packages/views/src/canvas-view/query-frames.tsx`), DOM-island rich cards. + +### 5. Agent retrieval — general-purpose, not social-aware + +`packages/brain` provides tiered retrieval (`hybrid-graph → bm25-graph → +bm25 → scan`) with BFS relation expansion and token-budget packing; +`packages/sqlite/src/fts.ts` maintains `nodes_fts`; the agent tool surface +(`packages/plugins/src/ai-surface/tools/`) exposes `xnet_search`, +`xnet_graph_expand`, `xnet_create_context_pack`, database query tools, and +canvas read tools. `SocialContent` deliberately denormalizes `searchText` +(20k chars) for exactly this purpose. So an agent *can* find a saved post by +keyword today. What it cannot do: + +- Read a **video transcript** — none exist. YouTube Takeout does not include + them, nothing fetches them, and `contentKind: 'transcript'` is emitted + only by the Claude-chat importer. +- Ask a social-shaped question cheaply ("what have I bookmarked about + fermentation across all platforms?") — that's a facet query over + `interactionKind` + full-text, which works via `xnet_database_query` but + isn't packaged as a retrieval profile or context pack. + +--- + +## External Research + +### Transcripts are the hard 20% + +There is **no official API** for arbitrary-video transcripts. The +[YouTube captions API](https://outlierkit.com/resources/youtube-transcript-api/) +only serves videos you own (owner OAuth). The de-facto standard is the +unofficial timedtext endpoint used by +[youtube-transcript-api](https://www.notelm.ai/blog/youtube-transcript-api) +(Python) and TypeScript equivalents like +[youtube-caption-extractor](https://github.com/devhims/youtube-caption-extractor) — +free, no key, works for any captioned video, **but IP-blocked from +datacenter ranges** ([developer guide](https://medium.com/@volods/how-to-get-youtube-transcripts-a-complete-developers-guide-b3f092eb0a96), +[production notes](https://transcriptapi.com/blog/extract-youtube-transcripts-programmatically)). +Commercial proxies ([ScrapingDog](https://www.scrapingdog.com/youtube-transcripts-api/), +[TranscriptFetch](https://transcriptfetch.com/)) exist for cloud workloads. + +> [!IMPORTANT] +> The datacenter IP block is the moat that makes every "second brain for +> social" SaaS pay for proxy fleets — and xNet is **local-first**, so it +> walks around it. Transcript fetches issued from the *user's own device*, +> at human-ish rates, against videos the user demonstrably saved, are +> indistinguishable from that user opening the video. The result is stored +> once as a node and syncs everywhere. Local-first turns the hardest part of +> this feature into the cheapest. + +Two hazards to respect: (a) exploration 0417 already confirmed **oEmbed +returns title only — never transcripts**; (b) rate discipline matters — a +2,000-video watch history must be a slow trickle queue (the +`SocialEnrichmentQueue` backoff pattern already models this), not a burst. + +### Prior art: the category exists, and it's cloud-shaped + +[Tavlo](https://www.tavlo.ca/) and [Second Brain](https://www.thesecondbrain.io/) +sell exactly this pitch — "your TikToks, Reels, YouTube saves, auto-organized +and searchable." Both are SaaS: your graph lives on their servers, is +enriched by their pipelines, and is gone when they are. +[Obsidian](https://www.kosmik.app/blog/best-second-brain-apps)-style graph +views prove the visualization appetite but have no social importers. Nothing +in the field combines **local-first ownership + archive-grade import + a +real canvas + agent tools**. This is differentiation xNet already paid for; +the atlas just makes it visible. + +--- + +## Key Findings + +1. **The storage shape is already the graph.** Collections, items, + interactions, and actors are real nodes with real relations — no ETL step + between "imported" and "visualizable" exists or is needed. +2. **The rendering switcher is shipped**; what's missing is *reach* — + canvas mode unwired in web, no TikTok feeds, no social calendar seed. +3. **Enrichment is the difference between an atlas and a wall of IDs** + (0170's core finding: YouTube playlist CSVs carry only video IDs). + Completing it (TikTok + Electron) is table stakes for this vision. +4. **Transcripts are the only genuinely new capability**, and local-first + makes them uniquely feasible for xNet vs. cloud competitors. +5. **Agent access needs packaging, not plumbing** — FTS + `searchText` + + `xnet_create_context_pack` exist; a social retrieval profile and a + transcript corpus make them useful. +6. **Wider retrieval widens the egress hole** (0379): once transcripts and + DM-adjacent content are agent-readable, the existing egress-budget and + approval-broker guardrails in `packages/plugins/src/ai-surface/` are + load-bearing, and sensitive buckets must stay opt-in as 0153 established. + +## Options And Tradeoffs + +### Option A — A bespoke "Social Atlas" app surface + +A dedicated route with its own graph renderer, timeline, and media browser. + +- ✅ Maximal control over the experience. +- ❌ Re-litigates 0158, which explicitly chose *presentation modes over a + bespoke social page* — and won: the switcher shipped and generalizes to + every schema. +- ❌ A second graph renderer and second feed to maintain. + +**Rejected.** 🛑 + +### Option B — Complete the existing seams (four work packages) + +Wire `createSocialCanvasProjectionPlan` into web; finish enrichment (TikTok +oEmbed, Electron parity); add a transcript enrichment stage on the user's +device; package agent access (social retrieval profile + seeded context +packs + TikTok/calendar view seeds). + +- ✅ Every piece lands in an existing file or registry; no new package. +- ✅ Each package ships independently and is useful alone. +- ❌ Transcript fetching is unofficial-API territory — needs graceful + degradation and per-platform honesty (YouTube: yes; Instagram/TikTok: + no captions endpoint → description text only). + +**Recommended.** ✅ + +### Option C — Official data-API integration (YouTube Data API, Meta Graph API) + +OAuth per platform, live sync instead of archive import, official metadata. + +- ✅ Richest, freshest metadata; durable against scraping countermeasures. +- ❌ API keys, quota management, app review processes (Meta especially); + contradicts the archive-first, own-your-data posture; still doesn't give + transcripts (captions API is owner-only). +- 🔁 Worth revisiting as an *optional* enrichment source (`data-api` is + already an `enrichmentSource` in `constants.ts`) once B ships. + +**Deferred.** + +> [!NOTE] +> No new revenue lane is proposed, so the Charter §6 "no ground rent" tests +> are not triggered. If a hosted transcript-proxy tier is ever proposed for +> convenience, it must pass them then (a local fetch path must always +> remain — the vanish test). + +## Recommendation + +Ship Option B as four independent work packages, in this order: + +```mermaid +flowchart LR + subgraph WP1["WP1 · Canvas wiring"] + A1[Wire social canvas projection\ninto web DataWorkspace] --> A2[Saved-view canvas mode\ninserts live query frame] + end + subgraph WP2["WP2 · Enrichment completion"] + B1[TikTok oEmbed enrichment] --> B2[TikTok feed view seeds] + B3[Electron enrichment parity] + B4[Social calendar seed\nover interaction timestamps] + end + subgraph WP3["WP3 · Transcripts"] + C1[Device-local timedtext fetcher] --> C2[Transcript stored as\nSocialContent kind=transcript] + C2 --> C3[FTS-indexed, linked to video] + end + subgraph WP4["WP4 · Agent context"] + D1[Social retrieval profile] --> D2[Seeded social context packs] + end + WP1 --> WP4 + WP2 --> WP3 --> WP4 +``` + +### The transcript pipeline (WP3), concretely + +Model it as a second enrichment stage riding the existing queue: same +deterministic-ID upsert pattern, same backoff, but **device-local fetch** +(no hub proxy — the hub's datacenter IP is exactly what gets blocked) and +the result stored as a linked `SocialContent` node +(`contentKind: 'transcript'`) rather than a field on `SocialEnrichment`, +because transcripts are large (10–100KB), deserve their own FTS row, and the +kind already exists in the vocabulary. + +```mermaid +sequenceDiagram + participant Q as SocialEnrichmentQueue (device) + participant YT as youtube.com timedtext + participant S as NodeStore + participant F as nodes_fts + participant AI as Agent (xnet_search) + + Q->>Q: pick captioned video, jittered delay + Q->>YT: fetch caption track (user's residential IP) + alt captions exist + YT-->>Q: timed text (VTT/JSON) + Q->>S: upsert SocialContent{kind: transcript,\nrelation → video, deterministic id} + S->>F: index transcript text + else no captions / blocked + YT-->>Q: 404 / block + Q->>S: mark enrichment status=exhausted\n(loud, distinguishable from "absent") + end + AI->>F: "what did I watch about fermentation?" + F-->>AI: transcript hits → linked videos → collections +``` + +Per the repo's error rule: "no captions available," "fetch blocked," and +"not yet attempted" must be three distinguishable states on the enrichment +node — a truncated trickle run is not a completed one. + +### What the user gets when all four land + +Open the Data Workspace → the YouTube Playlists lens → tap +canvas in the presentation switcher → playlists and their videos +land as thumbnail cards with edges on the infinite canvas, deduped against +what's already placed → tap any card for the live embed → flip the same +lens to timeline to see it by save-date, or open the seeded +watch-history calendar → ask the agent "summarize the themes in videos I +saved this spring" and it retrieves across transcripts, descriptions, and +collection structure — all local, all owned, all synced. + +## Risks And Open Questions + +- **Transcript fetch fragility** 🔶 — the timedtext surface is unofficial + and shifts. Mitigation: isolate behind a `TranscriptFetcher` seam, degrade + loudly to `status: 'exhausted'`, never block import or enrichment on it. +- **Volume** — a heavy account = thousands of videos × 30KB transcripts ≈ + tens of MB of text nodes. Fine for SQLite/FTS; keep transcripts out of Yjs + docs (they're structured nodes, LWW is fine) and respect the trickle cap. +- **Egress** ⚠️ — agent-readable transcripts + DMs is the 0379 hazard. + Sensitive buckets (messages, searches) must stay outside default retrieval + profiles; the egress budget and approval broker gate the rest. +- **Instagram/TikTok transcripts** — no caption endpoint exists; honest + scope is *description + hashtags* enrichment. On-device Whisper over saved + media is a separate future exploration (weight + ToS questions). +- **Canvas scale** — the 75-node projection cap is right for insertion; + "my entire watch history as a canvas" is not a projection, it's a graph + lens (`mode: 'graph'`), and the seeded atlas already handles that. +- **Open question** — should transcript fetch be opt-in per import run + (checkbox in the wizard) or ambient once enrichment is on? Leaning + per-run opt-in, consistent with 0153's privacy-forward staging UX. + +## Implementation Checklist + +**WP1 — Canvas wiring** + +- [x] Wire `createSocialCanvasProjectionPlan()` (or retire it in favor of + `createSavedViewCanvasProjectionNodes`) into the web + `DataWorkspaceCore` insert path — resolve the duplication between + `packages/social/src/projection/canvas.ts` and + `packages/react/src/components/savedViewVisualPreview.ts` first. +- [x] Bring `onInsertSavedLensAsCanvasFrame` to parity in `apps/web` + (Electron-only today). + +**WP2 — Enrichment completion** + +- [x] Add TikTok oEmbed to the enrichment fetch path (direct from client — + TikTok sends CORS headers; fall back to hub `/unfurl` otherwise). +- [x] Add TikTok feed view seeds (favorites, collections) to + `packages/social/src/feeds/defaults.ts`. +- [x] Extract the enrichment queue from `apps/web/src/hooks/` into a shared + package location and wire it in Electron. +- [x] Seed a social calendar/timeline saved view over + `SocialInteraction.occurredAt` (watch/like/save history). + +**WP3 — Transcripts** + +- [x] Add a `TranscriptFetcher` seam + device-local YouTube timedtext + implementation with jittered trickle scheduling on the enrichment + queue. +- [x] Store transcripts as `SocialContent` (`contentKind: 'transcript'`) + linked to the video node, deterministic ID, FTS-indexed. +- [x] Three distinguishable enrichment states: not-attempted / no-captions / + fetch-blocked; surface counts in the workspace privacy/status summary. +- [x] Per-import-run opt-in toggle in the import wizard. + +**WP4 — Agent context** + +- [x] Define a social `RetrievalProfile` (content + transcripts + collection + structure; sensitive buckets excluded by default). +- [x] Seed one or two social context packs via the existing + `xnet_create_context_pack` shape ("everything I saved about X"). +- [x] Changelog fragment ("Your imported social library now …") — user-facing + feature, no `skip-changelog`. + +## Validation Checklist + +- [ ] Import a real YouTube Takeout + Instagram + TikTok archive; every feed + view shows thumbnails and titles (no opaque-ID rows) on web **and** + Electron. +- [ ] Presentation switcher round-trip on one social lens: table → feed → + timeline → graph → canvas, each renders without error; canvas insert + places source-backed cards that open live embeds. +- [x] Transcript run over a 50-video playlist completes with per-state + counts (fetched / no-captions / blocked) that sum to 50. +- [ ] `xnet_search` from an agent session returns a transcript hit and can + `xnet_graph_expand` from transcript → video → playlist. +- [x] Sensitive buckets (DMs, search history) absent from default retrieval + profile results; present only after explicit opt-in. +- [x] `pnpm typecheck && pnpm test` green; new logic in `packages/social` + covered by unit tests. + +> [!IMPORTANT] +> **What the three open boxes need, and why they are still open.** +> All thirteen implementation items shipped; these three require inputs the +> implementation pass could not produce for itself, and checking them off +> without those inputs would be the exact "a truncated run is not a completed +> one" failure this exploration argues against. +> +> - **Real archives.** Boxes 1 and 2 need an actual YouTube Takeout, Instagram +> and TikTok export in hand. No fixture stands in for "does a real playlist +> CSV render with real thumbnails", which is the whole claim. +> - **A live agent session.** Box 4 needs an agent talking to a populated +> workspace. The blocker that would have made it fail is fixed — see below — +> but the round trip itself has not been run. +> +> The transcript box is checked on a 50-target pass through +> `runTranscriptFetchPass` with the **transport stubbed**: the accounting +> invariant (every target lands in exactly one terminal state, and the states +> sum to 50) is genuinely verified; a live network run against YouTube is not. + +> [!WARNING] +> **Found while validating: imported social content was never in the search +> index.** `extractSearchableContent` (`packages/sqlite/src/fts.ts`) read +> `content`, `description`, `body`, `name` and `note` — but not `searchText`, +> the property 22 importer call sites denormalize full text into *precisely so +> it can be searched* (0152). Every imported post, comment and — as of this +> exploration — every video transcript was absent from `nodes_fts` while the +> pipeline reported it indexed, so search returned a clean empty result rather +> than an error. This is the 0379 finding in a second disguise. Fixed, with +> `textPreview` as the fallback. **Existing databases need `rebuildFTS()`** to +> pick up already-imported rows; new writes are correct from here on. + +## References + +- Explorations: `0152_[x]_ACTUAL_SOCIAL_GRAPH_IMPORTER.md`, + `0153_[x]_SOCIAL_DATA_WORKSPACE_UI.md`, `0158` (visual data workspace), + `0170` (feed views + media cache), `0295` (URL up-res / unfurl), + `0379` (knowledge base on xNet primitives), `0374` (index pipeline). +- Code: `packages/social/src/` (importers, schemas, projection, feeds, + lenses), `packages/react/src/components/SavedViewRunner.tsx`, + `packages/views/src/data-workspace/DataWorkspaceCore.tsx`, + `packages/hub/src/routes/unfurl.ts`, + `apps/web/src/hooks/social-feed-enrichment.ts`, + `packages/brain/src/`, `packages/plugins/src/ai-surface/tools/`. +- Web: [YouTube transcript API landscape (OutlierKit)](https://outlierkit.com/resources/youtube-transcript-api/), + [youtube-transcript-api notes (NoteLM)](https://www.notelm.ai/blog/youtube-transcript-api), + [youtube-caption-extractor (TS)](https://github.com/devhims/youtube-caption-extractor), + [developer's guide to YouTube transcripts (Medium)](https://medium.com/@volods/how-to-get-youtube-transcripts-a-complete-developers-guide-b3f092eb0a96), + [production extraction notes (TranscriptAPI)](https://transcriptapi.com/blog/extract-youtube-transcripts-programmatically), + [Tavlo](https://www.tavlo.ca/), [Second Brain](https://www.thesecondbrain.io/). diff --git a/packages/data/etc/data.api.md b/packages/data/etc/data.api.md index c951d7a69..8fbf7091d 100644 --- a/packages/data/etc/data.api.md +++ b/packages/data/etc/data.api.md @@ -5021,6 +5021,9 @@ export const DEFAULT_EXTERNAL_REFERENCE_IFRAME_ALLOW = "accelerometer; autoplay; // @public (undocumented) export const DEFAULT_OFFLINE_POLICY: OfflineAuthPolicy; +// @public +export const DEFAULT_PROMOTION_THRESHOLD = 8; + // @public export const DEFAULT_ROW_HEIGHT: RowHeight; @@ -8744,6 +8747,14 @@ export type PortableChangeRecord = { batchSize?: number; }; +// @public +export type PortableHubAddress = { + name: string; + url: string; + resolverUrl?: string; + observedAt: number; +}; + // @public export type PortableYjsDocRecord = { nodeId: string; @@ -8955,6 +8966,16 @@ export const ProjectSchema: DefinedSchema<{ // @public export function promoteOverlay(authority: string, field: string, coreProp: string): LensOperation; +// @public (undocumented) +export interface PromotionProposal { + authority: string; + count: number; + coverage: number; + field: string; + lens: SchemaLens; + overlayKey: string; +} + // @public export interface PropertyBuilder { coerce(value: unknown): T | null; @@ -8989,6 +9010,18 @@ export interface PropertyTimestamp { // @public export type PropertyType = 'text' | 'number' | 'checkbox' | 'json' | 'date' | 'dateRange' | 'geo' | 'select' | 'multiSelect' | 'person' | 'relation' | 'rollup' | 'formula' | 'url' | 'email' | 'phone' | 'file' | 'created' | 'updated' | 'createdBy'; +// @public +export function proposePromotion(rows: ReadonlyArray>, overlayKey: string, from: SchemaIRI, to: SchemaIRI, options?: ProposePromotionOptions): PromotionProposal | null; + +// @public (undocumented) +export interface ProposePromotionOptions { + dismissed?: Iterable; + threshold?: number; +} + +// @public +export function proposePromotions(rows: ReadonlyArray>, from: SchemaIRI, to: SchemaIRI, options?: ProposePromotionOptions): PromotionProposal[]; + // @public export function pruneVersionHistory(history: SchemaVersionEntry[]): SchemaVersionEntry[]; @@ -11419,6 +11452,7 @@ export type WriteBundleOptions = { ownerDid: string; manifestSigner?: (bytes: Uint8Array) => Promise | Uint8Array; commitSigner?: (bytes: Uint8Array) => Promise | Uint8Array; + hubAddress?: PortableHubAddress; since?: BundleFrontier; blobPort?: BundleBlobPort; yjsPort?: BundleYjsPort; @@ -11456,6 +11490,7 @@ export type XnetpackManifest = { commits?: number; }; contentDigest: string; + hubAddress?: PortableHubAddress; signatureB64?: string; }; diff --git a/packages/react/etc/react.api.md b/packages/react/etc/react.api.md index 20e9717a7..70360fff0 100644 --- a/packages/react/etc/react.api.md +++ b/packages/react/etc/react.api.md @@ -54,6 +54,8 @@ import { FrontierEntry } from '@xnetjs/history'; import { HistoricalState } from '@xnetjs/history'; import { HistoryHorizon } from '@xnetjs/history'; import { HistoryTarget } from '@xnetjs/history'; +import { HubAddressOutcome } from '@xnetjs/runtime'; +import { HubAddressStorage } from '@xnetjs/runtime'; import { HybridKeyBundle } from '@xnetjs/identity'; import { Identity } from '@xnetjs/identity'; import { ImporterContribution } from '@xnetjs/plugins'; @@ -364,6 +366,7 @@ export function createSavedViewVisualCanvasProjectionRequest(input: { layout: SavedViewVisualLayoutOption; previews: readonly SavedViewVisualPreviewModel[]; nodes: readonly SavedViewCanvasProjectionNode[]; + edges?: readonly SavedViewCanvasProjectionEdge[]; }): SavedViewVisualCanvasProjectionRequest; // @public (undocumented) @@ -841,6 +844,14 @@ export function hasSavedViewVisualPreviewSensitiveData(preview: SavedViewVisualP export { HistoryHorizon } +// @public +export interface HubAddressConfig { + fetchImpl?: typeof fetch; + name: string; + resolverUrl: string; + storage?: HubAddressStorage; +} + // Warning: (ae-forgotten-export) The symbol "HubConnectScreenProps" needs to be exported by the entry point index.d.ts // // @public (undocumented) @@ -1630,6 +1641,15 @@ export interface ReplyContext { replyToUser?: string; } +// @public (undocumented) +export interface ResolvedHubUrl { + fallbacks: string[]; + // (undocumented) + outcome: HubAddressOutcome | null; + stale: boolean; + url: string | null; +} + export { RestoreResult } // @public @@ -1645,6 +1665,15 @@ export type RunAtprotoCeremony = (input: { handleOrPds: string; }) => Promise; +// @public +export type SavedViewCanvasProjectionEdge = { + id: string; + sourceId: string; + targetId: string; + relationshipKind: string; + label?: string; +}; + // @public (undocumented) export type SavedViewCanvasProjectionNode = { id: string; @@ -1875,6 +1904,7 @@ export type SavedViewVisualCanvasProjectionRequest = { projectionGroupBy: SavedViewVisualLayoutOption['projectionGroupBy']; }; nodes: SavedViewCanvasProjectionNode[]; + edges: SavedViewCanvasProjectionEdge[]; sourceNodeIds: string[]; omittedNodeCount: number; previewCount: number; @@ -3036,6 +3066,9 @@ export interface UseRelatedRowsResult { // @public (undocumented) export const useRemoteSchema: (iri: string | undefined) => RemoteSchemaState; +// @public +export function useResolvedHubUrl(address: HubAddressConfig | undefined, configuredHubUrl: string | null): ResolvedHubUrl; + // @public export function useReverseRelations(rowId: string, databaseId: string): UseReverseRelationsResult; @@ -3334,6 +3367,7 @@ export interface XNetConfig { disablePlugins?: boolean; disableSyncManager?: boolean; encryptionKey?: Uint8Array; + hubAddress?: HubAddressConfig; hubOptions?: { autoAuth?: boolean; authToken?: string; diff --git a/packages/react/src/components/SavedViewRunner.tsx b/packages/react/src/components/SavedViewRunner.tsx index 4188fb4a3..ad0ce4a1c 100644 --- a/packages/react/src/components/SavedViewRunner.tsx +++ b/packages/react/src/components/SavedViewRunner.tsx @@ -240,6 +240,21 @@ export type SavedViewVisualLayoutOption = { projectionGroupBy: 'platform' | 'kind' | 'creator' | 'privacy' } +/** + * A relationship between two projected nodes. + * + * `relationshipKind` stays an open string here rather than the social + * package's closed vocabulary: this package must not depend on that one, and + * the consumer that cares narrows it when it builds a layout plan. + */ +export type SavedViewCanvasProjectionEdge = { + id: string + sourceId: string + targetId: string + relationshipKind: string + label?: string +} + export type SavedViewVisualCanvasProjectionRequest = { id: string title: string @@ -254,6 +269,8 @@ export type SavedViewVisualCanvasProjectionRequest = { projectionGroupBy: SavedViewVisualLayoutOption['projectionGroupBy'] } nodes: SavedViewCanvasProjectionNode[] + /** Relationships between the projected nodes, when the result carries any. */ + edges: SavedViewCanvasProjectionEdge[] sourceNodeIds: string[] omittedNodeCount: number previewCount: number @@ -1027,6 +1044,18 @@ export function SavedViewRunner({ () => deriveSavedViewVisualGraphEdges(arrangedVisualPreviews), [arrangedVisualPreviews] ) + // The same relationships, in the shape a canvas projection consumes (0419). + const visualCanvasProjectionEdges = useMemo( + () => + visualGraphEdges.map((edge) => ({ + id: edge.id, + sourceId: edge.sourceNodeId, + targetId: edge.targetNodeId, + relationshipKind: edge.kind, + label: edge.kind + })), + [visualGraphEdges] + ) const presentationModeOptions = useMemo( () => createSavedViewPresentationModeOptions({ @@ -1494,6 +1523,7 @@ export function SavedViewRunner({ { + const placed = new Set(input.nodes.map((node) => node.id)) + return placed.has(edge.sourceId) && placed.has(edge.targetId) + }), sourceNodeIds: input.nodes.map((node) => node.id), omittedNodeCount: Math.max(0, input.previews.length - input.nodes.length), previewCount: input.previews.length @@ -2229,6 +2266,7 @@ function groupCanvasProjectionNodes( function SavedViewVisualCanvasProjectionPanel({ previews, projectionNodes, + projectionEdges, layout, descriptor, title, @@ -2239,6 +2277,7 @@ function SavedViewVisualCanvasProjectionPanel({ }: { previews: SavedViewVisualPreviewModel[] projectionNodes: SavedViewCanvasProjectionNode[] + projectionEdges?: SavedViewCanvasProjectionEdge[] layout: SavedViewVisualLayoutOption descriptor?: SavedViewDescriptor | string | null title: string @@ -2266,7 +2305,8 @@ function SavedViewVisualCanvasProjectionPanel({ sourceSchemaId, layout, previews, - nodes: projectionNodes + nodes: projectionNodes, + ...(projectionEdges ? { edges: projectionEdges } : {}) }) return ( diff --git a/packages/react/src/index.ts b/packages/react/src/index.ts index 6921da2d7..9558ffef9 100644 --- a/packages/react/src/index.ts +++ b/packages/react/src/index.ts @@ -86,6 +86,7 @@ export { type SavedViewRowInspectorModel, type SavedViewRunnerProps, type SavedViewSortDirection, + type SavedViewCanvasProjectionEdge, type SavedViewVisualCanvasProjectionRequest, type SavedViewVisualLayoutId, type SavedViewVisualLayoutOption, diff --git a/packages/social/package.json b/packages/social/package.json index 72113605b..58779d57f 100644 --- a/packages/social/package.json +++ b/packages/social/package.json @@ -68,10 +68,22 @@ "./connect": { "import": "./src/connect/index.ts", "types": "./src/connect/index.ts" + }, + "./enrichment": { + "import": "./src/enrichment/index.ts", + "types": "./src/enrichment/index.ts" + }, + "./transcripts": { + "import": "./src/transcripts/index.ts", + "types": "./src/transcripts/index.ts" + }, + "./retrieval": { + "import": "./src/retrieval/index.ts", + "types": "./src/retrieval/index.ts" } }, "scripts": { - "build": "tsup src/index.ts src/schemas/index.ts src/import/index.ts src/import/core.ts src/import/browser.ts src/import/node.ts src/importers/index.ts src/feeds/index.ts src/lenses/index.ts src/patterns/index.ts src/projection/index.ts src/publish/index.ts src/views/index.ts src/workspace/index.ts src/connect/index.ts --format esm --dts", + "build": "tsup src/index.ts src/schemas/index.ts src/import/index.ts src/import/core.ts src/import/browser.ts src/import/node.ts src/importers/index.ts src/feeds/index.ts src/lenses/index.ts src/patterns/index.ts src/projection/index.ts src/publish/index.ts src/views/index.ts src/workspace/index.ts src/connect/index.ts src/enrichment/index.ts src/transcripts/index.ts src/retrieval/index.ts --format esm --dts", "test": "vitest run", "typecheck": "tsc --noEmit", "clean": "rm -rf dist" diff --git a/packages/social/src/__tests__/views-lenses-projection.test.ts b/packages/social/src/__tests__/views-lenses-projection.test.ts index 12c8e59b4..ffa3fd01f 100644 --- a/packages/social/src/__tests__/views-lenses-projection.test.ts +++ b/packages/social/src/__tests__/views-lenses-projection.test.ts @@ -95,16 +95,22 @@ describe('social feed views', () => { 'social.feed.youtube-videos', 'social.feed.youtube-playlists', 'social.feed.instagram-saved', - 'social.feed.instagram-likes' + 'social.feed.instagram-likes', + 'social.feed.tiktok-videos', + 'social.feed.tiktok-collections', + 'social.feed.activity-timeline' ]) for (const feed of feeds) { expect(validateSavedViewDescriptor(feed.descriptor).valid).toBe(true) - expect(feed.descriptor.presentation).toEqual({ - mode: 'feed', - feedLayout: 'grid', - feedDensity: 'cozy' - }) + // Media feeds open as a thumbnail grid; the activity view is over + // interactions, which carry a timestamp and no thumbnail of their own, + // so it opens on the time axis instead (0419). + expect(feed.descriptor.presentation).toEqual( + feed.id === 'social.feed.activity-timeline' + ? { mode: 'timeline', feedLayout: 'list', feedDensity: 'compact' } + : { mode: 'feed', feedLayout: 'grid', feedDensity: 'cozy' } + ) expect(JSON.parse(feed.savedViewProperties.descriptor).presentation).toEqual( feed.descriptor.presentation ) @@ -134,6 +140,9 @@ describe('social workspace seeds', () => { 'social.feed.youtube-playlists', 'social.feed.instagram-saved', 'social.feed.instagram-likes', + 'social.feed.tiktok-videos', + 'social.feed.tiktok-collections', + 'social.feed.activity-timeline', 'social.lens.people-i-follow', 'social.lens.saved-content-by-creator', 'social.lens.conversation-references', @@ -143,7 +152,7 @@ describe('social workspace seeds', () => { repeated.map((seed) => seed.deterministicId) ) expect(seeds.filter((seed) => seed.seedKind === 'schema-view')).toHaveLength(6) - expect(seeds.filter((seed) => seed.seedKind === 'feed-view')).toHaveLength(4) + expect(seeds.filter((seed) => seed.seedKind === 'feed-view')).toHaveLength(7) expect(seeds.filter((seed) => seed.seedKind === 'graph-lens')).toHaveLength(4) for (const seed of seeds) { diff --git a/apps/web/src/hooks/social-feed-enrichment.test.ts b/packages/social/src/enrichment/enrichment.test.ts similarity index 77% rename from apps/web/src/hooks/social-feed-enrichment.test.ts rename to packages/social/src/enrichment/enrichment.test.ts index 7622be7d7..f31e4f70e 100644 --- a/apps/web/src/hooks/social-feed-enrichment.test.ts +++ b/packages/social/src/enrichment/enrichment.test.ts @@ -11,9 +11,11 @@ import { resolveHubAuthToken, socialEnrichmentKey, SocialEnrichmentQueue, + supportsDirectOEmbed, thumbnailContentTypeFor, + tiktokOEmbedUrl, type SocialEnrichmentTarget -} from './social-feed-enrichment' +} from './index' const target: SocialEnrichmentTarget = { key: socialEnrichmentKey('youtube', 'abc123'), @@ -269,18 +271,100 @@ describe('fetchEnrichmentForTarget', () => { }) }) +describe('direct TikTok oEmbed', () => { + const tiktokTarget: SocialEnrichmentTarget = { + key: socialEnrichmentKey('tiktok', '7123'), + platform: 'tiktok', + platformContentId: '7123', + url: 'https://www.tiktok.com/@chef/video/7123' + } + + it('marks TikTok as the one platform reachable without the hub', () => { + expect(supportsDirectOEmbed('tiktok')).toBe(true) + expect(supportsDirectOEmbed('youtube')).toBe(false) + expect(new URL(tiktokOEmbedUrl(tiktokTarget.url)).searchParams.get('url')).toBe( + tiktokTarget.url + ) + }) + + it('resolves TikTok metadata with no hub configured', async () => { + const requested: string[] = [] + const result = await fetchEnrichmentForTarget({ + target: tiktokTarget, + blobStore: null, + fetchImpl: (async (input: RequestInfo | URL) => { + requested.push(String(input)) + return new Response( + JSON.stringify({ + title: 'Sourdough in 60 seconds', + author_name: 'chef', + thumbnail_url: 'https://p16.tiktokcdn.com/thumb.jpg' + }), + { headers: { 'Content-Type': 'application/json' } } + ) + }) as typeof fetch + }) + + expect(requested[0]).toContain('tiktok.com/oembed') + expect(result.payload.status).toBe('resolved') + expect(result.payload.metadata?.title).toBe('Sourdough in 60 seconds') + expect(result.payload.metadata?.source).toBe('oembed') + }) + + it('treats a 200 with no metadata as unavailable rather than resolved', async () => { + const result = await fetchEnrichmentForTarget({ + target: tiktokTarget, + blobStore: null, + fetchImpl: (async () => + new Response('{}', { headers: { 'Content-Type': 'application/json' } })) as typeof fetch + }) + + expect(result.payload.status).toBe('unavailable') + }) + + it('separates a refusal from a missing video', async () => { + const blocked = await fetchEnrichmentForTarget({ + target: tiktokTarget, + blobStore: null, + fetchImpl: (async () => new Response('', { status: 429 })) as typeof fetch + }) + expect(blocked.payload.status).toBe('blocked') + + const missing = await fetchEnrichmentForTarget({ + target: tiktokTarget, + blobStore: null, + fetchImpl: (async () => new Response('', { status: 404 })) as typeof fetch + }) + expect(missing.payload.status).toBe('unavailable') + }) + + it('reports hub-only platforms as unavailable when there is no hub', async () => { + const result = await fetchEnrichmentForTarget({ + target, + blobStore: null, + fetchImpl: (async () => { + throw new Error('should not fetch') + }) as typeof fetch + }) + + expect(result.payload.status).toBe('unavailable') + expect(result.payload.reason).toContain('no hub available') + }) +}) + describe('loadMissingThumbnailBlobUrls', () => { it('creates object urls for uncached blob cids only', async () => { const added = await loadMissingThumbnailBlobUrls({ rows: [ - { id: 'a', thumbnailBlobCid: 'cid:blake3:one' }, - { id: 'b', thumbnailBlobCid: 'cid:blake3:cached' }, - { id: 'c', thumbnailBlobCid: 'not-a-cid' }, - { id: 'd' } + { thumbnailBlobCid: 'cid:blake3:one' }, + { thumbnailBlobCid: 'cid:blake3:cached' }, + { thumbnailBlobCid: 'not-a-cid' }, + {} ], blobStore: { get: async () => new Uint8Array([9]) }, hasUrl: (cid) => cid === 'cid:blake3:cached', - createUrl: () => 'blob:created' + createUrl: () => 'blob:created', + contentTypeFor: thumbnailContentTypeFor }) expect([...added.entries()]).toEqual([['cid:blake3:one', 'blob:created']]) @@ -288,18 +372,20 @@ describe('loadMissingThumbnailBlobUrls', () => { it('skips blobs that fail to load and stops when cancelled', async () => { const missing = await loadMissingThumbnailBlobUrls({ - rows: [{ id: 'a', thumbnailBlobCid: 'cid:blake3:gone' }], + rows: [{ thumbnailBlobCid: 'cid:blake3:gone' }], blobStore: { get: async () => null }, hasUrl: () => false, - createUrl: () => 'blob:never' + createUrl: () => 'blob:never', + contentTypeFor: thumbnailContentTypeFor }) expect(missing.size).toBe(0) const cancelled = await loadMissingThumbnailBlobUrls({ - rows: [{ id: 'a', thumbnailBlobCid: 'cid:blake3:one' }], + rows: [{ thumbnailBlobCid: 'cid:blake3:one' }], blobStore: { get: async () => new Uint8Array([1]) }, hasUrl: () => false, createUrl: () => 'blob:never', + contentTypeFor: thumbnailContentTypeFor, isCancelled: () => true }) expect(cancelled.size).toBe(0) diff --git a/packages/social/src/enrichment/fetch.ts b/packages/social/src/enrichment/fetch.ts new file mode 100644 index 000000000..9bf0684e0 --- /dev/null +++ b/packages/social/src/enrichment/fetch.ts @@ -0,0 +1,268 @@ +/** + * Where enrichment metadata actually comes from. + * + * Exploration 0170 found that CORS, not product design, dictates this layer: + * YouTube's oEmbed, Instagram and X all refuse a browser fetch, so the hub has + * to proxy them. TikTok is the exception — its oEmbed endpoint sends CORS + * headers, so the client can ask it directly. + * + * That exception is worth taking rather than routing everything through the + * hub for symmetry, because it is the only path that works with **no hub at + * all**. A desktop user with a purely local workspace gets TikTok titles and + * thumbnails; before this, they got nothing. + * + * Thumbnail *bytes* are a separate question from thumbnail *URLs*. The CDN + * that serves them does not send CORS headers, so bytes still need the hub — + * and when there is no hub we keep the URL and let the image element fetch it, + * which is the same request the page would make anyway. + */ + +import type { SocialEnrichmentTarget, SocialUnfurlMetadataPayload } from './targets' + +const TIKTOK_OEMBED_ENDPOINT = 'https://www.tiktok.com/oembed' + +/** Platforms whose oEmbed endpoint is reachable from a browser. */ +export const DIRECT_OEMBED_PLATFORMS: readonly string[] = ['tiktok'] + +export function supportsDirectOEmbed(platform: string): boolean { + return DIRECT_OEMBED_PLATFORMS.includes(platform) +} + +/** The subset of the TikTok oEmbed response we read. */ +type TikTokOEmbedResponse = { + title?: string + author_name?: string + author_url?: string + thumbnail_url?: string + provider_name?: string +} + +export function tiktokOEmbedUrl(contentUrl: string): string { + const url = new URL(TIKTOK_OEMBED_ENDPOINT) + url.searchParams.set('url', contentUrl) + return url.toString() +} + +/** + * Fetch TikTok metadata straight from the client. + * + * Returns the same payload shape the hub's `/unfurl/metadata` produces, so the + * caller does not branch on where the metadata came from — only on whether it + * arrived. + */ +export async function fetchTikTokOEmbed(input: { + target: SocialEnrichmentTarget + fetchImpl?: typeof fetch + signal?: AbortSignal +}): Promise { + const fetchImpl = input.fetchImpl ?? fetch + + let response: Response + try { + response = await fetchImpl( + tiktokOEmbedUrl(input.target.url), + input.signal ? { signal: input.signal } : {} + ) + } catch (error) { + return { + status: 'error', + reason: error instanceof Error ? error.message : 'oEmbed request threw' + } + } + + if (response.status === 404) { + return { status: 'unavailable', reason: 'video not found' } + } + if (!response.ok) { + const blocked = response.status === 403 || response.status === 429 + return { + status: blocked ? 'blocked' : 'error', + reason: `oEmbed request failed with ${response.status}` + } + } + + let body: TikTokOEmbedResponse + try { + body = (await response.json()) as TikTokOEmbedResponse + } catch { + return { status: 'error', reason: 'oEmbed response was not JSON' } + } + + // A 200 with no title is TikTok's way of saying the video is gone or + // private. Treating it as resolved would write an enrichment node that + // renders as a blank card forever. + if (!body.title && !body.thumbnail_url) { + return { status: 'unavailable', reason: 'oEmbed returned no metadata' } + } + + return { + status: 'resolved', + metadata: { + title: body.title ?? null, + description: null, + imageUrl: body.thumbnail_url ?? null, + authorName: body.author_name ?? null, + providerName: body.provider_name ?? 'TikTok', + source: 'oembed', + sourceUrl: input.target.url + } + } +} + +export type EnrichmentFetchResult = { + payload: SocialUnfurlMetadataPayload + thumbnailBlobCid?: string + thumbnailContentType?: string +} + +export type BlobPutSink = { put(data: Uint8Array): Promise } + +/** + * Capture a thumbnail into the blob store through the hub's image proxy. + * + * Returns nothing rather than throwing when the image cannot be captured: a + * missing local copy degrades to loading the URL directly, which is a slower + * card, not a broken one. + */ +async function captureThumbnail(input: { + httpUrl: string + headers: Record + imageUrl: string + blobStore: BlobPutSink + fetchImpl: typeof fetch +}): Promise<{ thumbnailBlobCid?: string; thumbnailContentType?: string }> { + const response = await input + .fetchImpl(`${input.httpUrl}/unfurl/image?url=${encodeURIComponent(input.imageUrl)}`, { + headers: input.headers + }) + .catch(() => null) + if (!response?.ok) return {} + + const bytes = new Uint8Array(await response.arrayBuffer()) + if (bytes.byteLength === 0) return {} + + return { + thumbnailBlobCid: await input.blobStore.put(bytes), + thumbnailContentType: response.headers.get('content-type') ?? undefined + } +} + +export type FetchEnrichmentInput = { + target: SocialEnrichmentTarget + blobStore: BlobPutSink | null + /** Hub base URL. When absent, only direct-oEmbed platforms can resolve. */ + httpUrl?: string + headers?: Record + fetchImpl?: typeof fetch + signal?: AbortSignal +} + +/** + * Resolve metadata for one target. + * + * Direct oEmbed first where it works, hub proxy otherwise. With no hub and no + * direct path the result is an explicit `unavailable` rather than a silent + * no-op, so the queue records the attempt and does not spin on it. + */ +export async function fetchEnrichmentForTarget( + input: FetchEnrichmentInput +): Promise { + const fetchImpl = input.fetchImpl ?? fetch + const headers = input.headers ?? {} + + if (supportsDirectOEmbed(input.target.platform)) { + const payload = await fetchTikTokOEmbed({ + target: input.target, + fetchImpl, + ...(input.signal ? { signal: input.signal } : {}) + }) + + const imageUrl = payload.metadata?.imageUrl + if (payload.status !== 'resolved' || !imageUrl || !input.blobStore || !input.httpUrl) { + return { payload } + } + + return { + payload, + ...(await captureThumbnail({ + httpUrl: input.httpUrl, + headers, + imageUrl, + blobStore: input.blobStore, + fetchImpl + })) + } + } + + if (!input.httpUrl) { + return { + payload: { + status: 'unavailable', + reason: `no hub available to unfurl ${input.target.platform}` + } + } + } + + const response = await fetchImpl( + `${input.httpUrl}/unfurl/metadata?url=${encodeURIComponent(input.target.url)}&provider=${encodeURIComponent(input.target.platform)}`, + { headers } + ) + if (!response.ok) { + throw new Error(`Unfurl request failed with ${response.status}`) + } + const payload = (await response.json()) as SocialUnfurlMetadataPayload + + const imageUrl = payload.metadata?.imageUrl + if (payload.status !== 'resolved' || !imageUrl || !input.blobStore) { + return { payload } + } + + return { + payload, + ...(await captureThumbnail({ + httpUrl: input.httpUrl, + headers, + imageUrl, + blobStore: input.blobStore, + fetchImpl + })) + } +} + +/** + * Materialize object URLs for blob-cached thumbnails that are not in the + * cache yet. Returns the new cid → object URL entries. + */ +export type ThumbnailBlobRow = { + thumbnailBlobCid?: string + metadataJson?: string +} + +export async function loadMissingThumbnailBlobUrls(input: { + rows: readonly ThumbnailBlobRow[] + blobStore: { get(cid: `cid:blake3:${string}`): Promise } + hasUrl: (cid: string) => boolean + createUrl: (blob: Blob) => string + contentTypeFor: (row: ThumbnailBlobRow) => string + limit?: number + isCancelled?: () => boolean +}): Promise> { + const added = new Map() + const missing = input.rows + .filter((row) => row.thumbnailBlobCid && !input.hasUrl(row.thumbnailBlobCid)) + .slice(0, input.limit ?? 200) + + for (const row of missing) { + const cid = row.thumbnailBlobCid as string + if (!cid.startsWith('cid:blake3:') || added.has(cid)) continue + + const bytes = await input.blobStore.get(cid as `cid:blake3:${string}`).catch(() => null) + if (input.isCancelled?.()) return added + if (!bytes) continue + + const blob = new Blob([bytes as BlobPart], { type: input.contentTypeFor(row) }) + added.set(cid, input.createUrl(blob)) + } + + return added +} diff --git a/packages/social/src/enrichment/index.ts b/packages/social/src/enrichment/index.ts new file mode 100644 index 000000000..20ce4b2af --- /dev/null +++ b/packages/social/src/enrichment/index.ts @@ -0,0 +1,35 @@ +/** + * Display enrichment for imported social content (explorations 0170, 0419). + */ + +export { + buildEnrichmentNodeData, + enrichmentTargetForPreview, + feedEnrichmentEntryFor, + hubAuthHeaders, + hubHttpUrlFor, + nextEnrichmentAttempt, + resolveHubAuthToken, + socialEnrichmentKey, + thumbnailContentTypeFor, + type EnrichmentRowLike, + type FeedEnrichmentEntry, + type SocialEnrichmentNodeData, + type SocialEnrichmentPreviewLike, + type SocialEnrichmentTarget, + type SocialUnfurlMetadataPayload +} from './targets' + +export { + fetchEnrichmentForTarget, + fetchTikTokOEmbed, + loadMissingThumbnailBlobUrls, + supportsDirectOEmbed, + tiktokOEmbedUrl, + DIRECT_OEMBED_PLATFORMS, + type BlobPutSink, + type EnrichmentFetchResult, + type FetchEnrichmentInput +} from './fetch' + +export { SocialEnrichmentQueue, DEFAULT_ENRICHMENT_INTERVAL_MS } from './queue' diff --git a/packages/social/src/enrichment/queue.ts b/packages/social/src/enrichment/queue.ts new file mode 100644 index 000000000..6cd0d7b02 --- /dev/null +++ b/packages/social/src/enrichment/queue.ts @@ -0,0 +1,81 @@ +/** + * The enrichment fetch queue. + * + * Session-scoped and paced: each key is attempted at most once per session, so + * a feed that scrolls back over the same rows does not re-ask, and the interval + * keeps provider traffic at a trickle while the first screen still fills in + * within a few seconds. + */ + +import type { SocialEnrichmentTarget } from './targets' + +export const DEFAULT_ENRICHMENT_INTERVAL_MS = 500 + +function defaultDelay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +export class SocialEnrichmentQueue { + private pending: SocialEnrichmentTarget[] = [] + private seen = new Set() + private running = false + private disposed = false + + constructor( + private readonly executor: (target: SocialEnrichmentTarget) => Promise, + private readonly intervalMs = DEFAULT_ENRICHMENT_INTERVAL_MS, + private readonly delayFn: (ms: number) => Promise = defaultDelay, + private readonly random: () => number = Math.random + ) {} + + /** Keys that already have enrichment nodes never enter the queue. */ + markKnown(keys: Iterable): void { + for (const key of keys) this.seen.add(key) + } + + enqueue(targets: readonly SocialEnrichmentTarget[]): void { + if (this.disposed) return + + for (const target of targets) { + if (this.seen.has(target.key)) continue + this.seen.add(target.key) + this.pending.push(target) + } + + void this.pump() + } + + get pendingCount(): number { + return this.pending.length + } + + dispose(): void { + this.disposed = true + this.pending = [] + } + + private async pump(): Promise { + if (this.running) return + this.running = true + + try { + while (!this.disposed && this.pending.length > 0) { + const target = this.pending.shift() + if (!target) break + + try { + await this.executor(target) + } catch { + // The executor records failures on the enrichment node; a key + // that threw stays in `seen` so this session will not retry it. + } + + if (this.pending.length > 0) { + await this.delayFn(this.intervalMs + Math.floor(this.random() * 200)) + } + } + } finally { + this.running = false + } + } +} diff --git a/packages/social/src/enrichment/run-options.test.ts b/packages/social/src/enrichment/run-options.test.ts new file mode 100644 index 000000000..105fdc9c1 --- /dev/null +++ b/packages/social/src/enrichment/run-options.test.ts @@ -0,0 +1,44 @@ +/** + * Per-run transcript consent (exploration 0419). + * + * Lives beside the enrichment tests rather than under `import/` because that + * directory's suites need package subpath resolution the shared pool does not + * provide; these are pure functions over a JSON string. + */ + +import { describe, expect, it } from 'vitest' +import { + parseSocialImportRunOptions, + runWantsTranscripts, + serializeSocialImportRunOptions, + DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS +} from '../import/run-options' + +describe('social import run options', () => { + it('defaults to not fetching transcripts', () => { + expect(DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS.fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions(undefined).fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions(null).fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions('').fetchTranscripts).toBe(false) + }) + + it('round-trips an explicit opt-in', () => { + const json = serializeSocialImportRunOptions({ fetchTranscripts: true }) + expect(parseSocialImportRunOptions(json).fetchTranscripts).toBe(true) + }) + + it('never reads corrupted or coerced values as consent', () => { + expect(parseSocialImportRunOptions('not json').fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions('{"fetchTranscripts":"yes"}').fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions('{"fetchTranscripts":1}').fetchTranscripts).toBe(false) + expect(parseSocialImportRunOptions('{}').fetchTranscripts).toBe(false) + }) + + it('reads consent off a stored run', () => { + expect(runWantsTranscripts({ optionsJson: '{"fetchTranscripts":true}' })).toBe(true) + expect(runWantsTranscripts({ optionsJson: '{"fetchTranscripts":false}' })).toBe(false) + // A run imported before this field existed has not consented. + expect(runWantsTranscripts({})).toBe(false) + expect(runWantsTranscripts(null)).toBe(false) + }) +}) diff --git a/apps/web/src/hooks/social-feed-enrichment.ts b/packages/social/src/enrichment/targets.ts similarity index 51% rename from apps/web/src/hooks/social-feed-enrichment.ts rename to packages/social/src/enrichment/targets.ts index 316c38fcd..eb6150d3f 100644 --- a/apps/web/src/hooks/social-feed-enrichment.ts +++ b/packages/social/src/enrichment/targets.ts @@ -1,10 +1,10 @@ /** - * Pure logic behind useSocialFeedEnrichment: keying previews to - * enrichment nodes, mapping hub unfurl responses onto node properties, - * and a rate-limited fetch queue so feeds enrich visible items first - * without hammering the hub or upstream providers. + * Enrichment targets and the mapping onto stored nodes. + * + * Moved out of `apps/web` (exploration 0419) so the desktop app gets the same + * pipeline instead of a second implementation. Nothing here touches React or + * the DOM — the surfaces differ, the rules do not. */ -import type { SavedViewVisualPreviewModel } from '@xnetjs/react' export type SocialEnrichmentTarget = { key: string @@ -13,6 +13,13 @@ export type SocialEnrichmentTarget = { url: string } +/** The fields an enrichment decision needs from a rendered preview. */ +export type SocialEnrichmentPreviewLike = { + platform?: string | null + platformContentId?: string | null + url?: string | null +} + export type SocialUnfurlMetadataPayload = { status?: string reason?: string @@ -51,10 +58,10 @@ export function socialEnrichmentKey(platform: string, platformContentId: string) /** * A preview is enrichable when it maps to a platform content node with a - * canonical URL the hub can unfurl. + * canonical URL some provider can resolve. */ export function enrichmentTargetForPreview( - preview: Pick + preview: SocialEnrichmentPreviewLike ): SocialEnrichmentTarget | null { if (!preview.platformContentId || !preview.url) return null if (!preview.platform || preview.platform === 'generic') return null @@ -196,163 +203,3 @@ export async function resolveHubAuthToken( export function nextEnrichmentAttempt(existing: { attemptCount?: number } | undefined): number { return (existing?.attemptCount ?? 0) + 1 } - -export type EnrichmentFetchResult = { - payload: SocialUnfurlMetadataPayload - thumbnailBlobCid?: string - thumbnailContentType?: string -} - -/** - * Fetch unfurled metadata for a target through the hub and, when a - * thumbnail is available, capture its bytes into the blob store so the - * feed can render it without any later network access. - */ -export async function fetchEnrichmentForTarget(input: { - httpUrl: string - headers: Record - target: SocialEnrichmentTarget - blobStore: { put(data: Uint8Array): Promise } | null - fetchImpl?: typeof fetch -}): Promise { - const fetchImpl = input.fetchImpl ?? fetch - const response = await fetchImpl( - `${input.httpUrl}/unfurl/metadata?url=${encodeURIComponent(input.target.url)}&provider=${encodeURIComponent(input.target.platform)}`, - { headers: input.headers } - ) - if (!response.ok) { - throw new Error(`Unfurl request failed with ${response.status}`) - } - const payload = (await response.json()) as SocialUnfurlMetadataPayload - - const imageUrl = payload.metadata?.imageUrl - if (payload.status !== 'resolved' || !imageUrl || !input.blobStore) { - return { payload } - } - - const imageResponse = await fetchImpl( - `${input.httpUrl}/unfurl/image?url=${encodeURIComponent(imageUrl)}`, - { headers: input.headers } - ).catch(() => null) - if (!imageResponse?.ok) { - return { payload } - } - - const bytes = new Uint8Array(await imageResponse.arrayBuffer()) - if (bytes.byteLength === 0) { - return { payload } - } - - return { - payload, - thumbnailBlobCid: await input.blobStore.put(bytes), - thumbnailContentType: imageResponse.headers.get('content-type') ?? undefined - } -} - -/** - * Materialize object URLs for blob-cached thumbnails that are not in the - * cache yet. Returns the new cid → object URL entries. - */ -export async function loadMissingThumbnailBlobUrls(input: { - rows: readonly EnrichmentRowLike[] - blobStore: { get(cid: `cid:blake3:${string}`): Promise } - hasUrl: (cid: string) => boolean - createUrl: (blob: Blob) => string - limit?: number - isCancelled?: () => boolean -}): Promise> { - const added = new Map() - const missing = input.rows - .filter((row) => row.thumbnailBlobCid && !input.hasUrl(row.thumbnailBlobCid)) - .slice(0, input.limit ?? 200) - - for (const row of missing) { - const cid = row.thumbnailBlobCid as string - if (!cid.startsWith('cid:blake3:') || added.has(cid)) continue - - const bytes = await input.blobStore.get(cid as `cid:blake3:${string}`).catch(() => null) - if (input.isCancelled?.()) return added - if (!bytes) continue - - const blob = new Blob([bytes as BlobPart], { type: thumbnailContentTypeFor(row) }) - added.set(cid, input.createUrl(blob)) - } - - return added -} - -const DEFAULT_ENRICHMENT_INTERVAL_MS = 500 - -function defaultDelay(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -/** - * Session-scoped fetch queue. Each key is attempted at most once per - * session; pacing keeps provider traffic at a polite trickle while the - * first screen of a feed still enriches within seconds. - */ -export class SocialEnrichmentQueue { - private pending: SocialEnrichmentTarget[] = [] - private seen = new Set() - private running = false - private disposed = false - - constructor( - private readonly executor: (target: SocialEnrichmentTarget) => Promise, - private readonly intervalMs = DEFAULT_ENRICHMENT_INTERVAL_MS, - private readonly delayFn: (ms: number) => Promise = defaultDelay - ) {} - - /** Keys that already have enrichment nodes never enter the queue. */ - markKnown(keys: Iterable): void { - for (const key of keys) this.seen.add(key) - } - - enqueue(targets: readonly SocialEnrichmentTarget[]): void { - if (this.disposed) return - - for (const target of targets) { - if (this.seen.has(target.key)) continue - this.seen.add(target.key) - this.pending.push(target) - } - - void this.pump() - } - - get pendingCount(): number { - return this.pending.length - } - - dispose(): void { - this.disposed = true - this.pending = [] - } - - private async pump(): Promise { - if (this.running) return - this.running = true - - try { - while (!this.disposed && this.pending.length > 0) { - const target = this.pending.shift() - if (!target) break - - try { - await this.executor(target) - } catch { - // The executor records failures on the enrichment node; a key - // that threw stays in `seen` so this session will not retry it. - } - - if (this.pending.length > 0) { - await this.delayFn(this.intervalMs + Math.floor(Math.random() * 200)) - } - } - } finally { - this.running = false - } - } -} diff --git a/packages/social/src/feeds/defaults.ts b/packages/social/src/feeds/defaults.ts index acf63a716..2a9c6babf 100644 --- a/packages/social/src/feeds/defaults.ts +++ b/packages/social/src/feeds/defaults.ts @@ -14,13 +14,16 @@ import { queryOperators, validateSavedViewDescriptor } from '@xnetjs/data' -import { SocialCollectionSchema, SocialContentSchema } from '../schemas' +import { SocialCollectionSchema, SocialContentSchema, SocialInteractionSchema } from '../schemas' export type SocialFeedViewId = | 'social.feed.youtube-videos' | 'social.feed.youtube-playlists' | 'social.feed.instagram-saved' | 'social.feed.instagram-likes' + | 'social.feed.tiktok-videos' + | 'social.feed.tiktok-collections' + | 'social.feed.activity-timeline' export type SocialFeedViewScope = NonNullable @@ -28,7 +31,7 @@ export type SocialFeedViewDefinition = { id: SocialFeedViewId title: string description: string - platform: 'youtube' | 'instagram' + platform: 'youtube' | 'instagram' | 'tiktok' | 'all' descriptor: SavedViewDescriptor savedViewProperties: { title: string @@ -51,6 +54,19 @@ const DEFAULT_FEED_PRESENTATION: SavedViewPresentationHint = { feedDensity: 'cozy' } +/** + * The activity view opens on the time axis rather than the grid. + * + * Interactions carry a timestamp and no thumbnail of their own, so a grid of + * them would be a grid of blank cards; the same records read well as a + * calendar of when things were watched, liked and saved. + */ +const TIMELINE_PRESENTATION: SavedViewPresentationHint = { + mode: 'timeline', + feedLayout: 'list', + feedDensity: 'compact' +} + function page(options: SocialFeedViewOptions) { return { first: options.pageSize ?? DEFAULT_FEED_PAGE_SIZE, count: 'estimate' as const } } @@ -101,6 +117,7 @@ export function createDefaultSocialFeedViews( const scope = options.scope ?? 'workspace' const content = queryOperators<(typeof SocialContentSchema)['_properties']>() const collection = queryOperators<(typeof SocialCollectionSchema)['_properties']>() + const interaction = queryOperators<(typeof SocialInteractionSchema)['_properties']>() return [ defineSocialFeedView({ @@ -160,6 +177,48 @@ export function createDefaultSocialFeedViews( orderBy: { observedAt: 'desc', importedAt: 'desc' }, page: page(options) }) + }), + defineSocialFeedView({ + id: 'social.feed.tiktok-videos', + title: 'TikTok Videos', + description: + 'Videos you liked or added to favourites on TikTok, newest first. TikTok records the like and the favourite as separate acts against the same video, so both land here.', + platform: 'tiktok', + scope, + query: defineNodeQueryAST(SocialContentSchema, { + where: and(content.eq('platform', 'tiktok'), content.eq('contentKind', 'video')), + orderBy: { observedAt: 'desc', publishedAt: 'desc', importedAt: 'desc' }, + page: page(options) + }) + }), + defineSocialFeedView({ + id: 'social.feed.tiktok-collections', + title: 'TikTok Collections', + description: 'Your TikTok favourite folders and topic collections, as browsable cards.', + platform: 'tiktok', + scope, + query: defineNodeQueryAST(SocialCollectionSchema, { + where: collection.eq('platform', 'tiktok'), + orderBy: { title: 'asc', observedAt: 'desc' }, + page: page(options) + }) + }), + defineSocialFeedView({ + id: 'social.feed.activity-timeline', + title: 'Activity Timeline', + description: + 'Everything you watched, liked, saved and bookmarked across platforms, on the time axis. Search history is deliberately absent — it is the most revealing bucket in the archive and the least useful as a calendar.', + platform: 'all', + scope, + presentation: TIMELINE_PRESENTATION, + query: defineNodeQueryAST(SocialInteractionSchema, { + where: and( + interaction.neq('interactionKind', 'search'), + interaction.neq('interactionKind', 'message') + ), + orderBy: { observedAt: 'desc', publishedAt: 'desc', importedAt: 'desc' }, + page: page(options) + }) }) ] } diff --git a/packages/social/src/feeds/tiktok-and-timeline.test.ts b/packages/social/src/feeds/tiktok-and-timeline.test.ts new file mode 100644 index 000000000..53bf99963 --- /dev/null +++ b/packages/social/src/feeds/tiktok-and-timeline.test.ts @@ -0,0 +1,80 @@ +/** + * TikTok feeds and the cross-platform activity timeline (exploration 0419). + */ + +import { describe, expect, it } from 'vitest' +import { createDefaultSocialFeedViews } from './defaults' + +function viewById(id: string) { + return createDefaultSocialFeedViews().find((view) => view.id === id) +} + +/** Collect every `{ field, value }` equality/inequality leaf in a descriptor. */ +function collectPredicates( + descriptor: unknown +): Array<{ op: string; field: string; value: unknown }> { + const found: Array<{ op: string; field: string; value: unknown }> = [] + const visit = (value: unknown): void => { + if (Array.isArray(value)) { + value.forEach(visit) + return + } + if (!value || typeof value !== 'object') return + + const record = value as Record + if (typeof record.op === 'string' && typeof record.field === 'string') { + found.push({ op: record.op, field: record.field, value: record.value }) + } + Object.values(record).forEach(visit) + } + visit(descriptor) + return found +} + +describe('TikTok feed views', () => { + it('seeds a videos feed and a collections feed', () => { + const videos = viewById('social.feed.tiktok-videos') + const collections = viewById('social.feed.tiktok-collections') + + expect(videos?.platform).toBe('tiktok') + expect(collections?.platform).toBe('tiktok') + }) + + it('scopes the videos feed to TikTok video content', () => { + const predicates = collectPredicates(viewById('social.feed.tiktok-videos')?.descriptor) + + expect(predicates).toContainEqual({ op: 'eq', field: 'platform', value: 'tiktok' }) + expect(predicates).toContainEqual({ op: 'eq', field: 'contentKind', value: 'video' }) + }) + + it('opens both TikTok feeds in the thumbnail grid', () => { + for (const id of ['social.feed.tiktok-videos', 'social.feed.tiktok-collections']) { + expect(viewById(id)?.descriptor.presentation).toEqual({ + mode: 'feed', + feedLayout: 'grid', + feedDensity: 'cozy' + }) + } + }) +}) + +describe('activity timeline', () => { + const timeline = viewById('social.feed.activity-timeline') + + it('opens on the time axis rather than as a grid of blank cards', () => { + expect(timeline?.descriptor.presentation?.mode).toBe('timeline') + }) + + it('spans every platform', () => { + const predicates = collectPredicates(timeline?.descriptor) + expect(predicates.some((predicate) => predicate.field === 'platform')).toBe(false) + expect(timeline?.platform).toBe('all') + }) + + it('excludes search history and message interactions', () => { + const predicates = collectPredicates(timeline?.descriptor) + + expect(predicates).toContainEqual({ op: 'neq', field: 'interactionKind', value: 'search' }) + expect(predicates).toContainEqual({ op: 'neq', field: 'interactionKind', value: 'message' }) + }) +}) diff --git a/packages/social/src/import/core.ts b/packages/social/src/import/core.ts index 34438036f..0efc65d8b 100644 --- a/packages/social/src/import/core.ts +++ b/packages/social/src/import/core.ts @@ -15,6 +15,13 @@ export { type SocialCommitSummary } from './commit' export { detectSocialArchive, probeSocialArchive, type SocialArchiveDetection } from './detector' +export { + parseSocialImportRunOptions, + runWantsTranscripts, + serializeSocialImportRunOptions, + DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS, + type SocialImportRunOptions +} from './run-options' export { clearCompletedSocialImportJobs, createSocialImportJob, diff --git a/packages/social/src/import/run-options.ts b/packages/social/src/import/run-options.ts new file mode 100644 index 000000000..fcb9ba6f1 --- /dev/null +++ b/packages/social/src/import/run-options.ts @@ -0,0 +1,50 @@ +/** + * Per-run import options (exploration 0419). + * + * Transcript fetching is a decision about *this archive*, not a global + * preference: consenting to fetch captions for a public YouTube library says + * nothing about whether the same should happen for an archive of private + * uploads. So the answer rides on the import run, and a run that never + * answered reads as "no" rather than as "not set". + */ + +export type SocialImportRunOptions = { + /** Fetch video transcripts for content this run imported. */ + fetchTranscripts: boolean +} + +export const DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS: SocialImportRunOptions = { + fetchTranscripts: false +} + +export function serializeSocialImportRunOptions(options: SocialImportRunOptions): string { + return JSON.stringify(options) +} + +/** + * Read the options off a stored run. + * + * Anything missing or unparseable falls back to the defaults, which are the + * conservative answers — a corrupted field must never read as consent. + */ +export function parseSocialImportRunOptions( + optionsJson: string | null | undefined +): SocialImportRunOptions { + if (!optionsJson) return DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS + + try { + const parsed = JSON.parse(optionsJson) as Partial + return { + fetchTranscripts: parsed?.fetchTranscripts === true + } + } catch { + return DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS + } +} + +/** Whether a stored run opted into transcript fetching. */ +export function runWantsTranscripts( + run: { optionsJson?: string | null } | null | undefined +): boolean { + return parseSocialImportRunOptions(run?.optionsJson).fetchTranscripts +} diff --git a/packages/social/src/import/stage-archive.ts b/packages/social/src/import/stage-archive.ts index 6b7517735..180e1c7fd 100644 --- a/packages/social/src/import/stage-archive.ts +++ b/packages/social/src/import/stage-archive.ts @@ -23,6 +23,11 @@ import { shouldCommitSourceRecordNodes, type SocialImportSourceRecordMode } from './policy' +import { + serializeSocialImportRunOptions, + DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS, + type SocialImportRunOptions +} from './run-options' import { createStagingSummaryAccumulator } from './staging' import { createLargeArchiveStoragePlan } from './storage' import { createSocialImportTelemetryEvents, type SocialImportTelemetryEvent } from './telemetry' @@ -69,6 +74,8 @@ export type SocialImportStageInput = { readTextEntry?: TextArchiveEntryReader buckets?: readonly string[] includeSensitive?: boolean + /** Fetch transcripts for what this run imports (0419). Off unless asked. */ + fetchTranscripts?: boolean importedAt?: string observedBy?: string onProgress?: (progress: SocialImportStageProgress) => void @@ -197,7 +204,8 @@ export async function createSocialImportStagePlan( selectedBuckets, selection: { buckets: selectedBuckets, - includeSensitive: Boolean(input.includeSensitive) + includeSensitive: Boolean(input.includeSensitive), + runOptions: { fetchTranscripts: input.fetchTranscripts === true } }, storagePlan: createLargeArchiveStoragePlan(input.manifest) } @@ -245,6 +253,7 @@ export async function stageSocialArchive( importRunId: plan.importRunId, importedAt: plan.importedAt, selectedBuckets: plan.selectedBuckets, + options: { fetchTranscripts: plan.selection.runOptions?.fetchTranscripts === true }, summary }), records: createSocialImportNodeDrafts(stagedRecords), @@ -319,6 +328,7 @@ export async function* streamSocialImportNodeDrafts( importRunId: plan.importRunId, importedAt: plan.importedAt, selectedBuckets: plan.selectedBuckets, + options: { fetchTranscripts: plan.selection.runOptions?.fetchTranscripts === true }, summary }) yield importRunNode @@ -502,6 +512,8 @@ export function createSocialImportRunDraft(input: { importedAt: string selectedBuckets: string[] summary: StagingSummary + /** Per-run choices such as transcript fetching (0419). */ + options?: SocialImportRunOptions }): SocialImportNodeDraft { return { kind: 'import-run', @@ -519,6 +531,9 @@ export function createSocialImportRunDraft(input: { startedAt: input.importedAt, completedAt: input.importedAt, selectedBucketsJson: JSON.stringify(input.selectedBuckets), + optionsJson: serializeSocialImportRunOptions( + input.options ?? DEFAULT_SOCIAL_IMPORT_RUN_OPTIONS + ), summaryJson: JSON.stringify(input.summary), warningCount: input.summary.totalWarnings, errorCount: 0 diff --git a/packages/social/src/import/types.ts b/packages/social/src/import/types.ts index d5a1d3631..1eb2d75e8 100644 --- a/packages/social/src/import/types.ts +++ b/packages/social/src/import/types.ts @@ -48,6 +48,11 @@ export type ImportProbe = { export type ImportSelection = { buckets?: readonly string[] includeSensitive?: boolean + /** + * Per-run choices recorded on the import run (0419) — currently whether to + * fetch video transcripts for what this archive brings in. + */ + runOptions?: { fetchTranscripts?: boolean } } export type JsonArchiveEntryReader = (path: string) => Promise diff --git a/packages/social/src/index.ts b/packages/social/src/index.ts index 2d47ffaad..12d9aced0 100644 --- a/packages/social/src/index.ts +++ b/packages/social/src/index.ts @@ -10,6 +10,9 @@ export * from './patterns' export * from './projection' export * from './publish' export * from './schemas' +export * from './transcripts' +export * from './retrieval' +export * from './enrichment' export * from './connect' export * from './views' export * from './workspace' diff --git a/packages/social/src/retrieval/context-packs.ts b/packages/social/src/retrieval/context-packs.ts new file mode 100644 index 000000000..b6dc48da0 --- /dev/null +++ b/packages/social/src/retrieval/context-packs.ts @@ -0,0 +1,105 @@ +/** + * Seeded context packs over the social graph (exploration 0419). + * + * `xnet_create_context_pack` already takes a query, a seed list and a budget. + * What was missing was anything to point it at: an agent asked "what has this + * person been watching about X" had to invent a query shape, guess which + * schemas mattered, and hope it did not walk into a DM. + * + * These are those questions, written down once — each with the retrieval scope + * it runs under, so the pack's boundary travels with the pack rather than + * being re-decided at every call site. + */ + +import type { SocialRetrievalScope } from './scope' +import { SocialContentSchema } from '../schemas' +import { createSocialRetrievalScope, SOCIAL_RETRIEVAL_SCOPE } from './scope' + +export type SocialContextPackId = 'social.pack.saved-library' | 'social.pack.watched-transcripts' + +/** A seed resource, matching the `xnet_create_context_pack` seed shape. */ +export type SocialContextPackSeed = { + kind: 'node' | 'saved-view' | 'schema' + id: string +} + +export type SocialContextPackDefinition = { + id: SocialContextPackId + title: string + /** What the pack is for, in the words an agent would use to pick it. */ + description: string + /** Default query text; callers normally replace or extend this. */ + query: string + seeds: SocialContextPackSeed[] + /** Maximum resources to pull. */ + limit: number + scope: SocialRetrievalScope +} + +export type SocialContextPackOptions = { + /** Search text to use instead of the pack's default. */ + query?: string + limit?: number + /** Widen the scope. Off by default for every pack. */ + scope?: SocialRetrievalScope +} + +const DEFAULT_PACK_LIMIT = 40 + +/** + * The default packs. + * + * Two, deliberately. A long menu of near-identical packs is a worse interface + * than two that clearly differ: one is "things I chose to keep", the other is + * "things that were said in videos I watched". + */ +export function createDefaultSocialContextPacks( + options: SocialContextPackOptions = {} +): SocialContextPackDefinition[] { + const scope = options.scope ?? SOCIAL_RETRIEVAL_SCOPE + const limit = options.limit ?? DEFAULT_PACK_LIMIT + + return [ + { + id: 'social.pack.saved-library', + title: 'Saved library', + description: + 'Posts, videos and links the user saved, liked or bookmarked across platforms, with the collections they were filed under.', + query: options.query ?? '', + seeds: [{ kind: 'schema', id: SocialContentSchema._schemaId }], + limit, + scope + }, + { + id: 'social.pack.watched-transcripts', + title: 'Watched transcripts', + description: + 'Spoken content from videos the user saved or watched, for answering what was actually said rather than what a title claims.', + query: options.query ?? '', + seeds: [{ kind: 'schema', id: SocialContentSchema._schemaId }], + limit, + scope + } + ] +} + +/** + * A pack that may read direct messages. + * + * Not in the default list, and it takes an explicit call to build — the point + * is that widening the boundary is a visible act, not a flag someone flips. + */ +export function createSensitiveSocialContextPack( + options: SocialContextPackOptions = {} +): SocialContextPackDefinition { + return { + id: 'social.pack.saved-library', + title: 'Saved library, including messages', + description: + 'The saved library widened to include direct messages and conversations. Only for an explicit, user-initiated request.', + query: options.query ?? '', + seeds: [{ kind: 'schema', id: SocialContentSchema._schemaId }], + limit: options.limit ?? DEFAULT_PACK_LIMIT, + scope: createSocialRetrievalScope({ includeMessages: true }) + } +} diff --git a/packages/social/src/retrieval/index.ts b/packages/social/src/retrieval/index.ts new file mode 100644 index 000000000..6cee9c43e --- /dev/null +++ b/packages/social/src/retrieval/index.ts @@ -0,0 +1,28 @@ +/** + * Agent-facing retrieval boundary for imported social data (exploration 0419). + */ + +export { + createSocialRetrievalScope, + filterSocialRetrievalCandidates, + isSocialNodeRetrievable, + socialRetrievalDecision, + DEFAULT_SOCIAL_RETRIEVAL_SCHEMA_IDS, + EXCLUDED_SOCIAL_PRIVACY_CLASSES, + SENSITIVE_SOCIAL_INTERACTION_KINDS, + SENSITIVE_SOCIAL_RETRIEVAL_SCHEMA_IDS, + SOCIAL_RETRIEVAL_SCOPE, + type SocialRetrievalCandidate, + type SocialRetrievalDecision, + type SocialRetrievalScope, + type SocialRetrievalScopeOptions +} from './scope' + +export { + createDefaultSocialContextPacks, + createSensitiveSocialContextPack, + type SocialContextPackDefinition, + type SocialContextPackId, + type SocialContextPackOptions, + type SocialContextPackSeed +} from './context-packs' diff --git a/packages/social/src/retrieval/retrieval.test.ts b/packages/social/src/retrieval/retrieval.test.ts new file mode 100644 index 000000000..b651aacb5 --- /dev/null +++ b/packages/social/src/retrieval/retrieval.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { + SocialContentSchema, + SocialConversationSchema, + SocialInteractionSchema, + SocialMessageSchema +} from '../schemas' +import { + createDefaultSocialContextPacks, + createSensitiveSocialContextPack, + createSocialRetrievalScope, + filterSocialRetrievalCandidates, + isSocialNodeRetrievable, + socialRetrievalDecision, + SOCIAL_RETRIEVAL_SCOPE +} from './index' + +describe('social retrieval scope', () => { + it('admits content, collections and interactions by default', () => { + expect(isSocialNodeRetrievable({ schemaId: SocialContentSchema._schemaId })).toBe(true) + expect( + isSocialNodeRetrievable({ + schemaId: SocialInteractionSchema._schemaId, + interactionKind: 'bookmark' + }) + ).toBe(true) + }) + + it('excludes direct messages and conversations by default', () => { + expect(socialRetrievalDecision({ schemaId: SocialMessageSchema._schemaId })).toEqual({ + eligible: false, + reason: 'schema-not-in-scope' + }) + expect(socialRetrievalDecision({ schemaId: SocialConversationSchema._schemaId }).eligible).toBe( + false + ) + }) + + it('excludes search history without excluding the rest of the interactions', () => { + const search = { + schemaId: SocialInteractionSchema._schemaId, + interactionKind: 'search' + } + expect(socialRetrievalDecision(search)).toEqual({ + eligible: false, + reason: 'interaction-kind-excluded' + }) + expect( + isSocialNodeRetrievable({ + schemaId: SocialInteractionSchema._schemaId, + interactionKind: 'like' + }) + ).toBe(true) + }) + + it('excludes account-security, billing and ad-targeting privacy classes', () => { + for (const privacyClass of ['third-party-private', 'account-security', 'billing', 'ads']) { + expect( + socialRetrievalDecision({ schemaId: SocialContentSchema._schemaId, privacyClass }) + ).toEqual({ eligible: false, reason: 'privacy-class-excluded' }) + } + expect( + isSocialNodeRetrievable({ schemaId: SocialContentSchema._schemaId, privacyClass: 'public' }) + ).toBe(true) + }) + + it('widens only when asked, and says that it was widened', () => { + expect(SOCIAL_RETRIEVAL_SCOPE.includesSensitive).toBe(false) + + const widened = createSocialRetrievalScope({ includeMessages: true }) + expect(widened.includesSensitive).toBe(true) + expect(isSocialNodeRetrievable({ schemaId: SocialMessageSchema._schemaId }, widened)).toBe(true) + + const withSearch = createSocialRetrievalScope({ includeSearchHistory: true }) + expect( + isSocialNodeRetrievable( + { schemaId: SocialInteractionSchema._schemaId, interactionKind: 'search' }, + withSearch + ) + ).toBe(true) + // Widening one axis must not widen the other. + expect(isSocialNodeRetrievable({ schemaId: SocialMessageSchema._schemaId }, withSearch)).toBe( + false + ) + }) + + it('tallies exclusions by reason so a surface can report them', () => { + const result = filterSocialRetrievalCandidates([ + { schemaId: SocialContentSchema._schemaId }, + { schemaId: SocialContentSchema._schemaId, privacyClass: 'billing' }, + { schemaId: SocialMessageSchema._schemaId }, + { schemaId: SocialInteractionSchema._schemaId, interactionKind: 'search' } + ]) + + expect(result.eligible).toHaveLength(1) + expect(result.excluded).toEqual({ + 'schema-not-in-scope': 1, + 'privacy-class-excluded': 1, + 'interaction-kind-excluded': 1 + }) + }) +}) + +describe('social context packs', () => { + it('ships two packs, both on the conservative scope', () => { + const packs = createDefaultSocialContextPacks() + expect(packs.map((pack) => pack.id)).toEqual([ + 'social.pack.saved-library', + 'social.pack.watched-transcripts' + ]) + for (const pack of packs) { + expect(pack.scope.includesSensitive).toBe(false) + expect(pack.limit).toBeGreaterThan(0) + } + }) + + it('carries the caller query and limit through', () => { + const [pack] = createDefaultSocialContextPacks({ query: 'fermentation', limit: 5 }) + expect(pack?.query).toBe('fermentation') + expect(pack?.limit).toBe(5) + }) + + it('keeps the message-reading pack out of the default list', () => { + const sensitive = createSensitiveSocialContextPack() + expect(sensitive.scope.includesSensitive).toBe(true) + expect(createDefaultSocialContextPacks().some((pack) => pack.scope.includesSensitive)).toBe( + false + ) + }) +}) diff --git a/packages/social/src/retrieval/scope.ts b/packages/social/src/retrieval/scope.ts new file mode 100644 index 000000000..228ec3d78 --- /dev/null +++ b/packages/social/src/retrieval/scope.ts @@ -0,0 +1,217 @@ +/** + * What an agent may read from the social graph (exploration 0419). + * + * Importing a decade of social archives and then pointing a retriever at the + * whole thing is not one feature, it is two — and the second one is the + * dangerous half. Exploration 0379 named the shape of it: better retrieval + * widens the egress hole, because the same index that answers "what did I save + * about fermentation" will just as happily answer with a private message. + * + * So the scope is an allowlist, not a denylist, and it is expressed over two + * independent axes that both have to pass: + * + * - **Schema** — direct messages and conversations are out. Not because the + * text is uninteresting, but because a DM is someone else's words held in + * confidence, and no default should put them in a prompt. + * - **Privacy class** — the importer already labels every record, and the + * classes it uses for account security, billing and ad targeting describe + * data that has no business being context for anything. + * + * Both axes are widenable by explicit opt-in, and nothing here reads a config + * file or an environment variable: a caller that wants private material has to + * ask for it in code the user can see. + * + * This is deliberately *not* the `RetrievalProfile` node from exploration 0415. + * That holds four tuning weights — how far to walk, how much to trust the + * vector side. This holds eligibility. Collapsing them would mean a knob + * labelled "make search better" could quietly change what search is allowed to + * see. + */ + +import { + SocialActorSchema, + SocialCollectionItemSchema, + SocialCollectionSchema, + SocialContentSchema, + SocialConversationSchema, + SocialEnrichmentSchema, + SocialInteractionSchema, + SocialMessageSchema +} from '../schemas' + +/** + * Schemas an agent may retrieve from by default. + * + * Content (including transcripts), the collections that give it structure, the + * actors who made it, the interactions that record what the user did with it, + * and the enrichment that makes it legible. + */ +export const DEFAULT_SOCIAL_RETRIEVAL_SCHEMA_IDS: readonly string[] = [ + SocialContentSchema._schemaId, + SocialCollectionSchema._schemaId, + SocialCollectionItemSchema._schemaId, + SocialActorSchema._schemaId, + SocialInteractionSchema._schemaId, + SocialEnrichmentSchema._schemaId +] + +/** + * Schemas held back unless a caller opts in. + * + * Messages and conversations carry other people's words; the import machinery + * (archives, runs, jobs, raw source records) is bookkeeping that would only + * dilute a context pack. + */ +export const SENSITIVE_SOCIAL_RETRIEVAL_SCHEMA_IDS: readonly string[] = [ + SocialMessageSchema._schemaId, + SocialConversationSchema._schemaId +] + +/** Privacy classes never eligible by default. */ +export const EXCLUDED_SOCIAL_PRIVACY_CLASSES: readonly string[] = [ + 'third-party-private', + 'account-security', + 'billing', + 'ads' +] + +/** + * Interaction kinds held back by default. + * + * A search history is a list of things the user typed when they thought no one + * was reading, which makes it the single most revealing bucket in the archive + * and the one least likely to be wanted as context. + */ +export const SENSITIVE_SOCIAL_INTERACTION_KINDS: readonly string[] = ['search', 'message'] + +export type SocialRetrievalScope = { + /** Schema ids eligible for retrieval. */ + schemaIds: readonly string[] + /** Privacy classes that disqualify a node regardless of schema. */ + excludedPrivacyClasses: readonly string[] + /** Interaction kinds that disqualify a `SocialInteraction`. */ + excludedInteractionKinds: readonly string[] + /** True when the scope was widened past the defaults. */ + includesSensitive: boolean +} + +export type SocialRetrievalScopeOptions = { + /** + * Include direct messages and conversations. + * + * Off by default, and the parameter is named for what it costs rather than + * what it enables. + */ + includeMessages?: boolean + /** Include search history and message interactions. */ + includeSearchHistory?: boolean + /** Privacy classes to admit despite the default exclusion. */ + allowPrivacyClasses?: readonly string[] +} + +/** + * Build a retrieval scope. + * + * Called with no arguments this is the conservative default, which is the only + * scope any automatic caller should use. + */ +export function createSocialRetrievalScope( + options: SocialRetrievalScopeOptions = {} +): SocialRetrievalScope { + const allowed = new Set(options.allowPrivacyClasses ?? []) + const schemaIds = [ + ...DEFAULT_SOCIAL_RETRIEVAL_SCHEMA_IDS, + ...(options.includeMessages ? SENSITIVE_SOCIAL_RETRIEVAL_SCHEMA_IDS : []) + ] + + return { + schemaIds, + excludedPrivacyClasses: EXCLUDED_SOCIAL_PRIVACY_CLASSES.filter( + (privacyClass) => !allowed.has(privacyClass) + ), + excludedInteractionKinds: options.includeSearchHistory + ? [] + : SENSITIVE_SOCIAL_INTERACTION_KINDS, + includesSensitive: Boolean( + options.includeMessages || options.includeSearchHistory || allowed.size > 0 + ) + } +} + +/** The default scope, precomputed for the common case. */ +export const SOCIAL_RETRIEVAL_SCOPE: SocialRetrievalScope = createSocialRetrievalScope() + +/** The minimum a node must carry for an eligibility decision. */ +export type SocialRetrievalCandidate = { + schemaId: string + privacyClass?: string + interactionKind?: string +} + +export type SocialRetrievalDecision = { + eligible: boolean + /** Why it was excluded; absent when eligible. */ + reason?: 'schema-not-in-scope' | 'privacy-class-excluded' | 'interaction-kind-excluded' +} + +/** + * Decide whether one node may be retrieved, and say why when it may not. + * + * The reason is part of the return value rather than a log line because the + * only honest way to show a user what their agent can see is to be able to + * explain each exclusion. + */ +export function socialRetrievalDecision( + candidate: SocialRetrievalCandidate, + scope: SocialRetrievalScope = SOCIAL_RETRIEVAL_SCOPE +): SocialRetrievalDecision { + if (!scope.schemaIds.includes(candidate.schemaId)) { + return { eligible: false, reason: 'schema-not-in-scope' } + } + if (candidate.privacyClass && scope.excludedPrivacyClasses.includes(candidate.privacyClass)) { + return { eligible: false, reason: 'privacy-class-excluded' } + } + if ( + candidate.interactionKind && + scope.excludedInteractionKinds.includes(candidate.interactionKind) + ) { + return { eligible: false, reason: 'interaction-kind-excluded' } + } + + return { eligible: true } +} + +/** Convenience predicate over {@link socialRetrievalDecision}. */ +export function isSocialNodeRetrievable( + candidate: SocialRetrievalCandidate, + scope: SocialRetrievalScope = SOCIAL_RETRIEVAL_SCOPE +): boolean { + return socialRetrievalDecision(candidate, scope).eligible +} + +/** + * Filter a candidate list, returning what passed and a per-reason tally of + * what did not. + * + * The tally exists so a surface can say "412 items, 9 held back as private" + * instead of silently returning 403. + */ +export function filterSocialRetrievalCandidates( + candidates: readonly T[], + scope: SocialRetrievalScope = SOCIAL_RETRIEVAL_SCOPE +): { eligible: T[]; excluded: Record, number> } { + const excluded = { + 'schema-not-in-scope': 0, + 'privacy-class-excluded': 0, + 'interaction-kind-excluded': 0 + } + const eligible: T[] = [] + + for (const candidate of candidates) { + const decision = socialRetrievalDecision(candidate, scope) + if (decision.eligible) eligible.push(candidate) + else if (decision.reason) excluded[decision.reason] += 1 + } + + return { eligible, excluded } +} diff --git a/packages/social/src/schemas/import.ts b/packages/social/src/schemas/import.ts index a1a540a69..8479d9ff7 100644 --- a/packages/social/src/schemas/import.ts +++ b/packages/social/src/schemas/import.ts @@ -43,6 +43,15 @@ export const SocialImportRunSchema = defineSchema({ startedAt: date({ required: true, includeTime: true }), completedAt: date({ includeTime: true }), selectedBucketsJson: text({ maxLength: 20000 }), + /** + * Per-run options the user chose, as JSON (exploration 0419). + * + * Transcript fetching is opt-in per run rather than a global setting, + * because the choice is about a specific archive: saying yes to captions + * for a public YouTube library is a different decision from saying yes for + * an archive full of private uploads. + */ + optionsJson: text({ maxLength: 5000 }), summaryJson: text({ maxLength: 50000 }), warningCount: number({ min: 0, integer: true }), errorCount: number({ min: 0, integer: true }) diff --git a/packages/social/src/transcripts/index.ts b/packages/social/src/transcripts/index.ts new file mode 100644 index 000000000..33f15268a --- /dev/null +++ b/packages/social/src/transcripts/index.ts @@ -0,0 +1,53 @@ +/** + * Video transcripts for imported social content (exploration 0419). + */ + +export { + enrichmentStatusForTranscriptOutcome, + isRetryableTranscriptStatus, + type TranscriptCue, + type TranscriptFetchOutcome, + type TranscriptFetchStatus, + type TranscriptFetcher, + type TranscriptTarget +} from './types' + +export { + createYouTubeTranscriptFetcher, + parseYouTubeJson3Transcript, + parseYouTubeTranscript, + parseYouTubeXmlTranscript, + youTubeTimedTextUrl, + type YouTubeTimedTextFormat, + type YouTubeTranscriptFetcherOptions +} from './youtube' + +export { + createSocialTranscriptId, + createTranscriptContentDrafts, + segmentTranscript, + transcriptText, + TRANSCRIPT_SEGMENT_MAX_CHARS, + type TranscriptContentDraft, + type TranscriptContentDraftInput, + type TranscriptSegment +} from './nodes' + +export { + describeTranscriptRun, + summarizeTranscriptRun, + transcriptStateForEnrichmentStatus, + transcriptStateForOutcome, + TRANSCRIPT_TARGET_STATES, + type TranscriptRunSummary, + type TranscriptTargetState +} from './states' + +export { + runTranscriptFetchPass, + DEFAULT_MAX_CONSECUTIVE_BLOCKS, + DEFAULT_TRANSCRIPT_INTERVAL_MS, + DEFAULT_TRANSCRIPT_JITTER_MS, + type TranscriptPassOptions, + type TranscriptPassResult +} from './schedule' diff --git a/packages/social/src/transcripts/nodes.ts b/packages/social/src/transcripts/nodes.ts new file mode 100644 index 000000000..867ffd905 --- /dev/null +++ b/packages/social/src/transcripts/nodes.ts @@ -0,0 +1,231 @@ +/** + * Turning fetched captions into stored nodes. + * + * A transcript is a `SocialContent` node with `contentKind: 'transcript'`, + * linked back to the video it describes — not a field on the enrichment node. + * Three reasons, in order of how much they matter: + * + * 1. It earns its own row in the full-text index, so an agent searching for a + * phrase lands on the transcript and can walk one relation to the video. + * 2. `searchText` is capped, and transcripts routinely exceed the cap. A field + * would have to truncate; a node can segment. + * 3. `transcript` was already in the content vocabulary — this is the kind + * finally having a producer, not a new concept. + * + * Segmentation is on cue boundaries, never mid-sentence, and every segment + * records its index and the total so a partial write is visibly partial. + */ + +import type { TranscriptCue } from './types' +import { createSocialNodeId } from '../import/ids' + +/** + * Characters of transcript text per node. + * + * `SocialContent.searchText` caps at 20 000; this leaves headroom so a segment + * that lands slightly over a cue boundary still fits without truncation. + */ +export const TRANSCRIPT_SEGMENT_MAX_CHARS = 18_000 + +/** `SocialContent.textPreview` cap. */ +const TEXT_PREVIEW_MAX_CHARS = 5_000 + +/** `SocialContent.title` cap. */ +const TITLE_MAX_CHARS = 1_000 + +export type TranscriptSegment = { + index: number + startMs: number + endMs: number + text: string + cueCount: number +} + +export type TranscriptContentDraft = { + /** Deterministic node id — the same transcript upserts, never duplicates. */ + id: string + properties: { + platform: string + contentKind: 'transcript' + platformContentKind: string + platformContentId: string + canonicalUrl?: string + parentContent?: string + title: string + textPreview: string + searchText: string + language?: string + publishedAt?: number + observedAt?: number + importedAt: number + privacyClass: string + visibility: string + metadataJson: string + } +} + +/** + * Deterministic id for one transcript segment. + * + * Keyed by platform, video and segment index so a re-fetch of the same video + * lands on the same nodes rather than accumulating copies beside them. + */ +export function createSocialTranscriptId( + platform: string, + platformContentId: string, + segmentIndex = 0 +): string { + return createSocialNodeId('content', [platform, 'transcript', platformContentId, segmentIndex]) +} + +/** Join cues into one continuous string, collapsing the gaps between them. */ +export function transcriptText(cues: readonly TranscriptCue[]): string { + return cues + .map((cue) => cue.text.trim()) + .filter(Boolean) + .join(' ') + .replace(/\s+/g, ' ') + .trim() +} + +/** + * Split cues into segments that each fit in one node. + * + * Splits happen between cues, so no segment ever ends mid-word. A single cue + * longer than the budget gets its own oversized segment rather than being cut — + * losing words to make a number fit is exactly the silent truncation this + * pipeline is supposed to avoid. + */ +export function segmentTranscript( + cues: readonly TranscriptCue[], + maxChars = TRANSCRIPT_SEGMENT_MAX_CHARS +): TranscriptSegment[] { + const usable = cues.filter((cue) => cue.text.trim().length > 0) + if (usable.length === 0) return [] + + const segments: TranscriptSegment[] = [] + let current: TranscriptCue[] = [] + let currentLength = 0 + + const flush = (): void => { + if (current.length === 0) return + + const first = current[0] + const last = current[current.length - 1] + if (!first || !last) return + + segments.push({ + index: segments.length, + startMs: first.startMs, + endMs: last.startMs + last.durationMs, + text: transcriptText(current), + cueCount: current.length + }) + current = [] + currentLength = 0 + } + + for (const cue of usable) { + const text = cue.text.trim() + const addition = currentLength === 0 ? text.length : text.length + 1 + + if (currentLength > 0 && currentLength + addition > maxChars) { + flush() + } + + current.push(cue) + currentLength += currentLength === 0 ? text.length : text.length + 1 + } + + flush() + return segments +} + +function clamp(value: string, max: number): string { + return value.length <= max ? value : value.slice(0, max) +} + +function assignDefined( + target: T, + key: K, + value: T[K] | undefined +): void { + if (value !== undefined) target[key] = value +} + +export type TranscriptContentDraftInput = { + platform: string + platformContentId: string + /** Node id of the video this transcript describes. */ + videoNodeId?: string + /** Title of the video, used to name the transcript node. */ + videoTitle?: string + canonicalUrl?: string + cues: readonly TranscriptCue[] + language?: string + autoGenerated?: boolean + /** Epoch ms the fetch completed. */ + fetchedAtMs: number + publishedAtMs?: number + /** + * Privacy class for the stored transcript. Defaults to `public`, which is + * correct for a published video's captions and wrong for anything else — + * callers importing from a private source must say so. + */ + privacyClass?: string +} + +/** + * Build the node drafts for one video's transcript. + * + * Returns one draft per segment, in order. An empty cue list returns `[]` + * rather than an empty node: "we fetched nothing" is not a transcript. + */ +export function createTranscriptContentDrafts( + input: TranscriptContentDraftInput +): TranscriptContentDraft[] { + const segments = segmentTranscript(input.cues) + if (segments.length === 0) return [] + + const baseTitle = input.videoTitle?.trim() || `${input.platform} ${input.platformContentId}` + + return segments.map((segment) => { + const suffix = segments.length > 1 ? ` (part ${segment.index + 1}/${segments.length})` : '' + const title = clamp(`Transcript — ${baseTitle}${suffix}`, TITLE_MAX_CHARS) + + const properties: TranscriptContentDraft['properties'] = { + platform: input.platform, + contentKind: 'transcript', + platformContentKind: input.autoGenerated ? 'transcript-asr' : 'transcript', + platformContentId: input.platformContentId, + title, + textPreview: clamp(segment.text, TEXT_PREVIEW_MAX_CHARS), + searchText: segment.text, + importedAt: input.fetchedAtMs, + observedAt: input.fetchedAtMs, + privacyClass: input.privacyClass ?? 'public', + visibility: 'private', + metadataJson: JSON.stringify({ + transcript: { + segmentIndex: segment.index, + segmentCount: segments.length, + startMs: segment.startMs, + endMs: segment.endMs, + cueCount: segment.cueCount, + autoGenerated: input.autoGenerated ?? false, + ...(input.language ? { language: input.language } : {}) + } + }) + } + + assignDefined(properties, 'canonicalUrl', input.canonicalUrl) + assignDefined(properties, 'parentContent', input.videoNodeId) + assignDefined(properties, 'language', input.language) + assignDefined(properties, 'publishedAt', input.publishedAtMs) + + return { + id: createSocialTranscriptId(input.platform, input.platformContentId, segment.index), + properties + } + }) +} diff --git a/packages/social/src/transcripts/schedule.ts b/packages/social/src/transcripts/schedule.ts new file mode 100644 index 000000000..2e5ff916c --- /dev/null +++ b/packages/social/src/transcripts/schedule.ts @@ -0,0 +1,112 @@ +/** + * Pacing a transcript pass. + * + * A watch history is thousands of videos. Fetched flat out from one address + * that is a scrape; fetched at a human pace it is a person catching up on + * their own library. The interval is the feature, not an apology for one. + * + * The other half is knowing when to stop. Once the endpoint starts refusing, + * every further request is both useless and evidence against the user, so a + * run gives up after a short streak of refusals — and reports itself as + * incomplete, with the untried targets counted, rather than presenting a + * truncated pass as a finished one. + */ + +import type { TranscriptFetchOutcome, TranscriptFetcher, TranscriptTarget } from './types' +import type { TranscriptRunSummary, TranscriptTargetState } from './states' +import { summarizeTranscriptRun, transcriptStateForOutcome } from './states' + +/** Milliseconds between attempts, before jitter. */ +export const DEFAULT_TRANSCRIPT_INTERVAL_MS = 1_500 + +/** Extra milliseconds, drawn per attempt, so the cadence is not a metronome. */ +export const DEFAULT_TRANSCRIPT_JITTER_MS = 1_000 + +/** Consecutive refusals after which a pass stops rather than pressing on. */ +export const DEFAULT_MAX_CONSECUTIVE_BLOCKS = 3 + +export type TranscriptPassResult = { + summary: TranscriptRunSummary + /** Targets that were refused or failed, in the order they were attempted. */ + retryable: TranscriptTarget[] + /** True when the pass gave up early on a streak of refusals. */ + stoppedEarly: boolean +} + +export type TranscriptPassOptions = { + targets: readonly TranscriptTarget[] + fetcher: TranscriptFetcher + /** Called for each attempt, in order. Errors here abort the pass. */ + onResult: (target: TranscriptTarget, outcome: TranscriptFetchOutcome) => void | Promise + intervalMs?: number + jitterMs?: number + maxConsecutiveBlocks?: number + /** Injected for tests; defaults to `setTimeout`. */ + delayFn?: (ms: number) => Promise + /** Injected for tests; defaults to `Math.random`. */ + random?: () => number + signal?: AbortSignal +} + +function defaultDelay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +/** + * Run one transcript pass over a target list. + * + * Unsupported targets are skipped without an attempt and stay `not-attempted`, + * which is the honest reading: we did not look, because this fetcher cannot. + */ +export async function runTranscriptFetchPass( + options: TranscriptPassOptions +): Promise { + const intervalMs = options.intervalMs ?? DEFAULT_TRANSCRIPT_INTERVAL_MS + const jitterMs = options.jitterMs ?? DEFAULT_TRANSCRIPT_JITTER_MS + const maxConsecutiveBlocks = options.maxConsecutiveBlocks ?? DEFAULT_MAX_CONSECUTIVE_BLOCKS + const delayFn = options.delayFn ?? defaultDelay + const random = options.random ?? Math.random + + const states: TranscriptTargetState[] = [] + const retryable: TranscriptTarget[] = [] + let consecutiveBlocks = 0 + let stoppedEarly = false + let attempted = 0 + + for (const target of options.targets) { + if (options.signal?.aborted) { + stoppedEarly = true + break + } + if (!options.fetcher.supports(target)) continue + + if (attempted > 0) { + await delayFn(intervalMs + Math.floor(random() * jitterMs)) + } + attempted += 1 + + const outcome = await options.fetcher.fetch(target, options.signal) + states.push(transcriptStateForOutcome(outcome.status)) + await options.onResult(target, outcome) + + if (outcome.status === 'blocked' || outcome.status === 'error') { + retryable.push(target) + } + + if (outcome.status === 'blocked') { + consecutiveBlocks += 1 + if (consecutiveBlocks >= maxConsecutiveBlocks) { + stoppedEarly = true + break + } + } else { + consecutiveBlocks = 0 + } + } + + return { + summary: summarizeTranscriptRun({ states, total: options.targets.length }), + retryable, + stoppedEarly + } +} diff --git a/packages/social/src/transcripts/states.ts b/packages/social/src/transcripts/states.ts new file mode 100644 index 000000000..bef44f180 --- /dev/null +++ b/packages/social/src/transcripts/states.ts @@ -0,0 +1,125 @@ +/** + * Counting what a transcript run actually did. + * + * The rule this module exists to enforce: a run is only "complete" when every + * target it was given is accounted for by exactly one terminal state. A pass + * that quietly stopped at video 40 of 200 must not read the same as one that + * finished, and a library that was rate-limited must not read the same as a + * library of videos that genuinely have no captions. + */ + +import type { TranscriptFetchStatus } from './types' + +/** + * The state a single target is in. + * + * `not-attempted` is a first-class value, not the absence of one. It is what + * separates "we have not looked yet" from "we looked and found nothing". + */ +export type TranscriptTargetState = + | 'not-attempted' + | 'fetched' + | 'no-captions' + | 'blocked' + | 'error' + +export const TRANSCRIPT_TARGET_STATES: readonly TranscriptTargetState[] = [ + 'not-attempted', + 'fetched', + 'no-captions', + 'blocked', + 'error' +] + +export type TranscriptRunSummary = { + /** How many targets the run was given. */ + total: number + notAttempted: number + fetched: number + noCaptions: number + blocked: number + errored: number + /** Every target reached a terminal state. */ + complete: boolean + /** + * A retry could change the outcome — something was refused or failed, as + * opposed to simply having no captions. + */ + retryable: boolean +} + +/** Map a stored `SocialEnrichment.status` back to a target state. */ +export function transcriptStateForEnrichmentStatus( + status: string | undefined +): TranscriptTargetState { + switch (status) { + case 'resolved': + return 'fetched' + case 'unavailable': + return 'no-captions' + case 'blocked': + return 'blocked' + case 'error': + return 'error' + default: + return 'not-attempted' + } +} + +/** Map a live fetch outcome to a target state. */ +export function transcriptStateForOutcome(status: TranscriptFetchStatus): TranscriptTargetState { + return status === 'fetched' ? 'fetched' : status +} + +/** + * Summarize a run. + * + * `total` is the size of the target list the run was handed, which may exceed + * the number of states supplied — anything unaccounted for counts as + * `not-attempted` rather than vanishing from the tally. That is what makes the + * counts sum to `total` and makes an interrupted run visible as one. + */ +export function summarizeTranscriptRun(input: { + states: readonly TranscriptTargetState[] + total?: number +}): TranscriptRunSummary { + const counts: Record = { + 'not-attempted': 0, + fetched: 0, + 'no-captions': 0, + blocked: 0, + error: 0 + } + + for (const state of input.states) counts[state] += 1 + + const total = Math.max(input.total ?? input.states.length, input.states.length) + counts['not-attempted'] += total - input.states.length + + return { + total, + notAttempted: counts['not-attempted'], + fetched: counts.fetched, + noCaptions: counts['no-captions'], + blocked: counts.blocked, + errored: counts.error, + complete: counts['not-attempted'] === 0, + retryable: counts.blocked > 0 || counts.error > 0 + } +} + +/** + * A one-line, human-readable account of a run. + * + * Written to be quotable in a status panel: it always names the incomplete and + * blocked cases rather than rounding them away. + */ +export function describeTranscriptRun(summary: TranscriptRunSummary): string { + const parts = [`${summary.fetched} fetched`, `${summary.noCaptions} without captions`] + if (summary.blocked > 0) parts.push(`${summary.blocked} blocked`) + if (summary.errored > 0) parts.push(`${summary.errored} failed`) + if (summary.notAttempted > 0) parts.push(`${summary.notAttempted} not attempted`) + + const status = summary.complete ? 'complete' : 'incomplete' + return `${summary.total} videos, ${status}: ${parts.join(', ')}.` +} diff --git a/packages/social/src/transcripts/transcripts.test.ts b/packages/social/src/transcripts/transcripts.test.ts new file mode 100644 index 000000000..1cabdfad0 --- /dev/null +++ b/packages/social/src/transcripts/transcripts.test.ts @@ -0,0 +1,383 @@ +import { describe, expect, it } from 'vitest' +import { + createSocialTranscriptId, + createTranscriptContentDrafts, + createYouTubeTranscriptFetcher, + describeTranscriptRun, + enrichmentStatusForTranscriptOutcome, + parseYouTubeJson3Transcript, + parseYouTubeXmlTranscript, + runTranscriptFetchPass, + segmentTranscript, + summarizeTranscriptRun, + transcriptStateForEnrichmentStatus, + youTubeTimedTextUrl, + type TranscriptCue, + type TranscriptTarget +} from './index' + +function cue(startMs: number, text: string, durationMs = 1000): TranscriptCue { + return { startMs, durationMs, text } +} + +function jsonResponse(body: string, status = 200): Response { + return new Response(body, { status }) +} + +describe('youtube caption parsing', () => { + it('joins json3 segments into one cue each', () => { + const body = JSON.stringify({ + events: [ + { tStartMs: 0, dDurationMs: 1500, segs: [{ utf8: 'hello ' }, { utf8: 'world' }] }, + { tStartMs: 1500, dDurationMs: 900, segs: [{ utf8: 'again' }] }, + { tStartMs: 2400, dDurationMs: 100, segs: [{ utf8: ' ' }] } + ] + }) + + expect(parseYouTubeJson3Transcript(body)).toEqual([ + { startMs: 0, durationMs: 1500, text: 'hello world' }, + { startMs: 1500, durationMs: 900, text: 'again' } + ]) + }) + + it('returns no cues for a malformed payload rather than throwing', () => { + expect(parseYouTubeJson3Transcript('not json')).toEqual([]) + }) + + it('decodes entities and strips markup from the xml format', () => { + const body = + 'Bread & butter' + + 'it's rising' + + expect(parseYouTubeXmlTranscript(body)).toEqual([ + { startMs: 0, durationMs: 1500, text: 'Bread & butter' }, + { startMs: 1500, durationMs: 2000, text: "it's rising" } + ]) + }) + + it('selects the auto-generated track with kind=asr', () => { + const url = new URL(youTubeTimedTextUrl({ videoId: 'abc123', autoGenerated: true })) + expect(url.searchParams.get('v')).toBe('abc123') + expect(url.searchParams.get('kind')).toBe('asr') + expect(new URL(youTubeTimedTextUrl({ videoId: 'abc123' })).searchParams.get('kind')).toBeNull() + }) +}) + +describe('youtube transcript fetcher', () => { + const target: TranscriptTarget = { platform: 'youtube', platformContentId: 'abc123' } + + it('falls back from the author track to the auto-generated one', async () => { + const urls: string[] = [] + const fetcher = createYouTubeTranscriptFetcher({ + fetchImpl: (async (input: string) => { + urls.push(String(input)) + if (!String(input).includes('kind=asr')) return jsonResponse('') + return jsonResponse( + JSON.stringify({ events: [{ tStartMs: 0, dDurationMs: 500, segs: [{ utf8: 'hi' }] }] }) + ) + }) as unknown as typeof fetch + }) + + const outcome = await fetcher.fetch(target) + expect(outcome.status).toBe('fetched') + expect(urls).toHaveLength(2) + if (outcome.status === 'fetched') { + expect(outcome.autoGenerated).toBe(true) + expect(outcome.cues).toEqual([{ startMs: 0, durationMs: 500, text: 'hi' }]) + } + }) + + it('reports an empty track as no-captions, not as an error', async () => { + const fetcher = createYouTubeTranscriptFetcher({ + fetchImpl: (async () => jsonResponse('')) as unknown as typeof fetch + }) + + expect((await fetcher.fetch(target)).status).toBe('no-captions') + }) + + it('reports a refusal as blocked and stops trying other tracks', async () => { + let calls = 0 + const fetcher = createYouTubeTranscriptFetcher({ + fetchImpl: (async () => { + calls += 1 + return jsonResponse('', 429) + }) as unknown as typeof fetch, + languages: ['en', 'de'] + }) + + const outcome = await fetcher.fetch(target) + expect(outcome.status).toBe('blocked') + expect(calls).toBe(1) + }) + + it('reports a thrown transport failure as an error', async () => { + const fetcher = createYouTubeTranscriptFetcher({ + fetchImpl: (async () => { + throw new Error('offline') + }) as unknown as typeof fetch + }) + + const outcome = await fetcher.fetch(target) + expect(outcome).toEqual({ status: 'error', reason: 'offline' }) + }) +}) + +describe('transcript segmentation', () => { + it('splits on cue boundaries and never mid-word', () => { + const cues = [cue(0, 'a'.repeat(30)), cue(1000, 'b'.repeat(30)), cue(2000, 'c'.repeat(30))] + const segments = segmentTranscript(cues, 70) + + expect(segments).toHaveLength(2) + expect(segments[0]?.text).toBe(`${'a'.repeat(30)} ${'b'.repeat(30)}`) + expect(segments[1]?.text).toBe('c'.repeat(30)) + expect(segments[1]?.startMs).toBe(2000) + }) + + it('keeps a single oversized cue whole rather than truncating it', () => { + const segments = segmentTranscript([cue(0, 'x'.repeat(500))], 100) + expect(segments).toHaveLength(1) + expect(segments[0]?.text).toHaveLength(500) + }) + + it('drops empty cues', () => { + expect(segmentTranscript([cue(0, ' ')])).toEqual([]) + }) +}) + +describe('transcript content drafts', () => { + const base = { + platform: 'youtube', + platformContentId: 'abc123', + videoNodeId: 'social:content:video', + videoTitle: 'Sourdough basics', + canonicalUrl: 'https://www.youtube.com/watch?v=abc123', + fetchedAtMs: 1_700_000_000_000 + } + + it('links each segment back to the video and stays deterministic', () => { + const drafts = createTranscriptContentDrafts({ ...base, cues: [cue(0, 'mix the flour')] }) + + expect(drafts).toHaveLength(1) + expect(drafts[0]?.id).toBe(createSocialTranscriptId('youtube', 'abc123', 0)) + expect(drafts[0]?.properties.parentContent).toBe('social:content:video') + expect(drafts[0]?.properties.contentKind).toBe('transcript') + expect(drafts[0]?.properties.searchText).toBe('mix the flour') + + const again = createTranscriptContentDrafts({ ...base, cues: [cue(0, 'mix the flour')] }) + expect(again[0]?.id).toBe(drafts[0]?.id) + }) + + it('numbers multi-part transcripts and records the segment count', () => { + const cues = Array.from({ length: 6 }, (_, index) => cue(index * 1000, 'word '.repeat(1000))) + const drafts = createTranscriptContentDrafts({ ...base, cues }) + + expect(drafts.length).toBeGreaterThan(1) + expect(drafts[0]?.properties.title).toContain(`part 1/${drafts.length}`) + for (const draft of drafts) { + expect(draft.properties.searchText.length).toBeLessThanOrEqual(20_000) + const metadata = JSON.parse(draft.properties.metadataJson) as { + transcript: { segmentCount: number } + } + expect(metadata.transcript.segmentCount).toBe(drafts.length) + } + }) + + it('returns nothing for an empty transcript', () => { + expect(createTranscriptContentDrafts({ ...base, cues: [] })).toEqual([]) + }) + + it('marks an auto-generated track in the platform content kind', () => { + const drafts = createTranscriptContentDrafts({ + ...base, + cues: [cue(0, 'hello')], + autoGenerated: true + }) + expect(drafts[0]?.properties.platformContentKind).toBe('transcript-asr') + }) +}) + +describe('run states', () => { + it('keeps no-captions, blocked and not-attempted distinguishable', () => { + expect(enrichmentStatusForTranscriptOutcome('no-captions')).toBe('unavailable') + expect(enrichmentStatusForTranscriptOutcome('blocked')).toBe('blocked') + expect(transcriptStateForEnrichmentStatus('unavailable')).toBe('no-captions') + expect(transcriptStateForEnrichmentStatus('blocked')).toBe('blocked') + expect(transcriptStateForEnrichmentStatus(undefined)).toBe('not-attempted') + }) + + it('counts unattempted targets so the states sum to the total', () => { + const summary = summarizeTranscriptRun({ + states: ['fetched', 'fetched', 'no-captions', 'blocked'], + total: 50 + }) + + expect(summary.total).toBe(50) + expect( + summary.fetched + + summary.noCaptions + + summary.blocked + + summary.errored + + summary.notAttempted + ).toBe(50) + expect(summary.notAttempted).toBe(46) + expect(summary.complete).toBe(false) + expect(summary.retryable).toBe(true) + }) + + it('reports a run that reached every target as complete', () => { + const summary = summarizeTranscriptRun({ states: ['fetched', 'no-captions'], total: 2 }) + expect(summary.complete).toBe(true) + expect(summary.retryable).toBe(false) + expect(describeTranscriptRun(summary)).toBe( + '2 videos, complete: 1 fetched, 1 without captions.' + ) + }) + + it('names the untried remainder in the description', () => { + const summary = summarizeTranscriptRun({ states: ['blocked'], total: 10 }) + expect(describeTranscriptRun(summary)).toContain('9 not attempted') + expect(describeTranscriptRun(summary)).toContain('incomplete') + }) +}) + +describe('transcript pass', () => { + const targets: TranscriptTarget[] = Array.from({ length: 5 }, (_, index) => ({ + platform: 'youtube', + platformContentId: `video-${index}` + })) + + function fetcherReturning(outcomes: readonly string[]) { + let index = 0 + return { + platform: 'youtube', + supports: () => true, + fetch: async () => { + const status = outcomes[Math.min(index++, outcomes.length - 1)] ?? 'no-captions' + if (status === 'fetched') { + return { status: 'fetched' as const, cues: [cue(0, 'hi')] } + } + return { status: status as 'no-captions' | 'blocked' | 'error', reason: 'test' } + } + } + } + + it('paces attempts and records every outcome', async () => { + const delays: number[] = [] + const seen: string[] = [] + + const result = await runTranscriptFetchPass({ + targets, + fetcher: fetcherReturning(['fetched']), + onResult: (target, outcome) => { + seen.push(`${target.platformContentId}:${outcome.status}`) + }, + delayFn: async (ms) => { + delays.push(ms) + }, + random: () => 0.5, + intervalMs: 1000, + jitterMs: 400 + }) + + expect(seen).toHaveLength(5) + expect(delays).toEqual([1200, 1200, 1200, 1200]) + expect(result.summary.complete).toBe(true) + expect(result.summary.fetched).toBe(5) + expect(result.stoppedEarly).toBe(false) + }) + + it('gives up after consecutive refusals and reports the run incomplete', async () => { + const result = await runTranscriptFetchPass({ + targets, + fetcher: fetcherReturning(['blocked']), + onResult: () => {}, + delayFn: async () => {}, + random: () => 0, + maxConsecutiveBlocks: 2 + }) + + expect(result.stoppedEarly).toBe(true) + expect(result.summary.blocked).toBe(2) + expect(result.summary.notAttempted).toBe(3) + expect(result.summary.complete).toBe(false) + expect(result.retryable).toHaveLength(2) + }) + + it('accounts for every video in a 50-item playlist run', async () => { + // The shape of a real pass, with the transport stubbed: a mixed playlist + // where some videos have captions, some do not, and one request fails. + const playlist: TranscriptTarget[] = Array.from({ length: 50 }, (_, index) => ({ + platform: 'youtube', + platformContentId: `playlist-video-${index}` + })) + const drafts: string[] = [] + let index = 0 + + const result = await runTranscriptFetchPass({ + targets: playlist, + fetcher: { + platform: 'youtube', + supports: () => true, + fetch: async () => { + const position = index++ + if (position % 10 === 7) return { status: 'no-captions' as const } + if (position === 23) return { status: 'error' as const, reason: 'timeout' } + return { status: 'fetched' as const, cues: [cue(0, `line ${position}`)] } + } + }, + onResult: (target, outcome) => { + if (outcome.status !== 'fetched') return + for (const draft of createTranscriptContentDrafts({ + platform: target.platform, + platformContentId: target.platformContentId, + cues: outcome.cues, + fetchedAtMs: 1_700_000_000_000 + })) { + drafts.push(draft.id) + } + }, + delayFn: async () => {}, + random: () => 0 + }) + + const { summary } = result + expect(summary.total).toBe(50) + expect( + summary.fetched + + summary.noCaptions + + summary.blocked + + summary.errored + + summary.notAttempted + ).toBe(50) + expect(summary.fetched).toBe(44) + expect(summary.noCaptions).toBe(5) + expect(summary.errored).toBe(1) + expect(summary.notAttempted).toBe(0) + expect(summary.complete).toBe(true) + expect(describeTranscriptRun(summary)).toContain('complete') + + // One transcript node per fetched video, each with a distinct id. + expect(drafts).toHaveLength(44) + expect(new Set(drafts).size).toBe(44) + }) + + it('leaves unsupported targets unattempted rather than counting them as misses', async () => { + const result = await runTranscriptFetchPass({ + targets: [ + { platform: 'youtube', platformContentId: 'a' }, + { platform: 'tiktok', platformContentId: 'b' } + ], + fetcher: { + platform: 'youtube', + supports: (target) => target.platform === 'youtube', + fetch: async () => ({ status: 'fetched', cues: [cue(0, 'hi')] }) + }, + onResult: () => {}, + delayFn: async () => {} + }) + + expect(result.summary.fetched).toBe(1) + expect(result.summary.notAttempted).toBe(1) + expect(result.summary.complete).toBe(false) + }) +}) diff --git a/packages/social/src/transcripts/types.ts b/packages/social/src/transcripts/types.ts new file mode 100644 index 000000000..445369b5d --- /dev/null +++ b/packages/social/src/transcripts/types.ts @@ -0,0 +1,105 @@ +/** + * The transcript fetch seam (exploration 0419). + * + * Transcripts are the one part of the social atlas that cannot come out of an + * archive: no platform export ships them, and the official caption APIs only + * serve videos you own. What is left is the unofficial caption endpoint each + * platform's own player uses — which is blocked from datacenter address ranges + * and is not blocked from the device the user watches on. + * + * That asymmetry is why this is a seam rather than a hub route. A fetch issued + * from the user's own machine, for a video that user demonstrably saved, at a + * human pace, is the same request their browser would make. Routing it through + * a shared server would be slower, more fragile, and would turn a local read + * into someone else's egress. + * + * The contract is deliberately total: every attempt ends in exactly one of four + * outcomes, and "no captions exist" is a different value from "we were refused" + * and from "we have not looked yet". A pipeline that collapsed those would + * report a blocked run as a complete one. + */ + +/** A video we can attempt a transcript for. */ +export type TranscriptTarget = { + /** Platform id, matching `socialPlatforms` (`youtube`, `tiktok`, …). */ + platform: string + /** The platform's own id for the video (not the xNet node id). */ + platformContentId: string + /** Canonical watch URL, when known. */ + url?: string + /** BCP-47 language preference, best-effort. */ + preferredLanguage?: string +} + +/** One timed line of a transcript. */ +export type TranscriptCue = { + /** Cue start in milliseconds from the beginning of the video. */ + startMs: number + /** Cue duration in milliseconds; 0 when the source does not say. */ + durationMs: number + text: string +} + +/** + * The result of one attempt. + * + * `no-captions` and `blocked` are both "we got nothing", and keeping them + * apart is the whole point: the first is a fact about the video and will not + * change on retry, the second is a fact about us and will. + */ +export type TranscriptFetchOutcome = + | { + status: 'fetched' + cues: TranscriptCue[] + /** Language actually returned, when the source declares one. */ + language?: string + /** True when the track is machine-generated rather than author-supplied. */ + autoGenerated?: boolean + } + | { status: 'no-captions'; reason?: string } + | { status: 'blocked'; reason?: string } + | { status: 'error'; reason: string } + +export type TranscriptFetchStatus = TranscriptFetchOutcome['status'] + +/** + * A per-platform transcript source. + * + * Implementations must not throw for an expected miss — a video without + * captions returns `no-captions`, and a refusal returns `blocked`. Throwing is + * reserved for programmer error. + */ +export interface TranscriptFetcher { + /** Platform this fetcher serves. */ + readonly platform: string + /** Whether this fetcher can attempt the given target at all. */ + supports(target: TranscriptTarget): boolean + fetch(target: TranscriptTarget, signal?: AbortSignal): Promise +} + +/** + * Map a fetch outcome onto the persisted `SocialEnrichment.status` vocabulary. + * + * The mapping is total and lossless in the direction that matters: each of the + * four outcomes lands on a distinct stored status, so a later count over stored + * nodes can reconstruct what happened without re-fetching. + */ +export function enrichmentStatusForTranscriptOutcome( + status: TranscriptFetchStatus +): 'resolved' | 'unavailable' | 'blocked' | 'error' { + switch (status) { + case 'fetched': + return 'resolved' + case 'no-captions': + return 'unavailable' + case 'blocked': + return 'blocked' + case 'error': + return 'error' + } +} + +/** True when another attempt could plausibly succeed. */ +export function isRetryableTranscriptStatus(status: TranscriptFetchStatus): boolean { + return status === 'blocked' || status === 'error' +} diff --git a/packages/social/src/transcripts/youtube.ts b/packages/social/src/transcripts/youtube.ts new file mode 100644 index 000000000..347f5b4c2 --- /dev/null +++ b/packages/social/src/transcripts/youtube.ts @@ -0,0 +1,215 @@ +/** + * YouTube transcript fetching over the player's own caption endpoint. + * + * Parsing is separated from fetching on purpose: the wire formats are stable + * and testable, the transport is neither. Everything below the fetcher is a + * pure function over a string, so the fragile part is one small surface. + */ + +import type { + TranscriptCue, + TranscriptFetchOutcome, + TranscriptFetcher, + TranscriptTarget +} from './types' + +const TIMEDTEXT_ENDPOINT = 'https://www.youtube.com/api/timedtext' + +/** Status codes that mean "you, not the video" — worth retrying later. */ +const BLOCKED_STATUS_CODES = new Set([401, 403, 407, 429, 500, 502, 503, 504]) + +export type YouTubeTimedTextFormat = 'json3' | 'xml' + +/** + * The `json3` payload shape, reduced to what we read. + * + * `segs` is the word/phrase split inside a cue; joining it is what turns an + * auto-generated track back into a sentence. + */ +type Json3Payload = { + events?: Array<{ + tStartMs?: number + dDurationMs?: number + segs?: Array<{ utf8?: string }> + }> +} + +function normalizeCueText(value: string): string { + return value.replace(/\s+/g, ' ').trim() +} + +/** Decode the small set of XML entities the caption track actually uses. */ +function decodeXmlEntities(value: string): string { + return value + .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-fA-F]+);/g, (_, code: string) => String.fromCodePoint(parseInt(code, 16))) + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&/g, '&') +} + +/** Parse the `fmt=json3` caption payload. Returns `[]` for an empty track. */ +export function parseYouTubeJson3Transcript(body: string): TranscriptCue[] { + let payload: Json3Payload + try { + payload = JSON.parse(body) as Json3Payload + } catch { + return [] + } + + const cues: TranscriptCue[] = [] + for (const event of payload.events ?? []) { + const text = normalizeCueText((event.segs ?? []).map((seg) => seg.utf8 ?? '').join('')) + if (!text) continue + + cues.push({ + startMs: Math.max(0, Math.round(event.tStartMs ?? 0)), + durationMs: Math.max(0, Math.round(event.dDurationMs ?? 0)), + text + }) + } + + return cues +} + +/** Parse the legacy `` caption payload. */ +export function parseYouTubeXmlTranscript(body: string): TranscriptCue[] { + const cues: TranscriptCue[] = [] + const pattern = /]*)>([\s\S]*?)<\/text>/g + + for (const match of body.matchAll(pattern)) { + const attributes = match[1] ?? '' + const text = normalizeCueText(decodeXmlEntities(match[2] ?? '').replace(/<[^>]+>/g, '')) + if (!text) continue + + const start = Number(/\bstart="([^"]*)"/.exec(attributes)?.[1] ?? '0') + const duration = Number(/\bdur="([^"]*)"/.exec(attributes)?.[1] ?? '0') + + cues.push({ + startMs: Number.isFinite(start) ? Math.max(0, Math.round(start * 1000)) : 0, + durationMs: Number.isFinite(duration) ? Math.max(0, Math.round(duration * 1000)) : 0, + text + }) + } + + return cues +} + +/** Parse either caption wire format, picking by content rather than by flag. */ +export function parseYouTubeTranscript(body: string): TranscriptCue[] { + const trimmed = body.trim() + if (!trimmed) return [] + if (trimmed.startsWith('{')) return parseYouTubeJson3Transcript(trimmed) + return parseYouTubeXmlTranscript(trimmed) +} + +export function youTubeTimedTextUrl(input: { + videoId: string + language?: string + format?: YouTubeTimedTextFormat + autoGenerated?: boolean +}): string { + const url = new URL(TIMEDTEXT_ENDPOINT) + url.searchParams.set('v', input.videoId) + url.searchParams.set('lang', input.language ?? 'en') + url.searchParams.set('fmt', input.format ?? 'json3') + // `kind=asr` selects the machine-generated track; author-supplied tracks are + // served without it, so the two are separate requests rather than a fallback + // inside one. + if (input.autoGenerated) url.searchParams.set('kind', 'asr') + return url.toString() +} + +export type YouTubeTranscriptFetcherOptions = { + fetchImpl?: typeof fetch + /** Languages to try, in order. Defaults to English. */ + languages?: readonly string[] +} + +/** + * Classify a transport failure. + * + * An empty 200 means the track does not exist; a 403/429 means we were refused. + * Conflating them would make a rate-limited run look like a library of videos + * that simply have no captions. + */ +function outcomeForResponse(status: number): TranscriptFetchOutcome | null { + if (status === 404) return { status: 'no-captions', reason: 'caption track not found' } + if (BLOCKED_STATUS_CODES.has(status)) { + return { status: 'blocked', reason: `caption request refused with ${status}` } + } + if (status >= 400) return { status: 'error', reason: `caption request failed with ${status}` } + return null +} + +/** + * Device-local YouTube transcript fetcher. + * + * Tries the author-supplied track first and the auto-generated one second, per + * language. The first non-empty track wins; a clean miss on every combination + * is `no-captions`, and a refusal anywhere short-circuits to `blocked` so the + * queue can back off instead of burning through the rest of the list. + */ +export function createYouTubeTranscriptFetcher( + options: YouTubeTranscriptFetcherOptions = {} +): TranscriptFetcher { + const fetchImpl = options.fetchImpl ?? fetch + const languages = options.languages?.length ? options.languages : ['en'] + + return { + platform: 'youtube', + + supports(target: TranscriptTarget): boolean { + return target.platform === 'youtube' && Boolean(target.platformContentId) + }, + + async fetch(target: TranscriptTarget, signal?: AbortSignal): Promise { + const preferred = target.preferredLanguage + const ordered = preferred + ? [preferred, ...languages.filter((l) => l !== preferred)] + : languages + + for (const language of ordered) { + for (const autoGenerated of [false, true]) { + const url = youTubeTimedTextUrl({ + videoId: target.platformContentId, + language, + autoGenerated + }) + + let response: Response + try { + response = await fetchImpl(url, signal ? { signal } : {}) + } catch (error) { + return { + status: 'error', + reason: error instanceof Error ? error.message : 'caption request threw' + } + } + + const failure = outcomeForResponse(response.status) + // A refusal is about us and applies to every remaining combination. + if (failure?.status === 'blocked') return failure + if (failure?.status === 'error') return failure + if (failure) continue + + const body = await response.text().catch(() => '') + const cues = parseYouTubeTranscript(body) + if (cues.length === 0) continue + + return { + status: 'fetched', + cues, + language, + autoGenerated + } + } + } + + return { status: 'no-captions', reason: 'no caption track for the requested languages' } + } + } +} diff --git a/packages/sqlite/src/fts.test.ts b/packages/sqlite/src/fts.test.ts new file mode 100644 index 000000000..5169acb2a --- /dev/null +++ b/packages/sqlite/src/fts.test.ts @@ -0,0 +1,68 @@ +/** + * What reaches the full-text index. + * + * The regression this guards (exploration 0419): imported social content puts + * its full text in `searchText`, and the extractor did not read that property. + * Every imported post, comment and video transcript was therefore absent from + * search while the pipeline reported them as indexed — the quiet kind of + * failure, where the query returns a clean empty result. + */ + +import { describe, expect, it } from 'vitest' +import { extractSearchableContent } from './fts' + +describe('extractSearchableContent', () => { + it('indexes the denormalized search text of imported social content', () => { + const content = extractSearchableContent({ + platform: 'youtube', + contentKind: 'transcript', + title: 'Transcript — Sourdough basics', + searchText: 'first you mix the flour and the water', + textPreview: 'first you mix the flour' + }) + + expect(content).toContain('first you mix the flour and the water') + }) + + it('prefers the full text over the truncated preview', () => { + const content = extractSearchableContent({ + searchText: 'the complete transcript body', + textPreview: 'the complete' + }) + + expect(content).toBe('the complete transcript body') + }) + + it('falls back to the preview when there is no full text', () => { + expect(extractSearchableContent({ textPreview: 'only a preview' })).toBe('only a preview') + expect(extractSearchableContent({ searchText: '', textPreview: 'only a preview' })).toBe( + 'only a preview' + ) + }) + + it('still indexes the properties it always did', () => { + expect(extractSearchableContent({ description: 'a description' })).toBe('a description') + expect(extractSearchableContent({ body: 'a body' })).toBe('a body') + expect(extractSearchableContent({ name: 'a name' })).toBe('a name') + expect(extractSearchableContent({ note: 'a note' })).toBe('a note') + expect(extractSearchableContent({ content: 'plain string content' })).toBe( + 'plain string content' + ) + }) + + it('extracts text from TipTap document content', () => { + const content = extractSearchableContent({ + content: { + type: 'doc', + content: [{ type: 'paragraph', content: [{ type: 'text', text: 'hello world' }] }] + } + }) + + expect(content).toContain('hello world') + }) + + it('returns null when a node carries no searchable text', () => { + expect(extractSearchableContent({})).toBeNull() + expect(extractSearchableContent({ platform: 'youtube', viewCount: 12 })).toBeNull() + }) +}) diff --git a/packages/sqlite/src/fts.ts b/packages/sqlite/src/fts.ts index dd4455a30..03ef5e431 100644 --- a/packages/sqlite/src/fts.ts +++ b/packages/sqlite/src/fts.ts @@ -270,6 +270,21 @@ export function extractSearchableContent(properties: Record): s parts.push(note) } + // Imported social content denormalizes its full text into `searchText` + // precisely so it can be searched (exploration 0152) — but nothing here read + // it, so every imported post, comment and video transcript was invisible to + // full-text search while appearing to be indexed. `textPreview` is the + // shorter fallback for records that carry no full text. + const searchText = properties.searchText + if (typeof searchText === 'string' && searchText.length > 0) { + parts.push(searchText) + } else { + const textPreview = properties.textPreview + if (typeof textPreview === 'string') { + parts.push(textPreview) + } + } + return parts.length > 0 ? parts.join(' ') : null } diff --git a/packages/views/src/canvas-view/index.ts b/packages/views/src/canvas-view/index.ts index 441e8136a..10cdeefab 100644 --- a/packages/views/src/canvas-view/index.ts +++ b/packages/views/src/canvas-view/index.ts @@ -63,6 +63,11 @@ export { type UseCanvasQueryFramesOptions, type UseCanvasQueryFramesResult } from './query-frames.js' +export { + applySocialCanvasProjectionPlan, + describeSocialCanvasProjection, + type ApplySocialCanvasProjectionResult +} from './social-projection.js' export { useCanvasSourceReferences, useSelectedSourceReferences, diff --git a/packages/views/src/canvas-view/query-frames.tsx b/packages/views/src/canvas-view/query-frames.tsx index be18f5691..a6b4f19e6 100644 --- a/packages/views/src/canvas-view/query-frames.tsx +++ b/packages/views/src/canvas-view/query-frames.tsx @@ -34,7 +34,12 @@ import { type SavedViewSchemaRegistry, type UseSavedViewResult } from '@xnetjs/react' +import type { SocialCanvasProjectionPlan } from '@xnetjs/social/projection' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { + applySocialCanvasProjectionPlan, + type ApplySocialCanvasProjectionResult +} from './social-projection.js' export type SavedViewCanvasQueryFrameInput = { viewId: string @@ -354,6 +359,13 @@ export interface UseCanvasQueryFramesResult { selectedQueryFrameDefinition: ReturnType | null createQueryFrameFromSavedView: (input: SavedViewCanvasQueryFrameInput) => boolean refreshSelectedQueryFrame: () => boolean + /** + * Place a laid-out social projection — individual source-backed cards and + * the connections between them — rather than a lens that re-runs (0419). + */ + applySocialCanvasProjection: ( + plan: SocialCanvasProjectionPlan + ) => ApplySocialCanvasProjectionResult | null } export function useCanvasQueryFrames({ @@ -422,6 +434,17 @@ export function useCanvasQueryFrames({ [onUndoBoundary, placePrimitiveObject] ) + const applySocialCanvasProjection = useCallback( + (plan: SocialCanvasProjectionPlan): ApplySocialCanvasProjectionResult | null => { + if (!doc || plan.nodes.length === 0) return null + + const result = applySocialCanvasProjectionPlan(doc, plan) + onUndoBoundary?.() + return result + }, + [doc, onUndoBoundary] + ) + const refreshSelectedQueryFrame = useCallback((): boolean => { if (!selectedQueryFrameNode) return false @@ -439,6 +462,7 @@ export function useCanvasQueryFrames({ selectedQueryFrameNode, selectedQueryFrameDefinition, createQueryFrameFromSavedView, - refreshSelectedQueryFrame + refreshSelectedQueryFrame, + applySocialCanvasProjection } } diff --git a/packages/views/src/canvas-view/social-projection.test.ts b/packages/views/src/canvas-view/social-projection.test.ts new file mode 100644 index 000000000..8b9815e46 --- /dev/null +++ b/packages/views/src/canvas-view/social-projection.test.ts @@ -0,0 +1,132 @@ +/** + * The two halves of canvas projection compose, and the plan lands on a doc + * (exploration 0419). + */ + +import type { SavedViewCanvasProjectionNode } from '@xnetjs/react' +import { getCanvasConnectorsMap, getCanvasObjectsMap } from '@xnetjs/canvas' +import { createSocialCanvasProjectionPlan } from '@xnetjs/social' +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { + applySocialCanvasProjectionPlan, + describeSocialCanvasProjection +} from './social-projection' + +function projectionNode(id: string, overrides: Partial = {}) { + return { + id, + schemaId: 'xnet://xnet.fyi/social/SocialContent@1.0.0', + kind: 'content' as const, + title: `Video ${id}`, + platform: 'youtube', + privacyClass: 'public', + groupKey: 'youtube', + ...overrides + } +} + +function planFor( + nodeIds: readonly string[], + edges: Parameters[0]['edges'] = [] +) { + return createSocialCanvasProjectionPlan({ + nodes: nodeIds.map((id) => projectionNode(id)), + edges, + options: { title: 'YouTube Playlists', lensId: 'social.feed.youtube-playlists' } + }) +} + +describe('social canvas projection', () => { + it('accepts the extractor output from @xnetjs/react without a shim', () => { + // The point of the pair: react extracts, social lays out. If these types + // ever drift apart this stops compiling. + const nodes: SavedViewCanvasProjectionNode[] = [projectionNode('a'), projectionNode('b')] + const plan = createSocialCanvasProjectionPlan({ + nodes, + options: { title: 'Lens' } + }) + + expect(plan.nodeCount).toBe(2) + expect(plan.nodes[0]?.sourceNodeId).toBe('a') + expect(plan.nodes[0]?.type).toBe('external-reference') + }) + + it('writes objects and connectors into the canvas doc', () => { + const doc = new Y.Doc() + const plan = planFor( + ['a', 'b'], + [{ sourceId: 'a', targetId: 'b', relationshipKind: 'contains' }] + ) + + const result = applySocialCanvasProjectionPlan(doc, plan) + + expect(result.nodeCount).toBe(2) + expect(result.edgeCount).toBe(1) + expect(getCanvasObjectsMap(doc).size).toBe(2) + expect(getCanvasConnectorsMap(doc).size).toBe(1) + }) + + it('keeps each card source-backed so it can resolve its own node', () => { + const doc = new Y.Doc() + applySocialCanvasProjectionPlan(doc, planFor(['a'])) + + const objects = getCanvasObjectsMap<{ properties: Record }>(doc) + const placed = [...objects.values()][0] + + expect(placed?.properties.sourceNodeId).toBe('a') + expect(placed?.properties.sourceSchemaId).toBe('xnet://xnet.fyi/social/SocialContent@1.0.0') + expect(placed?.properties.sourceCardRole).toBe('social-projection') + }) + + it('re-projecting the same lens overwrites rather than duplicating', () => { + const doc = new Y.Doc() + const plan = planFor(['a', 'b']) + + applySocialCanvasProjectionPlan(doc, plan) + applySocialCanvasProjectionPlan(doc, plan) + + expect(getCanvasObjectsMap(doc).size).toBe(2) + }) + + it('offsets the whole projection from a given origin', () => { + const doc = new Y.Doc() + applySocialCanvasProjectionPlan(doc, planFor(['a']), { origin: { x: 500, y: 250 } }) + + const objects = getCanvasObjectsMap<{ position: { x: number; y: number } }>(doc) + const placed = [...objects.values()][0] + + expect(placed?.position.x).toBe(500) + expect(placed?.position.y).toBe(250) + }) + + it('drops edges whose endpoints were not placed', () => { + const doc = new Y.Doc() + const plan = planFor( + ['a'], + [{ sourceId: 'a', targetId: 'missing', relationshipKind: 'related' }] + ) + + const result = applySocialCanvasProjectionPlan(doc, plan) + expect(result.edgeCount).toBe(0) + expect(getCanvasConnectorsMap(doc).size).toBe(0) + }) + + it('names what the caps left out instead of implying the graph is whole', () => { + const plan = createSocialCanvasProjectionPlan({ + nodes: Array.from({ length: 5 }, (_, index) => projectionNode(`n${index}`)), + options: { title: 'Lens', maxNodes: 2 } + }) + const doc = new Y.Doc() + const result = applySocialCanvasProjectionPlan(doc, plan) + + expect(result.omittedNodeCount).toBe(3) + expect(describeSocialCanvasProjection(result)).toContain('3 more not shown') + }) + + it('says nothing about omissions when nothing was omitted', () => { + const doc = new Y.Doc() + const result = applySocialCanvasProjectionPlan(doc, planFor(['a'])) + expect(describeSocialCanvasProjection(result)).toBe('Projected 1 cards, 0 connections.') + }) +}) diff --git a/packages/views/src/canvas-view/social-projection.ts b/packages/views/src/canvas-view/social-projection.ts new file mode 100644 index 000000000..a05968733 --- /dev/null +++ b/packages/views/src/canvas-view/social-projection.ts @@ -0,0 +1,120 @@ +/** + * Placing a projected social graph on a canvas (exploration 0419). + * + * ## Resolving the duplication + * + * Two functions looked like the same thing and were not: + * + * - `createSavedViewCanvasProjectionNodes` (`@xnetjs/react`) **extracts** — it + * turns rendered previews into `{ id, schemaId, kind, title, … }` node + * inputs. It knows about previews and nothing about geometry. + * - `createSocialCanvasProjectionPlan` (`@xnetjs/social`) **lays out** — it + * takes those inputs plus edges and produces positioned, source-backed + * drafts with connectors, bounded by node and edge caps. It knows about + * geometry and nothing about previews. + * + * They compose; neither was redundant. What was missing was this file — the + * step that writes a plan into a canvas document, which is why the layout half + * had no caller for so long. + * + * ## Idempotence + * + * Plan ids are deterministic in the lens and the source node, so projecting + * the same lens twice overwrites the same objects rather than stacking a + * second copy on top of the first. + */ + +import type { CanvasEdge, CanvasNode } from '@xnetjs/canvas' +import type { SocialCanvasProjectionPlan } from '@xnetjs/social/projection' +import type * as Y from 'yjs' +import { getCanvasConnectorsMap, getCanvasObjectsMap } from '@xnetjs/canvas' + +export type ApplySocialCanvasProjectionResult = { + /** Objects written. */ + nodeCount: number + /** Connectors written. */ + edgeCount: number + /** Nodes the plan's cap left out of the projection. */ + omittedNodeCount: number + /** Edges the plan's cap left out. */ + omittedEdgeCount: number + /** Ids of the objects written, for selection. */ + objectIds: string[] +} + +/** + * Write a projection plan into a canvas document. + * + * The whole plan lands in one transaction, so the canvas never renders a + * half-placed graph and undo treats the projection as a single act. + * + * The result reports what the plan's caps dropped. A caller that ignores those + * numbers is showing the user a partial graph as if it were the whole one. + */ +export function applySocialCanvasProjectionPlan( + doc: Y.Doc, + plan: SocialCanvasProjectionPlan, + options: { origin?: { x: number; y: number } } = {} +): ApplySocialCanvasProjectionResult { + const originX = options.origin?.x ?? 0 + const originY = options.origin?.y ?? 0 + const objects = getCanvasObjectsMap(doc) + const connectors = getCanvasConnectorsMap(doc) + + const nodes: CanvasNode[] = plan.nodes.map((draft) => ({ + id: draft.id, + type: draft.type, + position: { + x: originX + draft.position.x, + y: originY + draft.position.y, + width: draft.position.width, + height: draft.position.height, + zIndex: draft.position.zIndex + }, + properties: { + ...draft.properties, + sourceNodeId: draft.sourceNodeId, + sourceSchemaId: draft.sourceSchemaId + }, + locked: draft.locked + })) as CanvasNode[] + + const edges: CanvasEdge[] = plan.edges.map((draft) => ({ + id: draft.id, + sourceId: draft.sourceId, + targetId: draft.targetId, + source: { objectId: draft.source.objectId, placement: draft.source.placement }, + target: { objectId: draft.target.objectId, placement: draft.target.placement }, + ...(draft.label ? { label: draft.label } : {}), + relationship: draft.relationship + })) as CanvasEdge[] + + doc.transact(() => { + for (const node of nodes) objects.set(node.id, node) + for (const edge of edges) connectors.set(edge.id, edge) + }) + + return { + nodeCount: nodes.length, + edgeCount: edges.length, + omittedNodeCount: plan.omittedNodeCount, + omittedEdgeCount: plan.omittedEdgeCount, + objectIds: nodes.map((node) => node.id) + } +} + +/** + * A one-line account of what landed, naming anything the caps dropped. + * + * Bounded projection is the right default on a canvas — 10 000 cards is not a + * view of anything — but the bound has to be visible, or the board silently + * misrepresents the library it came from. + */ +export function describeSocialCanvasProjection(result: ApplySocialCanvasProjectionResult): string { + const parts = [`${result.nodeCount} cards`, `${result.edgeCount} connections`] + if (result.omittedNodeCount > 0) parts.push(`${result.omittedNodeCount} more not shown`) + if (result.omittedEdgeCount > 0) { + parts.push(`${result.omittedEdgeCount} connections not shown`) + } + return `Projected ${parts.join(', ')}.` +} diff --git a/packages/views/src/data-workspace/DataWorkspaceCore.tsx b/packages/views/src/data-workspace/DataWorkspaceCore.tsx index 65b055c1d..39d9f3487 100644 --- a/packages/views/src/data-workspace/DataWorkspaceCore.tsx +++ b/packages/views/src/data-workspace/DataWorkspaceCore.tsx @@ -38,6 +38,11 @@ import { type SocialImportJobProgress } from '@xnetjs/social/import/core' import { createDefaultSocialGraphAtlas, type SocialGraphAtlasEntry } from '@xnetjs/social/lenses' +import { + createSocialCanvasProjectionPlan, + type SocialCanvasProjectionPlan, + type SocialProjectionRelationshipKind +} from '@xnetjs/social/projection' import { createSocialPatternSavedViewDraft, detectSocialPatterns, @@ -89,6 +94,37 @@ export type SavedViewCanvasFrameInput = { title?: string description?: string descriptor?: string + /** + * A laid-out graph to place instead of a live query frame (0419). + * + * Present when the user asked for the *projection* — the individual cards + * and the connections between them — rather than a lens that re-runs. A host + * that does not understand it should still insert the frame, which is the + * weaker but never-wrong reading of the request. + */ + projection?: SocialCanvasProjectionPlan +} + +const SOCIAL_RELATIONSHIP_KINDS: readonly SocialProjectionRelationshipKind[] = [ + 'follows', + 'saved', + 'authored', + 'participated', + 'referenced', + 'cited', + 'contains', + 'related' +] + +/** + * Narrow the runner's open relationship string to the projection vocabulary. + * + * Anything unrecognized becomes `related` rather than being dropped: the two + * nodes really are connected, and losing the edge would understate the graph + * more than labelling it loosely does. + */ +function socialRelationshipKind(kind: string): SocialProjectionRelationshipKind { + return SOCIAL_RELATIONSHIP_KINDS.find((candidate) => candidate === kind) ?? 'related' } export type SavedViewRow = { @@ -602,11 +638,34 @@ export function useDataWorkspace({ ? JSON.stringify(request.descriptor) : selectedView?.descriptor + // The request already carries extracted projection nodes; laying them out + // is the other half of the pair (see canvas-view/social-projection.ts). + // A request with no nodes yields no plan, and the host falls back to + // inserting the lens as a live frame. + const projection = + request.nodes.length > 0 + ? createSocialCanvasProjectionPlan({ + nodes: request.nodes, + edges: (request.edges ?? []).map((edge) => ({ + id: edge.id, + sourceId: edge.sourceId, + targetId: edge.targetId, + relationshipKind: socialRelationshipKind(edge.relationshipKind), + ...(edge.label ? { label: edge.label } : {}) + })), + options: { + title: request.title, + lensId: selectedView?.id ?? request.id + } + }) + : null + onInsertSavedLensAsCanvasFrame({ id: selectedView?.id ?? request.id, title: request.title, ...(request.description ? { description: request.description } : {}), - ...(descriptorJson ? { descriptor: descriptorJson } : {}) + ...(descriptorJson ? { descriptor: descriptorJson } : {}), + ...(projection ? { projection } : {}) }) } diff --git a/packages/views/src/data-workspace/index.ts b/packages/views/src/data-workspace/index.ts index 0f4b75bfc..811e5c0fa 100644 --- a/packages/views/src/data-workspace/index.ts +++ b/packages/views/src/data-workspace/index.ts @@ -14,3 +14,9 @@ export { upsertDefaultSocialWorkspace, type SocialWorkspaceSeedSummary } from './social-workspace.js' +export { + clearPendingCanvasLens, + stashPendingCanvasLens, + takePendingCanvasLens, + type PendingCanvasLens +} from './pending-canvas-lens.js' diff --git a/packages/views/src/data-workspace/pending-canvas-lens.test.ts b/packages/views/src/data-workspace/pending-canvas-lens.test.ts new file mode 100644 index 000000000..2ed5225d8 --- /dev/null +++ b/packages/views/src/data-workspace/pending-canvas-lens.test.ts @@ -0,0 +1,68 @@ +/** + * Handing a lens from the Data Workspace to a canvas across a route change + * (exploration 0419). + */ + +import { beforeEach, describe, expect, it } from 'vitest' +import { + clearPendingCanvasLens, + stashPendingCanvasLens, + takePendingCanvasLens +} from './pending-canvas-lens' + +const lens = { + canvasId: 'canvas:desk:abc', + viewId: 'social.feed.youtube-videos', + title: 'YouTube Videos', + descriptorJson: '{"version":1}' +} + +describe('pending canvas lens', () => { + beforeEach(() => { + clearPendingCanvasLens() + }) + + it('hands the request to the canvas it was addressed to', () => { + stashPendingCanvasLens(lens) + expect(takePendingCanvasLens('canvas:desk:abc')).toEqual(lens) + }) + + it('is claimed exactly once, so revisiting does not re-insert', () => { + stashPendingCanvasLens(lens) + + expect(takePendingCanvasLens('canvas:desk:abc')).not.toBeNull() + expect(takePendingCanvasLens('canvas:desk:abc')).toBeNull() + }) + + it('leaves a request alone when a different canvas opens first', () => { + stashPendingCanvasLens(lens) + + expect(takePendingCanvasLens('canvas:other')).toBeNull() + expect(takePendingCanvasLens('canvas:desk:abc')).toEqual(lens) + }) + + it('carries a projection plan through the handoff', () => { + const withProjection = { + ...lens, + projection: { + commandId: 'social.canvasProjection.create' as const, + title: 'YouTube Videos', + nodeCount: 1, + edgeCount: 0, + omittedNodeCount: 0, + omittedEdgeCount: 0, + bounds: { x: 0, y: 0, width: 260, height: 132 }, + nodes: [], + edges: [] + } + } + + stashPendingCanvasLens(withProjection) + expect(takePendingCanvasLens('canvas:desk:abc')?.projection?.title).toBe('YouTube Videos') + }) + + it('returns nothing when the stored value is unusable', () => { + sessionStorage.setItem('xnet:views:pending-canvas-lens', 'not json') + expect(takePendingCanvasLens('canvas:desk:abc')).toBeNull() + }) +}) diff --git a/packages/views/src/data-workspace/pending-canvas-lens.ts b/packages/views/src/data-workspace/pending-canvas-lens.ts new file mode 100644 index 000000000..9cf054e07 --- /dev/null +++ b/packages/views/src/data-workspace/pending-canvas-lens.ts @@ -0,0 +1,109 @@ +/** + * Handing a saved lens from the Data Workspace to a canvas (exploration 0419). + * + * The desktop app could already do this because its canvas is always mounted: + * the workspace calls a ref and the frame appears. The web app routes between + * `/data` and `/canvas/:id`, so the canvas that should receive the frame does + * not exist yet at the moment the user asks for it. + * + * This is the smallest thing that closes that gap: park the request, navigate, + * and let the canvas claim it on arrival. Parking is in `sessionStorage` so a + * full page load on the way still finds it, with an in-memory fallback for + * environments that have no storage. + * + * A parked request is claimed exactly once. Leaving it in place would mean a + * frame reappearing every time the user revisited that canvas, which reads as + * the app duplicating their work. + */ + +import type { SocialCanvasProjectionPlan } from '@xnetjs/social/projection' + +const STORAGE_KEY = 'xnet:views:pending-canvas-lens' + +export type PendingCanvasLens = { + /** Canvas the lens should land on. */ + canvasId: string + /** Saved view id. */ + viewId: string + title: string + /** Serialized `SavedViewDescriptor`. */ + descriptorJson: string | null + /** + * A laid-out projection to place instead of a live query frame. + * + * Serializable by construction — the plan is plain data — which is what + * lets the request survive a full page load between the two routes. + */ + projection?: SocialCanvasProjectionPlan +} + +/** In-memory fallback for environments without `sessionStorage`. */ +let memoryPending: PendingCanvasLens | null = null + +function storage(): Storage | null { + try { + return typeof sessionStorage === 'undefined' ? null : sessionStorage + } catch { + // Storage access throws in some sandboxed contexts rather than being absent. + return null + } +} + +export function stashPendingCanvasLens(pending: PendingCanvasLens): void { + memoryPending = pending + try { + storage()?.setItem(STORAGE_KEY, JSON.stringify(pending)) + } catch { + // Memory already holds it; a quota or privacy-mode failure is not worth + // failing the user's action over. + } +} + +function readPending(): PendingCanvasLens | null { + const raw = (() => { + try { + return storage()?.getItem(STORAGE_KEY) ?? null + } catch { + return null + } + })() + + if (!raw) return memoryPending + + try { + const parsed = JSON.parse(raw) as PendingCanvasLens + if (typeof parsed?.canvasId !== 'string' || typeof parsed?.viewId !== 'string') return null + return parsed + } catch { + return null + } +} + +function clearPending(): void { + memoryPending = null + try { + storage()?.removeItem(STORAGE_KEY) + } catch { + // Nothing to do; the in-memory copy is already gone. + } +} + +/** + * Claim the parked lens for a canvas, if one is waiting for it. + * + * Returns `null` when nothing is parked or when what is parked belongs to a + * different canvas — in which case it is left alone, so opening an unrelated + * canvas on the way does not swallow the request. + */ +export function takePendingCanvasLens(canvasId: string): PendingCanvasLens | null { + const pending = readPending() + if (!pending || pending.canvasId !== canvasId) return null + + clearPending() + return pending +} + +/** Drop any parked lens. Exported for tests and for cancel paths. */ +export function clearPendingCanvasLens(): void { + clearPending() +} diff --git a/packages/views/src/index.ts b/packages/views/src/index.ts index 8b7caf0f7..13bd550db 100644 --- a/packages/views/src/index.ts +++ b/packages/views/src/index.ts @@ -267,6 +267,10 @@ export { useDataWorkspace, getDefaultSocialWorkspaceSeeds, upsertDefaultSocialWorkspace, + clearPendingCanvasLens, + stashPendingCanvasLens, + takePendingCanvasLens, + type PendingCanvasLens, type DataWorkspaceBodyProps, type GraphAtlasRow, type SavedViewCanvasFrameInput, @@ -277,6 +281,10 @@ export { type WorkspaceMetric } from './data-workspace/index.js' +// Social feed enrichment (0170/0419) — shared so the desktop app gets the +// same titles and thumbnails the web app has had since 0170. +export { useSocialFeedEnrichment } from './social-enrichment/index.js' + // Shared CanvasView core (exploration 0277 / 0230 Phase 5): canvas // capabilities both the web and desktop CanvasViews consume. export { @@ -294,6 +302,8 @@ export { CanvasWidgetNodeCard, getCanvasQueryFrameTargets, parseSavedViewDescriptorForCanvasFrame, + applySocialCanvasProjectionPlan, + describeSocialCanvasProjection, useCanvasQueryFrames, isPeekableCanvasDisplayType, shouldActivateDatabasePreviewSurface, diff --git a/packages/views/src/social-enrichment/index.ts b/packages/views/src/social-enrichment/index.ts new file mode 100644 index 000000000..e4fededdd --- /dev/null +++ b/packages/views/src/social-enrichment/index.ts @@ -0,0 +1,5 @@ +/** + * Social feed enrichment for every surface (exploration 0419). + */ + +export { useSocialFeedEnrichment } from './useSocialFeedEnrichment.js' diff --git a/apps/web/src/hooks/useSocialFeedEnrichment.ts b/packages/views/src/social-enrichment/useSocialFeedEnrichment.ts similarity index 78% rename from apps/web/src/hooks/useSocialFeedEnrichment.ts rename to packages/views/src/social-enrichment/useSocialFeedEnrichment.ts index cef9c88e2..002affedb 100644 --- a/apps/web/src/hooks/useSocialFeedEnrichment.ts +++ b/packages/views/src/social-enrichment/useSocialFeedEnrichment.ts @@ -1,21 +1,23 @@ /** - * useSocialFeedEnrichment — feed enrichment adapter backed by local - * SocialEnrichment nodes and the hub /unfurl proxy. + * Feed enrichment, shared by every surface (exploration 0419). * - * Lookups merge locally cached titles, descriptions, and thumbnails over - * imported preview rows; requestMany feeds a rate-limited queue that - * fetches metadata (and thumbnail bytes into the BlobStore) for the - * previews currently on screen, once, and persists the result so every - * later render works entirely from the local store. + * This hook lived in `apps/web` and so did the capability: the desktop app + * rendered the same imported feeds as a wall of opaque platform ids because + * the enrichment pipeline simply was not there. Moving it here is the whole + * fix — the logic never had anything web-specific in it. + * + * Lookups merge locally cached titles, descriptions and thumbnails over + * imported preview rows. `requestMany` feeds a paced queue that resolves the + * previews currently on screen, once, and persists the result so every later + * render works entirely from the local store. */ + import type { SavedViewFeedEnrichmentAdapter, SavedViewFeedEnrichmentEntry, SavedViewVisualPreviewModel } from '@xnetjs/react' import { useMutate, useQuery, useXNet } from '@xnetjs/react' -import { createSocialEnrichmentId, SocialEnrichmentSchema } from '@xnetjs/social/schemas' -import { useEffect, useMemo, useRef, useState } from 'react' import { buildEnrichmentNodeData, enrichmentTargetForPreview, @@ -28,9 +30,13 @@ import { resolveHubAuthToken, socialEnrichmentKey, SocialEnrichmentQueue, + supportsDirectOEmbed, + thumbnailContentTypeFor, type EnrichmentRowLike, type SocialEnrichmentTarget -} from './social-feed-enrichment' +} from '@xnetjs/social/enrichment' +import { createSocialEnrichmentId, SocialEnrichmentSchema } from '@xnetjs/social/schemas' +import { useEffect, useMemo, useRef, useState } from 'react' type EnrichmentRow = EnrichmentRowLike & { platform?: string @@ -74,6 +80,7 @@ export function useSocialFeedEnrichment(): SavedViewFeedEnrichmentAdapter { blobStore, hasUrl: (cid) => blobUrlsRef.current.has(cid), createUrl: (blob) => URL.createObjectURL(blob), + contentTypeFor: thumbnailContentTypeFor, isCancelled: () => cancelled }).then((added) => { if (cancelled || added.size === 0) return @@ -88,12 +95,9 @@ export function useSocialFeedEnrichment(): SavedViewFeedEnrichmentAdapter { const executorRef = useRef<(target: SocialEnrichmentTarget) => Promise>(async () => {}) executorRef.current = async (target) => { - if (!hubUrl) return - - const token = await resolveHubAuthToken(getHubAuthToken) + const token = hubUrl ? await resolveHubAuthToken(getHubAuthToken) : '' const result = await fetchEnrichmentForTarget({ - httpUrl: hubHttpUrlFor(hubUrl), - headers: hubAuthHeaders(token), + ...(hubUrl ? { httpUrl: hubHttpUrlFor(hubUrl), headers: hubAuthHeaders(token) } : {}), target, blobStore }) @@ -104,8 +108,8 @@ export function useSocialFeedEnrichment(): SavedViewFeedEnrichmentAdapter { payload: result.payload, attemptCount: nextEnrichmentAttempt(existing), fetchedAtMs: Date.now(), - thumbnailBlobCid: result.thumbnailBlobCid, - thumbnailContentType: result.thumbnailContentType + ...(result.thumbnailBlobCid ? { thumbnailBlobCid: result.thumbnailBlobCid } : {}), + ...(result.thumbnailContentType ? { thumbnailContentType: result.thumbnailContentType } : {}) }) await mutate([ @@ -155,12 +159,6 @@ export function useSocialFeedEnrichment(): SavedViewFeedEnrichmentAdapter { return feedEnrichmentEntryFor(row, blobUrl) } - if (!hubUrl) { - // Without a hub there is nothing to fetch; cached enrichment still - // renders, new fetches resume when a hub session exists. - return { lookup } - } - return { lookup, requestMany: (previews) => { @@ -168,6 +166,10 @@ export function useSocialFeedEnrichment(): SavedViewFeedEnrichmentAdapter { .map((preview) => enrichmentTargetForPreview(preview)) .filter((target): target is SocialEnrichmentTarget => Boolean(target)) .filter((target) => !rowsByKeyRef.current.has(target.key)) + // Without a hub only the direct-oEmbed platforms can resolve; + // queueing the rest would write a row of `unavailable` nodes that + // then never retry once a hub does appear. + .filter((target) => Boolean(hubUrl) || supportsDirectOEmbed(target.platform)) if (targets.length > 0) ensureQueue().enqueue(targets) } } diff --git a/site/src/data/changelog/2026-08-01-your-imported-social-library-is-now-legi.json b/site/src/data/changelog/2026-08-01-your-imported-social-library-is-now-legi.json new file mode 100644 index 000000000..6f0772ec5 --- /dev/null +++ b/site/src/data/changelog/2026-08-01-your-imported-social-library-is-now-legi.json @@ -0,0 +1,11 @@ +{ + "id": "2026-08-01-your-imported-social-library-is-now-legi", + "date": "August 1, 2026", + "title": "Your imported social library is now legible, projectable, and searchable by transcript", + "summary": "Imported TikTok favourites and collections get their own feeds, every surface now shows real titles and thumbnails instead of platform ids, a saved lens can be projected onto the canvas as cards with the connections between them, and YouTube transcripts can be fetched per import run so an AI agent can search what was actually said in videos you saved.", + "highlights": [], + "tags": [ + "app", + "ai" + ] +} diff --git a/vitest.config.ts b/vitest.config.ts index cff8c80d0..ae05bf6e1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -146,6 +146,15 @@ export default defineConfig({ // Community hosting primitives are pure and dependency-free // (welcome queue — exploration 0359), same reasoning as feeds. 'packages/social/src/community/**/*.test.ts', + // Transcript fetching, segmentation and run accounting (0419) — + // pure functions over strings plus an injectable fetch seam. + 'packages/social/src/transcripts/**/*.test.ts', + // Agent-facing retrieval scope for social data (0419) — pure + // predicates over schema ids and privacy classes. + 'packages/social/src/retrieval/**/*.test.ts', + // Display enrichment (0170/0419) — moved out of apps/web so the + // desktop app shares it; pure logic plus an injectable fetch. + 'packages/social/src/enrichment/**/*.test.ts', // The publish pipeline (0420) is pure too — relative imports plus // @xnetjs/data — so it runs here rather than only under the package // config, where CI would never see it. Named explicitly rather than