feat(omni): S2 input expansion — image/audio/URL sources and token-dimension transport guard

packages/acp-bridge/tsconfig.json

@@ -12,6 +12,7 @@
       "@qwen-code/qwen-code-core/transcriptRecords": [
         "../core/src/utils/transcript-records.ts"
       ],
+      "@qwen-code/qwen-code-core/omni": ["../core/src/omni/index.ts"],
       "@qwen-code/qwen-code-core/*": ["../core/src/*"]
     }
   },

packages/acp-bridge/vitest.config.ts

@@ -18,6 +18,10 @@ export default defineConfig({
         __dirname,
         '../core/src/utils/transcript-records.ts',
       ),
+      '@qwen-code/qwen-code-core/omni': path.resolve(
+        __dirname,
+        '../core/src/omni/index.ts',
+      ),
     },
   },
   test: {

packages/cli/src/acp-integration/session/Session.test.ts

@@ -461,6 +461,9 @@ describe('Session', () => {
   }
 
   beforeEach(() => {
+    // Self-hosted CI runners export QWEN_RUNTIME_DIR; it outranks
+    // Storage.setRuntimeBaseDir, breaking the runtime-pinning assertions.
+    vi.stubEnv('QWEN_RUNTIME_DIR', '');
     startToolSpanSpy.mockClear();
     addToolArgumentsAttributesSpy.mockClear();
     addToolCallResultAttributesSpy.mockClear();
@@ -662,6 +665,7 @@ describe('Session', () => {
   });
 
   afterEach(() => {
+    vi.unstubAllEnvs();
     // Reset global runtime base dir state to prevent state leakage between tests
     core.Storage.setRuntimeBaseDir(null);
     // Clear session reference to allow garbage collection

packages/cli/src/acp-integration/session/Session.ts

@@ -8263,6 +8263,22 @@ export class Session implements SessionContext {
             });
           }
 
+          // Omni second normalization trigger point (parity with
+          // CoreToolScheduler.processToolResultImages, design §8.2):
+          // inline tool-result media becomes oss:// fileData before the
+          // vision bridge runs, which then skips the converted parts.
+          {
+            // Dynamic import: keeps omni out of the ACP static closure
+            // (serve fast-path bundle-closure CI check).
+            const { processToolResultOmniMedia } = await import(
+              '@qwen-code/qwen-code-core/omni'
+            );
+            responseParts = await processToolResultOmniMedia(
+              responseParts,
+              this.config,
+              activeToolAbortSignal,
+            );
+          }
           const visionBridgeNotices: string[] = [];
           responseParts = await bridgeToolResultImages({
             config: this.config,

packages/cli/src/config/config.ts

@@ -2193,6 +2193,8 @@ export async function loadCliConfig(
       : undefined,
     omniEnabled: settings.omni?.enabled ?? false,
     omniUploadMaxFileBytes: settings.omni?.upload?.maxFileBytes,
+    omniMaxEstimatedTokens: settings.omni?.transport?.maxEstimatedTokens,
+    omniDownloadMaxFileBytes: settings.omni?.download?.maxFileBytes,
     // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes
     // through the CDP tunnel (far lighter than the OS-level computer-use
     // driver), so disable computer-use to keep the agent off that heavy path.

packages/cli/src/config/settingsSchema.ts

@@ -3677,6 +3677,67 @@ const SETTINGS_SCHEMA = {
           },
         },
       },
+      transport: {
+        type: 'object',
+        label: 'Omni Transport Guard',
+        category: 'Experimental',
+        requiresRestart: true,
+        default: {},
+        description:
+          'Transport guard dimensions beyond the byte ceiling for omni ' +
+          'media delivery.',
+        showInDialog: false,
+        properties: {
+          maxEstimatedTokens: {
+            type: 'number',
+            label: 'Max Estimated Tokens',
+            category: 'Experimental',
+            requiresRestart: true,
+            default: 0,
+            description:
+              'Estimated-token ceiling for a single omni media input, ' +
+              'checked before upload using the versioned raw-resource ' +
+              'estimator. 0 disables the token guard (estimates are still ' +
+              'attached for observability) — the estimation formula is ' +
+              'pending confirmation with the model provider; set a ' +
+              'positive threshold to enforce fail-closed rejection.',
+            showInDialog: false,
+            jsonSchemaOverride: {
+              type: 'number',
+              minimum: 0,
+              default: 0,
+            },
+          },
+        },
+      },
+      download: {
+        type: 'object',
+        label: 'Omni Download',
+        category: 'Experimental',
+        requiresRestart: true,
+        default: {},
+        description: 'URL media localization limits for omni delivery.',
+        showInDialog: false,
+        properties: {
+          maxFileBytes: {
+            type: 'number',
+            label: 'Max Download File Bytes',
+            category: 'Experimental',
+            requiresRestart: true,
+            default: 0,
+            description:
+              'Byte ceiling for downloading URL media inputs. 0 or unset ' +
+              'follows omni.upload.maxFileBytes (downloading more than the ' +
+              'upload channel can deliver is pointless).',
+            showInDialog: false,
+            jsonSchemaOverride: {
+              type: 'number',
+              minimum: 0,
+              default: 0,
+            },
+          },
+        },
+      },
     },
   },
 } as const satisfies SettingsSchema;

packages/cli/src/ui/hooks/atCommandProcessor.ts

@@ -251,6 +251,12 @@ export async function resolveAtCommandQuery({
     ref: { id?: string; title?: string };
   }> = [];
 
+  // URL media references (`@https://…`) collected during the loop and
+  // localized (downloaded → recognized → promoted into the omni object
+  // store) after it. Only active when omni delivery is on; otherwise URL
+  // tokens keep today's fall-through behavior (left as text).
+  const urlMediaRefs: Array<{ originalAtPath: string; url: string }> = [];
+
   for (const atPathPart of atPathCommandParts) {
     const originalAtPath = atPathPart.content; // e.g., "@file.txt" or "@"
 
@@ -263,6 +269,23 @@ export async function resolveAtCommandQuery({
 
     const pathName = originalAtPath.substring(1);
 
+    // URL media reference (`@https://…`): detected BEFORE every other
+    // parser — a URL can't be an extension/session/MCP ref, and without
+    // this branch it would fall through to filesystem resolution where the
+    // ENOENT skip silently drops it. Only intercepted when omni delivery
+    // is active; otherwise preserve the legacy text fall-through.
+    if (/^https?:\/\//i.test(pathName) && config.isOmniEnabled?.()) {
+      const omni = await import('@qwen-code/qwen-code-core/omni');
+      if (omni.isOmniDeliveryActive(config) && omni.parseHttpUrlRef(pathName)) {
+        if (!urlMediaRefs.some((r) => r.url === pathName)) {
+          urlMediaRefs.push({ originalAtPath, url: pathName });
+        }
+        // Keep the URL verbatim in the text sent to the model.
+        atPathToResolvedSpecMap.set(originalAtPath, pathName);
+        continue;
+      }
+    }
+
     // Extension reference (`@ext:<name>`): detected BEFORE MCP/filesystem
     // resolution. Only matches when the path starts with `ext:` and the name
     // corresponds to an active extension.
@@ -505,6 +528,99 @@ export async function resolveAtCommandQuery({
     ),
   );
 
+  // URL media localization: download → recognize → promote into the omni
+  // object store → upload → fileData part. Sequential (media downloads can
+  // be large; parallel GiB transfers would contend), failure-isolated per
+  // URL (an error card is shown and the turn continues — mirroring the MCP
+  // resource block's Promise.allSettled semantics).
+  const urlMediaParts: Part[] = [];
+  const urlMediaDisplays: IndividualToolCallDisplay[] = [];
+  const urlMediaLabels: string[] = [];
+  if (urlMediaRefs.length > 0) {
+    const core = await import('@qwen-code/qwen-code-core/omni');
+    for (let i = 0; i < urlMediaRefs.length; i++) {
+      const ref = urlMediaRefs[i];
+      const callId = `client-url-media-${userMessageTimestamp}-${i}`;
+      const hostLabel = (() => {
+        try {
+          return new URL(ref.url).hostname;
+        } catch {
+          return 'invalid-url';
+        }
+      })();
+      let tempPartPath: string | undefined;
+      try {
+        const store = new core.OmniObjectStore(config.storage.getQwenDir());
+        const downloadsDir = path.join(store.getOmniRootDir(), 'downloads');
+        const downloaded = await core.downloadMediaUrl({
+          url: ref.url,
+          downloadsDir,
+          maxBytes: core.effectiveMaxDownloadFileBytes(config),
+          signal,
+        });
+        tempPartPath = downloaded.partPath;
+        // Full local-file pipeline on the downloaded bytes: sniff/probe/
+        // hash again (the design requires re-recognition from the local
+        // file — never trust transfer-time observations), guard, store,
+        // upload. readMediaViaOmniDelivery needs a stable name for
+        // display; use the URL basename.
+        const urlBase = path.basename(new URL(ref.url).pathname) || hostLabel;
+        const delivery = await core.processMediaForOmniDelivery(
+          downloaded.partPath,
+          config,
+          { signal },
+        );
+        urlMediaParts.push({
+          fileData: {
+            fileUri: delivery.fileUri,
+            mimeType: delivery.mimeType,
+            displayName: urlBase,
+          },
+        });
+        urlMediaLabels.push(ref.url);
+        urlMediaDisplays.push({
+          callId,
+          name: 'Fetch Media URL',
+          description: `Downloaded ${ref.url}`,
+          status: ToolCallStatus.Success,
+          resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload.`,
+          confirmationDetails: undefined,
+        });
+      } catch (error) {
+        if (signal.aborted) {
+          // User cancelled mid-download: end the turn quietly instead of
+          // throwing — resolveAtCommandQuery has no throw contract, and a
+          // rejection here would surface as an unhandled-rejection banner
+          // (cancelOngoingRequest already handles the UI reset).
+          if (tempPartPath) {
+            await fs.rm(tempPartPath, { force: true }).catch(() => {});
+          }
+          return {
+            processedQuery: null,
+            shouldProceed: false,
+            toolDisplays: [...urlMediaDisplays],
+            filesRead: urlMediaLabels,
+            // No chat-recording entry for a user-cancelled resolution.
+          };
+        }
+        const reason = getErrorMessage(error);
+        onDebugMessage(`Failed to localize media URL ${ref.url}: ${reason}`);
+        urlMediaDisplays.push({
+          callId,
+          name: 'Fetch Media URL',
+          description: `Download ${ref.url}`,
+          status: ToolCallStatus.Error,
+          resultDisplay: `Failed to fetch media from ${hostLabel}: ${reason}`,
+          confirmationDetails: undefined,
+        });
+      } finally {
+        if (tempPartPath) {
+          await fs.rm(tempPartPath, { force: true }).catch(() => {});
+        }
+      }
+    }
+  }
+
   const resourceParts: Part[] = [];
   const resourceDisplays: IndividualToolCallDisplay[] = [];
   const resourceLabels: string[] = [];
@@ -565,7 +681,11 @@ export async function resolveAtCommandQuery({
     mcpResourceRefs.length === 0 &&
     mcpServerMentions.length === 0 &&
     extensionMentions.length === 0 &&
-    sessionMentions.length === 0
+    sessionMentions.length === 0 &&
+    // URL media refs count as content too: without this, a URL-only prompt
+    // would run the whole download→upload pipeline and then discard the
+    // delivered parts (and their display cards) via this early return.
+    urlMediaRefs.length === 0
   ) {
     onDebugMessage('No valid file paths found in @ commands to read.');
     if (initialQueryText === '@' && query.trim() === '@') {
@@ -852,13 +972,15 @@ export async function resolveAtCommandQuery({
         ...scopedMentionLabels,
         ...contentLabelsForDisplay,
         ...resourceLabels,
+        ...urlMediaLabels,
       ];
       return {
         processedQuery: null,
         shouldProceed: false,
         toolDisplays: [
           ...scopedMentionDisplays,
           ...resourceDisplays,
+          ...urlMediaDisplays,
           errorToolCallDisplay,
         ],
         filesRead: labelsOnError,
@@ -881,11 +1003,13 @@ export async function resolveAtCommandQuery({
     ...scopedMentionParts,
     ...fileParts,
     ...resourceParts,
+    ...urlMediaParts,
   ];
   const allLabels = [
     ...scopedMentionLabels,
     ...contentLabelsForDisplay,
     ...resourceLabels,
+    ...urlMediaLabels,
   ];
 
   return {
@@ -895,6 +1019,7 @@ export async function resolveAtCommandQuery({
       ...scopedMentionDisplays,
       ...fileDisplays,
       ...resourceDisplays,
+      ...urlMediaDisplays,
     ],
     filesRead: allLabels,
     recording: {

packages/cli/tsconfig.json

@@ -11,6 +11,7 @@
       "@qwen-code/qwen-code-core/transcriptRecords": [
         "../core/src/utils/transcript-records.ts"
       ],
+      "@qwen-code/qwen-code-core/omni": ["../core/src/omni/index.ts"],
       "@qwen-code/qwen-code-core/*": ["../core/src/*"],
       "@qwen-code/acp-bridge": ["../acp-bridge/src/index.ts"],
       "@qwen-code/acp-bridge/transcriptReplay": [

packages/cli/vitest.config.ts

@@ -19,6 +19,10 @@ export default defineConfig({
         __dirname,
         '../core/src/utils/transcript-records.ts',
       ),
+      '@qwen-code/qwen-code-core/omni': path.resolve(
+        __dirname,
+        '../core/src/omni/index.ts',
+      ),
       '@qwen-code/qwen-code-core': path.resolve(__dirname, '../core/index.ts'),
       // cli's daemon-status-provider.test.ts imports `FakeAgent` /
       // `makeChannel` from acp-bridge's package-private

packages/core/package.json

@@ -13,6 +13,10 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
+    "./omni": {
+      "types": "./dist/src/omni/index.d.ts",
+      "import": "./dist/src/omni/index.js"
+    },
     "./transcriptRecords": {
       "types": "./dist/src/utils/transcript-records.d.ts",
       "import": "./dist/src/utils/transcript-records.js"

packages/core/src/config/config.ts

@@ -1104,6 +1104,10 @@ export interface ConfigParameters {
   omniEnabled?: boolean;
   /** Per-file byte ceiling for omni media uploads (default 1 GiB). */
   omniUploadMaxFileBytes?: number;
+  /** Estimated-token ceiling for omni media (0/unset = guard disabled). */
+  omniMaxEstimatedTokens?: number;
+  /** Byte ceiling for omni URL downloads (unset = follow upload cap). */
+  omniDownloadMaxFileBytes?: number;
   /** Image generation model selected through `/model --image`. */
   imageModel?: string;
   /**
@@ -1931,6 +1935,8 @@ export class Config {
   private readonly artifactOss?: ArtifactOssConfig;
   private readonly omniEnabled: boolean = false;
   private readonly omniUploadMaxFileBytes?: number;
+  private readonly omniMaxEstimatedTokens?: number;
+  private readonly omniDownloadMaxFileBytes?: number;
   private workflowsEnabled = false;
   private readonly skipWorkflowUsageWarning: boolean = false;
   private readonly computerUseEnabled: boolean = true;
@@ -2208,6 +2214,8 @@ export class Config {
     this.artifactOss = params.artifactOss;
     this.omniEnabled = params.omniEnabled ?? false;
     this.omniUploadMaxFileBytes = params.omniUploadMaxFileBytes;
+    this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens;
+    this.omniDownloadMaxFileBytes = params.omniDownloadMaxFileBytes;
     this.workflowsEnabled = params.workflowsEnabled ?? false;
     this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false;
     this.computerUseEnabled = params.computerUseEnabled ?? true;
@@ -6372,6 +6380,14 @@ export class Config {
     return this.omniUploadMaxFileBytes;
   }
 
+  getOmniMaxEstimatedTokens(): number | undefined {
+    return this.omniMaxEstimatedTokens;
+  }
+
+  getOmniDownloadMaxFileBytes(): number | undefined {
+    return this.omniDownloadMaxFileBytes;
+  }
+
   resolveImageGenerationModel(
     setting: string | undefined,
   ): ImageGenerationConfig | undefined {