feat(core): resolve model modalities from API metadata

docs/design/model-modality-catalog.md

@@ -0,0 +1,84 @@
+# API-backed model modalities
+
+## Problem
+
+Qwen Code currently derives missing input modalities from model-id regular
+expressions. That works for known model families, but new models and provider
+aliases remain text-only until the table is updated manually.
+
+## Design
+
+Ship a compact snapshot containing only provider identity and input modalities,
+generated from models.dev with `npm run generate:model-modalities`. Load it
+during CLI configuration and cache the full live catalog at
+`~/.qwen/models-dev.json`.
+
+A valid disk cache is used immediately. When no cache exists, startup uses the
+built-in snapshot immediately. Network access is always a background refresh,
+so a cold start never waits for models.dev. The default loader refreshes stale
+data after one hour and continues refreshing hourly while the process remains
+alive. Refresh failures are non-fatal and retain the last usable disk cache or
+built-in snapshot.
+
+A successful refresh replaces the process-wide current catalog. Configurations
+created after that point use the new metadata, including later sessions in a
+long-running daemon. Existing sessions retain the catalog they were created
+with so their capabilities do not change midway through a conversation.
+
+Model modality resolution uses this precedence:
+
+1. Explicit `generationConfig.modalities` from settings or a provider entry.
+2. Exact model metadata from the models.dev provider catalog.
+3. The existing model-id heuristic.
+4. Text-only for an unknown model.
+
+Model-id rules, including MiniMax M3 defaults, are fallback behavior only and
+never override an explicit user value.
+
+Built-in `/auth` templates omit modalities and let the catalog resolve them.
+Modalities selected explicitly in the advanced setup flow continue to take
+precedence. Existing saved configurations are not migrated or rewritten.
+Their modalities remain explicit and take precedence over catalog metadata.
+
+Provider identity is resolved from the configured provider when possible and
+otherwise from its endpoint or credential environment key. The catalog lookup
+is exact and case-insensitive; it does not guess across similarly named models.
+An SDK protocol such as `openai` is not treated as provider identity when an
+unknown custom endpoint or credential key is present.
+
+Exact provider endpoints take precedence over local provider aliases so the
+China and international Alibaba catalogs remain distinct. A missing endpoint
+uses the local provider's default region, while an endpoint that models.dev
+does not list falls back to the model-id heuristic instead of borrowing another
+region's metadata. Idealab's `Qwen*-DogFooding` models and its listed
+`bailian/` aliases resolve by model family to the corresponding Alibaba China,
+DeepSeek, Moonshot, or MiniMax model IDs.
+
+Every resolved provider first uses its exact catalog entry. When that catalog
+does not list a recognized DeepSeek, Kimi, MiniMax, or GLM model, lookup falls
+back to the model family's original provider catalog. This applies to Alibaba
+and other gateways, while exact metadata from the current provider always
+takes precedence. Lookup never scans arbitrary providers for the first matching
+model ID; unrecognized families remain on the local fallback path.
+
+OpenRouter model variants first use an exact catalog entry. When models.dev
+does not list the variant, the recognized OpenRouter suffixes `:free`,
+`:extended`, `:thinking`, `:online`, `:nitro`, `:floor`, and `:exacto` fall
+back to the base model. Unknown suffixes remain unmatched.
+
+The loaded catalog is passed to `ModelsConfig` and `ModelRegistry`. This covers
+initial resolution, provider-backed models installed through `/auth`, manually
+edited model providers, runtime model switches, and model-provider hot reloads
+without persisting remotely supplied metadata into user settings.
+The daemon workspace-provider status path uses the same catalog so model lists
+shown before a session starts match the configuration used by live sessions.
+
+## Boundaries
+
+- This change reads input modalities only. It does not import pricing, token
+  limits, reasoning flags, or the remote model list.
+- It does not call provider-specific `/models` endpoints because their schemas
+  are not portable. Those endpoints can be added later as a higher-priority
+  source without changing the catalog interface.
+- Unit tests disable the automatic network load and exercise the loader with
+  injected cache paths and fetch functions.

package.json

@@ -34,6 +34,7 @@
     "dev:daemon": "node scripts/daemon-dev.js",
     "debug": "cross-env DEBUG=1 node --inspect-brk scripts/start.js",
     "generate": "node scripts/generate-git-commit-info.js",
+    "generate:model-modalities": "node scripts/generate-model-modality-snapshot.js",
     "generate:settings-schema": "node --import tsx/esm scripts/generate-settings-schema.ts",
     "build": "cross-env NODE_OPTIONS=\"--max-old-space-size=3072\" node scripts/build.js",
     "build-and-start": "npm run build && npm run start",

packages/cli/src/config/config.test.ts

@@ -1153,6 +1153,75 @@ describe('loadCliConfig', () => {
     ]);
   });
 
+  it('loads model metadata for production CLI sessions', async () => {
+    const originalNodeEnv = process.env['NODE_ENV'];
+    const originalVitest = process.env['VITEST'];
+    const originalVitestWorkerId = process.env['VITEST_WORKER_ID'];
+    vi.stubEnv('OPENAI_API_KEY', '');
+    vi.stubEnv('OPENAI_BASE_URL', '');
+    process.env['NODE_ENV'] = 'production';
+    delete process.env['VITEST'];
+    delete process.env['VITEST_WORKER_ID'];
+    const catalogSpy = vi
+      .spyOn(ServerConfig, 'loadModelMetadataCatalog')
+      .mockResolvedValue({
+        openai: {
+          models: {
+            'gpt-4o': {
+              modalities: { input: ['text', 'image', 'pdf'] },
+            },
+          },
+        },
+      });
+    process.argv = [
+      'node',
+      'script.js',
+      '--auth-type',
+      'openai',
+      '--model',
+      'gpt-4o',
+    ];
+
+    try {
+      const argv = await parseArguments();
+      const config = await loadCliConfig(
+        {
+          proxy: 'http://catalog-proxy.example:8080',
+          security: { auth: { selectedType: 'openai' } },
+          modelProviders: { openai: [{ id: 'gpt-4o' }] },
+        },
+        argv,
+      );
+
+      expect(catalogSpy).toHaveBeenCalledWith({
+        proxyUrl: 'http://catalog-proxy.example:8080',
+      });
+      expect(config.getModelsConfig().getGenerationConfig().modalities).toEqual(
+        { image: true, pdf: true },
+      );
+    } finally {
+      if (originalNodeEnv === undefined) delete process.env['NODE_ENV'];
+      else process.env['NODE_ENV'] = originalNodeEnv;
+      if (originalVitest === undefined) delete process.env['VITEST'];
+      else process.env['VITEST'] = originalVitest;
+      if (originalVitestWorkerId === undefined) {
+        delete process.env['VITEST_WORKER_ID'];
+      } else {
+        process.env['VITEST_WORKER_ID'] = originalVitestWorkerId;
+      }
+    }
+  });
+
+  it('does not load model metadata in test mode', async () => {
+    const catalogSpy = vi.spyOn(ServerConfig, 'loadModelMetadataCatalog');
+    process.argv = ['node', 'script.js'];
+
+    const argv = await parseArguments();
+    await loadCliConfig({}, argv);
+
+    expect(catalogSpy).not.toHaveBeenCalled();
+  });
+
   it('enables debug file logging for --debug when QWEN_DEBUG_LOG_FILE is unset', async () => {
     delete process.env['QWEN_DEBUG_LOG_FILE'];
     process.argv = ['node', 'script.js', '--debug'];

packages/cli/src/config/config.ts

@@ -43,6 +43,7 @@ import {
   type SkillLevel,
   type WebSearchSettings,
   MAX_SUBAGENT_DEPTH_LIMIT,
+  loadModelMetadataCatalog,
   addDaemonRequestAttribute,
 } from '@qwen-code/qwen-code-core';
 import { extensionsCommand } from '../commands/extensions.js';
@@ -1573,6 +1574,19 @@ export async function loadCliConfig(
   const bareMode = isBareMode(argv.bare);
   const safeMode =
     argv.safeMode !== undefined ? argv.safeMode : isSafeModeEnv();
+  const proxy =
+    argv.proxy ||
+    settings.proxy ||
+    process.env['HTTPS_PROXY'] ||
+    process.env['https_proxy'] ||
+    process.env['HTTP_PROXY'] ||
+    process.env['http_proxy'];
+  const modelMetadataCatalogPromise =
+    process.env['NODE_ENV'] === 'test' ||
+    process.env['VITEST'] !== undefined ||
+    process.env['VITEST_WORKER_ID'] !== undefined
+      ? Promise.resolve({})
+      : loadModelMetadataCatalog({ proxyUrl: proxy });
 
   // Surface `--insecure` as an env var so it reaches the undici dispatcher
   // layer (which controls TLS verification) without threading a flag through
@@ -2248,13 +2262,7 @@ export async function loadCliConfig(
     clearContextOnIdle: settings.context?.clearContextOnIdle,
     fileFiltering: settings.context?.fileFiltering,
     plansDirectory: settings.plansDirectory,
-    proxy:
-      argv.proxy ||
-      settings.proxy ||
-      process.env['HTTPS_PROXY'] ||
-      process.env['https_proxy'] ||
-      process.env['HTTP_PROXY'] ||
-      process.env['http_proxy'],
+    proxy,
     cwd,
     fileDiscoveryService: fileService,
     bugCommand: settings.advanced?.bugCommand,
@@ -2322,6 +2330,7 @@ export async function loadCliConfig(
     includePartialMessages,
     modelProvidersConfig,
     providerProtocolConfig,
+    modelMetadataCatalog: await modelMetadataCatalogPromise,
     generationConfigSources: resolvedCliConfig.sources,
     generationConfig: resolvedCliConfig.generationConfig,
     initialModelRegistryBaseUrl: resolvedCliConfig.registryBaseUrl,

packages/cli/src/serve/server/auth-provider-helpers.test.ts

@@ -0,0 +1,21 @@
+/**
+ * @license
+ * Copyright 2026 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, expect, it } from 'vitest';
+import { buildAuthProviderCatalog } from './auth-provider-helpers.js';
+
+describe('buildAuthProviderCatalog', () => {
+  it('exposes preset modalities to daemon clients', () => {
+    const catalog = buildAuthProviderCatalog('/workspace');
+    const minimax = catalog.providers.find(
+      (provider) => provider.id === 'minimax',
+    );
+
+    expect(
+      minimax?.models?.find((model) => model.id === 'MiniMax-M3')?.modalities,
+    ).toEqual({ image: true, video: true });
+  });
+});

packages/cli/src/serve/workspace-providers-status.test.ts

@@ -22,6 +22,7 @@ const coreMock = vi.hoisted(() => ({
     isEnabled: vi.fn(() => false),
     warn: vi.fn(),
   },
+  loadModelMetadataCatalog: vi.fn(async () => ({})),
 }));
 
 vi.mock('@qwen-code/qwen-code-core', async (importOriginal) => {
@@ -38,6 +39,7 @@ vi.mock('@qwen-code/qwen-code-core', async (importOriginal) => {
   return {
     ...actual,
     createDebugLogger: () => coreMock.debugLogger,
+    loadModelMetadataCatalog: coreMock.loadModelMetadataCatalog,
     ModelsConfig: TestModelsConfig,
   };
 });
@@ -71,6 +73,7 @@ describe('createWorkspaceProvidersStatusProvider', () => {
     coreMock.modelsConfigErrorMessage =
       'Failed loading provider https://user:secret@broken.example/v1';
     coreMock.debugLogger.warn.mockClear();
+    coreMock.loadModelMetadataCatalog.mockClear();
     resetHomeEnvBootstrapForTesting();
   });
 
@@ -131,6 +134,69 @@ describe('createWorkspaceProvidersStatusProvider', () => {
     expect(second.current?.modelId).toBe('model-b(openai)');
   });
 
+  it('loads the catalog with the workspace proxy outside test mode', async () => {
+    await writeUserSettings({ proxy: 'http://settings-proxy.example:8080' });
+    const provider = createWorkspaceProvidersStatusProvider({
+      env: { NODE_ENV: 'production' },
+    });
+
+    await provider(workspace, false);
+
+    expect(coreMock.loadModelMetadataCatalog).toHaveBeenCalledOnce();
+    expect(coreMock.loadModelMetadataCatalog).toHaveBeenCalledWith({
+      proxyUrl: 'http://settings-proxy.example:8080',
+    });
+  });
+
+  it('does not load the catalog when the explicit test gate fires', async () => {
+    const provider = createWorkspaceProvidersStatusProvider({
+      env: {
+        NODE_ENV: 'test',
+        HTTPS_PROXY: 'http://env-proxy.example:8080',
+      },
+    });
+
+    await provider(workspace, false);
+
+    expect(coreMock.loadModelMetadataCatalog).not.toHaveBeenCalled();
+  });
+
+  it('reports catalog modalities for configured workspace models', async () => {
+    const provider = createWorkspaceProvidersStatusProvider({
+      env: {},
+      modelMetadataCatalog: {
+        openrouter: {
+          api: 'https://openrouter.ai/api/v1',
+          models: {
+            'google/gemma-3-12b-it': {
+              modalities: { input: ['text', 'image'] },
+            },
+          },
+        },
+      },
+    });
+    await writeUserSettings({
+      security: { auth: { selectedType: 'openai' } },
+      model: { name: 'google/gemma-3-12b-it' },
+      modelProviders: {
+        openai: [
+          {
+            id: 'google/gemma-3-12b-it',
+            baseUrl: 'https://openrouter.ai/api/v1',
+            envKey: 'OPENROUTER_API_KEY',
+          },
+        ],
+      },
+    });
+
+    const result = await provider(workspace, false);
+    const model = result.providers
+      .flatMap((entry) => entry.models)
+      .find((entry) => entry.baseModelId === 'google/gemma-3-12b-it');
+
+    expect(model?.modalities).toEqual({ image: true });
+  });
+
   it('returns the workspace approval mode', async () => {
     const provider = createWorkspaceProvidersStatusProvider({ env: {} });
     await writeUserSettings({

packages/cli/src/serve/workspace-providers-status.ts

@@ -8,10 +8,11 @@ import {
   ApprovalMode,
   APPROVAL_MODES,
   createDebugLogger,
+  loadModelMetadataCatalog,
   ModelsConfig,
   tokenLimit,
 } from '@qwen-code/qwen-code-core';
-import type { AuthType } from '@qwen-code/qwen-code-core';
+import type { AuthType, ModelMetadataCatalog } from '@qwen-code/qwen-code-core';
 import type {
   ServeWorkspaceProviderCurrent,
   ServeWorkspaceProviderModel,
@@ -45,6 +46,7 @@ export interface WorkspaceProvidersStatusProviderOptions {
   argv?: Partial<CliGenerationConfigInputs['argv']>;
   env?: Record<string, string | undefined>;
   workspaceTrusted?: boolean;
+  modelMetadataCatalog?: ModelMetadataCatalog;
 }
 
 export function createWorkspaceProvidersStatusProvider(
@@ -54,11 +56,11 @@ export function createWorkspaceProvidersStatusProvider(
     buildWorkspaceProvidersStatus(workspaceCwd, acpChannelLive, options);
 }
 
-function buildWorkspaceProvidersStatus(
+async function buildWorkspaceProvidersStatus(
   workspaceCwd: string,
   acpChannelLive: boolean,
   options: WorkspaceProvidersStatusProviderOptions,
-): ServeWorkspaceProvidersStatus {
+): Promise<ServeWorkspaceProvidersStatus> {
   try {
     const loaded = loadSettings(
       workspaceCwd,
@@ -77,6 +79,20 @@ function buildWorkspaceProvidersStatus(
     );
     const settings = loaded.merged;
     const env = options.env ?? snapshotProcessEnv();
+    const modelMetadataCatalog =
+      options.modelMetadataCatalog ??
+      (env['NODE_ENV'] === 'test' ||
+      env['VITEST'] !== undefined ||
+      env['VITEST_WORKER_ID'] !== undefined
+        ? {}
+        : await loadModelMetadataCatalog({
+            proxyUrl:
+              settings.proxy ||
+              env['HTTPS_PROXY'] ||
+              env['https_proxy'] ||
+              env['HTTP_PROXY'] ||
+              env['http_proxy'],
+          }));
     const selectedAuthType =
       settings.security?.auth?.selectedType ?? getAuthTypeFromEnv(env);
     const argv: CliGenerationConfigInputs['argv'] = {
@@ -98,6 +114,7 @@ function buildWorkspaceProvidersStatus(
       providerProtocolConfig: settings.providerProtocol,
       generationConfig: resolvedCliConfig.generationConfig,
       generationConfigSources: resolvedCliConfig.sources,
+      modelMetadataCatalog,
     });
     const currentAuth = selectedAuthType;
     const currentModelId = (