feat(tools): add ToolSearch for on-demand loading of deferred tool schemas

integration-tests/cli/json-schema.test.ts

@@ -0,0 +1,247 @@
+/**
+ * @license
+ * Copyright 2025 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Integration tests for `--json-schema` headless structured output.
+ *
+ * Validates that:
+ *   - A valid schema makes the synthetic `structured_output` tool the only
+ *     way for the model to terminate, and the submitted args land in the
+ *     result message's `structured_result` field.
+ *   - Schema validation happens at CLI parse time; bad schemas fail fast
+ *     with a non-zero exit code instead of silently no-oping at runtime.
+ *   - File-based schemas (`@/path/to/schema.json`) are loaded and parsed.
+ */
+
+import { describe, it, expect, afterEach } from 'vitest';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { TestRig, validateModelOutput } from '../test-helper.js';
+
+interface ResultMessage {
+  type: string;
+  is_error: boolean;
+  result?: string;
+  structured_result?: unknown;
+  error?: { message: string };
+}
+
+function findResultMessage(parsed: unknown): ResultMessage | undefined {
+  if (!Array.isArray(parsed)) return undefined;
+  return parsed.find(
+    (msg): msg is ResultMessage =>
+      typeof msg === 'object' &&
+      msg !== null &&
+      (msg as { type?: unknown }).type === 'result',
+  );
+}
+
+describe('--json-schema headless structured output', () => {
+  let rig: TestRig;
+
+  afterEach(async () => {
+    if (rig) await rig.cleanup();
+  });
+
+  it('emits structured_result when the model fills the schema', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-inline');
+
+    const schema = JSON.stringify({
+      type: 'object',
+      required: ['answer'],
+      properties: {
+        answer: { type: 'number' },
+      },
+      additionalProperties: false,
+    });
+
+    const stdout = await rig.run(
+      'What is 2 + 2? Submit it via the structured_output tool.',
+      '--output-format',
+      'json',
+      '--json-schema',
+      schema,
+    );
+
+    const parsed = JSON.parse(stdout);
+    const result = findResultMessage(parsed);
+    expect(result, 'expected a result message').toBeDefined();
+    expect(result!.is_error).toBe(false);
+    expect(result, 'expected structured_result on success').toHaveProperty(
+      'structured_result',
+    );
+
+    const structured = result!.structured_result as { answer?: unknown };
+    expect(structured).toBeTypeOf('object');
+    expect(structured.answer).toBe(4);
+
+    // The `result` string must be the JSON-stringified payload (contract).
+    expect(typeof result!.result).toBe('string');
+    expect(JSON.parse(result!.result!)).toEqual(structured);
+
+    // The structured_output tool must have been invoked.
+    const toolLogs = rig.readToolLogs();
+    const found = toolLogs.find(
+      (l) => l.toolRequest.name === 'structured_output',
+    );
+    expect(
+      found,
+      `expected structured_output tool call, saw: ${toolLogs.map((l) => l.toolRequest.name).join(', ')}`,
+    ).toBeTruthy();
+
+    validateModelOutput(stdout, null, 'json-schema inline');
+  });
+
+  it('loads a schema from disk via the @path syntax', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-file');
+
+    const schemaPath = join(rig.testDir!, 'schema.json');
+    writeFileSync(
+      schemaPath,
+      JSON.stringify({
+        type: 'object',
+        required: ['city', 'country'],
+        properties: {
+          city: { type: 'string' },
+          country: { type: 'string' },
+        },
+        additionalProperties: false,
+      }),
+    );
+
+    const stdout = await rig.run(
+      'What is the capital of France and what country is it in? Submit via structured_output.',
+      '--output-format',
+      'json',
+      '--json-schema',
+      `@${schemaPath}`,
+    );
+
+    const result = findResultMessage(JSON.parse(stdout));
+    expect(result?.is_error).toBe(false);
+    const structured = result!.structured_result as {
+      city?: unknown;
+      country?: unknown;
+    };
+    expect(structured).toBeTypeOf('object');
+    expect(typeof structured.city).toBe('string');
+    expect(typeof structured.country).toBe('string');
+    expect(String(structured.city).toLowerCase()).toContain('paris');
+  });
+
+  it('fails fast at CLI parse time on invalid JSON', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-bad-json');
+
+    let thrown: Error | undefined;
+    try {
+      await rig.run('hi', '--json-schema', '{not valid json');
+      expect.fail('expected non-zero exit on invalid JSON');
+    } catch (e) {
+      thrown = e as Error;
+    }
+
+    expect(thrown).toBeDefined();
+    expect(thrown!.message).toMatch(/--json-schema is not valid JSON/i);
+  });
+
+  it('fails fast at CLI parse time on invalid JSON Schema', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-bad-schema');
+
+    // Ajv strict-compile will reject `type: "this-is-not-a-real-type"`.
+    let thrown: Error | undefined;
+    try {
+      await rig.run(
+        'hi',
+        '--json-schema',
+        JSON.stringify({ type: 'this-is-not-a-real-type' }),
+      );
+      expect.fail('expected non-zero exit on invalid schema');
+    } catch (e) {
+      thrown = e as Error;
+    }
+
+    expect(thrown).toBeDefined();
+    expect(thrown!.message).toMatch(
+      /--json-schema is not a valid JSON Schema/i,
+    );
+  });
+
+  it('rejects a missing schema file', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-missing-file');
+
+    let thrown: Error | undefined;
+    try {
+      await rig.run('hi', '--json-schema', '@/tmp/__does_not_exist__.json');
+      expect.fail('expected non-zero exit on missing file');
+    } catch (e) {
+      thrown = e as Error;
+    }
+
+    expect(thrown).toBeDefined();
+    expect(thrown!.message).toMatch(/--json-schema could not read/i);
+  });
+
+  it('exits 1 with is_error=true when the model emits plain text instead of calling structured_output', async () => {
+    rig = new TestRig();
+    await rig.setup('json-schema-plain-text-error');
+
+    const schema = JSON.stringify({
+      type: 'object',
+      required: ['answer'],
+      properties: { answer: { type: 'string' } },
+      additionalProperties: false,
+    });
+
+    // Force the model down the plain-text path deterministically by
+    // excluding the synthetic tool from the registry. Without
+    // structured_output available, the model has no choice but to emit
+    // plain text, which is exactly the failure mode this branch handles
+    // (`config.getJsonSchema()` set + no submission == exit 1 + isError).
+    let thrown: Error | undefined;
+    try {
+      await rig.run(
+        'Reply with the literal text "ok".',
+        '--output-format',
+        'json',
+        '--json-schema',
+        schema,
+        '--exclude-tools',
+        'structured_output',
+      );
+      expect.fail('expected non-zero exit when model emits plain text');
+    } catch (e) {
+      thrown = e as Error;
+    }
+
+    expect(thrown).toBeDefined();
+
+    // Stdout (containing the JSON result array) is captured in the error
+    // body in JSON-output mode.
+    const stdoutMatch = thrown!.message.match(
+      /Stdout:\n([\s\S]*?)(?:\n\nStderr:|$)/,
+    );
+    expect(
+      stdoutMatch,
+      `expected JSON stdout in error body, got: ${thrown!.message.slice(0, 400)}`,
+    ).toBeTruthy();
+
+    const parsed = JSON.parse(stdoutMatch![1]);
+    const result = findResultMessage(parsed);
+    expect(result).toBeDefined();
+    expect(result!.is_error).toBe(true);
+    expect(result!.error?.message).toMatch(/Model produced plain text/i);
+
+    // structured_output must NOT have been called (otherwise the success
+    // branch would have terminated and we'd never hit this code path).
+    const calls = rig.readToolLogs().map((l) => l.toolRequest.name);
+    expect(calls).not.toContain('structured_output');
+  });
+});

integration-tests/cli/tool-search.test.ts

@@ -0,0 +1,139 @@
+/**
+ * @license
+ * Copyright 2025 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Integration tests for ToolSearch / deferred-tool flow.
+ *
+ * Validates the core contract: tools flagged `shouldDefer=true` are NOT in
+ * the initial function-declaration list, but the model can reach them via
+ * `tool_search` (either by `select:Name` lookup or keyword query) and then
+ * invoke them in the same session.
+ *
+ * Cron tools (cron_create, cron_list, cron_delete) are convenient deferred
+ * targets: deterministic, side-effect-free in -p mode, and gated behind the
+ * `experimental.cron` setting so we control when they're registered.
+ */
+
+import { describe, it, expect, afterEach } from 'vitest';
+import {
+  TestRig,
+  printDebugInfo,
+  validateModelOutput,
+} from '../test-helper.js';
+
+describe('tool-search / deferred tools', () => {
+  let rig: TestRig;
+
+  afterEach(async () => {
+    if (rig) await rig.cleanup();
+  });
+
+  it('reveals a deferred tool via select: and lets the model invoke it', async () => {
+    rig = new TestRig();
+    await rig.setup('tool-search-select-then-invoke', {
+      settings: { experimental: { cron: true } },
+    });
+
+    // Force the model down the select: path so the assertion isn't dependent
+    // on whether the model spontaneously chose keyword search vs. select.
+    const result = await rig.run(
+      'Step 1: call the tool_search tool with query "select:cron_list". ' +
+        'Step 2: call cron_list with no arguments. ' +
+        'Step 3: reply with just the word "done".',
+    );
+
+    const foundSearch = await rig.waitForToolCall('tool_search');
+    const foundList = await rig.waitForToolCall('cron_list');
+
+    if (!foundSearch || !foundList) {
+      printDebugInfo(rig, result, {
+        'tool_search found': foundSearch,
+        'cron_list found': foundList,
+      });
+    }
+
+    expect(foundSearch, 'expected tool_search to be called').toBeTruthy();
+    expect(
+      foundList,
+      'cron_list must succeed after tool_search reveals it',
+    ).toBeTruthy();
+
+    // Order matters: tool_search must come before cron_list. If cron_list
+    // were called first, the API would have rejected it (schema not loaded).
+    const calls = rig.readToolLogs().map((l) => l.toolRequest.name);
+    const searchIdx = calls.indexOf('tool_search');
+    const listIdx = calls.indexOf('cron_list');
+    expect(searchIdx).toBeGreaterThanOrEqual(0);
+    expect(listIdx).toBeGreaterThan(searchIdx);
+
+    validateModelOutput(result, null, 'select-then-invoke');
+  });
+
+  it('finds deferred tools via keyword search', async () => {
+    rig = new TestRig();
+    await rig.setup('tool-search-keyword', {
+      settings: { experimental: { cron: true } },
+    });
+
+    // The tool_search response is a synthetic <functions>...</functions>
+    // block; we check the ARGS the model sent (a keyword query, not select:)
+    // and trust the schema-loading behavior covered above.
+    const result = await rig.run(
+      'Use the tool_search tool with the keyword query "cron schedule" ' +
+        '(no select: prefix). Then reply with just the word "ok".',
+    );
+
+    const foundSearch = await rig.waitForToolCall('tool_search');
+    expect(foundSearch, 'expected tool_search to be called').toBeTruthy();
+
+    const searchCalls = rig
+      .readToolLogs()
+      .filter((l) => l.toolRequest.name === 'tool_search');
+    expect(searchCalls.length).toBeGreaterThan(0);
+
+    // At least one tool_search call must have used a keyword query.
+    const usedKeyword = searchCalls.some((c) => {
+      try {
+        const args = JSON.parse(c.toolRequest.args || '{}');
+        const q = String(args.query ?? '');
+        return q.length > 0 && !q.toLowerCase().startsWith('select:');
+      } catch {
+        return false;
+      }
+    });
+    expect(
+      usedKeyword,
+      `expected at least one keyword tool_search; saw args: ${searchCalls
+        .map((c) => c.toolRequest.args)
+        .join(' | ')}`,
+    ).toBeTruthy();
+
+    validateModelOutput(result, null, 'keyword search');
+  });
+
+  it('does not register deferred tools when their feature flag is off', async () => {
+    rig = new TestRig();
+    // No experimental.cron setting → cron_* tools must not be registered at
+    // all (deferred or otherwise). tool_search has nothing to surface.
+    await rig.setup('tool-search-no-cron');
+
+    const result = await rig.run(
+      'Call tool_search with query "select:cron_list". ' +
+        'Then reply with the literal text "missing" if cron_list was not in the result, ' +
+        'or "found" if it was.',
+    );
+
+    const foundSearch = await rig.waitForToolCall('tool_search');
+    expect(foundSearch, 'tool_search should still be available').toBeTruthy();
+
+    // cron_list must NOT have been invoked — it was never registered.
+    const calls = rig.readToolLogs().map((l) => l.toolRequest.name);
+    expect(calls).not.toContain('cron_list');
+    expect(calls).not.toContain('cron_create');
+
+    validateModelOutput(result, null, 'no-cron');
+  });
+});