diff --git a/.aoneci/npm-publish.yml b/.aoneci/npm-publish.yml new file mode 100644 index 00000000000..82b878ff83f --- /dev/null +++ b/.aoneci/npm-publish.yml @@ -0,0 +1,180 @@ +# .aoneci/workflows/npm-publish.yml +# 发布 npm 包到 anpm +# +# 版本策略: +# - main 分支 + release_mode=true:x.y.z-dataworks.N + dist-tag=latest(正式版) +# - 其他所有情况: x.y.z-beta.N + dist-tag=beta(预发布版) +# - 版本号自动查询 registry 递增 N +# +# 流程: +# 1. npm ci — 安装依赖 +# 2. npm run build — tsc 编译所有 workspace 包 +# 3. npm run bundle — esbuild 首次打包 CLI(使用 package.json 原始版本) +# 4. publish-packages.js: +# a. 写入 .npmrc 认证 +# b. auto-version → 更新所有 package.json 版本号 +# c. re-bundle → 重新 esbuild 打包(使用新版本号,嵌入 CLI_VERSION) +# d. npm publish --workspaces → 发布所有非 private 包 +# 5. 打包 bundle tarball 并上传 OSS +# - latest 发布更新 latest/dataworks 指针 +# - beta 发布更新 beta 指针 + +name: 'NPM Publish' + +# 仅手动触发,避免误发布。在 Aone CI 平台手动选择分支和参数执行。 +# 默认 dry-run,真实发布需手动关闭。 + +params: + dry_run: + description: '是否 dry-run(勾选=仅模拟,不勾选=真实发布)' + type: boolean + default: true + release_mode: + description: '正式版发布(仅 main 分支生效,开启后发布 dataworks + latest)' + type: boolean + default: false + +jobs: + npm-build-publish: + runs-on: + - 32-128Gi + name: 'Build & Publish' + image: alios-8u + envs: + BOOTSTRAP_TOKEN: ${{secrets.BOOTSTRAP_TOKEN}} + OSS_UPLOAD_TARGETS: "public finance" + steps: + - uses: checkout + - uses: setup-env + inputs: + node-version: '22' + tnpm-version: '10' + python-version: '3.10' + rust-toolchain-version: stable + tnpm-cache: true + + - name: 'Install dependencies' + run: 'npm ci' + + - name: 'Build project (tsc)' + run: 'npm run build' + + - name: 'Run tests' + run: | + # Root `npm run test` runs workspaces in parallel, which makes AoneCI + # failure logs hard to locate. Run publish checks sequentially so the + # first failing package is visible in the job output. + set -e + npm -w packages/cli run test -- \ + --testTimeout=30000 \ + --hookTimeout=30000 + npm -w packages/core run test -- \ + --testTimeout=30000 + npm -w packages/sdk-typescript run test + npm -w packages/vscode-ide-companion run test + + - name: 'Bundle CLI (esbuild)' + run: 'npm run bundle' + + - name: 'Verify bundle' + run: | + echo "=== dist/cli.js ===" + ls -lh dist/cli.js + echo "" + echo "=== dist/ structure ===" + ls dist/ + + - name: 'Publish packages' + run: | + # 仅 main 分支 + release_mode=true 才发正式版,其他一律 beta + if [ "${{git.branch}}" = "main" ] && [ "${{ params.release_mode }}" = "true" ]; then + PUBLISH_TAG="latest" + PUBLISH_PRE_ID="dataworks" + else + PUBLISH_TAG="beta" + PUBLISH_PRE_ID="beta" + fi + echo "Branch: ${{git.branch}} release_mode=${{params.release_mode}} → tag=$PUBLISH_TAG, pre_id=$PUBLISH_PRE_ID" + ARGS="--token ${{ secrets.inner_npm_publish_token }} --tag $PUBLISH_TAG --pre-id $PUBLISH_PRE_ID --auto-version" + [ "${{ params.dry_run }}" = "true" ] && ARGS="$ARGS --dry-run" + node scripts/publish-packages.js $ARGS + + - name: 'Verify published CLI package' + run: | + DRY_RUN="${{ params.dry_run }}" + if [ "$DRY_RUN" = "true" ]; then + echo "Skipped (dry-run mode)" + exit 0 + fi + sleep 5 + echo "=== @alife/dataworks-qwen-code ===" + npm view @alife/dataworks-qwen-code --registry=https://registry.anpm.alibaba-inc.com/ --json 2>&1 | node -e " + const j = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); + const tags = j['dist-tags'] || {}; + for (const [tag, ver] of Object.entries(tags)) { + console.log(tag + ':', ver); + } + " || echo "View failed" + + # ── OSS Bundle 发布 ── + + - name: 'Resolve version & OSS policy' + run: | + VERSION=$(node -e "process.stdout.write(require('./package.json').version)") + mkdir -p /workspace + echo "${VERSION}" > /workspace/.resolved_version + date -u +%Y-%m-%dT%H:%M:%SZ > /workspace/.build_time + if [ "${{git.branch}}" = "main" ] && [ "${{ params.release_mode }}" = "true" ]; then + OSS_RELEASE_CHANNELS="dataworks" + SKIP_LATEST_POINTER="" + else + OSS_RELEASE_CHANNELS="beta" + SKIP_LATEST_POINTER="1" + fi + { + printf 'OSS_RELEASE_CHANNELS=%s\n' "${OSS_RELEASE_CHANNELS}" + printf 'SKIP_LATEST_POINTER=%s\n' "${SKIP_LATEST_POINTER}" + printf 'SKIP_METADATA=%s\n' "" + printf 'SKIP_ROOT_SCRIPTS=%s\n' "" + } > /workspace/.oss_policy.env + echo "version=${VERSION} channel=${OSS_RELEASE_CHANNELS}" + + - name: 'Build bundle tarball' + run: | + # dist/ 已由 publish-packages.js 构建完成(含正确版本号) + # 直接打包为可部署的 tarball + # STANDALONE_DIR 和 ARTIFACT_DIR 必须与 prepare-artifact.sh 的查找路径一致 + export ARCH="amd64" + export WORKSPACE_DIR="/workspace" + export SOURCE_DIR="." + export STANDALONE_DIR="/workspace/build/qwen-code" + export ARTIFACT_DIR="/workspace/build" + bash .aoneci/scripts/build-standalone-ci.sh + + - name: 'Prepare artifact' + run: | + ARTIFACT_DIR="/workspace/artifact" \ + ARCH="amd64" \ + WORKSPACE_DIR="/workspace" \ + SOURCE_DIR="." \ + bash .aoneci/scripts/prepare-artifact.sh + + - name: 'Upload to OSS' + run: | + DRY_RUN="${{ params.dry_run }}" + if [ "$DRY_RUN" = "true" ]; then + echo "Skipped (dry-run mode)" + exit 0 + fi + set -a; source /workspace/.oss_policy.env; set +a + ARTIFACT_DIR="/workspace/artifact" \ + ARCH="amd64" \ + SOURCE_DIR="." \ + WORKSPACE_DIR="/workspace" \ + OSS_GROUP="alishu" \ + OSS_PROJECT="qwen-code" \ + SKIP_METADATA="${SKIP_METADATA:-}" \ + SKIP_LATEST_POINTER="${SKIP_LATEST_POINTER:-}" \ + SKIP_ROOT_SCRIPTS="${SKIP_ROOT_SCRIPTS:-}" \ + OSS_RELEASE_CHANNELS="${OSS_RELEASE_CHANNELS:-}" \ + bash .aoneci/scripts/upload-oss.sh diff --git a/.aoneci/release-rollback.yml b/.aoneci/release-rollback.yml new file mode 100644 index 00000000000..e6dbac1811f --- /dev/null +++ b/.aoneci/release-rollback.yml @@ -0,0 +1,217 @@ +# .aoneci/release-rollback.yml +# 回滚发布:将 OSS channel 指针回退到指定版本,并 deprecate 指定的 npm 包版本。 +# +# 使用场景: +# - 发布了一个有问题的版本,需要让下游自动更新机制回退到旧版本 +# - 废弃某个 npm 版本,安装时显示警告信息 +# +# 操作: +# 1. 将指定 channel(beta/dataworks/latest)的 metadata.json 回退到 rollback_to 版本 +# 2. (可选)npm deprecate 指定版本的所有非 private workspace 包 +# +# 仅手动触发,需要明确确认。 + +name: 'Release Rollback' + +params: + rollback_to: + description: '回退到的目标版本号(OSS 上必须已存在该版本的 metadata.json),如 0.15.11-dataworks.2' + type: string + required: true + channels: + description: '要回退的 channel(逗号分隔),如 beta 或 dataworks,latest' + type: string + default: 'beta' + deprecate_version: + description: '要废弃的 npm 版本号(留空则不执行 npm deprecate),如 0.17.0-beta.3' + type: string + deprecate_message: + description: 'npm deprecate 的警告信息' + type: string + default: 'This version has been rolled back. Please use an earlier version.' + dry_run: + description: '勾选=仅模拟,不真正执行' + type: boolean + default: true + +jobs: + rollback: + runs-on: + - 4-8Gi + name: 'Rollback Release' + image: alios-8u + envs: + BOOTSTRAP_TOKEN: ${{secrets.BOOTSTRAP_TOKEN}} + OSS_UPLOAD_TARGETS: "public finance" + steps: + - uses: checkout + - uses: setup-env + inputs: + node-version: '22' + + - name: 'Validate inputs' + run: | + ROLLBACK_TO="${{ params.rollback_to }}" + CHANNELS="${{ params.channels }}" + DRY_RUN="${{ params.dry_run }}" + + if [ -z "$ROLLBACK_TO" ]; then + echo "❌ rollback_to 版本号不能为空" + exit 1 + fi + + echo "=== Rollback Plan ===" + echo " Target version: $ROLLBACK_TO" + echo " Channels: $CHANNELS" + echo " Deprecate version: ${{ params.deprecate_version }}" + echo " Dry run: $DRY_RUN" + echo "" + + - name: 'Rollback OSS channel pointers' + run: | + set -euo pipefail + + ROLLBACK_TO="${{ params.rollback_to }}" + CHANNELS="${{ params.channels }}" + DRY_RUN="${{ params.dry_run }}" + + SCRIPT_DIR="${AONE_CI_SOURCE:-.}/.aoneci/scripts" + OSS_PROJECT_ROOT="public-datasets/aone-release/alishu/qwen-code" + OSS_PREFIX="${OSS_PROJECT_ROOT}/${ROLLBACK_TO}" + + TMPDIR=$(mktemp -d) + trap 'rm -rf "${TMPDIR}"' EXIT + + # shellcheck disable=SC1091 + . "${SCRIPT_DIR}/oss-targets.sh" + + CHANNEL_LIST=$(printf '%s' "${CHANNELS}" | tr ',[:space:]' '\n' | sed '/^$/d') + TARGETS="$(oss_upload_targets)" + + printf '%s\n' "${TARGETS}" | while IFS= read -r TARGET; do + [ -n "${TARGET}" ] || continue + oss_configure_target "${TARGET}" + + echo "" + echo "📋 [${OSS_TARGET}] 验证目标版本 ${ROLLBACK_TO} 的 metadata.json..." + METADATA_URL="oss://${OSS_BUCKET}/${OSS_PREFIX}/metadata.json" + TARGET_METADATA="${TMPDIR}/${OSS_TARGET}_metadata.json" + if ! ${OSSUTIL} stat "$METADATA_URL" &>/dev/null; then + echo "❌ [${OSS_TARGET}] 目标版本 ${ROLLBACK_TO} 的 metadata.json 不存在于 OSS" + echo " 路径: ${METADATA_URL}" + exit 1 + fi + + ${OSSUTIL} cp -f "$METADATA_URL" "${TARGET_METADATA}" + echo "✅ [${OSS_TARGET}] 目标版本 metadata.json:" + cat "${TARGET_METADATA}" + echo "" + + for CHANNEL in ${CHANNEL_LIST}; do + case "${CHANNEL}" in + *[!A-Za-z0-9._-]*|.|..|"") + echo "Invalid OSS release channel: ${CHANNEL}" >&2 + exit 1 + ;; + esac + CHANNEL_PATH="oss://${OSS_BUCKET}/${OSS_PROJECT_ROOT}/${CHANNEL}/metadata.json" + + echo "" + echo "🔄 [${OSS_TARGET}] Channel: ${CHANNEL}" + echo " 当前指向:" + ${OSSUTIL} cp -f "$CHANNEL_PATH" "${TMPDIR}/current_${OSS_TARGET}_${CHANNEL}.json" 2>/dev/null || true + cat "${TMPDIR}/current_${OSS_TARGET}_${CHANNEL}.json" 2>/dev/null || echo " (不存在)" + + if [ "$DRY_RUN" = "true" ]; then + echo " [DRY-RUN] 将回退到 ${ROLLBACK_TO}" + echo " 路径: ${CHANNEL_PATH}" + else + ${OSSUTIL} cp -f "${TARGET_METADATA}" "$CHANNEL_PATH" + echo " ✅ 已回退到 ${ROLLBACK_TO}" + fi + done + done + + - name: 'Deprecate npm version' + run: | + set -euo pipefail + + DEPRECATE_VERSION="${{ params.deprecate_version }}" + DEPRECATE_MSG="${{ params.deprecate_message }}" + DRY_RUN="${{ params.dry_run }}" + REGISTRY="https://registry.anpm.alibaba-inc.com/" + + if [ -z "$DEPRECATE_VERSION" ]; then + echo "⏭️ deprecate_version 为空,跳过 npm deprecate" + exit 0 + fi + + # 配置 npm auth + NPM_TOKEN="${{ secrets.inner_npm_publish_token }}" + echo "//registry.anpm.alibaba-inc.com/:_authToken=${NPM_TOKEN}" > ~/.npmrc + + # 读取所有需要 deprecate 的包 + PACKAGES=$(node -e " + const fs = require('fs'); + const path = require('path'); + const glob = require('glob') || { sync: () => [] }; + const root = JSON.parse(fs.readFileSync('package.json', 'utf-8')); + const ws = root.workspaces || []; + const pkgs = []; + for (const pattern of ws) { + if (pattern.includes('*')) { + const base = pattern.split('*')[0]; + if (!fs.existsSync(base)) continue; + for (const d of fs.readdirSync(base, { withFileTypes: true })) { + if (!d.isDirectory()) continue; + const p = path.join(base, d.name, 'package.json'); + if (fs.existsSync(p)) { + const pkg = JSON.parse(fs.readFileSync(p, 'utf-8')); + if (!pkg.private) pkgs.push(pkg.name); + } + } + } else { + const p = path.join(pattern, 'package.json'); + if (fs.existsSync(p)) { + const pkg = JSON.parse(fs.readFileSync(p, 'utf-8')); + if (!pkg.private) pkgs.push(pkg.name); + } + } + } + // root package + const rootPkg = JSON.parse(fs.readFileSync('package.json', 'utf-8')); + if (!rootPkg.private) pkgs.unshift(rootPkg.name); + console.log(JSON.stringify([...new Set(pkgs)])); + ") + + echo "📦 要 deprecate 的包 (版本 ${DEPRECATE_VERSION}):" + echo "$PACKAGES" | node -e "JSON.parse(require('fs').readFileSync(0,'utf-8')).forEach(p => console.log(' - ' + p))" + + echo "" + echo "$PACKAGES" | node -e " + const pkgs = JSON.parse(require('fs').readFileSync(0, 'utf-8')); + const { execSync } = require('child_process'); + const version = '${DEPRECATE_VERSION}'; + const msg = '${DEPRECATE_MSG}'; + const dryRun = '${DRY_RUN}' === 'true'; + const registry = '${REGISTRY}'; + + for (const pkg of pkgs) { + const spec = pkg + '@' + version; + if (dryRun) { + console.log('[DRY-RUN] npm deprecate ' + spec); + } else { + try { + execSync('npm deprecate \"' + spec + '\" \"' + msg + '\" --registry ' + registry, { + stdio: 'inherit', + }); + console.log('✅ deprecated: ' + spec); + } catch (err) { + console.error('⚠️ failed to deprecate ' + spec + ' (may not exist on registry, skipping)'); + } + } + } + " + + echo "" + echo "=== Rollback Complete ===" diff --git a/.aoneci/scripts/build-standalone-ci.sh b/.aoneci/scripts/build-standalone-ci.sh new file mode 100755 index 00000000000..dd7ab316f49 --- /dev/null +++ b/.aoneci/scripts/build-standalone-ci.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# build-standalone-ci.sh — 构建 standalone 产物 +# +# 环境变量: +# SOURCE_DIR - 源码目录 (默认 AONE_CI_SOURCE 或 .) +# VERSION - 版本号 (默认从 package.json 读取) +# STANDALONE_DIR - standalone 产物目录 (默认 $PWD/.standalone) + +set -euo pipefail + +SOURCE_DIR="${AONE_CI_SOURCE:-.}" +cd "$SOURCE_DIR" + +# ── 确定版本号 ── +if [ -z "${VERSION:-}" ]; then + VERSION=$(node -e "console.log(require('./package.json').version || '')" 2>/dev/null || echo "") + if [ -z "$VERSION" ]; then + echo "⚠️ Warning: package.json 解析失败或 version 字段为空,VERSION 将回退为空值" >&2 + fi +fi +if [ -z "$VERSION" ] || [ "$VERSION" = "undefined" ]; then + echo "❌ 无法确定版本号" >&2 + exit 1 +fi +echo "📦 Version: $VERSION" + +# ── 确定 standalone 目录 ── +STANDALONE_DIR="${STANDALONE_DIR:-${PWD}/.standalone}" +if [ "$(dirname "$STANDALONE_DIR")" = "/" ]; then + echo "❌ 工作目录在根层级,请显式设置 STANDALONE_DIR 环境变量" >&2 + exit 1 +fi +echo "📁 Standalone dir: $STANDALONE_DIR" + +# ── Step 0: 确保已构建 ── +if [ ! -d "dist" ]; then + echo "[0/4] dist/ 不存在,执行构建..." + npm run build +else + echo "[0/4] dist/ 已存在,跳过构建" +fi + +# ── Step 1: 准备 standalone 目录 ── +echo "[1/4] 准备 standalone 目录..." +rm -rf "$STANDALONE_DIR" +mkdir -p "$STANDALONE_DIR" + +# 复制 dist +cp -r "${SOURCE_DIR}/dist" "${STANDALONE_DIR}/" + +# 复制 vendor(如果存在) +if [ -d "${SOURCE_DIR}/vendor" ]; then + cp -r "${SOURCE_DIR}/vendor" "${STANDALONE_DIR}/vendor" +fi + +# 复制 node_modules 中需要的 native 依赖(如果有) +if [ -d "${SOURCE_DIR}/node_modules" ]; then + # 只复制必要的 native binding(例如 @aspect 相关) + if [ -d "${SOURCE_DIR}/node_modules/@aspect" ]; then + mkdir -p "${STANDALONE_DIR}/node_modules" + cp -r "${SOURCE_DIR}/node_modules/@aspect" "${STANDALONE_DIR}/node_modules/" + fi +fi + +# ── Step 2: 注入版本号 ── +echo "[2/4] 注入版本号 ${VERSION}..." +if [ -f "${STANDALONE_DIR}/dist/cli.js" ]; then + STANDALONE_DIR="$STANDALONE_DIR" VERSION="$VERSION" node -e ' + const fs = require("fs"); + const path = require("path"); + const p = path.join(process.env.STANDALONE_DIR, "dist", "cli.js"); + let c = fs.readFileSync(p, "utf8"); + if (c.includes("__QWEN_VERSION__")) { + c = c.replace(/__QWEN_VERSION__/g, () => process.env.VERSION); + } + fs.writeFileSync(p, c); + ' || echo " -> 版本注入跳过(无 __QWEN_VERSION__ 占位符)" +fi + +# ── Node package scope ── +STANDALONE_DIR="$STANDALONE_DIR" SOURCE_DIR="$SOURCE_DIR" VERSION="$VERSION" node -e ' + const fs = require("fs"); + const path = require("path"); + let name = "@alife/dataworks-qwen-code"; + try { + const rootPkg = JSON.parse(fs.readFileSync(path.join(process.env.SOURCE_DIR, "package.json"), "utf8")); + if (rootPkg.name) name = rootPkg.name; + } catch {} + const pkg = { + name, + version: process.env.VERSION, + type: "module", + private: true, + }; + fs.writeFileSync(path.join(process.env.STANDALONE_DIR, "package.json"), JSON.stringify(pkg, null, 2) + "\n"); +' + +# ── Step 3: Launcher + Metadata ── +echo "[3/4] 生成 launcher 脚本和 metadata..." +mkdir -p "${STANDALONE_DIR}/bin" + +cat > "${STANDALONE_DIR}/bin/qwen" << 'LAUNCHER_EOF' +#!/usr/bin/env bash +set -euo pipefail + +# 解析 symlink 找到真实安装目录 +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +BIN_DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" +ROOT_DIR="$(dirname "$BIN_DIR")" + +exec node "${ROOT_DIR}/dist/cli.js" "$@" +LAUNCHER_EOF +chmod +x "${STANDALONE_DIR}/bin/qwen" + +# metadata (保持 metadata.json 文件名 + 兼容新旧字段名) +SHORT_SHA="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" +GIT_SHA="$(git rev-parse HEAD 2>/dev/null || echo unknown)" +BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +STANDALONE_DIR="$STANDALONE_DIR" VERSION="$VERSION" BUILD_TIME="$BUILD_TIME" \ + SHORT_SHA="$SHORT_SHA" GIT_SHA="$GIT_SHA" node -e ' + const fs = require("fs"); + const path = require("path"); + const meta = { + version: process.env.VERSION, + build_time: process.env.BUILD_TIME, + git_short_sha: process.env.SHORT_SHA, + git_sha: process.env.GIT_SHA, + }; + fs.writeFileSync( + path.join(process.env.STANDALONE_DIR, "metadata.json"), + JSON.stringify(meta, null, 2) + "\n" + ); +' + +# ── Step 4: 打包 ── +echo "[4/4] 打包..." +ARTIFACT_DIR="${ARTIFACT_DIR:-${PWD}/.artifacts}" +if [ "$ARTIFACT_DIR" = "/" ]; then + echo "❌ 工作目录在根层级,请显式设置 ARTIFACT_DIR 环境变量" >&2 + exit 1 +fi +mkdir -p "$ARTIFACT_DIR" +TARBALL="${ARTIFACT_DIR}/qwen-code-standalone-${VERSION}.tar.gz" +tar -czf "$TARBALL" -C "$(dirname "$STANDALONE_DIR")" "$(basename "$STANDALONE_DIR")" +echo "✅ Standalone artifact: $TARBALL" +echo " Size: $(du -h "$TARBALL" | cut -f1)" + +# 兼容旧格式:下游依赖 qwen-code-{version}-linux-amd64.tar.gz +ARCH="${ARCH:-amd64}" +TARBALL_COMPAT="${ARTIFACT_DIR}/qwen-code-${VERSION}-linux-${ARCH}.tar.gz" +cp "$TARBALL" "$TARBALL_COMPAT" +echo "✅ Compat artifact: $TARBALL_COMPAT" diff --git a/.aoneci/scripts/deploy-qwen.sh b/.aoneci/scripts/deploy-qwen.sh new file mode 100755 index 00000000000..b3634575c5d --- /dev/null +++ b/.aoneci/scripts/deploy-qwen.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────── +# deploy-qwen.sh +# +# 一键部署 / 升级 qwen-code 到 Linux 服务器。 +# +# 全流程: +# 1. 从 OSS 下载指定版本的 bundle 包 +# 2. 校验 SHA256 +# 3. 解压到安装目录 +# 4. 创建 /usr/local/bin/qwen 软链接 +# 5. 验证安装 +# +# 前提条件:系统已安装 Node.js >= 20 +# +# 用法: +# curl -fsSL /deploy-qwen.sh | bash +# curl -fsSL /deploy-qwen.sh | bash -s -- --version 0.14.8-dataworks.3 +# bash deploy-qwen.sh --version 0.14.8-dataworks.3 +# bash deploy-qwen.sh --install-dir /opt/qwen-code +# +# 环境变量 (均可通过命令行参数覆盖): +# QWEN_VERSION - 版本号(可通过 --version 传入) +# QWEN_ARCH - 架构 (amd64|arm64),默认自动检测 +# QWEN_INSTALL_DIR - 安装目录,默认 /usr/local/qwen-code +# QWEN_ARCHIVE - 已下载的本地 tar.gz 路径,存在时跳过下载 +# QWEN_OSS_BASE_URL - OSS 根地址覆盖,格式为 https://host/prefix +# ────────────────────────────────────────────────────────── +set -euo pipefail + +# ── 默认配置 ── +OSS_BUCKET="dataworks-notebook-cn-shanghai" +OSS_HOST="${OSS_BUCKET}.oss-cn-shanghai.aliyuncs.com" +OSS_PREFIX="public-datasets/aone-release/alishu/qwen-code" +EMBEDDED_QWEN_OSS_BASE_URL="__QWEN_OSS_BASE_URL__" +if [ "${EMBEDDED_QWEN_OSS_BASE_URL}" = "__QWEN_OSS_BASE_URL__" ]; then + EMBEDDED_QWEN_OSS_BASE_URL="https://${OSS_HOST}/${OSS_PREFIX}" +fi +OSS_BASE_URL="${QWEN_OSS_BASE_URL:-${EMBEDDED_QWEN_OSS_BASE_URL}}" + +VERSION="${QWEN_VERSION:-}" +ARCH="${QWEN_ARCH:-}" +INSTALL_DIR="${QWEN_INSTALL_DIR:-/usr/local/qwen-code}" +ARCHIVE="${QWEN_ARCHIVE:-}" +CREATE_SYMLINK="true" + +# ── 解析命令行参数 ── +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --arch) ARCH="$2"; shift 2 ;; + --install-dir) INSTALL_DIR="$2"; shift 2 ;; + --archive) ARCHIVE="$2"; shift 2 ;; + --oss-base-url) OSS_BASE_URL="$2"; shift 2 ;; + --no-symlink) CREATE_SYMLINK="false"; shift ;; + -h|--help) + sed -n '2,/^[^#]/{ /^#/s/^# \{0,1\}//p }' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +OSS_BASE_URL="${OSS_BASE_URL%/}" +case "${OSS_BASE_URL}" in + https://*) ;; + *) + echo "Invalid --oss-base-url: ${OSS_BASE_URL}" >&2 + exit 1 + ;; +esac +case "${OSS_BASE_URL}" in + *"'"*|*" "*|*";"*|*"|"*|*"&"*|*"\\"*) + echo "Unsafe --oss-base-url: ${OSS_BASE_URL}" >&2 + exit 1 + ;; +esac + +# ── 颜色输出 ── +if [ -t 1 ]; then + GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' +else + GREEN=''; YELLOW=''; RED=''; NC='' +fi +info() { echo -e "${GREEN}>>>${NC} $*"; } +warn() { echo -e "${YELLOW}>>> WARNING:${NC} $*"; } +error() { echo -e "${RED}>>> ERROR:${NC} $*" >&2; } + +# ── 检查 Node.js ── +if ! command -v node &>/dev/null; then + error "Node.js not found. Please install Node.js >= 20 first." + exit 1 +fi + +NODE_MAJOR=$(node -e "process.stdout.write(String(process.versions.node.split('.')[0]))") +if [ "${NODE_MAJOR}" -lt 20 ]; then + error "Node.js >= 20 required, current: $(node --version)" + exit 1 +fi +info "Node.js $(node --version) detected" + +# ── 自动检测架构 ── +if [ -z "${ARCH}" ]; then + case "$(uname -m)" in + x86_64|amd64) ARCH="amd64" ;; + aarch64|arm64) ARCH="arm64" ;; + *) + error "Unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +fi + +echo "" +echo "============================================" +echo " Qwen Code Deploy" +echo " Arch: ${ARCH}" +echo " Install Dir: ${INSTALL_DIR}" +echo "============================================" +echo "" + +# ════════════════════════════════════════════════════════════ +# Step 1: 确定版本号 +# ════════════════════════════════════════════════════════════ +if [ -z "${VERSION}" ]; then + info "No version specified, fetching latest..." + LATEST_URL="${OSS_BASE_URL}/latest/metadata.json" + if curl -fsSL --head "${LATEST_URL}" >/dev/null 2>&1; then + VERSION=$(curl -fsSL "${LATEST_URL}" 2>/dev/null \ + | grep -o '"version"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' || true) + fi + + if [ -z "${VERSION}" ]; then + error "Could not discover latest version. Please specify: --version " + exit 1 + fi +fi + +info "Target version: ${VERSION}" + +# ── 构造下载 URL ── +TARBALL="qwen-code-${VERSION}-linux-${ARCH}.tar.gz" +DOWNLOAD_URL="${OSS_BASE_URL}/${VERSION}/${TARBALL}" +SHA256_URL="${OSS_BASE_URL}/${VERSION}/SHA256SUMS" +METADATA_URL="${OSS_BASE_URL}/${VERSION}/metadata.json" + +RELEASES_DIR="${INSTALL_DIR}/releases" +CURRENT_LINK="${INSTALL_DIR}/current" +PREV_REF="" + +TMP_DIR=$(mktemp -d) +ARCHIVE_TMP="${TMP_DIR}/${TARBALL}" +STAGE_DIR="${TMP_DIR}/stage" +trap 'rm -rf "${TMP_DIR}"' EXIT + +# ── 检查版本是否存在 ── +if [ -n "${ARCHIVE}" ]; then + info "Using local archive: ${ARCHIVE}" + if [ ! -f "${ARCHIVE}" ]; then + error "Archive not found: ${ARCHIVE}" + exit 1 + fi +else + info "Checking version availability..." + if ! curl -fsSL --head "${DOWNLOAD_URL}" >/dev/null 2>&1; then + error "Version ${VERSION} not found at ${DOWNLOAD_URL}" + exit 1 + fi + info "Version ${VERSION} found" +fi + +# ════════════════════════════════════════════════════════════ +# Step 2: 下载并校验 +# ════════════════════════════════════════════════════════════ +echo "" +info "Step 2: Downloading and validating..." + +if [ -n "${ARCHIVE}" ]; then + cp "${ARCHIVE}" "${ARCHIVE_TMP}" +else + curl -fsSL "${DOWNLOAD_URL}" -o "${ARCHIVE_TMP}" + info "Downloaded ${TARBALL}" +fi + +# SHA256 校验 +if curl -fsSL "${SHA256_URL}" -o "${TMP_DIR}/SHA256SUMS" 2>/dev/null; then + EXPECTED=$(grep "${TARBALL}" "${TMP_DIR}/SHA256SUMS" | awk '{print $1}') + ACTUAL=$(sha256sum "${ARCHIVE_TMP}" | awk '{print $1}') + if [ -n "${EXPECTED}" ] && [ "${EXPECTED}" = "${ACTUAL}" ]; then + info "Checksum verified: ${ACTUAL:0:16}..." + elif [ -n "${EXPECTED}" ]; then + error "Checksum mismatch! Expected: ${EXPECTED}, Got: ${ACTUAL}" + exit 1 + fi +else + warn "SHA256SUMS not available, skipping checksum verification" +fi + +# ════════════════════════════════════════════════════════════ +# Step 3: 解压并验证 +# ════════════════════════════════════════════════════════════ +echo "" +info "Step 3: Extracting and verifying..." + +mkdir -p "${STAGE_DIR}" +tar -xzf "${ARCHIVE_TMP}" -C "${STAGE_DIR}" --strip-components=1 + +chmod +x "${STAGE_DIR}/bin/qwen" + +# 验证 qwen 命令能运行 +"${STAGE_DIR}/bin/qwen" --version 2>/dev/null \ + && info "qwen verified" \ + || warn "qwen --version check did not succeed (non-fatal for initial install)" + +# 下载 metadata 到 staging +curl -fsSL "${METADATA_URL}" -o "${STAGE_DIR}/metadata.json" 2>/dev/null || true + +# ════════════════════════════════════════════════════════════ +# Step 4: 安装(原子切换) +# ════════════════════════════════════════════════════════════ +echo "" +info "Step 4: Installing..." + +# 记录旧版本 +if [ -L "${CURRENT_LINK}" ]; then + PREV_REF=$(readlink "${CURRENT_LINK}" || true) +fi + +mkdir -p "${RELEASES_DIR}" + +NEXT_REF="releases/${VERSION}" +NEXT_DIR="${INSTALL_DIR}/${NEXT_REF}" + +if [ -e "${NEXT_DIR}" ]; then + if [ "${PREV_REF}" = "${NEXT_REF}" ]; then + PREV_REF="releases/${VERSION}.old.$(date +%Y%m%d%H%M%S)" + mv "${NEXT_DIR}" "${INSTALL_DIR}/${PREV_REF}" + else + rm -rf "${NEXT_DIR}" + fi +fi + +mv "${STAGE_DIR}" "${NEXT_DIR}" + +# 原子切换 current symlink +ln -sfn "${NEXT_REF}" "${CURRENT_LINK}" + +info "Installed to ${NEXT_DIR}" + +# ════════════════════════════════════════════════════════════ +# Step 5: 创建全局软链接 +# ════════════════════════════════════════════════════════════ +if [ "${CREATE_SYMLINK}" = "true" ]; then + echo "" + info "Step 5: Creating symlinks..." + + QWEN_BIN="${CURRENT_LINK}/bin/qwen" + + # 查找所有已存在的 qwen 命令并逐个替换 + ALL_QWEN_PATHS="" + if command -v qwen &>/dev/null; then + ALL_QWEN_PATHS=$(which -a qwen 2>/dev/null || true) + fi + + # 确保 /usr/local/bin 在列表中 + if ! echo "${ALL_QWEN_PATHS}" | grep -qx "/usr/local/bin/qwen"; then + ALL_QWEN_PATHS="/usr/local/bin/qwen +${ALL_QWEN_PATHS}" + fi + + STANDALONE_TARGET="${INSTALL_DIR}/current/bin/qwen" + + echo "${ALL_QWEN_PATHS}" | while IFS= read -r OLD_PATH; do + [ -z "${OLD_PATH}" ] && continue + + OLD_DIR=$(dirname "${OLD_PATH}") + + # 如果已经是指向当前版本的 symlink,跳过 + if [ -L "${OLD_PATH}" ]; then + LINK_TARGET=$(readlink -f "${OLD_PATH}" 2>/dev/null || true) + EXPECTED_TARGET=$(readlink -f "${STANDALONE_TARGET}" 2>/dev/null || true) + if [ "${LINK_TARGET}" = "${EXPECTED_TARGET}" ]; then + info "Already up to date: ${OLD_PATH}" + continue + fi + fi + + # 检查目录是否可写 + if [ ! -w "${OLD_DIR}" ]; then + warn "No write permission, skipping: ${OLD_PATH}" + continue + fi + + # 备份旧文件(非 symlink 才备份) + if [ -f "${OLD_PATH}" ] && [ ! -L "${OLD_PATH}" ]; then + mv "${OLD_PATH}" "${OLD_PATH}.old-backup" 2>/dev/null || true + info "Backed up: ${OLD_PATH} -> ${OLD_PATH}.old-backup" + fi + + # 创建 symlink + ln -sf "${STANDALONE_TARGET}" "${OLD_PATH}" 2>/dev/null \ + && info "Linked: ${OLD_PATH} -> ${STANDALONE_TARGET}" \ + || warn "Failed to link: ${OLD_PATH}" + done + + # metadata 软链接便于升级脚本读取 + ln -sfn "current/metadata.json" "${INSTALL_DIR}/metadata.json" 2>/dev/null || true + + hash -r 2>/dev/null || true +fi + +# ════════════════════════════════════════════════════════════ +# 完成 +# ════════════════════════════════════════════════════════════ +echo "" +echo "============================================" +echo " Qwen Code Deploy Complete" +echo "" +echo " Version: ${VERSION}" +echo " Arch: ${ARCH}" +echo " Node.js: $(node --version)" +echo " Binary: ${INSTALL_DIR}/current/bin/qwen" +echo "" +echo " Usage:" +echo " qwen # if /usr/local/bin is in PATH" +echo " ${INSTALL_DIR}/current/bin/qwen" +echo "" +echo " Upgrade:" +echo " curl -fsSL ${OSS_BASE_URL}/upgrade-qwen.sh | bash" +echo "" +if [ -n "${PREV_REF}" ] && [ "${PREV_REF}" != "${NEXT_REF}" ]; then +echo " Rollback to previous:" +echo " ln -sfn ${PREV_REF} ${CURRENT_LINK}" +echo "" +fi + +# 检查最终生效的 qwen 是否是新版本 +FINAL_QWEN="$(command -v qwen 2>/dev/null || true)" +if [ -n "${FINAL_QWEN}" ]; then + FINAL_REAL="$(readlink -f "${FINAL_QWEN}" 2>/dev/null || echo "${FINAL_QWEN}")" + EXPECTED_REAL="$(readlink -f "${INSTALL_DIR}/current/bin/qwen" 2>/dev/null || true)" + if [ "${FINAL_REAL}" != "${EXPECTED_REAL}" ]; then + echo " WARNING: qwen still points to old version!" + echo " Current: ${FINAL_QWEN} -> ${FINAL_REAL}" + echo " Expected: ${EXPECTED_REAL}" + echo "" + echo " Fix:" + echo " npm uninstall -g @qwen-code/qwen-code && hash -r" + echo "" + fi +fi +echo "============================================" diff --git a/.aoneci/scripts/oss-targets.sh b/.aoneci/scripts/oss-targets.sh new file mode 100644 index 00000000000..73b90ff4975 --- /dev/null +++ b/.aoneci/scripts/oss-targets.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Shared OSS target and credential helpers for qwen-code AoneCI uploads. + +oss_die() { + echo "ERROR: $*" >&2 + exit 1 +} + +oss_helper_dir() { + cd "$(dirname "${BASH_SOURCE[0]}")" && pwd +} + +oss_upload_targets() { + local raw="${OSS_UPLOAD_TARGETS:-public finance}" + printf '%s\n' "${raw}" \ + | tr ',;' ' ' \ + | tr '[:space:]' '\n' \ + | while IFS= read -r target; do + [ -n "${target}" ] && printf '%s\n' "${target}" + done \ + | awk '!seen[$0]++' +} + +oss_target_endpoint() { + case "$1" in + public) printf '%s' "${PUBLIC_OSS_ENDPOINT:-https://oss-cn-shanghai.aliyuncs.com}" ;; + finance) printf '%s' "${FINANCE_OSS_ENDPOINT:-https://oss-cn-shanghai-finance-1.aliyuncs.com}" ;; + *) oss_die "unsupported OSS upload target: $1" ;; + esac +} + +oss_target_bucket() { + case "$1" in + public) printf '%s' "${PUBLIC_OSS_BUCKET:-dataworks-notebook-cn-shanghai}" ;; + finance) printf '%s' "${FINANCE_OSS_BUCKET:-dataworks-notebook-cn-shanghai-finance-1}" ;; + *) oss_die "unsupported OSS upload target: $1" ;; + esac +} + +oss_endpoint_host() { + local endpoint="$1" + endpoint="${endpoint#https://}" + endpoint="${endpoint#http://}" + printf '%s' "${endpoint}" +} + +oss_http_url() { + local bucket="$1" + local endpoint="$2" + local key="$3" + printf 'https://%s.%s/%s' "${bucket}" "$(oss_endpoint_host "${endpoint}")" "${key}" +} + +oss_current_http_url() { + oss_http_url "${OSS_BUCKET}" "${OSS_ENDPOINT}" "$1" +} + +oss_credential_mode() { + local mode="${OSS_CREDENTIAL_MODE:-zerotrust}" + case "${mode}" in + zerotrust | zero-trust | zero_trust | sts | "") + printf 'zerotrust' + ;; + aksk | ak-sk | ak_sk | access-key | access_key) + printf 'aksk' + ;; + *) + oss_die "unsupported OSS credential mode: ${mode}" + ;; + esac +} + +oss_ensure_ossutil() { + if ! command -v ossutil64 >/dev/null 2>&1 && ! command -v ossutil >/dev/null 2>&1; then + echo ">>> Installing ossutil..." >&2 + curl -fsSL "https://gosspublic.alicdn.com/ossutil/1.7.18/ossutil-v1.7.18-linux-amd64.zip" -o /tmp/ossutil.zip + unzip -o /tmp/ossutil.zip -d /tmp/ossutil >/dev/null + chmod +x /tmp/ossutil/ossutil-v1.7.18-linux-amd64/ossutil64 + cp /tmp/ossutil/ossutil-v1.7.18-linux-amd64/ossutil64 /usr/local/bin/ossutil64 + fi + command -v ossutil64 || command -v ossutil +} + +oss_configure_target() { + local target="$1" + local credential_mode env_file resolver had_xtrace="" + + case "$-" in + *x*) had_xtrace="1"; set +x ;; + esac + + credential_mode="$(oss_credential_mode)" + echo ">>> Configuring OSS target ${target} with credential mode: ${credential_mode}" >&2 + + if [ "${credential_mode}" = "zerotrust" ]; then + [ -n "${BOOTSTRAP_TOKEN:-}" ] || oss_die "BOOTSTRAP_TOKEN is required when OSS_CREDENTIAL_MODE=zerotrust" + resolver="${OSS_STS_RESOLVER:-$(oss_helper_dir)/resolve-oss-sts.sh}" + [ -f "${resolver}" ] || oss_die "OSS STS resolver not found: ${resolver}" + env_file="$(mktemp)" + chmod 600 "${env_file}" + BOOTSTRAP_TOKEN="${BOOTSTRAP_TOKEN}" \ + ZERO_TRUST_TOOL_URL="${ZERO_TRUST_TOOL_URL:-}" \ + PUBLIC_OSS_STS_ROLE_ARN="${PUBLIC_OSS_STS_ROLE_ARN:-}" \ + FINANCE_OSS_STS_ROLE_ARN="${FINANCE_OSS_STS_ROLE_ARN:-}" \ + PUBLIC_OSS_ENDPOINT="${PUBLIC_OSS_ENDPOINT:-}" \ + PUBLIC_OSS_BUCKET="${PUBLIC_OSS_BUCKET:-}" \ + FINANCE_OSS_ENDPOINT="${FINANCE_OSS_ENDPOINT:-}" \ + FINANCE_OSS_BUCKET="${FINANCE_OSS_BUCKET:-}" \ + bash "${resolver}" "${target}" > "${env_file}" + # shellcheck disable=SC1090 + . "${env_file}" + rm -f "${env_file}" + else + echo ">>> Using explicit legacy AK/SK OSS credential mode." >&2 + OSS_TARGET="${target}" + OSS_ENDPOINT="$(oss_target_endpoint "${target}")" + OSS_BUCKET="$(oss_target_bucket "${target}")" + OSS_SECURITY_TOKEN="" + [ -n "${OSS_ACCESS_KEY_ID:-}" ] || oss_die "OSS_ACCESS_KEY_ID is required when OSS_CREDENTIAL_MODE=aksk" + [ -n "${OSS_ACCESS_KEY_SECRET:-}" ] || oss_die "OSS_ACCESS_KEY_SECRET is required when OSS_CREDENTIAL_MODE=aksk" + fi + + OSSUTIL="$(oss_ensure_ossutil)" + if [ -n "${OSS_SECURITY_TOKEN:-}" ]; then + "${OSSUTIL}" config -e "${OSS_ENDPOINT}" -i "${OSS_ACCESS_KEY_ID}" -k "${OSS_ACCESS_KEY_SECRET}" -t "${OSS_SECURITY_TOKEN}" >/dev/null + else + "${OSSUTIL}" config -e "${OSS_ENDPOINT}" -i "${OSS_ACCESS_KEY_ID}" -k "${OSS_ACCESS_KEY_SECRET}" >/dev/null + fi + + if [ -n "${had_xtrace}" ]; then + set -x + fi +} + +oss_targets_one_line() { + oss_upload_targets | paste -sd ' ' - +} diff --git a/.aoneci/scripts/prepare-artifact.sh b/.aoneci/scripts/prepare-artifact.sh new file mode 100755 index 00000000000..953e95d80f0 --- /dev/null +++ b/.aoneci/scripts/prepare-artifact.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────── +# prepare-artifact.sh +# +# 整理构建产物到最终上传目录: +# - 查找 tarball(支持新旧命名格式) +# - 从 standalone 目录拷贝 metadata.json +# - 生成 SHA256SUMS 校验文件 +# +# 环境变量: +# ARTIFACT_DIR - 产物输出根目录 +# ARCH - 目标架构 (amd64 | arm64),可选 +# WORKSPACE_DIR - CI 工作目录,默认 /workspace +# SOURCE_DIR - 源码根目录 +# SKIP_METADATA - 非空时跳过 metadata.json +# ────────────────────────────────────────────────────────── +set -eu + +ARTIFACT_DIR="${ARTIFACT_DIR:?ARTIFACT_DIR is required}" +ARCH="${ARCH:-amd64}" +WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}" +SOURCE_DIR="${SOURCE_DIR:-${AONE_CI_SOURCE:-.}}" +SKIP_METADATA="${SKIP_METADATA:-}" + +VERSION=$(cat "${WORKSPACE_DIR}/.resolved_version" 2>/dev/null || \ + node -e "const v=require('${SOURCE_DIR}/package.json').version; if(!v) process.exit(1); console.log(v)" 2>/dev/null || \ + echo "") + +if [ -z "${VERSION}" ]; then + echo "ERROR: cannot determine version" >&2 + exit 1 +fi + +BUILD_DIR="${WORKSPACE_DIR}/build" +STANDALONE_DIR="${BUILD_DIR}/qwen-code" + +# 支持新旧两种 tarball 命名格式 +TARBALL_NEW="qwen-code-standalone-${VERSION}.tar.gz" +TARBALL_OLD="qwen-code-${VERSION}-linux-${ARCH}.tar.gz" + +echo "=== VERSION: ${VERSION} ===" + +rm -rf "${ARTIFACT_DIR}" +mkdir -p "${ARTIFACT_DIR}" + +# 查找并拷贝 tarball(新格式 + 兼容旧格式) +TARBALL_NAME="" +for candidate in "${BUILD_DIR}/${TARBALL_NEW}" "${ARTIFACT_DIR}/../${TARBALL_NEW}" \ + "${BUILD_DIR}/${TARBALL_OLD}" "${ARTIFACT_DIR}/../${TARBALL_OLD}"; do + if [ -f "${candidate}" ]; then + TARBALL_NAME="$(basename "${candidate}")" + cp "${candidate}" "${ARTIFACT_DIR}/${TARBALL_NAME}" + break + fi +done + +if [ -z "${TARBALL_NAME}" ]; then + echo "ERROR: no tarball found (tried ${TARBALL_NEW} and ${TARBALL_OLD})" >&2 + exit 1 +fi + +# 拷贝兼容格式(如果存在且和主 tarball 不同名) +for candidate in "${BUILD_DIR}/${TARBALL_OLD}" "${ARTIFACT_DIR}/../${TARBALL_OLD}"; do + if [ -f "${candidate}" ] && [ "${TARBALL_NAME}" != "${TARBALL_OLD}" ]; then + cp "${candidate}" "${ARTIFACT_DIR}/${TARBALL_OLD}" + echo ">>> Compat tarball copied: ${TARBALL_OLD}" + break + fi +done + +# 生成 SHA256(包含所有 tarball) +(cd "${ARTIFACT_DIR}" && sha256sum *.tar.gz > SHA256SUMS) + +# metadata.json +METADATA_FILE="" +for candidate in "${STANDALONE_DIR}/metadata.json" "${STANDALONE_DIR}/META.json"; do + if [ -f "${candidate}" ]; then + METADATA_FILE="${candidate}" + break + fi +done + +if [ -z "${SKIP_METADATA}" ] && [ -n "${METADATA_FILE}" ]; then + cp "${METADATA_FILE}" "${ARTIFACT_DIR}/metadata.json" +else + echo ">>> Skipping metadata.json" +fi + +echo "=== artifact contents ===" +ls -lh "${ARTIFACT_DIR}" +if [ -f "${ARTIFACT_DIR}/metadata.json" ]; then + echo "=== metadata ===" + cat "${ARTIFACT_DIR}/metadata.json" +fi +echo "=== SHA256SUMS ===" +cat "${ARTIFACT_DIR}/SHA256SUMS" diff --git a/.aoneci/scripts/resolve-oss-sts.sh b/.aoneci/scripts/resolve-oss-sts.sh new file mode 100644 index 00000000000..c917bf0c95c --- /dev/null +++ b/.aoneci/scripts/resolve-oss-sts.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Resolve temporary OSS STS credentials from the AoneCI zero-trust bootstrap token. +# +# This script writes shell assignments to stdout for callers to source. Do not +# print the output directly to CI logs. + +set -euo pipefail + +target="${1:-${OSS_STS_TARGET:-public}}" + +ZERO_TRUST_APP_NAME="${ZERO_TRUST_APP_NAME:-lsp-server-outside}" +ZERO_TRUST_APP_GROUP="${ZERO_TRUST_APP_GROUP:-any-value}" +ZERO_TRUST_APP_ENV="${ZERO_TRUST_APP_ENV:-testing}" +ZERO_TRUST_APP_REGION="${ZERO_TRUST_APP_REGION:-cn-hangzhou}" +ZERO_TRUST_TOOL_URL="${ZERO_TRUST_TOOL_URL:-https://apsara-release-build.oss-cn-hangzhou-zmf.aliyuncs.com/aliyun-zerotrust-credential-provider/8654515/zero-trust-credentials-linux-amd64-1.2.5.zip}" + +BOOTSTRAP_TOKEN="${BOOTSTRAP_TOKEN:?BOOTSTRAP_TOKEN is required}" + +case "${target}" in + public) + OSS_STS_ROLE_ARN="${PUBLIC_OSS_STS_ROLE_ARN:-acs:ram::1200759642363824:role/lsp-aoneci-dataworks-datagovernance}" + OSS_ENDPOINT="${PUBLIC_OSS_ENDPOINT:-https://oss-cn-shanghai.aliyuncs.com}" + OSS_BUCKET="${PUBLIC_OSS_BUCKET:-dataworks-notebook-cn-shanghai}" + ;; + finance) + OSS_STS_ROLE_ARN="${FINANCE_OSS_STS_ROLE_ARN:-acs:ram::1797822531535220:role/lsp-aoneci-dataworks-finance}" + OSS_ENDPOINT="${FINANCE_OSS_ENDPOINT:-https://oss-cn-shanghai-finance-1.aliyuncs.com}" + OSS_BUCKET="${FINANCE_OSS_BUCKET:-dataworks-notebook-cn-shanghai-finance-1}" + ;; + *) + echo "ERROR: unsupported OSS_STS_TARGET: ${target}" >&2 + exit 2 + ;; +esac + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT +config_file="${tmp_dir}/zero-trust.yaml" +provider_zip="${tmp_dir}/zero-trust-credentials.zip" + +curl -fsSL "${ZERO_TRUST_TOOL_URL}" -o "${provider_zip}" +unzip -q "${provider_zip}" -d "${tmp_dir}" +provider_bin="$(find "${tmp_dir}" -type f -name aliyun-zerotrust-credential-provider | head -n 1)" +[ -n "${provider_bin}" ] || { + echo "ERROR: aliyun-zerotrust-credential-provider not found in archive" >&2 + exit 2 +} +chmod 755 "${provider_bin}" + +cat > "${config_file}" <&2 + exit 2 +fi + +shell_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +printf 'OSS_TARGET=%s\n' "$(shell_quote "${target}")" +printf 'OSS_ENDPOINT=%s\n' "$(shell_quote "${OSS_ENDPOINT}")" +printf 'OSS_BUCKET=%s\n' "$(shell_quote "${OSS_BUCKET}")" +printf 'OSS_ACCESS_KEY_ID=%s\n' "$(shell_quote "${access_key_id}")" +printf 'OSS_ACCESS_KEY_SECRET=%s\n' "$(shell_quote "${access_key_secret}")" +printf 'OSS_SECURITY_TOKEN=%s\n' "$(shell_quote "${security_token}")" + +echo "Resolved ${target} OSS STS credentials via zero-trust provider" >&2 diff --git a/.aoneci/scripts/send-dingtalk-alert.js b/.aoneci/scripts/send-dingtalk-alert.js new file mode 100644 index 00000000000..d20c5aeb3d5 --- /dev/null +++ b/.aoneci/scripts/send-dingtalk-alert.js @@ -0,0 +1,120 @@ +#!/usr/bin/env node +/* eslint-disable @typescript-eslint/no-require-imports */ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +const crypto = require('node:crypto'); + +function parseArgs(argv) { + const args = { + title: '', + content: '', + url: '', + dryRun: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case '--title': + args.title = argv[++i] ?? ''; + break; + case '--content': + args.content = argv[++i] ?? ''; + break; + case '--url': + args.url = argv[++i] ?? ''; + break; + case '--dry-run': + args.dryRun = true; + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + return args; +} + +function buildSignedUrl(webhook, secret) { + const url = new URL(webhook); + if (!secret) { + return url.toString(); + } + + const timestamp = process.env.DINGTALK_TIMESTAMP || Date.now().toString(); + const sign = crypto + .createHmac('sha256', secret) + .update(`${timestamp}\n${secret}`) + .digest('base64'); + + // sign is raw base64 here; URLSearchParams.set() handles percent-encoding + url.searchParams.set('timestamp', timestamp); + url.searchParams.set('sign', sign); + return url.toString(); +} + +function buildPayload({ title, content, url }) { + const text = [`### ${title}`, '', content, url ? `\n[查看详情](${url})` : ''] + .filter(Boolean) + .join('\n'); + + return { + msgtype: 'markdown', + markdown: { + title, + text, + }, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const webhook = process.env.CI_DINGTALK_WEBHOOK_URL || ''; + const secret = process.env.CI_DINGTALK_WEBHOOK_SECRET || ''; + + if (!args.title || !args.content) { + throw new Error('--title and --content are required'); + } + + if (!webhook) { + console.log('DingTalk webhook is not configured; skipping notification.'); + process.exit(0); + } + + const targetUrl = buildSignedUrl(webhook, secret); + const payload = buildPayload(args); + + if (args.dryRun) { + console.log(JSON.stringify({ url: targetUrl, payload })); + process.exit(0); + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 10000); + + const response = await fetch(targetUrl, { + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + clearTimeout(timeout); + + if (!response.ok) { + const body = await response.text(); + throw new Error(`DingTalk notification failed: ${response.status} ${body}`); + } + + console.log('DingTalk notification sent.'); +} + +main().catch((err) => { + console.error(err.message || err); + process.exit(1); +}); diff --git a/.aoneci/scripts/upgrade-qwen.sh b/.aoneci/scripts/upgrade-qwen.sh new file mode 100755 index 00000000000..cd4393399be --- /dev/null +++ b/.aoneci/scripts/upgrade-qwen.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────── +# upgrade-qwen.sh +# +# 一键升级 qwen-code 到最新版本。 +# +# 自动从 OSS 获取最新版本号,然后调用 deploy-qwen.sh 完成 +# 下载、安装全流程。 +# +# 用法: +# curl -fsSL /upgrade-qwen.sh | bash +# curl -fsSL /upgrade-qwen.sh | bash -s -- --force +# +# 参数: +# --force 跳过版本比较,强制重新安装 +# --dry-run 仅显示版本信息,不执行安装 +# --oss-base-url OSS 根地址覆盖,格式为 https://host/prefix +# 其他参数透传给 deploy-qwen.sh +# +# 环境变量: +# QWEN_OSS_BASE_URL - OSS 根地址覆盖,格式为 https://host/prefix +# ────────────────────────────────────────────────────────── +set -euo pipefail + +# ── OSS 配置 ── +OSS_BUCKET="dataworks-notebook-cn-shanghai" +OSS_HOST="${OSS_BUCKET}.oss-cn-shanghai.aliyuncs.com" +OSS_PREFIX="public-datasets/aone-release/alishu/qwen-code" +EMBEDDED_QWEN_OSS_BASE_URL="__QWEN_OSS_BASE_URL__" +if [ "${EMBEDDED_QWEN_OSS_BASE_URL}" = "__QWEN_OSS_BASE_URL__" ]; then + EMBEDDED_QWEN_OSS_BASE_URL="https://${OSS_HOST}/${OSS_PREFIX}" +fi +OSS_BASE_URL="${QWEN_OSS_BASE_URL:-${EMBEDDED_QWEN_OSS_BASE_URL}}" + +# ── 颜色输出 ── +if [ -t 1 ]; then + GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; CYAN='\033[0;36m'; NC='\033[0m' +else + GREEN=''; YELLOW=''; RED=''; CYAN=''; NC='' +fi +info() { echo -e "${GREEN}>>>${NC} $*"; } +warn() { echo -e "${YELLOW}>>> WARNING:${NC} $*"; } +error() { echo -e "${RED}>>> ERROR:${NC} $*" >&2; } + +# ── 解析参数 ── +FORCE="false" +DRY_RUN="false" +EXTRA_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --force) FORCE="true"; shift ;; + --dry-run) DRY_RUN="true"; shift ;; + --oss-base-url) OSS_BASE_URL="$2"; shift 2 ;; + -h|--help) + sed -n '2,/^[^#]/{ /^#/s/^# \{0,1\}//p }' "$0" + exit 0 + ;; + *) EXTRA_ARGS+=("$1"); shift ;; + esac +done + +OSS_BASE_URL="${OSS_BASE_URL%/}" +case "${OSS_BASE_URL}" in + https://*) ;; + *) + error "Invalid --oss-base-url: ${OSS_BASE_URL}" + exit 1 + ;; +esac +case "${OSS_BASE_URL}" in + *"'"*|*" "*|*";"*|*"|"*|*"&"*|*"\\"*) + error "Unsafe --oss-base-url: ${OSS_BASE_URL}" + exit 1 + ;; +esac +METADATA_URL="${OSS_BASE_URL}/latest/metadata.json" +DEPLOY_URL="${OSS_BASE_URL}/deploy-qwen.sh" + +# ── 获取最新版本号 ── +info "Fetching latest version from OSS..." +METADATA=$(curl -fsSL "${METADATA_URL}" 2>/dev/null) || { + error "Failed to fetch metadata from ${METADATA_URL}" + exit 1 +} + +LATEST_VERSION="" +BUILD_TIME="" +GIT_SHA="" + +parse_json_field() { + echo "${METADATA}" | grep -o "\"$1\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" \ + | head -1 | sed "s/.*\"$1\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/" +} + +if command -v node &>/dev/null; then + LATEST_VERSION=$(echo "${METADATA}" | node -e "process.stdout.write(JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')).version)") + BUILD_TIME=$(echo "${METADATA}" | node -e "const m=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); process.stdout.write(m.build_time||m.builtAt||'')") + GIT_SHA=$(echo "${METADATA}" | node -e "const m=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); process.stdout.write(m.git_short_sha||m.gitSha||'')") +elif command -v python3 &>/dev/null; then + LATEST_VERSION=$(echo "${METADATA}" | python3 -c "import sys,json; print(json.load(sys.stdin)['version'],end='')") + BUILD_TIME=$(echo "${METADATA}" | python3 -c "import sys,json; m=json.load(sys.stdin); print(m.get('build_time',m.get('builtAt','')),end='')") + GIT_SHA=$(echo "${METADATA}" | python3 -c "import sys,json; m=json.load(sys.stdin); print(m.get('git_short_sha',m.get('gitSha','')),end='')") +else + LATEST_VERSION=$(parse_json_field "version") + BUILD_TIME=$(parse_json_field "build_time") + [ -z "${BUILD_TIME}" ] && BUILD_TIME=$(parse_json_field "builtAt") + GIT_SHA=$(parse_json_field "git_short_sha") + [ -z "${GIT_SHA}" ] && GIT_SHA=$(parse_json_field "gitSha") +fi + +if [ -z "${LATEST_VERSION}" ]; then + error "Failed to parse version from metadata" + echo "${METADATA}" + exit 1 +fi + +echo "" +echo -e "${CYAN}============================================${NC}" +echo -e "${CYAN} Qwen Code Upgrade${NC}" +echo -e " Latest: ${GREEN}${LATEST_VERSION}${NC}" +[ -n "${BUILD_TIME}" ] && echo -e " Built at: ${BUILD_TIME}" +[ -n "${GIT_SHA}" ] && echo -e " Git SHA: ${GIT_SHA}" + +# ── 获取当前版本号 ── +INSTALL_DIR="${QWEN_INSTALL_DIR:-/usr/local/qwen-code}" +CURRENT_VERSION="" +LOCAL_META="" +for f in "${INSTALL_DIR}/metadata.json" "${INSTALL_DIR}/META.json"; do + [ -f "$f" ] && LOCAL_META="$f" && break +done +if [ -n "${LOCAL_META}" ]; then + if command -v node &>/dev/null; then + CURRENT_VERSION=$(LOCAL_META="$LOCAL_META" node -e "process.stdout.write(JSON.parse(require('fs').readFileSync(process.env.LOCAL_META,'utf8')).version||'')" 2>/dev/null || true) + elif command -v python3 &>/dev/null; then + CURRENT_VERSION=$(LOCAL_META="$LOCAL_META" python3 -c "import os,json; print(json.load(open(os.environ['LOCAL_META']))['version'],end='')" 2>/dev/null || true) + else + CURRENT_VERSION=$(grep -o '"version"[[:space:]]*:[[:space:]]*"[^"]*"' "${LOCAL_META}" \ + | head -1 | sed 's/.*"version"[[:space:]]*:[[:space:]]*"//;s/".*//' || true) + fi +fi + +if [ -n "${CURRENT_VERSION}" ]; then + echo -e " Current: ${CURRENT_VERSION}" +fi +echo -e "${CYAN}============================================${NC}" +echo "" + +# ── Dry run 模式 ── +if [ "${DRY_RUN}" = "true" ]; then + info "Dry run mode, not installing." + exit 0 +fi + +# ── 版本比较 ── +if [ "${FORCE}" != "true" ] && [ "${CURRENT_VERSION}" = "${LATEST_VERSION}" ]; then + info "Already running the latest version (${CURRENT_VERSION}). Use --force to reinstall." + exit 0 +fi + +# ── 执行升级 ── +info "Upgrading to ${LATEST_VERSION}..." +curl -fsSL "${DEPLOY_URL}" | bash -s -- --version "${LATEST_VERSION}" --oss-base-url "${OSS_BASE_URL}" "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}" +info "Upgrade complete!" diff --git a/.aoneci/scripts/upload-oss.sh b/.aoneci/scripts/upload-oss.sh new file mode 100755 index 00000000000..c4ee9e24b3b --- /dev/null +++ b/.aoneci/scripts/upload-oss.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────── +# upload-oss.sh +# +# 上传 qwen-code standalone 构建产物到 OSS。默认使用 AoneCI zero-trust +# BOOTSTRAP_TOKEN 换取目标云临时 STS 凭证,并同时写入公网与上海金融云。 +# +# 最终链接格式: +# https://./public-datasets/aone-release//// +# +# 环境变量: +# ARTIFACT_DIR - 产物根目录 +# ARCH - 目标架构 +# SOURCE_DIR - 源码根目录(定位脚本) +# WORKSPACE_DIR - CI 工作目录,默认 /workspace +# OSS_GROUP - OSS 路径中的 group +# OSS_PROJECT - OSS 路径中的 project +# BOOTSTRAP_TOKEN - AoneCI 项目级 zero-trust bootstrap token +# OSS_UPLOAD_TARGETS - 上传目标,默认 "public finance" +# OSS_CREDENTIAL_MODE - 默认 zerotrust;显式 aksk 时使用旧 AK/SK 回退 +# SKIP_METADATA - 非空时跳过 metadata 上传和指针更新 +# SKIP_LATEST_POINTER - 非空时只跳过 latest metadata 指针更新 +# SKIP_ROOT_SCRIPTS - 非空时不覆盖项目根目录下的 deploy/upgrade 脚本 +# OSS_RELEASE_CHANNELS - 可选,逗号/空格分隔的 metadata 指针,如 beta,dataworks +# ────────────────────────────────────────────────────────── +set -eu + +ARTIFACT_DIR="${ARTIFACT_DIR:?ARTIFACT_DIR is required}" +ARCH="${ARCH:?ARCH is required}" +SOURCE_DIR="${SOURCE_DIR:-${AONE_CI_SOURCE:-.}}" +WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OSS_GROUP="${OSS_GROUP:?OSS_GROUP is required}" +OSS_PROJECT="${OSS_PROJECT:?OSS_PROJECT is required}" +OSS_ACCESS_KEY_ID="${OSS_ACCESS_KEY_ID:-}" +OSS_ACCESS_KEY_SECRET="${OSS_ACCESS_KEY_SECRET:-}" +OSS_UPLOAD_TARGETS="${OSS_UPLOAD_TARGETS:-public finance}" +SKIP_METADATA="${SKIP_METADATA:-}" +SKIP_LATEST_POINTER="${SKIP_LATEST_POINTER:-}" +SKIP_ROOT_SCRIPTS="${SKIP_ROOT_SCRIPTS:-}" +OSS_RELEASE_CHANNELS="${OSS_RELEASE_CHANNELS:-}" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/oss-targets.sh" + +case "${ARCH}" in + amd64) TARGET_PLATFORM="linux" ;; + arm64) TARGET_PLATFORM="linux" ;; + *) echo "unsupported arch: ${ARCH}" >&2; exit 1 ;; +esac + +VERSION=$(cat "${WORKSPACE_DIR}/.resolved_version") +OSS_PREFIX="public-datasets/aone-release/${OSS_GROUP}/${OSS_PROJECT}/${VERSION}" +OSS_PROJECT_ROOT="public-datasets/aone-release/${OSS_GROUP}/${OSS_PROJECT}" + +# 支持新旧 tarball 命名(新格式不含 arch) +TARBALL_NEW="qwen-code-standalone-${VERSION}.tar.gz" +TARBALL_OLD="qwen-code-${VERSION}-${TARGET_PLATFORM}-${ARCH}.tar.gz" +if [ -f "${ARTIFACT_DIR}/${TARBALL_NEW}" ]; then + TARBALL="${TARBALL_NEW}" +elif [ -f "${ARTIFACT_DIR}/${TARBALL_OLD}" ]; then + TARBALL="${TARBALL_OLD}" +else + echo "ERROR: tarball not found (tried ${TARBALL_NEW} and ${TARBALL_OLD})" >&2 + exit 1 +fi + +find_script() { + local name="$1" + for candidate in \ + "${SOURCE_DIR:+${SOURCE_DIR}/.aoneci/scripts/${name}}" \ + "${AONE_CI_SOURCE:+${AONE_CI_SOURCE}/.aoneci/scripts/${name}}" \ + "${SCRIPT_DIR}/${name}"; do + if [ -n "${candidate}" ] && [ -f "${candidate}" ]; then + echo "${candidate}" + return + fi + done +} + +DEPLOY_SCRIPT="$(find_script "deploy-qwen.sh" || true)" +UPGRADE_SCRIPT="$(find_script "upgrade-qwen.sh" || true)" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +target_script_copy() { + local source_file="$1" + local name="$2" + local base_url="$3" + local output_file="${TMP_DIR}/${OSS_TARGET}-${name}" + local escaped_base="${base_url//&/\\&}" + sed "s#__QWEN_OSS_BASE_URL__#${escaped_base}#g" "${source_file}" > "${output_file}" + printf '%s' "${output_file}" +} + +upload_metadata_pointer() { + local channel="$1" + case "${channel}" in + *[!A-Za-z0-9._-]*|.|..|"") + echo "Invalid OSS release channel: ${channel}" >&2 + exit 1 + ;; + esac + ${OSSUTIL} cp -f "${ARTIFACT_DIR}/metadata.json" "oss://${OSS_BUCKET}/${OSS_PROJECT_ROOT}/${channel}/metadata.json" + echo ">>> [${OSS_TARGET}] ${channel} pointer updated" +} + +TARGETS="$(oss_upload_targets)" +TARGET_SUMMARY="$(oss_targets_one_line)" +PRIMARY_OSS_BASE="" +PRIMARY_OSS_ROOT="" + +while IFS= read -r target; do + [ -n "${target}" ] || continue + oss_configure_target "${target}" + OSS_BASE="$(oss_current_http_url "${OSS_PREFIX}")" + OSS_ROOT="$(oss_current_http_url "${OSS_PROJECT_ROOT}")" + [ -n "${PRIMARY_OSS_BASE}" ] || PRIMARY_OSS_BASE="${OSS_BASE}" + [ -n "${PRIMARY_OSS_ROOT}" ] || PRIMARY_OSS_ROOT="${OSS_ROOT}" + + echo ">>> [${OSS_TARGET}] uploading build artifacts" + ${OSSUTIL} cp -f "${ARTIFACT_DIR}/${TARBALL}" "oss://${OSS_BUCKET}/${OSS_PREFIX}/${TARBALL}" + ${OSSUTIL} cp -f "${ARTIFACT_DIR}/SHA256SUMS" "oss://${OSS_BUCKET}/${OSS_PREFIX}/SHA256SUMS" + + # 兼容旧格式:下游依赖 qwen-code-{version}-linux-{arch}.tar.gz + if [ -f "${ARTIFACT_DIR}/${TARBALL_OLD}" ] && [ "${TARBALL}" != "${TARBALL_OLD}" ]; then + ${OSSUTIL} cp -f "${ARTIFACT_DIR}/${TARBALL_OLD}" "oss://${OSS_BUCKET}/${OSS_PREFIX}/${TARBALL_OLD}" + echo ">>> [${OSS_TARGET}] compat tarball uploaded: ${TARBALL_OLD}" + fi + + if [ -z "${SKIP_METADATA}" ] && [ -f "${ARTIFACT_DIR}/metadata.json" ]; then + ${OSSUTIL} cp -f "${ARTIFACT_DIR}/metadata.json" "oss://${OSS_BUCKET}/${OSS_PREFIX}/metadata.json" + else + echo ">>> [${OSS_TARGET}] skipping metadata.json upload" + fi + + if [ -n "${DEPLOY_SCRIPT}" ]; then + TARGET_DEPLOY_SCRIPT="$(target_script_copy "${DEPLOY_SCRIPT}" "deploy-qwen.sh" "${OSS_ROOT}")" + ${OSSUTIL} cp -f "${TARGET_DEPLOY_SCRIPT}" "oss://${OSS_BUCKET}/${OSS_PREFIX}/deploy-qwen.sh" + if [ -z "${SKIP_ROOT_SCRIPTS}" ]; then + ${OSSUTIL} cp -f "${TARGET_DEPLOY_SCRIPT}" "oss://${OSS_BUCKET}/${OSS_PROJECT_ROOT}/deploy-qwen.sh" + echo ">>> [${OSS_TARGET}] deploy-qwen.sh uploaded to version dir and project root" + else + echo ">>> [${OSS_TARGET}] SKIP_ROOT_SCRIPTS set, deploy-qwen.sh only in version dir" + fi + else + echo ">>> [${OSS_TARGET}] WARNING: deploy-qwen.sh not found, skipping" + fi + + if [ -n "${UPGRADE_SCRIPT}" ]; then + TARGET_UPGRADE_SCRIPT="$(target_script_copy "${UPGRADE_SCRIPT}" "upgrade-qwen.sh" "${OSS_ROOT}")" + if [ -z "${SKIP_ROOT_SCRIPTS}" ]; then + ${OSSUTIL} cp -f "${TARGET_UPGRADE_SCRIPT}" "oss://${OSS_BUCKET}/${OSS_PROJECT_ROOT}/upgrade-qwen.sh" + echo ">>> [${OSS_TARGET}] upgrade-qwen.sh uploaded" + else + echo ">>> [${OSS_TARGET}] SKIP_ROOT_SCRIPTS set, skipping upgrade-qwen.sh" + fi + fi + + if [ -z "${SKIP_METADATA}" ] && [ -z "${SKIP_LATEST_POINTER}" ] && [ -f "${ARTIFACT_DIR}/metadata.json" ]; then + upload_metadata_pointer "latest" + else + echo ">>> [${OSS_TARGET}] skipping latest pointer update" + fi + + if [ -z "${SKIP_METADATA}" ] && [ -n "${OSS_RELEASE_CHANNELS}" ] && [ -f "${ARTIFACT_DIR}/metadata.json" ]; then + CHANNELS=$(printf '%s' "${OSS_RELEASE_CHANNELS}" | tr ',[:space:]' '\n' | sed '/^$/d') + for CHANNEL in ${CHANNELS}; do + upload_metadata_pointer "${CHANNEL}" + done + else + echo ">>> [${OSS_TARGET}] skipping channel pointer update" + fi + + echo ">>> [${OSS_TARGET}] uploaded ${OSS_BASE}/${TARBALL}" +done <&2 +} + +die() { + echo "[upstream-sync-domain-auth] ERROR: $*" >&2 + exit 1 +} + +require_var() { + local name="$1" + local value="$2" + if [ -z "$value" ]; then + die "missing required variable: $name" + fi +} + +has_git_repo() { + git rev-parse --git-dir >/dev/null 2>&1 +} + +urlencode() { + node -e "console.log(encodeURIComponent(process.argv[1] || ''))" "$1" +} + +derive_repo_path_from_origin() { + local remote_url="" + if ! remote_url="$(git remote get-url origin 2>/dev/null)"; then + return 1 + fi + + remote_url="${remote_url%.git}" + case "$remote_url" in + git@*:*) + remote_url="${remote_url#git@}" + remote_url="${remote_url#*:}" + ;; + ssh://git@*/*) + remote_url="${remote_url#ssh://git@}" + remote_url="${remote_url#*/}" + ;; + https://*/*) + remote_url="${remote_url#https://}" + remote_url="${remote_url#*/}" + ;; + http://*/*) + remote_url="${remote_url#http://}" + remote_url="${remote_url#*/}" + ;; + esac + + if [ -z "$remote_url" ] || [ "$remote_url" = "origin" ]; then + return 1 + fi + + printf '%s\n' "$remote_url" +} + +resolve_repo_path() { + if [ -n "$REPO_PATH" ]; then + return 0 + fi + + if has_git_repo; then + REPO_PATH="$(derive_repo_path_from_origin || true)" + fi + + require_var "REPO_PATH" "$REPO_PATH" +} + +resolve_auth() { + if [ -n "$AUTH_USERNAME" ] && [ -n "$PRIVATE_TOKEN" ]; then + AUTH_USER="$AUTH_USERNAME" + AUTH_TOKEN="$PRIVATE_TOKEN" + return 0 + fi + + if [ -n "$PRIVATE_TOKEN" ]; then + AUTH_USER="oauth2" + AUTH_TOKEN="$PRIVATE_TOKEN" + return 0 + fi + + if [ -n "$LEGACY_GIT_TOKEN" ]; then + AUTH_USER="oauth2" + AUTH_TOKEN="$LEGACY_GIT_TOKEN" + return 0 + fi + + die "missing auth credentials; set PRIVATE_TOKEN (optionally with AUTH_USERNAME) or LEGACY_GIT_TOKEN" +} + +log_auth_inputs() { + log "auth diag: AUTH_USERNAME=$([ -n "$AUTH_USERNAME" ] && echo set || echo unset), PRIVATE_TOKEN=$([ -n "$PRIVATE_TOKEN" ] && echo set || echo unset), LEGACY_GIT_TOKEN=$([ -n "$LEGACY_GIT_TOKEN" ] && echo set || echo unset)" + log "repo diag: REPO_PATH=${REPO_PATH:-}, SOURCE_BRANCH=${SOURCE_BRANCH:-}, TARGET_BRANCH=${TARGET_BRANCH:-}, WORK_DIR=${WORK_DIR}" +} + +origin_url() { + printf 'https://%s:%s@%s/%s.git' \ + "$(urlencode "$AUTH_USER")" \ + "$(urlencode "$AUTH_TOKEN")" \ + "$REMOTE_HOST" \ + "$REPO_PATH" +} + +project_encoded() { + node -e "console.log(encodeURIComponent(process.argv[1] || ''))" "$REPO_PATH" +} + +extract_json_field() { + local key="$1" + node -e " + const raw = require('fs').readFileSync(0, 'utf8').trim(); + if (!raw) { + process.exit(1); + } + const data = JSON.parse(raw); + const value = data?.[process.argv[1]]; + if (!value) { + process.exit(1); + } + console.log(value); + " "$key" +} + +extract_first_mr_url() { + node -e " + const remoteHost = process.argv[1] || 'code.alibaba-inc.com'; + const repoPath = process.argv[2] || ''; + const raw = require('fs').readFileSync(0, 'utf8').trim(); + if (!raw) { + process.exit(1); + } + + const data = JSON.parse(raw); + const items = Array.isArray(data) + ? data + : Array.isArray(data?.data?.list) + ? data.data.list + : Array.isArray(data?.list) + ? data.list + : data?.data?.mergeRequest + ? [data.data.mergeRequest] + : data?.mergeRequest + ? [data.mergeRequest] + : [data]; + const keys = [ + 'detail_url', + 'detailUrl', + 'web_url', + 'webUrl', + 'html_url', + 'url', + ]; + for (const item of items) { + if (!item || typeof item !== 'object') { + continue; + } + for (const key of keys) { + if (typeof item[key] === 'string' && item[key].startsWith('http')) { + console.log(item[key]); + process.exit(0); + } + } + if (repoPath && item.id !== undefined && item.id !== null) { + const id = String(item.id); + if (/^[0-9]+$/.test(id)) { + console.log('https://' + remoteHost + '/' + repoPath + '/codereview/' + id); + process.exit(0); + } + } + } + process.exit(1); + " "$REMOTE_HOST" "$REPO_PATH" +} + +write_conflict_files_from_mr_response() { + local response="$1" + if [ -z "$MR_CONFLICT_FILES_OUTPUT_PATH" ]; then + return 0 + fi + + printf '%s' "$response" | node -e " + const raw = require('fs').readFileSync(0, 'utf8').trim(); + if (!raw) { + process.exit(0); + } + + const data = JSON.parse(raw); + const items = Array.isArray(data) + ? data + : Array.isArray(data?.data?.list) + ? data.data.list + : Array.isArray(data?.list) + ? data.list + : data?.data?.mergeRequest + ? [data.data.mergeRequest] + : data?.mergeRequest + ? [data.mergeRequest] + : [data]; + const description = items + .map((item) => (item && typeof item.description === 'string' ? item.description : '')) + .find(Boolean); + if (!description) { + process.exit(0); + } + + const files = []; + const seen = new Set(); + const tick = String.fromCharCode(96); + for (const line of description.split(/\\r?\\n/)) { + const trimmed = line.trim(); + if (!trimmed.startsWith('- ' + tick) || !trimmed.endsWith(tick)) { + continue; + } + const file = trimmed.slice(3, -1); + if (file && !seen.has(file)) { + seen.add(file); + files.push(file); + } + } + if (files.length > 0) { + console.log(files.join('\\n')); + } + " > "$MR_CONFLICT_FILES_OUTPUT_PATH" +} + +ensure_repo() { + cd "$WORK_DIR" + + if ! has_git_repo; then + resolve_auth + resolve_repo_path + log "initializing git repository in $WORK_DIR" + git init + git remote remove origin >/dev/null 2>&1 || true + git remote add origin "$(origin_url)" + else + resolve_repo_path + fi + + git config user.name "$GIT_USER_NAME" + git config user.email "$GIT_USER_EMAIL" + + local git_dir + if git_dir="$(git rev-parse --git-dir 2>/dev/null)" && [ -e "$git_dir/hooks/pre-push" ]; then + log "removing stale .git/hooks/pre-push (CI image installs a git-lfs hook without git-lfs binary)" + rm -f "$git_dir/hooks/pre-push" + fi +} + +configure_authenticated_origin() { + resolve_auth + resolve_repo_path + git remote remove origin >/dev/null 2>&1 || true + git remote add origin "$(origin_url)" +} + +prepare_repo() { + log_auth_inputs + ensure_repo + + if [ -z "$SOURCE_BRANCH" ]; then + log "prepare: SOURCE_BRANCH is unset; skip branch checkout" + return 0 + fi + + if git ls-remote --heads origin "$SOURCE_BRANCH" | grep -q "$SOURCE_BRANCH"; then + git fetch origin "$SOURCE_BRANCH" + git checkout -B "$SOURCE_BRANCH" "origin/$SOURCE_BRANCH" + else + git checkout -B "$SOURCE_BRANCH" + fi +} + +query_existing_mrs() { + local include_target="${1:-1}" + if [ "$include_target" = "1" ]; then + curl -fsS -G \ + -H "PRIVATE-TOKEN: ${AUTH_TOKEN}" \ + --data-urlencode "state=opened" \ + --data-urlencode "source_branch=${SOURCE_BRANCH}" \ + --data-urlencode "target_branch=${TARGET_BRANCH}" \ + "${API_BASE}/projects/$(project_encoded)/merge_requests" + else + curl -fsS -G \ + -H "PRIVATE-TOKEN: ${AUTH_TOKEN}" \ + --data-urlencode "state=opened" \ + --data-urlencode "source_branch=${SOURCE_BRANCH}" \ + "${API_BASE}/projects/$(project_encoded)/merge_requests" + fi +} + +query_existing_code_reviews() { + curl -fsS -G \ + -H "PRIVATE-TOKEN: ${AUTH_TOKEN}" \ + --data-urlencode "order_by=updated_at" \ + --data-urlencode "page=1" \ + --data-urlencode "per_page=20" \ + --data-urlencode "q=repo:${REPO_PATH} AND state:opened,reopened AND target_branch:${TARGET_BRANCH} AND source_branch:${SOURCE_BRANCH}" \ + --data-urlencode "sort=desc" \ + --data-urlencode "v2=true" \ + "${CODE_API_BASE}/code_review/search" +} + +find_existing_mr_url() { + local response mr_url + if response="$(query_existing_mrs 1)" && \ + mr_url="$(printf '%s' "$response" | extract_first_mr_url)"; then + write_conflict_files_from_mr_response "$response" + printf '%s\n' "$mr_url" + return 0 + fi + + log "existing MR exact lookup returned no URL; retrying by source branch only" + if response="$(query_existing_mrs 0)" && \ + mr_url="$(printf '%s' "$response" | extract_first_mr_url)"; then + write_conflict_files_from_mr_response "$response" + printf '%s\n' "$mr_url" + return 0 + fi + + log "existing MR source lookup returned no URL; retrying code review search" + if response="$(query_existing_code_reviews)" && \ + mr_url="$(printf '%s' "$response" | extract_first_mr_url)"; then + write_conflict_files_from_mr_response "$response" + printf '%s\n' "$mr_url" + return 0 + fi + + return 1 +} + +publish_mr() { + local response http_code body mr_url + + log_auth_inputs + require_var "SOURCE_BRANCH" "$SOURCE_BRANCH" + require_var "TARGET_BRANCH" "$TARGET_BRANCH" + require_var "MR_TITLE" "$MR_TITLE" + + ensure_repo + configure_authenticated_origin + + if [ "$SKIP_PUSH" != "1" ]; then + git push origin "$SOURCE_BRANCH" --force + else + log "publish: skip push and reuse existing remote branch ${SOURCE_BRANCH}" + fi + + response="$( + curl -sS -w '\n%{http_code}' -X POST \ + -H "PRIVATE-TOKEN: ${AUTH_TOKEN}" \ + --data-urlencode "source_branch=${SOURCE_BRANCH}" \ + --data-urlencode "target_branch=${TARGET_BRANCH}" \ + --data-urlencode "title=${MR_TITLE}" \ + --data-urlencode "description=${MR_DESCRIPTION}" \ + "${API_BASE}/projects/$(project_encoded)/merge_requests" + )" + + http_code="$(printf '%s' "$response" | tail -n 1)" + body="$(printf '%s' "$response" | sed '$d')" + + case "$http_code" in + 201) + mr_url="$(printf '%s' "$body" | extract_json_field web_url)" || \ + die "merge request created but web_url missing: $body" + ;; + 409) + mr_url="$(printf '%s' "$body" | extract_first_mr_url 2>/dev/null || true)" + if [ -z "$mr_url" ]; then + mr_url="$(find_existing_mr_url)" || \ + die "merge request already exists but failed to query existing MR URL" + fi + ;; + *) + die "merge request creation failed (HTTP ${http_code}): ${body}" + ;; + esac + + if [ -n "$MR_URL_OUTPUT_PATH" ]; then + printf '%s\n' "$mr_url" > "$MR_URL_OUTPUT_PATH" + fi + + printf '%s\n' "$mr_url" +} + +case "$COMMAND" in + prepare) + prepare_repo + ;; + publish) + publish_mr + ;; + *) + die "usage: $0 " + ;; +esac diff --git a/.aoneci/upstream-sync-analyze.yml b/.aoneci/upstream-sync-analyze.yml new file mode 100644 index 00000000000..2f04419b1c6 --- /dev/null +++ b/.aoneci/upstream-sync-analyze.yml @@ -0,0 +1,90 @@ +# .aoneci/upstream-sync-analyze.yml +# 每日分析 upstream (QwenLM/qwen-code) 的变更,生成 sync report 并通过钉钉通知 +# +# 所需 CI 变量(在 Aone CI 平台配置): +# CI_DINGTALK_WEBHOOK_URL - 钉钉机器人 webhook 地址(可选) +# CI_DINGTALK_WEBHOOK_SECRET - 钉钉机器人加签密钥(可选) + +name: Upstream Sync 每日分析 + +triggers: + schedule: + - cron: '0 9 * * 1-5' # 工作日每天 9:00 执行 + push: + branches: ['trigger/upstream-analyze'] + +strategy: + fast-fail: true + +jobs: + analyze: + name: '分析 upstream 变更并通知' + image: node:20 + timeout: 15m + steps: + - uses: 'checkout' + + - uses: 'setup-github-proxy' + + - name: 'Fetch upstream' + run: | + git remote add upstream https://github.com/QwenLM/qwen-code.git 2>/dev/null || true + # CI 默认 shallow clone,需要 unshallow 才能让 merge-base 找到共同祖先。 + # 仅在仓库实际为 shallow 时执行,避免在普通仓库上 unshallow 报错被静默吞掉。 + if [ -f "$(git rev-parse --git-dir)/shallow" ]; then + git fetch --unshallow origin || git fetch --deepen=2000 origin + fi + git fetch upstream main --tags + echo "✅ upstream/main fetched: $(git rev-parse --short upstream/main)" + + - name: '运行分析脚本' + run: | + MERGE_BASE=$(git merge-base HEAD upstream/main) + NEW_COMMITS=$(git log --oneline "$MERGE_BASE..upstream/main" --no-merges | wc -l | tr -d ' ') + + if [ "$NEW_COMMITS" -eq "0" ]; then + echo "✅ 无新的 upstream 提交,已是最新状态。" | tee sync-report.md + else + echo "发现 $NEW_COMMITS 个新的 upstream 提交" + UPSTREAM_REMOTE="upstream" UPSTREAM_BRANCH="main" \ + bash scripts/upstream-sync-analyze.sh > sync-report.md + fi + + LATEST_TAG=$(git tag -l --sort=-v:refname 'v*' | head -1) + LAST_SYNCED_TAG=$(cat .last-synced-upstream-tag 2>/dev/null || echo "none") + if [ "$LATEST_TAG" != "$LAST_SYNCED_TAG" ] && [ -n "$LATEST_TAG" ]; then + echo "" >> sync-report.md + echo "## ⚠️ 新 Release 检测" >> sync-report.md + echo "发现新 tag: **$LATEST_TAG**(上次同步: $LAST_SYNCED_TAG)" >> sync-report.md + fi + + echo "--- 报告预览 ---" + head -20 sync-report.md + echo "--- 报告共 $(wc -l < sync-report.md) 行 ---" + + - name: '准备通知' + id: notify + run: | + TITLE="qwen-code upstream sync" + CONTENT="检查上游变更..." + if [ -f sync-report.md ]; then + FIRST=$(head -1 sync-report.md) + if echo "$FIRST" | grep -q "无新的 upstream"; then + TITLE="✅ Already Latest" + CONTENT="没有新的 upstream 提交" + else + NEW_COUNT=$(echo "$FIRST" | grep -oP '\d+' | head -1 || echo "?") + TITLE="📦 ${NEW_COUNT} new upstream commits" + CONTENT="$FIRST" + fi + fi + echo "$TITLE" > "${{outputs.title.path}}" + echo "$CONTENT" > "${{outputs.content.path}}" + + - uses: dingtalk-bot + inputs: + webhook: ${{secrets['CI_DINGTALK_WEBHOOK_URL']}} + msgtype: link + title: ${{steps.notify.outputs.title}} + content: ${{steps.notify.outputs.content}} + messageUrl: https://gitlab.alibaba-inc.com/alishu/qwen-code diff --git a/.aoneci/upstream-sync-merge.yml b/.aoneci/upstream-sync-merge.yml new file mode 100644 index 00000000000..8d20a37dfc4 --- /dev/null +++ b/.aoneci/upstream-sync-merge.yml @@ -0,0 +1,832 @@ +# .aoneci/upstream-sync-merge.yml +# 合并 upstream (QwenLM/qwen-code) 代码到内网仓库,执行验证,并创建 MR +# +# 所需 CI 变量(在 Aone CI 平台配置): +# vars.username - 域账号用户名(优先,用于 HTTPS push / MR) +# secrets.privateToken - 域账号 private token(优先,用于 HTTPS push / MR) +# vars.repoPath - 仓库路径(可选,未配置时回退到 Aone/CI 环境变量) +# CI_AONE_CODE_PRIVATE_TOKEN_{employeeId} - 旧 token 变量(兼容回退) +# CI_QWEN_API_KEY - Qwen API Key(用于 LLM 辅助冲突解决,可选) +# CI_QWEN_BASE_URL - Qwen API Base URL(默认 DashScope,可选) +# CI_DINGTALK_WEBHOOK_URL - 钉钉机器人 webhook 地址(可选) +# CI_DINGTALK_WEBHOOK_SECRET - 钉钉机器人加签密钥(可选) +# +# 触发方式: +# 1. 手动推送到 feat/upstream-sync-automation 分支 +# 2. 在 Aone CI 平台手动触发 + +name: Upstream Sync 合并 + +triggers: + schedule: + - cron: '20 22 * * *' # 每天 22:20 执行 + always: true # 即使没有新提交也触发 + push: + branches: ['feat/upstream-sync-automation', 'fix/ci-yaml-parse-error'] + +strategy: + fast-fail: true + +traits: + - type: notification + properties: + when: + - success + - fail + types: + - dingtalk + users: + - ${{git.employeeId}} + content: | + *** + ${{jobs.merge.outputs.summary}} + +jobs: + merge: + name: '合并 upstream 并验证' + image: node:22 + timeout: 30m + outputs: + summary: ${{steps.createMR.outputs.result}} + steps: + - uses: 'checkout' + + - uses: 'setup-github-proxy' + + - uses: 'setup-env' + inputs: + node-version: '20' + + - name: '初始化当前仓库的 Git 上下文' + env: + AUTH_USERNAME: ${{vars.username}} + PRIVATE_TOKEN: ${{secrets.privateToken}} + LEGACY_GIT_TOKEN: ${{secrets['CI_AONE_CODE_PRIVATE_TOKEN_' + git.employeeId]}} + REPO_PATH: ${{vars.repoPath}} + GIT_REPO_FULL_NAME: ${{git.repo.fullName}} + SOURCE_BRANCH: ${{git.branch}} + GIT_USER_NAME: aone-ci-bot + GIT_USER_EMAIL: ci-bot@alibaba-inc.com + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + mkdir -p "$STATE_DIR" + HELPER_SCRIPT="$STATE_DIR/upstream-sync-domain-auth.sh" + cp "$SOURCE_DIR/.aoneci/scripts/upstream-sync-domain-auth.sh" "$HELPER_SCRIPT" + chmod +x "$HELPER_SCRIPT" + echo "[auth] token=$( [ -n \"${PRIVATE_TOKEN:-}${LEGACY_GIT_TOKEN:-}\" ] && echo set || echo unset ) repo=${GIT_REPO_FULL_NAME:-(unset)}" + AUTH_USERNAME="${AUTH_USERNAME:-}" \ + PRIVATE_TOKEN="${PRIVATE_TOKEN:-}" \ + LEGACY_GIT_TOKEN="${LEGACY_GIT_TOKEN:-}" \ + REPO_PATH="${REPO_PATH:-}" \ + GIT_REPO_FULL_NAME="${GIT_REPO_FULL_NAME:-}" \ + SOURCE_BRANCH="${SOURCE_BRANCH:-}" \ + GIT_USER_NAME="${GIT_USER_NAME:-aone-ci-bot}" \ + GIT_USER_EMAIL="${GIT_USER_EMAIL:-ci-bot@alibaba-inc.com}" \ + WORK_DIR="$SOURCE_DIR" \ + bash "$HELPER_SCRIPT" prepare + + - name: '配置 Git 身份' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + mkdir -p "$STATE_DIR" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + CONFLICT_FILES_FILE="$STATE_DIR/conflict-files" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + NEW_COMMITS_FILE="$STATE_DIR/new-commits" + SKIP_PUSH_FILE="$STATE_DIR/skip-push" + VERIFY_SUMMARY_FILE="$STATE_DIR/verify-summary" + VERIFY_STATUS_FILE="$STATE_DIR/verify-status" + RISK_FILES_FILE="$STATE_DIR/risk-files" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + SYNC_CHECKPOINT_FILE="$STATE_DIR/sync-checkpoint" + MERGE_BASE_FILE="$STATE_DIR/merge-base" + + git config user.name "aone-ci-bot" + git config user.email "ci-bot@alibaba-inc.com" + + # 初始化状态文件,确保后续步骤即使前面步骤失败也能正常读取 + echo "skip" > "$CONFLICT_STATUS_FILE" + echo "" > "$CONFLICT_FILES_FILE" + echo "" > "$BRANCH_NAME_FILE" + echo "0" > "$NEW_COMMITS_FILE" + echo "0" > "$SKIP_PUSH_FILE" + echo "" > "$VERIFY_SUMMARY_FILE" + echo "skip" > "$VERIFY_STATUS_FILE" + echo "" > "$RISK_FILES_FILE" + cp .aoneci/scripts/send-dingtalk-alert.js "$DINGTALK_SCRIPT_FILE" + echo "" > "$SYNC_CHECKPOINT_FILE" + echo "" > "$MERGE_BASE_FILE" + + - name: 'Fetch upstream' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + TARGET_BRANCH="${CI_DEFAULT_BRANCH:-main}" + + git remote add upstream https://github.com/QwenLM/qwen-code.git 2>/dev/null || true + # CI 默认 shallow clone,需要 unshallow 才能让 merge-base 找到共同祖先。 + # 仅在仓库实际为 shallow 时执行,避免在普通仓库上 unshallow 报错被静默吞掉。 + if [ -f "$(git rev-parse --git-dir)/shallow" ]; then + git fetch --unshallow origin || git fetch --deepen=2000 origin + fi + git fetch upstream main --tags + git fetch origin "$TARGET_BRANCH" 2>/dev/null || true + + echo "📋 upstream: $(git log --oneline -1 upstream/main) | local HEAD: $(git log --oneline -1 HEAD)" + git push origin upstream/main:refs/heads/github-qwen-code-main --force 2>/dev/null || true + + - name: '检查是否有新变更' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + NEW_COMMITS_FILE="$STATE_DIR/new-commits" + TARGET_BRANCH="${CI_DEFAULT_BRANCH:-main}" + + # 用默认目标分支作为同步基准,而非 CI 触发分支的 HEAD + if git show-ref --verify --quiet "refs/remotes/origin/$TARGET_BRANCH"; then + SYNC_REF="origin/$TARGET_BRANCH" + else + SYNC_REF="HEAD" + fi + echo "📌 对比基准: $SYNC_REF ($(git rev-parse --short $SYNC_REF))" + + MERGE_BASE=$(git merge-base "$SYNC_REF" upstream/main 2>/dev/null || echo "") + if [ -z "$MERGE_BASE" ]; then + echo "⚠️ 无法计算 merge-base($SYNC_REF 与 upstream/main 无共同祖先),视为全量同步" + # 用 cherry 精确统计对方独有的提交数,避免把整个 upstream 历史都算进去。 + # 注意:grep -c 在 0 匹配时退出码为 1,必须用 awk 计数避免 fall-through 到 rev-list。 + CHERRY_OUT=$(git cherry "$SYNC_REF" upstream/main 2>/dev/null || true) + if [ -n "$CHERRY_OUT" ]; then + NEW_COMMITS=$(printf '%s\n' "$CHERRY_OUT" | awk '/^\+/ {n++} END {print n+0}') + else + NEW_COMMITS=$(git rev-list --count "$SYNC_REF..upstream/main" 2>/dev/null || echo "0") + fi + else + NEW_COMMITS=$(git log --oneline "$MERGE_BASE..upstream/main" --no-merges | wc -l | tr -d ' ') + echo "📊 merge-base: $(git rev-parse --short $MERGE_BASE) | upstream/main: $(git rev-parse --short upstream/main) | 新提交数: $NEW_COMMITS" + fi + + if [ "$NEW_COMMITS" -eq "0" ]; then + echo "✅ 无新 upstream 提交,无需合并。" + exit 0 + fi + echo "$NEW_COMMITS" > "$NEW_COMMITS_FILE" + echo "📦 $NEW_COMMITS commits → merging" + if [ -n "$MERGE_BASE" ]; then + git log --oneline "$MERGE_BASE..upstream/main" --no-merges | head -3 + else + git log --oneline upstream/main --no-merges | head -3 + fi + + - name: '基于 upstream/main 构建 sync 分支(patch-apply 模式)' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + mkdir -p "$STATE_DIR" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + CONFLICT_FILES_FILE="$STATE_DIR/conflict-files" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + NEW_COMMITS_FILE="$STATE_DIR/new-commits" + SKIP_PUSH_FILE="$STATE_DIR/skip-push" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + SYNC_CHECKPOINT_FILE="$STATE_DIR/sync-checkpoint" + MERGE_BASE_FILE="$STATE_DIR/merge-base" + TARGET_BRANCH="${CI_DEFAULT_BRANCH:-main}" + + NEW_COMMITS=$(cat "$NEW_COMMITS_FILE" 2>/dev/null || echo "0") + if [ "$NEW_COMMITS" = "0" ]; then + echo "✅ 无新 upstream 提交,跳过 sync" + exit 0 + fi + + SYNC_BRANCH="sync/upstream-$(date +%Y%m%d)" + echo "0" > "$SKIP_PUSH_FILE" + + # 检查远端是否已有同日 sync 分支且是最新的 + git fetch origin "$SYNC_BRANCH" 2>/dev/null || true + if git show-ref --verify --quiet "refs/remotes/origin/$SYNC_BRANCH"; then + if git merge-base --is-ancestor upstream/main "origin/$SYNC_BRANCH"; then + echo "♻️ 远端已有同日 sync 分支,检查 fork patches 是否完整..." + git checkout -B "$SYNC_BRANCH" "origin/$SYNC_BRANCH" + if bash "$FORK_DIR/verify-patches.sh"; then + echo "✅ fork patches 验证通过,复用 sync 分支" + echo "$SYNC_BRANCH" > "$BRANCH_NAME_FILE" + echo "1" > "$SKIP_PUSH_FILE" + echo "clean" > "$CONFLICT_STATUS_FILE" + echo "" > "$CONFLICT_FILES_FILE" + echo "$(git rev-parse HEAD)" > "$SYNC_CHECKPOINT_FILE" + echo "$(git rev-parse upstream/main)" > "$MERGE_BASE_FILE" + echo "📋 reusing sync branch | HEAD=$(git log --oneline -1 HEAD)" + exit 0 + else + echo "⚠️ 远端 sync 分支 fork patches 不完整,重新构建" + git checkout -B "$SYNC_BRANCH" "origin/$TARGET_BRANCH" + fi + fi + fi + + # ── 核心逻辑:基于 fork/main 创建 sync 分支,替换为 upstream + patches ── + echo "📦 基于 fork/main 创建 sync 分支,替换内容为 upstream/main + patches" + git checkout -B "$SYNC_BRANCH" "origin/$TARGET_BRANCH" + + # 用 upstream/main 的文件内容替换工作区,然后恢复 fork 独有文件 + git rm -rf --quiet . 2>/dev/null || true + git checkout upstream/main -- . + git checkout "origin/$TARGET_BRANCH" -- .fork/ .aoneci/ .qwen/ 2>/dev/null || true + + # 恢复 fork 独有文件(存在于 main 但不存在于 upstream 的文件) + # 排除 patch 涉及的文件,避免 "already exists" 导致 patch apply 失败 + echo "🔄 恢复 fork 独有文件..." + PATCH_FILES_TMP=$(mktemp) + grep -h '^diff --git' .fork/patches/*.patch 2>/dev/null \ + | sed 's|diff --git a/||;s| b/.*||' | sort -u > "$PATCH_FILES_TMP" || true + comm -23 \ + <(git ls-tree -r --name-only "origin/$TARGET_BRANCH" | sort) \ + <(git ls-tree -r --name-only upstream/main | sort) \ + | grep -v "^\.fork/\|^\.aoneci/\|^\.qwen/" \ + | comm -23 - "$PATCH_FILES_TMP" \ + | xargs -I{} git checkout "origin/$TARGET_BRANCH" -- "{}" 2>/dev/null || true + rm -f "$PATCH_FILES_TMP" + + SYNC_CHECKPOINT=$(git rev-parse upstream/main) + echo "$SYNC_CHECKPOINT" > "$SYNC_CHECKPOINT_FILE" + echo "$SYNC_CHECKPOINT" > "$MERGE_BASE_FILE" + echo "$SYNC_BRANCH" > "$BRANCH_NAME_FILE" + echo "pending" > "$CONFLICT_STATUS_FILE" + + # Step 1: Apply fork patches (--continue: don't stop on first failure) + APPLY_RC=0 + echo "🩹 执行 bash .fork/apply.sh --continue ..." + bash .fork/apply.sh --continue 2>&1 || APPLY_RC=$? + + if [ "$APPLY_RC" -ne 0 ]; then + echo "⚠️ patch apply 失败(exit $APPLY_RC),尝试 LLM 辅助修复..." + # 用 --check-applied 找出未完整应用的 patch;正向 --check 在已打补丁的树上会把成功的 patch 也误报为失败 + FAILED_PATCHES=$(bash .fork/apply.sh --check-applied 2>&1 | grep "^FAIL:" | awk '{print $2}' || true) + echo " 失败的 patches: $FAILED_PATCHES" + + # 尝试 LLM 修复:让它读 .rej 文件并手动应用 + QWEN_API_KEY="${{secrets['CI_QWEN_API_KEY']}}" + if [ -n "$QWEN_API_KEY" ]; then + QWEN_BASE_URL="${{secrets['CI_QWEN_BASE_URL']}}" + QWEN_BASE_URL="${QWEN_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1}" + LLM_PROMPT="Some fork patches failed to apply cleanly. For each .rej file in the working tree: 1) Read the .rej file to understand the intended change. 2) Read the target source file. 3) Apply the intended change manually (the context lines shifted but the intent is clear). 4) Delete the .rej file after successful application. 5) Run: git add the modified file. After all .rej files are resolved, verify none remain with: find . -name '*.rej'" + npm_config_registry="https://registry.anpm.alibaba-inc.com/" \ + OPENAI_API_KEY="$QWEN_API_KEY" \ + OPENAI_BASE_URL="$QWEN_BASE_URL" \ + npm exec --yes --package=@alife/dataworks-qwen-code@latest -- qwen \ + --auth-type openai \ + --prompt "$LLM_PROMPT" \ + --yolo || true + + # 全量验证所有 patch 是否已正确 apply(不仅检查 .rej 文件) + # "already exists" 等错误不产生 .rej,仅靠 .rej 数量会误判成功 + # 注意:此时多数 patch 已经打上,正向 --check 必然全部 FAIL, + # 必须用 --check-applied(反向 dry-run)验证"已应用"状态 + REJ_REMAINING=$(find . -name "*.rej" | wc -l | tr -d ' ') + STILL_FAILING=$(bash .fork/apply.sh --check-applied 2>&1 | grep -c "^FAIL:" || true) + if [ "$REJ_REMAINING" -eq 0 ] && [ "$STILL_FAILING" -eq 0 ]; then + echo "✅ LLM 成功修复所有 patch apply 失败" + APPLY_RC=0 + else + echo "⚠️ 修复后仍有问题: $REJ_REMAINING 个 .rej 文件, $STILL_FAILING 个 patch 未通过 --check-applied" + fi + fi + + if [ "$APPLY_RC" -ne 0 ]; then + echo "has_conflicts" > "$CONFLICT_STATUS_FILE" + echo "$FAILED_PATCHES" > "$CONFLICT_FILES_FILE" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync Patch Apply 失败" \ + --content "fork patches 无法干净 apply 到 upstream/main。需要人工 refresh patches。失败: $FAILED_PATCHES" \ + || true + echo "📋 patch-apply failed | branch=$SYNC_BRANCH | HEAD=$(git log --oneline -1 HEAD)" + exit 0 + fi + fi + + # Step 2: Rewrite package identity (name → @alife/..., add registry) + echo "📦 执行 rewrite-package-identity.js ..." + if [ -f ".fork/rewrite-package-identity.js" ]; then + node .fork/rewrite-package-identity.js + fi + + # Step 2.5: Regenerate package-lock.json to match rewritten names + echo "🔄 同步 package-lock.json ..." + npm install --package-lock-only --ignore-scripts 2>/dev/null || true + + # Step 3: Stage all changes + git add -A + echo "clean" > "$CONFLICT_STATUS_FILE" + echo "" > "$CONFLICT_FILES_FILE" + + echo "📋 patch-apply done | status=clean | branch=$SYNC_BRANCH | HEAD=$(git log --oneline -1 HEAD)" + + + - name: '提交 clean sync 结果' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + SYNC_BRANCH=$(cat "$BRANCH_NAME_FILE" 2>/dev/null || echo "") + if [ "$SYNC_STATUS" != "clean" ] || [ -z "$SYNC_BRANCH" ]; then + echo "📋 sync 状态为 $SYNC_STATUS,跳过 clean commit" + exit 0 + fi + + git add -A + git diff --cached --quiet || git commit -m "chore: sync upstream $(date +%Y-%m-%d)" + echo "📋 clean sync ready | branch=$SYNC_BRANCH | HEAD=$(git log --oneline -1 HEAD)" + + - name: '验证 fork patch 内容完整性 (gate)' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + if [ "$SYNC_STATUS" != "clean" ]; then + echo "📋 sync 状态为 $SYNC_STATUS,跳过 patch 内容验证" + exit 0 + fi + if [ ! -f .fork/verify-patches.sh ]; then + echo "⚠️ .fork/verify-patches.sh 不存在,跳过" + exit 0 + fi + + echo "🔍 验证 fork patch 内容是否完整..." + VERIFY_RC=0 + bash .fork/verify-patches.sh --verbose 2>&1 || VERIFY_RC=$? + + if [ "$VERIFY_RC" -ne 0 ]; then + echo "" + echo "❌ fork patch 内容验证失败,部分 patch 丢失" + echo "has_conflicts" > "$CONFLICT_STATUS_FILE" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync Fork Patch 内容丢失" \ + --content "sync 分支构建完成但 fork patch 内容验证失败,部分定制代码丢失。需要人工检查并修复。" \ + || true + exit 1 + fi + echo "✅ fork patch 内容验证通过" + + - name: '运行 fork patch verify (gate)' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + VERIFY_SUMMARY_FILE="$STATE_DIR/verify-summary" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + if [ "$SYNC_STATUS" = "skip" ]; then + echo "✅ 无 sync 变更,跳过 verify" + echo "skipped" > "$VERIFY_SUMMARY_FILE" + exit 0 + fi + if [ "$SYNC_STATUS" = "clean" ]; then + echo "✅ patch-apply 成功,跳过 verify.sh(patch apply 成功即证明 fork 定制完整)" + echo "skipped" > "$VERIFY_SUMMARY_FILE" + exit 0 + fi + if [ ! -f .fork/verify.sh ]; then + echo "⚠️ .fork/verify.sh 不存在,跳过" + echo "missing" > "$VERIFY_SUMMARY_FILE" + exit 0 + fi + + echo "🔍 运行 .fork/verify.sh 验证 fork patches 是否丢失" + echo " sync 状态: $SYNC_STATUS" + VERIFY_OUT="$STATE_DIR/verify-output" + JSON_OUT="$STATE_DIR/verify-result.json" + VERIFY_RC=0 + JSON_OUTPUT="$JSON_OUT" bash .fork/verify.sh > "$VERIFY_OUT" 2>&1 || VERIFY_RC=$? + + tail -30 "$VERIFY_OUT" > "$VERIFY_SUMMARY_FILE" + echo "--- verify.sh 摘要 ---" + cat "$VERIFY_SUMMARY_FILE" + echo "----------------------" + + if [ "$SYNC_STATUS" = "has_conflicts" ]; then + # 冲突未解决时分支已切到 upstream/main,verify 结果仅供 MR 描述参考 + echo "⚠️ 合并有未解决冲突,verify 结果为 advisory(不阻塞 MR 创建)" + echo " 人工或 AI agent 解决冲突后需重新运行 verify" + elif [ "$VERIFY_RC" -ne 0 ]; then + echo "" + echo "❌ fork patch verify FAILED (exit $VERIFY_RC)" + echo " 存在疑似丢失的 fork 定制。自动合并中止,需人工介入或 AI agent 修复后重试。" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync Fork Patch 验证失败" \ + --content "fork patch verify 失败,疑似存在 fork 定制丢失。摘要: $(tr '\n' ' ' < "$VERIFY_SUMMARY_FILE")" \ + || true + exit 1 + else + echo "✅ fork patch verify passed" + fi + + - name: '校验 patches.md 与 git 历史同步' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + if [ "$SYNC_STATUS" = "skip" ]; then + exit 0 + fi + if [ ! -f scripts/regen-fork-patches.sh ]; then + echo "⚠️ regen-fork-patches.sh 不存在,跳过同步检查" + exit 0 + fi + + echo "🔍 检查 .fork/patches.md 是否与合并后的 git 历史一致" + # --check 模式:不写盘,diff 不为空则 exit 1 + ORIGIN_REF=HEAD bash scripts/regen-fork-patches.sh --check || { + echo "" + echo "⚠️ patches.md 与当前 git 历史不一致(advisory,不阻塞)" + echo " 合并后请运行: bash scripts/regen-fork-patches.sh --write" + } + + - name: '计算 fork patch 影响清单' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + RISK_FILES_FILE="$STATE_DIR/risk-files" + TARGET_BRANCH="${CI_DEFAULT_BRANCH:-main}" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + SYNC_BRANCH=$(cat "$BRANCH_NAME_FILE" 2>/dev/null || echo "") + if [ -z "$SYNC_BRANCH" ] || [ "$SYNC_STATUS" = "skip" ]; then + echo "" > "$RISK_FILES_FILE" + exit 0 + fi + + # 计算:"本次 sync 改动文件" ∩ "fork-only 提交触动过的文件"。 + # 第二个集合:从 git log 取所有 fork commit 的 file 并集(排除 release/sync 类型, + # 与 verify.sh 的 classify_subject 规则保持一致)。 + CHANGED_FILES=$(mktemp) + FORK_FILES=$(mktemp) + INTERSECT=$(mktemp) + trap 'rm -f "$CHANGED_FILES" "$FORK_FILES" "$INTERSECT"' EXIT + + if git show-ref --verify --quiet "refs/remotes/origin/$TARGET_BRANCH"; then + BASE_REF="origin/$TARGET_BRANCH" + else + BASE_REF="$TARGET_BRANCH" + fi + git diff --name-only "$BASE_REF...HEAD" 2>/dev/null | sort -u > "$CHANGED_FILES" + + # 用 awk 内联跑 classify_subject 规则;只接受 fork 类的 commit。 + git log --no-merges --format='%H%x09%s' "upstream/main..origin/$TARGET_BRANCH" 2>/dev/null \ + | awk -F '\t' ' + { + s = $2 + if (s ~ /^Merge commit /) next + if (s ~ /^Merge branch sync\//) next + if (s ~ /^chore: sync upstream/) next + if (s ~ /sync\/resolve-upstream/) next + if (s ~ /^fix: align with upstream/) next + if (s ~ /^chore\(release\)/) next + if (s ~ /^chore: release/) next + if (s ~ /^chore: bump version/) next + if (s ~ /^chore: rebase/) next + if (s ~ /^build: bump version/) next + if (s ~ /^ci\(release\)/) next + if (s ~ /^ci: publish/) next + print $1 + } + ' \ + | while IFS= read -r sha; do + git show --name-only --format= "$sha" 2>/dev/null + done \ + | sort -u > "$FORK_FILES" + + comm -12 "$CHANGED_FILES" "$FORK_FILES" > "$INTERSECT" || true + RISK_COUNT=$(grep -c . "$INTERSECT" || true) + CHANGED_COUNT=$(grep -c . "$CHANGED_FILES" || true) + echo "📊 本次 sync 改动 $CHANGED_COUNT 个文件,其中 $RISK_COUNT 个属于 fork patch 触动过的高风险文件" + cp "$INTERSECT" "$RISK_FILES_FILE" + if [ "$RISK_COUNT" -gt 0 ]; then + echo "--- 高风险文件清单(前 30)---" + head -30 "$RISK_FILES_FILE" + echo "------------------------------" + fi + + - name: '检查 sync 分支发布状态' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + SKIP_PUSH_FILE="$STATE_DIR/skip-push" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + SYNC_BRANCH=$(cat "$BRANCH_NAME_FILE" 2>/dev/null || echo "") + SKIP_PUSH=$(cat "$SKIP_PUSH_FILE" 2>/dev/null || echo "0") + echo "📋 推送步骤 | sync 状态: $SYNC_STATUS | 分支: $SYNC_BRANCH" + if [ "$SYNC_STATUS" = "skip" ] || [ -z "$SYNC_BRANCH" ]; then + echo "✅ 无需同步,跳过推送" + exit 0 + fi + if [ "$SKIP_PUSH" = "1" ]; then + echo "✅ 远端已有可复用的 sync 分支,后续直接创建/复用 MR" + exit 0 + fi + echo "✅ sync 分支已就绪,下一步统一执行推送和 MR 创建" + + - id: createMR + name: '创建 Merge Request' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + NEW_COMMITS_FILE="$STATE_DIR/new-commits" + CONFLICT_FILES_FILE="$STATE_DIR/conflict-files" + MR_CONFLICT_FILES_FILE="$STATE_DIR/mr-conflict-files" + HELPER_SCRIPT="$STATE_DIR/upstream-sync-domain-auth.sh" + SKIP_PUSH_FILE="$STATE_DIR/skip-push" + VERIFY_SUMMARY_FILE="$STATE_DIR/verify-summary" + RISK_FILES_FILE="$STATE_DIR/risk-files" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + SYNC_BRANCH=$(cat "$BRANCH_NAME_FILE" 2>/dev/null || echo "") + SKIP_PUSH=$(cat "$SKIP_PUSH_FILE" 2>/dev/null || echo "0") + echo "📋 创建 MR | sync 状态: $SYNC_STATUS | 分支: $SYNC_BRANCH" + if [ "$SYNC_STATUS" = "skip" ] || [ -z "$SYNC_BRANCH" ]; then + SUMMARY="✅ 无新 upstream 提交,无需同步" + echo "$SUMMARY" + echo "$SUMMARY" > "${{outputs.result.path}}" + exit 0 + fi + TARGET_BRANCH="${CI_DEFAULT_BRANCH:-main}" + NEW_COMMITS=$(cat "$NEW_COMMITS_FILE" 2>/dev/null || echo "?") + echo "📊 target: $TARGET_BRANCH | 新提交: $NEW_COMMITS" + TODAY=$(date +%Y-%m-%d) + CONFLICT_STATUS="$SYNC_STATUS" + CONFLICT_FILES=$(cat "$CONFLICT_FILES_FILE" 2>/dev/null || echo "") + + # 根据冲突状态生成不同的 MR 标题 + if [ "$CONFLICT_STATUS" = "has_conflicts" ]; then + CONFLICT_COUNT=$(echo "$CONFLICT_FILES" | grep -c . || true) + if [ "$CONFLICT_COUNT" -gt 0 ]; then + CONFLICT_TITLE="$CONFLICT_COUNT conflicts" + else + CONFLICT_TITLE="conflicts" + fi + MR_TITLE="⚠️ chore: upstream sync $TODAY ($NEW_COMMITS commits, $CONFLICT_TITLE)" + else + MR_TITLE="chore: upstream sync $TODAY ($NEW_COMMITS commits)" + fi + + # 生成 MR 描述 + { + echo "## Upstream Sync $TODAY" + echo "" + echo "### 概要" + echo "- 合并了 **$NEW_COMMITS** 个 upstream 提交" + echo "- 源: QwenLM/qwen-code main" + echo "- Sync 分支: \`$SYNC_BRANCH\`" + + if [ "$CONFLICT_STATUS" = "has_conflicts" ]; then + echo "" + echo "### ⚠️ 未解决的冲突" + echo "" + echo "以下文件包含冲突标记(\`<<<<<<<\` / \`=======\` / \`>>>>>>>\`),需要人工解决:" + echo "" + echo "$CONFLICT_FILES" | while read -r f; do + echo "- \`$f\`" + done + echo "" + echo "> **注意**: 验证步骤(build/typecheck/test/lint)已跳过,请在解决冲突后本地验证。" + else + echo "" + echo "### 验证" + echo "- ⏳ 验证将在 MR 创建后异步执行" + fi + + # ── fork patch 高风险文件(影响清单) ── + RISK_LINES=$(grep -c . "$RISK_FILES_FILE" 2>/dev/null || true) + RISK_LINES="${RISK_LINES:-0}" + if [ "$RISK_LINES" -gt 0 ]; then + echo "" + echo "### 🎯 高风险文件(fork patch 触动过且本次 sync 也修改)" + echo "" + echo "**reviewer 重点 review 这 $RISK_LINES 个文件**(与 \`.fork/patches.md\` 中 fork 类 commit 触动过的文件交集):" + echo "" + head -50 "$RISK_FILES_FILE" | while IFS= read -r f; do + [ -z "$f" ] && continue + echo "- \`$f\`" + done + if [ "$RISK_LINES" -gt 50 ]; then + echo "" + echo "> 共 $RISK_LINES 个文件,仅展示前 50 个;完整清单见 CI artifact 或重跑流水线。" + fi + fi + + # ── verify.sh 摘要 ── + VERIFY_CONTENT=$(cat "$VERIFY_SUMMARY_FILE" 2>/dev/null || echo "") + if [ -n "$VERIFY_CONTENT" ] && [ "$VERIFY_CONTENT" != "skipped" ] && [ "$VERIFY_CONTENT" != "missing" ]; then + echo "" + if [ "$CONFLICT_STATUS" = "has_conflicts" ]; then + echo "### 🔍 Fork patch verify (advisory — 冲突未解决)" + echo "" + echo "\`bash .fork/verify.sh\` 在 upstream/main 上的参考结果(冲突未解决,不阻塞 MR 创建;解决后需重跑验证):" + else + echo "### ✅ Fork patch verify (gate passed)" + echo "" + echo "\`bash .fork/verify.sh\` 在合并结果上的验证通过(FAIL 会阻塞自动合并):" + fi + echo "" + echo '```text' + echo "$VERIFY_CONTENT" + echo '```' + fi + + echo "" + echo "### Review 要点" + echo "请查看每日 Upstream Sync 分析 CI 了解完整差异状态。" + } > /tmp/mr-body.md + MR_BODY=$(cat /tmp/mr-body.md) + rm -f "$MR_CONFLICT_FILES_FILE" + + # 安全检查:如果 sync 分支相对 target 无差异,跳过 MR 创建 + git fetch origin "$TARGET_BRANCH" 2>/dev/null || true + if git diff --quiet "origin/$TARGET_BRANCH" HEAD 2>/dev/null; then + SUMMARY="⚠️ Upstream Sync $TODAY | 合并失败且 sync 分支无差异(无法创建有效 MR)| 冲突文件: $(echo "$CONFLICT_FILES" | tr '\n' ' ') | 请手动合并: git fetch origin $SYNC_BRANCH && git checkout -b resolve/upstream-sync origin/$TARGET_BRANCH && git merge origin/$SYNC_BRANCH" + echo "$SUMMARY" + echo "$SUMMARY" > "${{outputs.result.path}}" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync 需要人工处理" \ + --content "$SUMMARY" \ + || true + exit 0 + fi + + MR_URL=$( + AUTH_USERNAME="${{vars.username}}" \ + PRIVATE_TOKEN="${{secrets.privateToken}}" \ + LEGACY_GIT_TOKEN="${{secrets['CI_AONE_CODE_PRIVATE_TOKEN_' + git.employeeId]}}" \ + REPO_PATH="${{vars.repoPath}}" \ + GIT_REPO_FULL_NAME="${{git.repo.fullName}}" \ + SOURCE_BRANCH="$SYNC_BRANCH" \ + TARGET_BRANCH="$TARGET_BRANCH" \ + MR_TITLE="$MR_TITLE" \ + MR_DESCRIPTION="$MR_BODY" \ + MR_CONFLICT_FILES_OUTPUT_PATH="$MR_CONFLICT_FILES_FILE" \ + SKIP_PUSH="$SKIP_PUSH" \ + WORK_DIR="$SOURCE_DIR" \ + bash "$HELPER_SCRIPT" publish + ) || { + echo "❌ sync 分支发布或 MR 创建失败" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync MR 创建失败" \ + --content "sync 分支发布或 MR 创建失败。分支: $SYNC_BRANCH,目标分支: $TARGET_BRANCH,请查看 CI 日志。" \ + || true + exit 1 + } + if [ -z "$MR_URL" ]; then + echo "❌ MR URL 为空,视为创建失败" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync MR 创建失败" \ + --content "sync 分支发布后返回的 MR URL 为空。分支: $SYNC_BRANCH,目标分支: $TARGET_BRANCH,请查看 CI 日志。" \ + || true + exit 1 + fi + EXISTING_CONFLICT_COUNT=$(echo "$CONFLICT_FILES" | grep -c . || true) + if [ "$EXISTING_CONFLICT_COUNT" -eq 0 ] && [ -s "$MR_CONFLICT_FILES_FILE" ]; then + CONFLICT_FILES=$(cat "$MR_CONFLICT_FILES_FILE") + EXISTING_CONFLICT_COUNT=$(echo "$CONFLICT_FILES" | grep -c . || true) + echo "📎 从已有 MR 读取到 $EXISTING_CONFLICT_COUNT 个冲突文件" + fi + + if [ "$CONFLICT_STATUS" = "has_conflicts" ]; then + CONFLICT_COUNT=$(echo "$CONFLICT_FILES" | grep -c . || true) + CONFLICT_SUMMARY="存在冲突待人工处理" + if [ "$CONFLICT_COUNT" -gt 0 ]; then + CONFLICT_SUMMARY="$CONFLICT_COUNT 个冲突待人工解决" + fi + SUMMARY="⚠️ Upstream Sync $TODAY | $NEW_COMMITS 个提交 | $CONFLICT_SUMMARY | MR: $MR_URL" + DINGTALK_CONTENT="$SUMMARY" + if [ "$CONFLICT_COUNT" -gt 0 ]; then + DINGTALK_FILES=$(echo "$CONFLICT_FILES" | sed 's/^/- /') + DINGTALK_CONTENT=$(printf '%s\n\n冲突文件:\n%s' "$SUMMARY" "$DINGTALK_FILES") + fi + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync 需要人工处理" \ + --content "$DINGTALK_CONTENT" \ + --url "$MR_URL" \ + || true + else + SUMMARY="✅ Upstream Sync $TODAY | $NEW_COMMITS 个提交 | MR: $MR_URL" + fi + echo "$SUMMARY" + echo "$SUMMARY" > "${{outputs.result.path}}" + + - name: '安装依赖并验证(非阻塞)' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + VERIFY_STATUS_FILE="$STATE_DIR/verify-status" + DINGTALK_SCRIPT_FILE="$STATE_DIR/send-dingtalk-alert.js" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + echo "📋 验证步骤 | sync 状态: $SYNC_STATUS" + if [ "$SYNC_STATUS" != "clean" ]; then + echo "⚠️ 状态为 $SYNC_STATUS,跳过验证" + echo "skipped" > "$VERIFY_STATUS_FILE" + exit 0 + fi + + echo "📦 安装依赖..." + if ! npm install --ignore-scripts; then + echo "⚠️ npm install 失败,跳过后续验证" + echo "skipped" > "$VERIFY_STATUS_FILE" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync 验证失败" \ + --content "sync MR 已创建,但 npm install 失败,后续 build/typecheck/test 验证已跳过。请查看 CI 日志。" \ + || true + exit 0 + fi + echo "✅ npm install 完成" + + if [ ! -f scripts/upstream-sync-verify.sh ]; then + echo "⚠️ scripts/upstream-sync-verify.sh 不存在,跳过验证" + echo "skipped" > "$VERIFY_STATUS_FILE" + exit 0 + fi + echo "🔍 开始验证..." + if bash scripts/upstream-sync-verify.sh; then + echo "✅ 验证通过" + echo "passed" > "$VERIFY_STATUS_FILE" + else + echo "⚠️ 验证失败(状态门禁将失败)" + echo "failed" > "$VERIFY_STATUS_FILE" + CI_DINGTALK_WEBHOOK_URL="${{secrets['CI_DINGTALK_WEBHOOK_URL']}}" \ + CI_DINGTALK_WEBHOOK_SECRET="${{secrets['CI_DINGTALK_WEBHOOK_SECRET']}}" \ + node "$DINGTALK_SCRIPT_FILE" \ + --title "Upstream Sync 验证失败" \ + --content "sync MR 已创建,但 scripts/upstream-sync-verify.sh 验证失败。请查看 CI 日志和 MR diff。" \ + || true + fi + + - name: '上游同步状态门禁' + run: | + SOURCE_DIR="${AONE_CI_SOURCE:-.}" + cd "$SOURCE_DIR" + STATE_DIR="$(cd "$SOURCE_DIR/.." && pwd)/.aoneci-upstream-sync" + CONFLICT_STATUS_FILE="$STATE_DIR/conflict-status" + VERIFY_STATUS_FILE="$STATE_DIR/verify-status" + BRANCH_NAME_FILE="$STATE_DIR/branch-name" + + SYNC_STATUS=$(cat "$CONFLICT_STATUS_FILE" 2>/dev/null || echo "skip") + VERIFY_STATUS=$(cat "$VERIFY_STATUS_FILE" 2>/dev/null || echo "skip") + SYNC_BRANCH=$(cat "$BRANCH_NAME_FILE" 2>/dev/null || echo "") + echo "📋 final gate | sync=$SYNC_STATUS | verify=$VERIFY_STATUS | branch=$SYNC_BRANCH" + + if [ "$SYNC_STATUS" = "has_conflicts" ]; then + echo "❌ upstream sync 需要人工处理:存在未解决冲突或疑似 fork 定制回退" + exit 1 + fi + if [ "$VERIFY_STATUS" = "failed" ]; then + echo "❌ upstream sync 验证失败,请查看前置验证 step 和 MR diff" + exit 1 + fi + echo "✅ upstream sync 状态门禁通过" diff --git a/.fork/SYNC-PLAN.md b/.fork/SYNC-PLAN.md new file mode 100644 index 00000000000..144c711ddef --- /dev/null +++ b/.fork/SYNC-PLAN.md @@ -0,0 +1,125 @@ +# Fork Upstream Sync - 设计方案与分支状态 + +## 概述 + +本 fork (`gitlab.alibaba-inc.com/alishu/qwen-code`) 的定义: + +``` +fork/main = upstream/main + .fork/patches/* + package identity rewrite +``` + +每日 CI 定时任务负责自动同步 upstream 的最新代码,并重新应用所有 patch。 + +## 同步策略:Patch-Apply 模式 + +### 核心流程 + +``` +upstream/main (最新) + │ + ├── git checkout -B sync/upstream-YYYYMMDD upstream/main + │ + ├── 从 fork/main 取回基础设施文件:.fork/ .aoneci/ .qwen/ + │ + ├── bash .fork/apply.sh (按 series 顺序 apply 所有 patch) + │ + ├── node .fork/rewrite-package-identity.js (改写 package name/registry) + │ + └── push → 创建 MR 到 main +``` + +### 为什么不用 Merge 模式 + +Merge 模式(把 upstream merge 进 fork/main)必然产生冲突,因为 fork/main 中已经 bake 了 patch 的修改。例如 `channel-registry.ts` 里的 import 路径已被改写,merge 时 upstream 版本和 fork 版本总是不同的。 + +Patch-Apply 模式每次从全新 upstream 基础出发,不存在"两边都改了"的问题。 + +## 当前分支状态 + +| 分支 | 基于 | 状态 | 说明 | +| -------------------------------- | ----------------------- | -------------------- | ------------------------------------ | +| `origin/main` (fork main) | — | 已合并到 `ab38e03e7` | Fork 主线,含 patch infrastructure | +| `fix/ci-remove-lfs-prepush-hook` | fork main (`ab38e03e7`) | **活跃,已推送** | 本次修复分支,2 个新 commit | +| `codex/fork-sync-guard` | fork main | 已推送 | CI sync 的额外防护(代码审查修复等) | +| `origin/sync/upstream-20260529` | upstream/main | 失败的 sync 分支 | 旧 merge 模式产生的,可废弃 | +| `resolve/upstream-sync-20260529` | fork main | 本地临时 | 同上,用于手动解冲突,可删除 | +| `inspect/sync-conflicts` | upstream/main | 本地临时 | 调试用,可删除 | + +## fix/ci-remove-lfs-prepush-hook 分支改动 + +### Commit 1: `b9b6d69ad` + +**fix(ci): drop stale .git/hooks/pre-push before upstream sync push** + +解决 CI runner 上残留的 git-lfs pre-push hook 导致 push 失败。 + +### Commit 2: `a8af9472d` + +**fix(ci): rewrite upstream sync to patch-apply model and refresh patches** + +核心改动: + +1. **`.aoneci/upstream-sync-merge.yml`** — 重写 CI 脚本 + - 移除 merge 逻辑(~340 行) + - 新增 patch-apply 逻辑 + - 修复 `qwen: not found`:`npx --registry` → `npm_config_registry` env + `npm exec` + +2. **`.fork/patches/0003-i18n-dataworks.patch`** — 更新 zh.js context anchor + - Upstream 在 `'Long conversation...'` 和 `// Exit Screen` 间插入了新行 + +3. **`.fork/patches/0007-feishu-channel.patch`** — 1924 行 → 22 行 + - Upstream PR #4379 已添加 feishu 源码 + - 仅保留 import path 改写(`@qwen-code/channel-feishu` → `@alife/...`) + +4. **`.fork/patches/0010-build-single-bundle.patch`** — 移除 feishu build order hunk + - Upstream 已在 build order 中包含 feishu + +5. **`.fork/patches/0011-test-fork-adaptations.patch`** — 移除 252 行 + - Upstream 已做相同的 static import 重构(detect-terminal-theme.test.ts) + +6. **`.fork/manifest.json`** — 对应更新 paths 和 metadata + +## Patch 清单(10 个) + +| # | 文件 | 行数 | 说明 | +| ---- | ----------------------------------- | ---- | --------------------------------------------------------------------------------------------- | +| 0001 | branding-header.patch | 263 | DataWorks branding | +| 0002 | branding-tips.patch | 124 | 启动提示 | +| 0003 | i18n-dataworks.patch | 78 | i18n 占位符 | +| 0004 | dsw-oauth-redirect.patch | 74 | DSW OAuth 代理 | +| 0005 | osc8-internal.patch | 115 | 终端超链接适配 | +| 0006 | dingtalk-channel-enhancements.patch | 649 | 钉钉 channel 增强 | +| 0007 | feishu-channel.patch | 22 | 飞书 import path | +| 0009 | claude-websearch-compat.patch | 155 | WebSearch 兼容 | +| 0010 | build-single-bundle.patch | 96 | 单文件打包 + 移除 acp-bridge 显式构建步骤(fork 通过 tsconfig project references 传递性编译) | +| 0011 | test-fork-adaptations.patch | 91 | 测试适配 | + +全部 10 个 patch 已验证可在 upstream/main (`c699738f9`) 上 clean apply。 + +## 后续操作 + +1. **合并此 MR** — 将 `fix/ci-remove-lfs-prepush-hook` 合入 fork main +2. **等待次日 CI cron** — 验证 sync 流程正常运行(无冲突、无 command not found) +3. **清理废弃分支** — `sync/upstream-20260529`、`resolve/upstream-sync-20260529`、`inspect/sync-conflicts` +4. **`codex/fork-sync-guard`** — 评估是否需要合并其额外防护逻辑 + +## CI YAML 关键片段 + +```yaml +# patch-apply 核心逻辑 +git checkout -B "$SYNC_BRANCH" upstream/main +git checkout "origin/$TARGET_BRANCH" -- .fork/ .aoneci/ .qwen/ 2>/dev/null || true +bash .fork/apply.sh 2>&1 || APPLY_RC=$? + +# 如果 apply 失败,尝试 LLM 修复 +npm_config_registry="https://registry.anpm.alibaba-inc.com/" \ + OPENAI_API_KEY="$QWEN_API_KEY" \ + OPENAI_BASE_URL="$QWEN_BASE_URL" \ + npm exec --yes --package=@alife/dataworks-qwen-code@latest -- qwen \ + --auth-type openai \ + --prompt "$LLM_PROMPT" \ + --yolo + +# package identity 改写 +node .fork/rewrite-package-identity.js +``` diff --git a/.fork/apply.sh b/.fork/apply.sh new file mode 100755 index 00000000000..07233237e2a --- /dev/null +++ b/.fork/apply.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# .fork/apply.sh — Apply all fork patches in series order. +# +# Usage: +# bash .fork/apply.sh # stop on first failure +# bash .fork/apply.sh --continue # apply remaining patches after failure +# bash .fork/apply.sh --check # dry-run: check if patches apply cleanly +# bash .fork/apply.sh --check-applied # verify patches are ALREADY applied +# # (reverse dry-run; forward --check always +# # fails on an already-patched tree) +# +# Exit codes: +# 0 all patches applied (or check passed) +# 1 one or more patches failed + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +SERIES="$FORK_DIR/patches/series" +PATCH_DIR="$FORK_DIR/patches" +MODE="${1:-apply}" + +if [ ! -f "$SERIES" ]; then + echo "❌ series file not found: $SERIES" >&2 + exit 1 +fi + +APPLIED=0 +FAILED=0 +FAILED_LIST=() + +while IFS= read -r patch; do + [[ -z "$patch" || "$patch" == \#* ]] && continue + PATCH_FILE="$PATCH_DIR/$patch" + + if [ ! -f "$PATCH_FILE" ]; then + echo "MISSING: $patch" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch (file not found)") + continue + fi + + if [ "$MODE" = "--check" ]; then + if git apply --check "$PATCH_FILE" 2>/dev/null; then + echo "OK: $patch ($(wc -l < "$PATCH_FILE" | tr -d ' ') lines)" + else + echo "FAIL: $patch ($(wc -l < "$PATCH_FILE" | tr -d ' ') lines)" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + fi + continue + fi + + # 已应用校验:patch 能干净地反向 apply,说明其全部 hunk 都已存在于工作区 + if [ "$MODE" = "--check-applied" ]; then + if git apply --reverse --check "$PATCH_FILE" 2>/dev/null; then + echo "OK: $patch (already applied)" + else + echo "FAIL: $patch (not fully applied)" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + fi + continue + fi + + if git apply --check "$PATCH_FILE" 2>/dev/null; then + if git apply "$PATCH_FILE"; then + echo "APPLIED: $patch" + APPLIED=$((APPLIED + 1)) + else + echo "FAILED: $patch (apply failed after --check passed)" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + fi + else + echo "FAILED: $patch" + git apply --reject "$PATCH_FILE" 2>/dev/null || true + REJ_COUNT=$(find . -name "*.rej" -newer "$PATCH_FILE" 2>/dev/null | wc -l | tr -d ' ') + if [ "$REJ_COUNT" -gt 0 ]; then + echo " $REJ_COUNT .rej file(s) generated. Locations:" + find . -name "*.rej" -newer "$PATCH_FILE" 2>/dev/null | while read -r f; do echo " $f"; done + fi + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + if [ "$MODE" != "--continue" ]; then + echo "" + echo "Use --continue to apply remaining patches after fixing .rej files" + break + fi + fi +done < "$SERIES" + +echo "" +if [ "$MODE" = "--check" ] || [ "$MODE" = "--check-applied" ]; then + echo "Check complete: $FAILED failed" +else + echo "Applied: $APPLIED Failed: $FAILED" +fi + +if [ "$FAILED" -gt 0 ]; then + echo "Failed patches:" + for p in "${FAILED_LIST[@]}"; do + echo " - $p" + done + exit 1 +fi +exit 0 diff --git a/.fork/create-patch.sh b/.fork/create-patch.sh new file mode 100755 index 00000000000..68b0c746c4d --- /dev/null +++ b/.fork/create-patch.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# .fork/create-patch.sh — Create a new patch from current modifications. +# +# Usage: +# bash .fork/create-patch.sh [file2 ...] +# +# Example: +# bash .fork/create-patch.sh my-feature packages/core/src/foo.ts packages/core/src/bar.ts +# +# Environment: +# UPSTREAM_REF default upstream/main +# FORK_REF optional committed fork ref to diff, for example origin/main +# PATCH_BASE_REF optional explicit base; defaults to merge-base(FORK_REF|HEAD, UPSTREAM_REF) + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH_DIR="$FORK_DIR/patches" +SERIES="$PATCH_DIR/series" +UPSTREAM_REF="${UPSTREAM_REF:-upstream/main}" + +if [ $# -lt 2 ]; then + echo "Usage: $0 [file2 ...]" >&2 + exit 1 +fi + +if ! git rev-parse --verify --quiet "${UPSTREAM_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ $UPSTREAM_REF not available. Run: git fetch upstream main" >&2 + exit 2 +fi +if [ -n "${FORK_REF:-}" ] && ! git rev-parse --verify --quiet "${FORK_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ FORK_REF not available: $FORK_REF" >&2 + exit 2 +fi + +if [ -n "${PATCH_BASE_REF:-}" ]; then + if ! git rev-parse --verify --quiet "${PATCH_BASE_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ PATCH_BASE_REF not available: $PATCH_BASE_REF" >&2 + exit 2 + fi + PATCH_BASE=$(git rev-parse "$PATCH_BASE_REF") +else + PATCH_HEAD="${FORK_REF:-HEAD}" + PATCH_BASE=$(git merge-base "$PATCH_HEAD" "$UPSTREAM_REF") +fi + +NAME="$1"; shift +FILES=("$@") + +LAST_NUM=$(ls "$PATCH_DIR"/*.patch 2>/dev/null | sort | tail -1 | xargs -I{} basename {} | grep -oE '^[0-9]{4}' || echo "0000") +NEXT_NUM=$(printf "%04d" $(( 10#$LAST_NUM + 1 ))) +PATCH_FILE="$PATCH_DIR/${NEXT_NUM}-${NAME}.patch" + +if [ -n "${FORK_REF:-}" ]; then + git diff --binary --no-color "$PATCH_BASE" "$FORK_REF" -- "${FILES[@]}" > "$PATCH_FILE" +else + git diff --binary --no-color "$PATCH_BASE" -- "${FILES[@]}" > "$PATCH_FILE" +fi + +if [ -s "$PATCH_FILE" ]; then + echo "${NEXT_NUM}-${NAME}.patch" >> "$SERIES" + echo "✅ Created: $(basename "$PATCH_FILE") ($(wc -l < "$PATCH_FILE") lines)" + echo " Base: $(git rev-parse --short=9 "$PATCH_BASE")" + echo " Added to series file" + echo "" + echo "Files in patch:" + printf '%s\n' "${FILES[@]}" | while read -r f; do echo " $f"; done +else + rm "$PATCH_FILE" + echo "❌ No diff found for specified files" >&2 + exit 1 +fi diff --git a/.fork/generate-patches.js b/.fork/generate-patches.js new file mode 100755 index 00000000000..804d5fdeeee --- /dev/null +++ b/.fork/generate-patches.js @@ -0,0 +1,256 @@ +#!/usr/bin/env node +// .fork/generate-patches.js +// +// Regenerates the ordered fork patch stack from .fork/manifest.json. +// The diff base is the fork/upstream merge-base by default, not upstream HEAD. + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +const repoRoot = process.cwd(); +const forkDir = path.join(repoRoot, '.fork'); +const manifestPath = path.join(forkDir, 'manifest.json'); +const args = new Set(process.argv.slice(2)); +const action = args.has('--check') ? 'check' : 'write'; + +function fail(message, code = 1) { + console.error(message); + process.exit(code); +} + +function git(gitArgs) { + return execFileSync('git', gitArgs, { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function gitOutput(gitArgs) { + return execFileSync('git', gitArgs, { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function verifyRef(ref, label) { + try { + git(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`]); + } catch { + fail(`ERROR: ${label} ref is not available: ${ref}`, 2); + } +} + +function shortRef(ref) { + return git(['rev-parse', '--short=9', ref]); +} + +function readManifest() { + if (!fs.existsSync(manifestPath)) { + fail(`ERROR: manifest not found: ${manifestPath}`, 2); + } + return JSON.parse(fs.readFileSync(manifestPath, 'utf8')); +} + +function resolveBase({ forkRef, upstreamRef }) { + if (process.env.PATCH_BASE_REF) { + verifyRef(process.env.PATCH_BASE_REF, 'PATCH_BASE_REF'); + return git(['rev-parse', process.env.PATCH_BASE_REF]); + } + return git(['merge-base', forkRef, upstreamRef]); +} + +function renderHeader(definition, context) { + const lines = [ + `Subject: ${definition.title ?? definition.file}`, + `Reason: ${definition.reason ?? 'Long-lived fork customization.'}`, + `Owner: ${definition.owner ?? 'DataWorks Qwen Code maintainers'}`, + `Patch-Base: ${context.patchBaseSha}`, + `Fork-Ref: ${context.forkRef} (${context.forkSha})`, + `Upstream-Ref: ${context.upstreamRef}`, + 'Paths:', + ...definition.paths.map((filePath) => ` - ${filePath}`), + ]; + + if (Array.isArray(definition.tests) && definition.tests.length > 0) { + lines.push('Tests:', ...definition.tests.map((test) => ` - ${test}`)); + } + + // Blank line separates header from diff body (standard patch format) + lines.push(''); + return `${lines.join('\n')}\n`; +} + +function normalizeContent(content) { + return content.endsWith('\n') ? content : `${content}\n`; +} + +function buildPatch(definition, context) { + const diff = gitOutput([ + 'diff', + '--binary', + '--no-color', + context.patchBaseSha, + context.forkRef, + '--', + ...definition.paths, + ]); + + if (diff.trim().length === 0) { + return ''; + } + return `${renderHeader(definition, context)}${normalizeContent(diff)}`; +} + +function validateDefinitions(definitions) { + const seen = new Set(); + for (const definition of definitions) { + if (!definition.file || typeof definition.file !== 'string') { + fail('ERROR: every patch definition requires a string "file"', 2); + } + if (seen.has(definition.file)) { + fail(`ERROR: duplicate patch file in manifest: ${definition.file}`, 2); + } + seen.add(definition.file); + if (!Array.isArray(definition.paths) || definition.paths.length === 0) { + fail(`ERROR: ${definition.file} requires a non-empty paths array`, 2); + } + } +} + +const manifest = readManifest(); +const patchConfig = manifest.patches ?? {}; +const definitions = patchConfig.definitions ?? []; + +if (!Array.isArray(definitions) || definitions.length === 0) { + fail('ERROR: .fork/manifest.json has no patches.definitions entries', 2); +} + +validateDefinitions(definitions); + +const upstreamRef = process.env.UPSTREAM_REF ?? 'upstream/main'; +const forkRef = process.env.FORK_REF ?? 'origin/main'; +verifyRef(upstreamRef, 'UPSTREAM_REF'); +verifyRef(forkRef, 'FORK_REF'); + +const patchBaseSha = resolveBase({ forkRef, upstreamRef }); +const context = { + upstreamRef, + upstreamSha: git(['rev-parse', upstreamRef]), + forkRef, + forkSha: git(['rev-parse', forkRef]), + patchBaseSha, +}; + +const patchDirRel = patchConfig.directory ?? 'patches/'; +const seriesRel = patchConfig.seriesFile ?? path.join(patchDirRel, 'series'); +const patchDir = path.join(forkDir, patchDirRel); +const seriesPath = path.join(forkDir, seriesRel); +const generated = []; +const retired = []; +const empty = []; + +for (const definition of definitions) { + const status = definition.status ?? 'active'; + if (status === 'retired') { + retired.push(definition.file); + continue; + } + if (status !== 'active') { + fail(`ERROR: ${definition.file} has unsupported status: ${status}`, 2); + } + + const content = buildPatch(definition, context); + if (!content) { + empty.push(definition.file); + continue; + } + + if (definition.file.includes('..')) { + fail(`ERROR: ${definition.file} contains path traversal sequence (..)`, 2); + } + const resolvedPath = path.resolve(patchDir, definition.file); + if ( + !resolvedPath.startsWith(patchDir + path.sep) && + resolvedPath !== patchDir + ) { + fail(`ERROR: ${definition.file} escapes patch directory`, 2); + } + + generated.push({ + file: definition.file, + path: resolvedPath, + content, + }); +} + +if (empty.length > 0) { + fail( + [ + 'ERROR: active patch definitions produced empty diffs.', + 'Retire them explicitly in .fork/manifest.json if upstream now covers them:', + ...empty.map((file) => ` - ${file}`), + ].join('\n'), + ); +} + +const seriesContent = `${generated.map((entry) => entry.file).join('\n')}\n`; +const mismatches = []; + +function currentFileContent(filePath) { + return fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf8') : null; +} + +if (action === 'check') { + for (const entry of generated) { + if (currentFileContent(entry.path) !== entry.content) { + mismatches.push(entry.file); + } + } + if (currentFileContent(seriesPath) !== seriesContent) { + mismatches.push(path.relative(forkDir, seriesPath)); + } + for (const file of retired) { + const retiredPath = path.join(patchDir, file); + if (fs.existsSync(retiredPath)) { + mismatches.push(file); + } + } + + if (mismatches.length > 0) { + fail( + [ + 'ERROR: generated fork patches are out of date:', + ...mismatches.map((file) => ` - ${file}`), + 'Run: node .fork/generate-patches.js --write', + ].join('\n'), + ); + } +} else { + fs.mkdirSync(patchDir, { recursive: true }); + for (const entry of generated) { + fs.writeFileSync(entry.path, entry.content); + } + fs.writeFileSync(seriesPath, seriesContent); + for (const file of retired) { + const retiredPath = path.join(patchDir, file); + if (fs.existsSync(retiredPath)) { + fs.rmSync(retiredPath); + } + } +} + +console.log(`patch_base: ${shortRef(patchBaseSha)}`); +console.log(`fork_ref: ${forkRef} (${shortRef(context.forkSha)})`); +console.log(`upstream_ref: ${upstreamRef} (${shortRef(context.upstreamSha)})`); +console.log( + `${action === 'check' ? 'checked' : 'wrote'} ${generated.length} patch(es)`, +); +for (const entry of generated) { + console.log(` - ${entry.file}`); +} +for (const file of retired) { + console.log(`retired: ${file}`); +} diff --git a/.fork/generate-patches.sh b/.fork/generate-patches.sh new file mode 100755 index 00000000000..efb656a3b07 --- /dev/null +++ b/.fork/generate-patches.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# .fork/generate-patches.sh — Regenerate patch files from .fork/manifest.json. + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +node "$FORK_DIR/generate-patches.js" "$@" diff --git a/.fork/manifest.json b/.fork/manifest.json new file mode 100644 index 00000000000..73accdaac4b --- /dev/null +++ b/.fork/manifest.json @@ -0,0 +1,141 @@ +{ + "version": 1, + "upstream": { + "remote": "https://github.com/QwenLM/qwen-code.git", + "branch": "main" + }, + "patches": { + "directory": "patches/", + "seriesFile": "patches/series", + "definitions": [ + { + "file": "0001-branding-header.patch", + "title": "DataWorks branding header", + "reason": "Keep DataWorks DataAgent branding in the CLI header and startup display.", + "paths": [ + "packages/cli/src/ui/components/AppHeader.test.tsx", + "packages/cli/src/ui/components/AsciiArt.ts", + "packages/cli/src/ui/components/ConfigInitDisplay.tsx", + "packages/cli/src/ui/components/Header.test.tsx", + "packages/cli/src/ui/components/Header.tsx" + ], + "tests": [ + "cd packages/cli && npx vitest run src/ui/components/Header.test.tsx src/ui/components/AppHeader.test.tsx" + ] + }, + { + "file": "0002-branding-tips.patch", + "title": "DataWorks startup tips", + "reason": "Keep startup tips and beta guidance tailored to DataWorks usage.", + "paths": [ + "packages/cli/src/services/tips/tipRegistry.ts", + "packages/cli/src/ui/components/Tips.test.ts", + "packages/cli/src/ui/components/Tips.tsx" + ], + "tests": [ + "cd packages/cli && npx vitest run src/ui/components/Tips.test.ts" + ] + }, + { + "file": "0003-i18n-dataworks.patch", + "title": "DataWorks i18n strings", + "reason": "Keep DataWorks-specific placeholders and usage examples.", + "paths": [ + "packages/cli/src/i18n/locales/en.js", + "packages/cli/src/i18n/locales/zh.js" + ] + }, + { + "file": "0004-dsw-oauth-redirect.patch", + "title": "DSW OAuth redirect rewrite", + "reason": "Rewrite MCP OAuth redirect URLs for the DSW proxy environment.", + "paths": [ + "packages/core/src/mcp/constants.test.ts", + "packages/core/src/mcp/constants.ts", + "packages/core/src/mcp/oauth-provider.test.ts", + "packages/core/src/mcp/oauth-provider.ts" + ], + "tests": [ + "cd packages/core && npx vitest run src/mcp/constants.test.ts src/mcp/oauth-provider.test.ts" + ] + }, + { + "file": "0005-osc8-internal.patch", + "title": "Internal OSC8 terminal links", + "reason": "Keep terminal hyperlink handling compatible with internal terminal environments.", + "paths": [ + "packages/cli/src/ui/utils/osc8.test.ts", + "packages/cli/src/ui/utils/osc8.ts" + ], + "tests": ["cd packages/cli && npx vitest run src/ui/utils/osc8.test.ts"] + }, + { + "file": "0006-dingtalk-channel-enhancements.patch", + "title": "DingTalk channel enhancements", + "reason": "Keep DingTalk channel card, markdown, and message routing behavior.", + "paths": [ + "packages/channels/base/src/ChannelBase.ts", + "packages/channels/dingtalk/src/DingtalkAdapter.ts", + "packages/channels/dingtalk/src/markdown.ts" + ] + }, + { + "file": "0007-feishu-channel.patch", + "title": "Feishu channel integration", + "reason": "Rewrite feishu channel import to fork package name in CLI registry.", + "paths": ["packages/cli/src/commands/channel/channel-registry.ts"] + }, + { + "file": "0009-claude-websearch-compat.patch", + "title": "Claude WebSearch compatibility", + "reason": "Keep fork behavior around Claude WebSearch conversion and integration coverage.", + "paths": [ + "packages/core/src/extension/claude-converter.ts", + "integration-tests/cli/web_search.test.ts" + ] + }, + { + "file": "0010-build-single-bundle.patch", + "title": "Single-file bundle output", + "reason": "Fork uses single outfile (dist/cli.js) without code-splitting; build.js omits acp-bridge from the explicit build order because the fork relies on cli's tsconfig project references to compile it transitively.", + "paths": ["esbuild.config.js", "scripts/build.js"] + }, + { + "file": "0011-test-fork-adaptations.patch", + "title": "Test adaptations for fork UI and CI", + "reason": "Adapt tests for fork OAuth menu structure, Aone CI runner detection, and static imports.", + "paths": [ + "packages/cli/src/ui/auth/AuthDialog.test.tsx", + "packages/cli/src/utils/doctorChecks.test.ts", + "packages/core/src/skills/skill-activation.test.ts" + ], + "tests": [ + "cd packages/cli && npx vitest run src/ui/auth/AuthDialog.test.tsx src/utils/doctorChecks.test.ts", + "cd packages/core && npx vitest run src/skills/skill-activation.test.ts" + ] + } + ] + }, + "packageIdentity": { + "registry": "https://registry.anpm.alibaba-inc.com", + "excludeRegistry": [ + "package.json", + "packages/channels/plugin-example/package.json" + ], + "mappings": { + "package.json": "@alife/dataworks-qwen-code", + "packages/cli/package.json": "@alife/dataworks-qwen-code", + "packages/core/package.json": "@alife/dataworks-qwen-code-core", + "packages/sdk-typescript/package.json": "@alife/dataworks-qwen-code-sdk", + "packages/acp-bridge/package.json": "@alife/dataworks-qwen-code-acp-bridge", + "packages/webui/package.json": "@alife/dataworks-qwen-code-webui", + "packages/web-templates/package.json": "@alife/dataworks-qwen-code-web-templates", + "packages/channels/base/package.json": "@alife/dataworks-qwen-code-channel-base", + "packages/channels/dingtalk/package.json": "@alife/dataworks-qwen-code-channel-dingtalk", + "packages/channels/feishu/package.json": "@alife/dataworks-qwen-code-channel-feishu", + "packages/channels/telegram/package.json": "@alife/dataworks-qwen-code-channel-telegram", + "packages/channels/weixin/package.json": "@alife/dataworks-qwen-code-channel-weixin", + "packages/channels/plugin-example/package.json": "@alife/dataworks-qwen-code-channel-plugin-example" + } + } +} diff --git a/.fork/patches.md b/.fork/patches.md new file mode 100644 index 00000000000..9275df46509 --- /dev/null +++ b/.fork/patches.md @@ -0,0 +1,306 @@ +# Fork Patch Manifest + +This file records the internal fork changes that exist in `alishu/qwen-code` +but are not part of the upstream `QwenLM/qwen-code` history. It is used as the +audit source for upstream sync reviews: when `upstream/main` is merged into the +fork, these entries should either still be present, be intentionally migrated, +or be explicitly retired because upstream now contains an equivalent change. + +## Snapshot + +> 由 `scripts/regen-fork-patches.sh --write` 自动维护。修改 SNAPSHOT 区段内容 +> 不会被保留,请改脚本而非手改。 + + + +- generated_at: 2026-05-23 +- fork_ref: `origin/main` +- fork_head: `c6b168ec0` +- upstream_ref: `upstream/main` +- upstream_head: `0cb9ff0a2` +- patch_base: `cc800d013` +- diff_range: `cc800d013..origin/main` +- first_parent_landing_commits: 48 +- patch_bearing_commits: 175 + + + +Commands used for this snapshot: + +```bash +git fetch origin main +git fetch upstream main --tags +PATCH_BASE_REF=$(git merge-base origin/main upstream/main) +git diff --name-status "$PATCH_BASE_REF..origin/main" +git log --first-parent --reverse --format='%h %s' "$PATCH_BASE_REF..origin/main" +git log --reverse --no-merges --format='%h %s' "$PATCH_BASE_REF..origin/main" +git cherry -v "$PATCH_BASE_REF" origin/main +node .fork/generate-patches.js --write +``` + +Do not use the current `upstream/main` head directly as the diff base for patch +generation. `patch_base` is the last shared upstream sync point for the fork +main branch; using it avoids mixing future upstream commits into fork patches. + +## Maintenance Rules + +- Add a new entry after every internal fork MR is merged into `main`. +- Keep `sync` entries for audit context, but do not use them as guard targets. +- Keep `release` entries separate from functional fork changes; version bumps + can be retired or replaced when the next release bump lands. +- If upstream later contains an equivalent patch, mark the commit as retired + before deleting it from this manifest. +- During upstream sync, review this file together with the generated sync MR + diff. Any guard-relevant entry that disappears from the fork must be restored, + migrated, or explicitly retired in the sync MR description. + +## PR/MR Landing Commits + +These are the first-parent commits by which internal fork changes landed on +`origin/main`. They are the audit layer for Code Review / MR history. + +> 由 `scripts/regen-fork-patches.sh --write` 自动维护。Type 列由 commit subject +> 启发式分类(fork / sync / sync-fix / release / upstream-equivalent);如需 +> 覆盖(如 test、upstream-equivalent),请改脚本中的 `classify_subject()`。 + + + +| Commit | Type | CR | Title | +| --- | --- | --- | --- | +| `8c8151af6` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585259 | feat: customize branding for DataWorks DataAgent | +| `34165d8ec` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585513 | fix: remove trailing space in header | +| `20b06f215` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585642 | publish dataworks scope npm | +| `19f048a0d` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585660 | feat: update ASCII logo for DataWorks branding | +| `f87780bf3` | sync | - | Merge commit '73042e3e68cfb9098e0db1a9af9de26a0cfe1ba7' into 'main' | +| `3a240e41a` | sync | - | Merge commit '9034663bbc7080b85b627029537b6394ea90de89' into 'main' | +| `b57ec053f` | sync | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26819487 | chore: sync upstream QwenLM/qwen-code to latest (399 commits - 0.14.3) | +| `67ee5fc8d` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26767752 | [to #80958901] fix(vscode-ide-companion): unblock test suite (postcss ESM + Storage deep-path mock) | +| `34b328af0` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26837934 | fix(core): fallback after empty stream retries | +| `488ce5444` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26741792 | 恢复部分合并丢失的 qwen-code支持双输出模式 代码 | +| `fc1e209a2` | fork | - | 恢复部分合并丢失的 qwen code cli 代码 | +| `0ce14f9de` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26881335 | 构建 qwen code 打包的二进制脚本 * wip: 构建二进制压缩包 | +| `0b8ed080f` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26843438 | build: add npm publish workflow for CI/CD pipeline | +| `116f798d3` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26889902 | fix(dingtalk): prioritize senderStaffId over senderId and add debug log | +| `ca172b61e` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26907000 | fix(i18n): restore DataWorks input placeholder and usage example tips * fix(i18n): restore DataWorks input placeholder and usage example tips | +| `be2e07469` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26904912 | refactor: clean up bundle-publish branch * chore: bump version to 0.14.6 across multiple package.json files | +| `7f7648125` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26921615 | fix(core): allow thought-only responses in GeminiChat stream validation * fix(core): auto-continue on mid-stream cut-off; classify empty streams as EMPTY_STREAM | +| `be928ad95` | sync | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26938960 | chore: sync upstream QwenLM/qwen-code to latest (56 commits — 0.14.5) | +| `683d7d7bf` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26940966 | refactor(mcp-oauth): move copy hint directly under the auth URL * feat(mcp): rewrite OAuth redirect URI for DSW proxy environment | +| `405901d78` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26940867 | feat(core): integrate upstream agent features with retry mechanism | +| `98695c409` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26959860 | feat(cli): Add OAuth flags to mcp add command * feat(mcp): rewrite OAuth redirect URI for DSW proxy environment | +| `124cd12bb` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26819585 | style: quote workflow job names and actions for consistency * feat: add feature flags for DataWorks branding and upstream sync automation | +| `4fc49738d` | sync-fix | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26974885 | fix: align StreamJsonOutputAdapter, DingtalkAdapter, WebViewProvider.test with upstream | +| `e3566cb10` | sync-fix | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26741792 | fix: align DualOutputBridge and RemoteInputWatcher with upstream (PR #3352) | +| `99d0ba4cb` | sync-fix | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26974885 | fix: align cli/config.ts and PanelManager.ts with upstream | +| `aeb95e37d` | sync-fix | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26976368 | fix: align gemini.tsx and DualOutputBridge.test.ts with upstream | +| `a7ddd8500` | sync-fix | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26976669 | fix: align mcp/add.test.ts type cast and core/config.ts JSDoc with upstream | +| `a49ee09fa` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26978689 | feat(ui): 在 Header 信息面板中展示当前 model 名称 | +| `0d064548f` | sync | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26977267 | chore: sync upstream QwenLM/qwen-code 2026-04-20 (48 commits, conflicts resolved) | +| `6fccf403b` | fork | https://code.alibaba-inc.com/alishu/opencode/codereview/26975666 | fix(build): bundle i18n locales and extension examples into dist/ | +| `54d3a11d3` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/26996251 | fix(mcp): make the OAuth authorization URL clickable when wrapped * fix(mcp): make the OAuth authorization URL clickable when wrapped | +| `8ed429500` | release | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27016717 | chore(release): bump version to 0.14.7 across all packages * chore(release): bump version to 0.14.7 across all packages | +| `d939701be` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27071384 | refactor: add BFF endpoint logic for OAuth redirect URI generation * refactor: add BFF endpoint logic for OAuth redirect URI generation | +| `91125d478` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27074746 | fix(cli): stabilize startup tip across Static remounts | +| `340070331` | release | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27202215 | chore(release): bump version to 0.14.8 | +| `3ce1b1b8e` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27202180 | test(cli): 精简 CLI 定制测试修复 | +| `7073c3460` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27240813 | test(cli): pre-resolve AppContainer sync conflict | +| `d4cbb7c11` | sync | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27382776 | Merge branch sync/upstream-20260511 into dataworks-20260511 Title: Sync QwenLM/qwen-code main 20260511 | +| `4ca5a58aa` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27384835 | feat(cli): wrap markdown links in OSC 8 so wrapped URLs stay clickable (#4037) * feat(cli): wrap markdown links in OSC 8 so wrapped URLs stay clickable (#4037) | +| `bc3703a37` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27386500 | ⚠️ chore: upstream sync 2026-05-14 (40 commits, 12 conflicts) | +| `bc3152001` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27399741 | 优化发布脚本 * chore(ci): remove deprecated Aone CI pipelines and optimize remaining ones | +| `aa94e5194` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27385182 | fix:update card bug and add stop btn with new module * fix:update card bug and add stop btn with new module | +| `c2bf54e8e` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27413710 | feat: add default OAuth redirect URI builder * feat: add default OAuth redirect URI builder | +| `6b37a472c` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27416044 | fix(cli): restore alishu / internal-deployment OSC 8 signals * fix(cli): restore alishu / internal-deployment OSC 8 signals | +| `498267a86` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27523343 | fix(ci): add always:true to schedule trigger for upstream sync pipeline | +| `f607af2d9` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27524077 | fix: remove built-in web_search tool, align with upstream MCP-based approach | +| `9e4b33fe7` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27532149 | feishu channel * fix(ci): add package scope to standalone artifact | +| `c6b168ec0` | fork | https://code.alibaba-inc.com/alishu/qwen-code/codereview/27524072 | fix(core): extend DashScope provider detection & remove broken remoteInput test | + + +## Patch-Bearing Commit Inventory + +These are the non-merge commits reachable from `origin/main` but not from +`upstream/main` at the snapshot above. This is the raw commit inventory used to +avoid losing fork-side patches during upstream sync. + + + +```text +2fa50a88c feat: customize branding for DataWorks DataAgent +27a44cbd9 feat: add DataWorks DataAgent branding in header +b8585686a fix: remove trailing space in header +62a2cbaf7 build: update package names and versions for publishing +129f91840 feat: update ASCII logo for DataWorks branding +ae0c47f06 build: bump cli package version to 0.0.3 +800506010 docs: add verbose/compact mode implementation plan +64862a9f5 feat: add VerboseModeContext for compact/verbose toggle +1577d4491 feat: add VerboseModeContext for compact/verbose toggle +1d9fc0ec4 feat: add TOGGLE_VERBOSE_MODE command and Ctrl+O key binding +90fe1fbce feat: add ui.verboseMode setting to schema +124d8065b feat: hide tool result display in compact mode +ce4d70be9 feat: hide thinking chain in compact mode +7eddfa4ed feat: wire VerboseModeContext into AppContainer with Ctrl+O toggle and settings persistence +836f8e174 feat: add verbose mode indicator to Footer +c1d22b6dc feat: add i18n keys for verbose/compact mode messages +6d4a8eaeb docs: update Ctrl+O keyboard shortcut description for verbose mode +b7e208d71 docs: add verbose design doc +593abf62e refactor: update verbose mode and docks +b1ed0ab71 build: sync package versions to 0.13.2 and fix repository URLs +791a6b28b refactor: update intl messages +879c6e896 build: bump versions to 0.13.2-dataworks.1 +f1ead3634 chore: remove package-lock.json from git tracking (already in .gitignore) +8c566af14 refactor: fix cr comments +f1b34290f build: bump versions to 0.13.2-dataworks.2 +0ecd5bd6c docs: capitalize Ctrl+O in settings schema description +b621fe82d feat: dataworks tips +f0a84ee0e feat: powered by +e7f251d3e build: bump versions to 0.13.2-dataworks.3 +aa99b2a40 refactor: fix build error +0b25af94e build: bump versions to 0.13.2-dataworks.4 +bdbe67c9f refactor: compact tool group display +a159cbc63 ci: puhlish v0.13.2-dataworks.5 +f316099e2 refactor: update tips message Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26675235 * refactor: update tips message +aa41e6000 update message folding style +2a8f3fa75 refactor: update tool call label +8f92fa373 ci: publish v0.13.2-dataworks.6 +a72d6fe66 ci: publish v0.13.2-dataworks.6 +ca924796d feat: squash merge QwenLM/qwen-code#2525 +017b556fc build: bump cli package version to 0.13.2-dataworks.7 +9dcb52787 chore(channels): update package names and publish config for dataworks +a54bec143 chore(core,cli): bump version to 0.13.2-dataworks.7/8 +9266b3633 feat(vscode-ide-companion): add fastModel config and core dist alias +3e2ab7e92 fix(webui): update types path and add css module declaration +a35c97106 fix(core): filter thinking/reasoning parts from followup suggestion text +8d65b63ac chore(core,cli): bump versions and publish +ef4328ec2 build: bump cli package version to 0.13.2-dataworks.10 +1e554bf28 feat(cli): keep user shell commands expanded in compact mode +0d11c073b build: fix build error +286e862b9 fix(core): support dedicated fast model generator and streaming for followup suggestions +eaea0b98f bump cli package version to 0.14.0-dataworks.2 +074af7baf feat(core): implement mid-turn queue drain for agent execution +e34d3f270 feat(cli): add mid-turn queue drain to main session +cd212672b fix: address Copilot review feedback on mid-turn drain +2f7792289 refactor: scope mid-turn drain to main session only +92a14fa8f fix: address Copilot review on main session mid-turn drain +bcf44c821 fix: guard mid-turn drain against cancelled turns +725ca9918 fix(permissions): match env-prefixed shell commands against saved permission rules Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26706519 * fix(permissions): match env-prefixed shell commands +1d7b246b9 build: bump cli package version to 0.14.0-dataworks.4 +845f6ef09 feat: delele model show +0fc44d466 refactor: remove unused imports and props in Header component +47a8ec3d2 [to #80958901] fix(cli): get all packages/cli unit tests passing +a5440168a refactor: update copy script path structure and timestamps Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26744727 * chore(packages): mark docs site and vscode companion private +1d687e21b [to #80958901] fix(core): convert brittle vi.mock factories to importOriginal mode +ff83fd967 fix(cli): cherry-pick verbose/compact mode improvements from QwenLM/qwen-code#2770 +1098cec72 【Github DDAR】 feat(cli): add queue input editing via Up arrow key +9e5a2ee57 【Github DDAR】 feat(core): intelligent tool parallelism with Kind-based batching and shell read-only detection +e0841ec0b fix(core): accept partial stream content when finish reason is missing +53e839b1e 【Github DDAR】 feat(core): implement mid-turn queue drain for agent execution +950589ca9 【Github DDAR】 fix(followup): prevent tool call UI leak and Enter accept buffer race +d3873ae00 【Github DDAR】 feat(prompt): add dangerous actions behavior guidance in system prompt +be73fba0e docs(core): add root cause analysis comments to stream validation logic +3afd17382 fix: guard mid-turn drain against cancelled turns Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26710182 * feat(core): implement mid-turn queue drain for agent execution +8e22c2371 fix: restore verbose/compact mode i18n keys for future use +cfbf53852 feat(core): add retry logic for subagent transient stream errors +f735292f8 qwen-code支持双输出模式 Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26741792 * chore(packages): mark docs site and vscode companion private +c9216139c fix: remove duplicate imports and exports causing build failure +2b162d5e2 fix(core): increase retry backoff to survive DashScope degradation storms +209a380cf fix(core): increase GeminiChat internal stream retries from 2 to 3 +fdfbb51e7 fix(build): fix webui types resolution and eslint no-internal-modules errors +81267f196 fix(build): fix webui types resolution and eslint no-internal-modules errors +73042e3e6 fix(build): fix webui dist/index.d.ts empty export for NodeNext consumers +9ce415280 [to #80958901] fix(core): import ApprovalMode directly to bypass barrel load-order race +b77d4e811 [to #80958901] fix(cli): regenerate Footer/HistoryItemDisplay snapshots after verboseMode default flip +2dc0799ec [to #80958901] fix(vscode-ide-companion): unblock test suite (postcss ESM + Storage deep-path mock) +625c0b376 refactor: enhance release mode logic in copy script +95b00eb56 修复 compact mode 下选择 "Allow always" 后权限不持久化的 bug +7cee461e3 build: switch package dependencies from npm to local file references +e3edf05ef chore: bump package versions to 0.14.2 across multiple packages +9d105025a chore: bump package versions to 0.14.2 +4d8a56153 build: update package dependencies to use npm registry instead of file references +accea6aee fix(webui): restore rolled-up type declarations +3ab9f838f fix(core): fallback after empty stream retries +dee9dcecb test: add verification for telemetry events in geminiChat test case +d2c25bb44 fix(build): restore workspace file deps and add lockfile +4a52f3266 refactor: remove unused fs import and cleanup vite config comments +a4c6fb376 refactor(vite.config): remove unused imports and simplify path handling +1f149c764 Revert "refactor(vite.config): remove unused imports and simplify path handling" +a15d8ca88 build: add npm publish workflow for CI/CD pipeline +fde65e74e fix: sync package-lock.json with @alife workspace package names +5540ac3b3 fix: resolve 3 CI test failures and add skip_tests option to publish workflow +e982bd532 ci: rename parameters to +ee428592d ci: update npm publish config parameters format +af7459161 fix: correct skip_tests condition syntax for AoneCI +33ce472f2 fix(sdk): clean up process exit listeners in ProcessTransport tests +a627407ee fix: use shell-level check for skip_tests instead of step if +d1bac6589 ci: comment out test step in npm publish workflow +f632f8f74 build: update package.json workspaces list +a7cde18cc test: remove unnecessary mocks and skips in tests +8351df2f3 build: add publishConfig.registry to all @alife packages +45967da30 build: pass npm token from secrets to npm-publisher +e2fa1f8a6 build: update package names and ci config for publishing +3ac560deb build: update package names and ci config for publishing +a7f362e82 ci: update npm publish config with new token format +68408a72f build: write .npmrc with auth token before npm publish +0729cd353 fix: simplify .npmrc to single line with registry + authToken +77c8ceaf3 ci: update npm publish config with new token format +0ba10b8ec fix: write .npmrc to project directory instead of home +353353a55 ci: enable npm publish token in ci config +87605a317 chore: bump version to 0.14.4 across multiple package.json files +488ce5444 恢复部分合并丢失的 qwen-code支持双输出模式 代码 +af8684ce0 ci: update npm publish config for internal registry +5d0b4feaa ci: comment out npm publish trigger branches +fc1e209a2 恢复部分合并丢失的 qwen code cli 代码 +0ce14f9de 构建 qwen code 打包的二进制脚本 Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26881335 * wip: 构建二进制压缩包 +b1f64553e fix(dingtalk): prioritize senderStaffId over senderId and add debug log +ca172b61e fix(i18n): restore DataWorks input placeholder and usage example tips Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26907000 * fix(i18n): restore DataWorks input placeholder and usage example tips +be2e07469 refactor: clean up bundle-publish branch Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26904912 * chore: bump version to 0.14.6 across multiple package.json files +7f7648125 fix(core): allow thought-only responses in GeminiChat stream validation Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26921615 * fix(core): auto-continue on mid-stream cut-off; classify empty streams as EMPTY_STREAM +a8f9a4f3e feat(subagents): propagate approval mode to sub-agents (#3066) +f33e231c0 feat(core): implement fork subagent for context sharing (#2936) +5274e8e07 fix(core): add retry mechanism for subagent stream errors + retryNote +683d7d7bf refactor(mcp-oauth): move copy hint directly under the auth URL Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26940966 * feat(mcp): rewrite OAuth redirect URI for DSW proxy environment +98695c409 feat(cli): Add OAuth flags to mcp add command Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26959860 * feat(mcp): rewrite OAuth redirect URI for DSW proxy environment +124cd12bb style: quote workflow job names and actions for consistency Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26819585 * feat: add feature flags for DataWorks branding and upstream sync automation +4fc49738d fix: align StreamJsonOutputAdapter, DingtalkAdapter, WebViewProvider.test with upstream +e3566cb10 fix: align DualOutputBridge and RemoteInputWatcher with upstream (PR #3352) +99d0ba4cb fix: align cli/config.ts and PanelManager.ts with upstream +aeb95e37d fix: align gemini.tsx and DualOutputBridge.test.ts with upstream +a7ddd8500 fix: align mcp/add.test.ts type cast and core/config.ts JSDoc with upstream +a49ee09fa feat(ui): 在 Header 信息面板中展示当前 model 名称 +6fccf403b fix(build): bundle i18n locales and extension examples into dist/ +54d3a11d3 fix(mcp): make the OAuth authorization URL clickable when wrapped Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/26996251 * fix(mcp): make the OAuth authorization URL clickable when wrapped +8ed429500 chore(release): bump version to 0.14.7 across all packages Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27016717 * chore(release): bump version to 0.14.7 across all packages +d939701be refactor: add BFF endpoint logic for OAuth redirect URI generation Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27071384 * refactor: add BFF endpoint logic for OAuth redirect URI generation +91125d478 fix(cli): stabilize startup tip across Static remounts +340070331 chore(release): bump version to 0.14.8 +3ce1b1b8e test(cli): 精简 CLI 定制测试修复 +7073c3460 test(cli): pre-resolve AppContainer sync conflict +b71663197 fix(cli): validate model slash command arguments Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27270879 * fix(cli): validate model slash command arguments +ffebdd3a8 fix(cli): unfreeze Ctrl+O compact-mode toggle on long conversations +369af25d3 Merge branch 'dataworks-20260508' of gitlab.alibaba-inc.com:alishu/qwen-code into feat/test-release Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27272361 * fix(ci): 修复上游同步合并中的历史追溯和定制校验问题 +40f349f8b fix(core): restore internal web search tool glue +be160f699 fix(cli): stabilize remote input bridge tests +cecd29374 test(cli): stabilize auth and theme CI tests +7f474d83d test(core): avoid cold import timeout in skill activation +279f76bf7 fix(build): restore dataworks npm publish metadata +6db7947a4 ci: update OSS endpoint and bucket config +ffa135639 ci: update oss secrets in ci workflows +74325e428 ci: remove oss upload smoke workflow +fc3bb356b fix(ci): publish qwen oss channel metadata +4a2524062 fix:put ding talk card +ab0edd331 fix(cli): restore DataWorks DataAgent branding lost during upstream sync +07246a1bd fix(cli): restore DataWorks tips and i18n translations lost during upstream sync +1b9d1b288 fix(cli): prioritize DataWorks tips over qwen-code native tips +f4b624da1 fix(cli): fix failing test assertions for DataWorks branding +4ca5a58aa feat(cli): wrap markdown links in OSC 8 so wrapped URLs stay clickable (#4037) Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27384835 * feat(cli): wrap markdown links in OSC 8 so wrapped URLs stay clickable (#4037) +bc3152001 优化发布脚本 Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27399741 * chore(ci): remove deprecated Aone CI pipelines and optimize remaining ones +aa94e5194 fix:update card bug and add stop btn with new module Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27385182 * fix:update card bug and add stop btn with new module +c2bf54e8e feat: add default OAuth redirect URI builder Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27413710 * feat: add default OAuth redirect URI builder +6b37a472c fix(cli): restore alishu / internal-deployment OSC 8 signals Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27416044 * fix(cli): restore alishu / internal-deployment OSC 8 signals +498267a86 fix(ci): add always:true to schedule trigger for upstream sync pipeline +f607af2d9 fix: remove built-in web_search tool, align with upstream MCP-based approach +9e4b33fe7 feishu channel Link: https://code.alibaba-inc.com/alishu/qwen-code/codereview/27532149 * fix(ci): add package scope to standalone artifact +c6b168ec0 fix(core): extend DashScope provider detection & remove broken remoteInput test +``` + + diff --git a/.fork/patches/0001-branding-header.patch b/.fork/patches/0001-branding-header.patch new file mode 100644 index 00000000000..12f5796d646 --- /dev/null +++ b/.fork/patches/0001-branding-header.patch @@ -0,0 +1,263 @@ +Subject: DataWorks branding header +Reason: Keep DataWorks DataAgent branding in the CLI header and startup display. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/cli/src/ui/components/AppHeader.test.tsx + - packages/cli/src/ui/components/AsciiArt.ts + - packages/cli/src/ui/components/ConfigInitDisplay.tsx + - packages/cli/src/ui/components/Header.test.tsx + - packages/cli/src/ui/components/Header.tsx +Tests: + - cd packages/cli && npx vitest run src/ui/components/Header.test.tsx src/ui/components/AppHeader.test.tsx + +diff --git a/packages/cli/src/ui/components/AppHeader.test.tsx b/packages/cli/src/ui/components/AppHeader.test.tsx +index 392d9f74b..e50998099 100644 +--- a/packages/cli/src/ui/components/AppHeader.test.tsx ++++ b/packages/cli/src/ui/components/AppHeader.test.tsx +@@ -105,7 +105,7 @@ describe('', () => { + + it('shows the header with all info when banner is visible', () => { + const { lastFrame } = renderWithProviders(createMockUIState()); +- expect(lastFrame()).toContain('>_ Qwen Code'); ++ expect(lastFrame()).toContain('>_ DataWorks DataAgent'); + expect(lastFrame()).toContain('Gemini Pro'); + expect(lastFrame()).toContain('/projects/qwen-code'); + }); +@@ -115,8 +115,8 @@ describe('', () => { + createMockUIState(), + createSettings({ hideTips: false, hideBanner: true }), + ); +- expect(lastFrame()).not.toContain('>_ Qwen Code'); +- expect(lastFrame()).not.toContain('██╔═══██╗'); ++ expect(lastFrame()).not.toContain('>_ DataWorks DataAgent'); ++ expect(lastFrame()).not.toContain('██████╗'); + }); + + it('renders the custom subtitle end-to-end through resolveCustomBanner (replaces the blank spacer between title and auth line)', () => { +diff --git a/packages/cli/src/ui/components/AsciiArt.ts b/packages/cli/src/ui/components/AsciiArt.ts +index c70a38f4c..e78127c3d 100644 +--- a/packages/cli/src/ui/components/AsciiArt.ts ++++ b/packages/cli/src/ui/components/AsciiArt.ts +@@ -5,10 +5,12 @@ + */ + + export const shortAsciiLogo = ` +- ▄▄▄▄▄▄ ▄▄ ▄▄ ▄▄▄▄▄▄▄ ▄▄▄ ▄▄ +-██╔═══██╗██║ ██║██╔════╝████╗ ██║ +-██║ ██║██║ █╗ ██║█████╗ ██╔██╗ ██║ +-██║▄▄ ██║██║███╗██║██╔══╝ ██║╚██╗██║ +-╚██████╔╝╚███╔███╔╝███████╗██║ ╚████║ +- ╚══▀▀═╝ ╚══╝╚══╝ ╚══════╝╚═╝ ╚═══╝ +-`; ++██████╗ █████╗ ████████╗ █████╗ █████╗ ██████╗ ███████╗███╗ ██╗████████╗ ++██╔══██╗██╔══██╗╚══██╔══╝██╔══██╗ ██╔══██╗██╔════╝ ██╔════╝████╗ ██║╚══██╔══╝ ++██║ ██║███████║ ██║ ███████║ ███████║██║ ███╗█████╗ ██╔██╗ ██║ ██║ ++██║ ██║██╔══██║ ██║ ██╔══██║ ██╔══██║██║ ██║██╔══╝ ██║╚██╗██║ ██║ ++██████╔╝██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝███████╗██║ ╚████║ ██║ ++╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═╝ ++ ++ ++ `; +diff --git a/packages/cli/src/ui/components/ConfigInitDisplay.tsx b/packages/cli/src/ui/components/ConfigInitDisplay.tsx +new file mode 100644 +index 000000000..264eeeafa +--- /dev/null ++++ b/packages/cli/src/ui/components/ConfigInitDisplay.tsx +@@ -0,0 +1,53 @@ ++/** ++ * @license ++ * Copyright 2025 Google LLC ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++import { useEffect, useState } from 'react'; ++import { appEvents } from './../../utils/events.js'; ++import { Box, Text } from 'ink'; ++import { useConfig } from '../contexts/ConfigContext.js'; ++import { type McpClient, MCPServerStatus } from '@qwen-code/qwen-code-core'; ++import { GeminiSpinner } from './GeminiRespondingSpinner.js'; ++import { theme } from '../semantic-colors.js'; ++import { t } from '../../i18n/index.js'; ++ ++export const ConfigInitDisplay = () => { ++ const config = useConfig(); ++ const [message, setMessage] = useState(t('Initializing...')); ++ ++ useEffect(() => { ++ const onChange = (clients?: Map) => { ++ if (!clients || clients.size === 0) { ++ setMessage(t('Initializing...')); ++ return; ++ } ++ let connected = 0; ++ for (const client of clients.values()) { ++ if (client.getStatus() === MCPServerStatus.CONNECTED) { ++ connected++; ++ } ++ } ++ setMessage( ++ t('Connecting to MCP servers... ({{connected}}/{{total}})', { ++ connected: String(connected), ++ total: String(clients.size), ++ }), ++ ); ++ }; ++ ++ appEvents.on('mcp-client-update', onChange); ++ return () => { ++ appEvents.off('mcp-client-update', onChange); ++ }; ++ }, [config]); ++ ++ return ( ++ ++ ++ {message} ++ ++ ++ ); ++}; +diff --git a/packages/cli/src/ui/components/Header.test.tsx b/packages/cli/src/ui/components/Header.test.tsx +index 833774d1f..11962e1cb 100644 +--- a/packages/cli/src/ui/components/Header.test.tsx ++++ b/packages/cli/src/ui/components/Header.test.tsx +@@ -36,15 +36,16 @@ describe('
', () => { + }); + + it('renders the ASCII logo on wide terminal', () => { ++ useTerminalSizeMock.mockReturnValue({ columns: 150, rows: 24 }); + const { lastFrame } = render(
); +- expect(lastFrame()).toContain('██╔═══██╗'); ++ expect(lastFrame()).toContain('██████╗'); + }); + + it('hides the ASCII logo on narrow terminal', () => { + useTerminalSizeMock.mockReturnValue({ columns: 60, rows: 24 }); + const { lastFrame } = render(
); +- expect(lastFrame()).not.toContain('██╔═══██╗'); +- expect(lastFrame()).toContain('>_ Qwen Code'); ++ expect(lastFrame()).not.toContain('██████╗'); ++ expect(lastFrame()).toContain('>_ DataWorks DataAgent'); + }); + + it('displays the version number', () => { +@@ -104,9 +105,10 @@ describe('
', () => { + it('renders plain text when NO_COLOR disables gradient colors', () => { + process.env['NO_COLOR'] = '1'; + ++ useTerminalSizeMock.mockReturnValue({ columns: 150, rows: 24 }); + const { lastFrame } = render(
); + +- expect(lastFrame()).toContain('██╔═══██╗'); ++ expect(lastFrame()).toContain('██████╗'); + }); + + it('renders the custom subtitle in place of the blank spacer row', () => { +@@ -119,7 +121,7 @@ describe('
', () => { + const frame = lastFrame() ?? ''; + expect(frame).toContain('Built-in DataWorks Official Skills'); + // Subtitle sits between the title and the auth line. +- const titleIdx = frame.indexOf('>_ Qwen Code'); ++ const titleIdx = frame.indexOf('>_ DataWorks DataAgent'); + const subtitleIdx = frame.indexOf('Built-in DataWorks Official Skills'); + const authIdx = frame.indexOf('Qwen OAuth'); + expect(titleIdx).toBeLessThan(subtitleIdx); +@@ -132,7 +134,7 @@ describe('
', () => { + // Title and auth still both render at their usual positions; the + // spacer between them is just whitespace-padding, so we assert the + // visible chrome the user sees. +- expect(frame).toContain('>_ Qwen Code'); ++ expect(frame).toContain('>_ DataWorks DataAgent'); + expect(frame).toContain('Qwen OAuth'); + }); + +@@ -141,7 +143,7 @@ describe('
', () => { +
, + ); + expect(lastFrame()).toContain('Acme CLI'); +- expect(lastFrame()).not.toContain('>_ Qwen Code'); ++ expect(lastFrame()).not.toContain('>_ DataWorks DataAgent'); + // version suffix is still appended + expect(lastFrame()).toContain('v1.0.0'); + }); +@@ -154,7 +156,7 @@ describe('
', () => { + />, + ); + expect(lastFrame()).toContain('LARGE_LOGO'); +- expect(lastFrame()).not.toContain('██╔═══██╗'); ++ expect(lastFrame()).not.toContain('██████╗'); + }); + + it('falls back to the small tier when the large one does not fit', () => { +@@ -179,20 +181,20 @@ describe('
', () => { + customAsciiArt={{ small: 'X'.repeat(150), large: 'Y'.repeat(150) }} + />, + ); +- expect(lastFrame()).not.toContain('██╔═══██╗'); ++ expect(lastFrame()).not.toContain('██████╗'); + expect(lastFrame()).not.toContain('X'.repeat(150)); + expect(lastFrame()).not.toContain('Y'.repeat(150)); + // Info panel still renders. + expect(lastFrame()).toContain('Qwen OAuth'); + }); + +- it('falls back to the default Qwen logo when no custom art was provided at all', () => { ++ it('falls back to the default logo when no custom art was provided at all', () => { + useTerminalSizeMock.mockReturnValue({ columns: 60, rows: 24 }); + const { lastFrame } = render(
); +- // With no customAsciiArt, narrow widths still hide the QWEN logo, but a ++ // With no customAsciiArt, narrow widths still hide the default logo, but a + // wide enough terminal would show it — the previous test already covers + // the wide case. This one just confirms the no-custom-art path doesn't + // incidentally hide the logo. +- expect(lastFrame()).toContain('>_ Qwen Code'); ++ expect(lastFrame()).toContain('>_ DataWorks DataAgent'); + }); + }); +diff --git a/packages/cli/src/ui/components/Header.tsx b/packages/cli/src/ui/components/Header.tsx +index 94ec21937..243c71f75 100644 +--- a/packages/cli/src/ui/components/Header.tsx ++++ b/packages/cli/src/ui/components/Header.tsx +@@ -58,10 +58,9 @@ interface HeaderProps { + */ + customAsciiArt?: { small?: string; large?: string }; + /** +- * Sanitized replacement for the bold ">_ Qwen Code" title in the info +- * panel. The version suffix is always appended. When undefined or empty +- * the default title is used; the leading `>_` glyph is part of the +- * default brand and is dropped when a custom title is set. ++ * Sanitized replacement for the bold ">_ DataWorks DataAgent (Powered by ++ * Qwen Code)" title in the info panel. The version suffix is always ++ * appended. When undefined or empty the default title is used. + */ + customBannerTitle?: string; + /** +@@ -204,10 +203,12 @@ export const Header: React.FC = ({ + width={showLogo ? availableInfoPanelWidth : undefined} + > + {/* Title line: customBannerTitle (already sanitized) or the default +- ">_ Qwen Code" brand. Version suffix is always appended. */} ++ DataWorks DataAgent brand. Version suffix is always appended. */} + + +- {customBannerTitle ? customBannerTitle : '>_ Qwen Code'} ++ {customBannerTitle ++ ? customBannerTitle ++ : '>_ DataWorks DataAgent (Powered by Qwen Code)'} + + (v{version}) + +@@ -217,7 +218,9 @@ export const Header: React.FC = ({ + {customBannerSubtitle ? ( + {customBannerSubtitle} + ) : ( +- ++ ++ Built-in DataWorks Official Skills ++ + )} + {/* Auth and Model line */} + diff --git a/.fork/patches/0002-branding-tips.patch b/.fork/patches/0002-branding-tips.patch new file mode 100644 index 00000000000..901d35bfcf0 --- /dev/null +++ b/.fork/patches/0002-branding-tips.patch @@ -0,0 +1,124 @@ +Subject: DataWorks startup tips +Reason: Keep startup tips and beta guidance tailored to DataWorks usage. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/cli/src/services/tips/tipRegistry.ts + - packages/cli/src/ui/components/Tips.test.ts + - packages/cli/src/ui/components/Tips.tsx +Tests: + - cd packages/cli && npx vitest run src/ui/components/Tips.test.ts + +diff --git a/packages/cli/src/services/tips/tipRegistry.ts b/packages/cli/src/services/tips/tipRegistry.ts +index cb655783b..d19007377 100644 +--- a/packages/cli/src/services/tips/tipRegistry.ts ++++ b/packages/cli/src/services/tips/tipRegistry.ts +@@ -184,4 +184,51 @@ export const tipRegistry: ContextualTip[] = [ + cooldownPrompts: 0, + priority: 50, + }, ++ ++ // DataWorks usage examples (priority 75 to show before qwen-code native tips) ++ { ++ id: 'dw-identity', ++ content: ++ '👤 Identity: "Help me verify my identity and permissions in DataWorks?"', ++ trigger: 'startup', ++ isRelevant: () => true, ++ cooldownPrompts: 0, ++ priority: 75, ++ }, ++ { ++ id: 'dw-analysis', ++ content: ++ '📊 Analysis: "Analyze the newly created nodes in the dataworks_analyze workspace in the past week and what they are doing?"', ++ trigger: 'startup', ++ isRelevant: () => true, ++ cooldownPrompts: 0, ++ priority: 75, ++ }, ++ { ++ id: 'dw-governance', ++ content: ++ '🧹 Governance: "In the dataworks_analyze workspace, help me find nodes that were created long ago but have never been published."', ++ trigger: 'startup', ++ isRelevant: () => true, ++ cooldownPrompts: 0, ++ priority: 75, ++ }, ++ { ++ id: 'dw-troubleshooting', ++ content: ++ '🔍 Troubleshooting: "The data in dwd_is_it_software_released_df and ads_is_it_sfw_moni_key_released_recycled_df are inconsistent, both have upstream ods_ism_it_software_key_released_df. Help me check what is different in their logic?"', ++ trigger: 'startup', ++ isRelevant: () => true, ++ cooldownPrompts: 0, ++ priority: 75, ++ }, ++ { ++ id: 'dw-fix', ++ content: ++ '🛠️ Fix: "In the employee table my_project.ods_emp_info_d, the department data for employee EMP001 is empty. Help me troubleshoot the cause and provide fix suggestions."', ++ trigger: 'startup', ++ isRelevant: () => true, ++ cooldownPrompts: 0, ++ priority: 75, ++ }, + ]; +diff --git a/packages/cli/src/ui/components/Tips.test.ts b/packages/cli/src/ui/components/Tips.test.ts +index 9a93d7d2f..8ce19ed0a 100644 +--- a/packages/cli/src/ui/components/Tips.test.ts ++++ b/packages/cli/src/ui/components/Tips.test.ts +@@ -129,9 +129,9 @@ describe('selectTip', () => { + const ctx = createContext({ sessionCount: 1 }); + const history = createHistory(); + const tip = selectTip('startup', ctx, tipRegistry, history); +- // New user tips have priority 70, so one of them should be selected ++ // DataWorks tips have priority 75, so one of them should be selected + expect(tip).not.toBeNull(); +- expect(tip!.priority).toBe(70); ++ expect(tip!.priority).toBe(75); + }); + + it('rotates startup tips across sessions via LRU', () => { +@@ -149,12 +149,12 @@ describe('selectTip', () => { + expect(tip2!.id).not.toBe(tip1!.id); + }); + +- it('returns a priority-70 tip for experienced users with insight available', () => { ++ it('returns a priority-75 tip for experienced users with insight available', () => { + const ctx = createContext({ sessionCount: 25 }); + const history = createHistory(); + const tip = selectTip('startup', ctx, tipRegistry, history); +- // insight-command has priority 70, same as other new-user tips ++ // DataWorks tips have priority 75, so one of them should be selected + expect(tip).not.toBeNull(); +- expect(tip!.priority).toBe(70); ++ expect(tip!.priority).toBe(75); + }); + }); +diff --git a/packages/cli/src/ui/components/Tips.tsx b/packages/cli/src/ui/components/Tips.tsx +index 0c259b623..21b70f543 100644 +--- a/packages/cli/src/ui/components/Tips.tsx ++++ b/packages/cli/src/ui/components/Tips.tsx +@@ -43,9 +43,16 @@ export const Tips: React.FC = () => { + const selectedTip = useMemo(() => pickStartupTip(), []); + + return ( +- ++ + +- {t('Tips:')} {t(selectedTip)} ++ {t('Example: ')} ++ {t(selectedTip)} ++ ++ ++ ++ {t( ++ 'This is a Beta version. Chat history will be lost after the personal development environment instance is deleted.', ++ )} + + + ); diff --git a/.fork/patches/0003-i18n-dataworks.patch b/.fork/patches/0003-i18n-dataworks.patch new file mode 100644 index 00000000000..7a415be2277 --- /dev/null +++ b/.fork/patches/0003-i18n-dataworks.patch @@ -0,0 +1,78 @@ +Subject: DataWorks i18n strings +Reason: Keep DataWorks-specific placeholders and usage examples. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/cli/src/i18n/locales/en.js + - packages/cli/src/i18n/locales/zh.js + +diff --git a/packages/cli/src/i18n/locales/en.js b/packages/cli/src/i18n/locales/en.js +index ecf686549..ae06c67ec 100644 +--- a/packages/cli/src/i18n/locales/en.js ++++ b/packages/cli/src/i18n/locales/en.js +@@ -1343,6 +1343,10 @@ export default { + // Startup Tips + // ============================================================================ + 'Tips:': 'Tips:', ++ 'Example: ': 'Example: ', ++ 'Example: /language output Português': 'Example: /language output Português', ++ 'This is a Beta version. Chat history will be lost after the personal development environment instance is deleted.': ++ 'This is a Beta version. Chat history will be lost after the personal development environment instance is deleted.', + 'Use /compress when the conversation gets long to summarize history and free up context.': + 'Use /compress when the conversation gets long to summarize history and free up context.', + 'Start a fresh idea with /clear or /new; the previous session stays available in history.': +@@ -1376,6 +1380,18 @@ export default { + 'Long conversation? /compress summarizes history to free context.': + 'Long conversation? /compress summarizes history to free context.', + ++ // DataWorks usage examples ++ '👤 Identity: "Help me verify my identity and permissions in DataWorks?"': ++ '👤 Identity: "Help me verify my identity and permissions in DataWorks?"', ++ '📊 Analysis: "Analyze the newly created nodes in the dataworks_analyze workspace in the past week and what they are doing?"': ++ '📊 Analysis: "Analyze the newly created nodes in the dataworks_analyze workspace in the past week and what they are doing?"', ++ '🧹 Governance: "In the dataworks_analyze workspace, help me find nodes that were created long ago but have never been published."': ++ '🧹 Governance: "In the dataworks_analyze workspace, help me find nodes that were created long ago but have never been published."', ++ '🔍 Troubleshooting: "The data in dwd_is_it_software_released_df and ads_is_it_sfw_moni_key_released_recycled_df are inconsistent, both have upstream ods_ism_it_software_key_released_df. Help me check what is different in their logic?"': ++ '🔍 Troubleshooting: "The data in dwd_is_it_software_released_df and ads_is_it_sfw_moni_key_released_recycled_df are inconsistent, both have upstream ods_ism_it_software_key_released_df. Help me check what is different in their logic?"', ++ '🛠️ Fix: "In the employee table my_project.ods_emp_info_d, the department data for employee EMP001 is empty. Help me troubleshoot the cause and provide fix suggestions."': ++ '🛠️ Fix: "In the employee table my_project.ods_emp_info_d, the department data for employee EMP001 is empty. Help me troubleshoot the cause and provide fix suggestions."', ++ + // ============================================================================ + // Exit Screen / Stats + // ============================================================================ +diff --git a/packages/cli/src/i18n/locales/zh.js b/packages/cli/src/i18n/locales/zh.js +index fd430b8c5..8f1a4c784 100644 +--- a/packages/cli/src/i18n/locales/zh.js ++++ b/packages/cli/src/i18n/locales/zh.js +@@ -1259,6 +1259,10 @@ export default { + // Startup Tips + // ============================================================================ + 'Tips:': '提示:', ++ 'Example: ': '示例:', ++ 'Example: /language output Português': '示例:/language output Português', ++ 'This is a Beta version. Chat history will be lost after the personal development environment instance is deleted.': ++ '这是 Beta 版本,个人开发环境实例删除后聊天记录会丢失。', + 'Use /compress when the conversation gets long to summarize history and free up context.': + '对话变长时用 /compress,总结历史并释放上下文。', + 'Start a fresh idea with /clear or /new; the previous session stays available in history.': +@@ -1345,6 +1349,18 @@ export default { + 'Show context window usage breakdown. Use "/context detail" for per-item breakdown.': + '显示上下文窗口使用情况明细。使用 "/context detail" 查看逐项明细。', + ++ // DataWorks usage examples ++ '👤 Identity: "Help me verify my identity and permissions in DataWorks?"': ++ '👤 身份确认:"帮我确认下在 DataWorks 的身份和权限?"', ++ '📊 Analysis: "Analyze the newly created nodes in the dataworks_analyze workspace in the past week and what they are doing?"': ++ '📊 任务分析:"帮我分析下 dataworks_analyze 这个工作空间最近一周新建的节点有哪些,并分析下具体在做什么?"', ++ '🧹 Governance: "In the dataworks_analyze workspace, help me find nodes that were created long ago but have never been published."': ++ '🧹 任务治理:"在 dataworks_analyze 工作空间中,帮我找出创建时间长但一直没有发布的节点。"', ++ '🔍 Troubleshooting: "The data in dwd_is_it_software_released_df and ads_is_it_sfw_moni_key_released_recycled_df are inconsistent, both have upstream ods_ism_it_software_key_released_df. Help me check what is different in their logic?"': ++ '🔍 问题定位:"现在发现 dwd_is_it_software_released_df 和 ads_is_it_sfw_moni_key_released_recycled_df 的数据不一致,他们上游都是 ods_ism_it_software_key_released_df。帮我看一下他们的逻辑有什么不一样?"', ++ '🛠️ Fix: "In the employee table my_project.ods_emp_info_d, the department data for employee EMP001 is empty. Help me troubleshoot the cause and provide fix suggestions."': ++ '🛠️ 问题修复:"员工信息表 my_project.ods_emp_info_d 中,工号 EMP001 的部门数据为空。请帮我排查原因并提供修复建议。"', ++ + // ============================================================================ + // Exit Screen / Stats + // ============================================================================ diff --git a/.fork/patches/0004-dsw-oauth-redirect.patch b/.fork/patches/0004-dsw-oauth-redirect.patch new file mode 100644 index 00000000000..b6844f477e9 --- /dev/null +++ b/.fork/patches/0004-dsw-oauth-redirect.patch @@ -0,0 +1,385 @@ +Subject: DSW OAuth redirect rewrite +Reason: Rewrite MCP OAuth redirect URLs for the DSW proxy environment. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/core/src/mcp/constants.test.ts + - packages/core/src/mcp/constants.ts + - packages/core/src/mcp/oauth-provider.test.ts + - packages/core/src/mcp/oauth-provider.ts +Tests: + - cd packages/core && npx vitest run src/mcp/constants.test.ts src/mcp/oauth-provider.test.ts + +diff --git a/packages/core/src/mcp/constants.test.ts b/packages/core/src/mcp/constants.test.ts +new file mode 100644 +index 000000000..7bd0fbcf6 +--- /dev/null ++++ b/packages/core/src/mcp/constants.test.ts +@@ -0,0 +1,130 @@ ++/** ++ * @license ++ * Copyright 2025 Qwen ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++import { describe, it, expect, beforeEach, afterEach } from 'vitest'; ++import { ++ getOAuthRedirectUri, ++ OAUTH_REDIRECT_PORT, ++ OAUTH_REDIRECT_PATH, ++} from './constants.js'; ++ ++describe('getOAuthRedirectUri', () => { ++ const ENV_KEYS = [ ++ 'BFF_ENDPOINT', ++ 'DATA_AGENT_INSTANCE_ID', ++ 'DA_RUNTIME_TYPE', ++ 'dsw_baseUrl', ++ ] as const; ++ ++ let savedEnv: Record; ++ ++ beforeEach(() => { ++ // Snapshot then clear all env vars that influence the result so each ++ // test starts from a known-empty state. ++ savedEnv = {}; ++ for (const key of ENV_KEYS) { ++ savedEnv[key] = process.env[key]; ++ delete process.env[key]; ++ } ++ }); ++ ++ afterEach(() => { ++ // Restore the original environment. ++ for (const key of ENV_KEYS) { ++ const value = savedEnv[key]; ++ if (value === undefined) { ++ delete process.env[key]; ++ } else { ++ process.env[key] = value; ++ } ++ } ++ }); ++ ++ it('falls back to localhost when no environment hints are set', () => { ++ expect(getOAuthRedirectUri()).toBe( ++ `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ describe('BFF proxy', () => { ++ beforeEach(() => { ++ process.env['BFF_ENDPOINT'] = 'https://bff.example.com'; ++ process.env['DATA_AGENT_INSTANCE_ID'] = 'inst-123'; ++ }); ++ ++ it('uses the kxuth segment for legacy instances', () => { ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/kxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('uses the bxkxuth segment when DA_RUNTIME_TYPE is ACS_SANDBOX', () => { ++ process.env['DA_RUNTIME_TYPE'] = 'ACS_SANDBOX'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/bxkxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('keeps the kxuth segment for any other DA_RUNTIME_TYPE value', () => { ++ process.env['DA_RUNTIME_TYPE'] = 'DSW'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/kxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('keeps the kxuth segment for an empty DA_RUNTIME_TYPE', () => { ++ process.env['DA_RUNTIME_TYPE'] = ''; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/kxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('strips trailing slashes from BFF_ENDPOINT to avoid a double slash', () => { ++ process.env['BFF_ENDPOINT'] = 'https://bff.example.com///'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/kxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('takes priority over dsw_baseUrl when both are present', () => { ++ process.env['dsw_baseUrl'] = 'https://dw.aliyun.com/dsw-380036'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://bff.example.com/skwacb/kxuth/inst-123${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('falls through when BFF_ENDPOINT is set but the instance id is missing', () => { ++ delete process.env['DATA_AGENT_INSTANCE_ID']; ++ expect(getOAuthRedirectUri()).toBe( ++ `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('treats empty-string env values as unset and falls through to localhost', () => { ++ process.env['BFF_ENDPOINT'] = ''; ++ process.env['DATA_AGENT_INSTANCE_ID'] = ''; ++ expect(getOAuthRedirectUri()).toBe( ++ `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ }); ++ ++ describe('DSW proxy', () => { ++ it('builds a proxy path from dsw_baseUrl', () => { ++ process.env['dsw_baseUrl'] = 'https://dw.aliyun.com/dsw-380036'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://dw.aliyun.com/dsw-380036/proxy/${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ ++ it('strips trailing slashes from dsw_baseUrl', () => { ++ process.env['dsw_baseUrl'] = 'https://dw.aliyun.com/dsw-380036///'; ++ expect(getOAuthRedirectUri()).toBe( ++ `https://dw.aliyun.com/dsw-380036/proxy/${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`, ++ ); ++ }); ++ }); ++}); +diff --git a/packages/core/src/mcp/constants.ts b/packages/core/src/mcp/constants.ts +index ca9c27f3c..2c0467eed 100644 +--- a/packages/core/src/mcp/constants.ts ++++ b/packages/core/src/mcp/constants.ts +@@ -25,3 +25,45 @@ export const OAUTH_REDIRECT_PORT = 7777; + * Path for OAuth redirect callback. + */ + export const OAUTH_REDIRECT_PATH = '/oauth/callback'; ++ ++/** ++ * Build the default OAuth redirect URI. ++ * ++ * The local callback server (localhost:) is not always reachable from ++ * the user's browser, so depending on the runtime the redirect URI must point ++ * at a reverse proxy instead: ++ * ++ * 1. BFF proxy (newer Data Agent runtime) — when BFF_ENDPOINT and ++ * DATA_AGENT_INSTANCE_ID are set: ++ * /skwacb// ++ * where is `bxkxuth` for ACS_SANDBOX instances ++ * (DA_RUNTIME_TYPE=ACS_SANDBOX) and `kxuth` otherwise. ++ * ++ * 2. DSW proxy (legacy) — when dsw_baseUrl is set, e.g. ++ * https://dw.aliyun.com/dsw-380036: ++ * /proxy/ ++ * ++ * 3. Local dev — neither is set: fall back to localhost. ++ */ ++export function getOAuthRedirectUri(): string { ++ // 新版,走 bff 代理转发的逻辑 ++ const bffEndpoint = process.env['BFF_ENDPOINT']; ++ const dataAgentInstanceId = process.env['DATA_AGENT_INSTANCE_ID']; ++ if (bffEndpoint && dataAgentInstanceId) { ++ // 新版 Data Agent 实例(DA_RUNTIME_TYPE=ACS_SANDBOX)使用 bxkxuth 代理段, ++ // 旧实例仍使用 kxuth。 ++ const proxySegment = ++ process.env['DA_RUNTIME_TYPE'] === 'ACS_SANDBOX' ? 'bxkxuth' : 'kxuth'; ++ const base = bffEndpoint.replace(/\/+$/, ''); ++ const bffOAuthProxyPath = `/skwacb/${proxySegment}/`; ++ return `${base}${bffOAuthProxyPath}${dataAgentInstanceId}${OAUTH_REDIRECT_PATH}`; ++ } ++ ++ // 兼容旧版 DSW 实例 ++ const dswBaseUrl = process.env['dsw_baseUrl']; ++ if (dswBaseUrl) { ++ const base = dswBaseUrl.replace(/\/+$/, ''); ++ return `${base}/proxy/${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`; ++ } ++ return `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`; ++} +diff --git a/packages/core/src/mcp/oauth-provider.test.ts b/packages/core/src/mcp/oauth-provider.test.ts +index 2edf45860..97ac0f0ba 100644 +--- a/packages/core/src/mcp/oauth-provider.test.ts ++++ b/packages/core/src/mcp/oauth-provider.test.ts +@@ -246,6 +246,120 @@ describe('MCPOAuthProvider', () => { + ); + }); + ++ it('uses an identical redirect_uri across registration, authorization, and token exchange when none is configured', async () => { ++ // Force getOAuthRedirectUri() to return a BFF proxy URL so the assertion ++ // is meaningful: the original bug hardcoded localhost at the registration ++ // and token-exchange sites while the authorization URL used the proxy, so ++ // the three only diverged in a proxied environment. ++ const envKeys = [ ++ 'BFF_ENDPOINT', ++ 'DATA_AGENT_INSTANCE_ID', ++ 'DA_RUNTIME_TYPE', ++ ] as const; ++ const savedEnv: Record = {}; ++ for (const key of envKeys) { ++ savedEnv[key] = process.env[key]; ++ } ++ process.env['BFF_ENDPOINT'] = 'https://bff.example.com'; ++ process.env['DATA_AGENT_INSTANCE_ID'] = 'inst-xyz'; ++ process.env['DA_RUNTIME_TYPE'] = 'ACS_SANDBOX'; ++ const expectedRedirectUri = ++ 'https://bff.example.com/skwacb/bxkxuth/inst-xyz/oauth/callback'; ++ ++ try { ++ // No redirectUri (exercises the fallback) and no clientId (forces ++ // dynamic client registration, so all three sites run). ++ const config: MCPOAuthConfig = { ++ enabled: true, ++ authorizationUrl: 'https://auth.example.com/authorize', ++ tokenUrl: 'https://auth.example.com/token', ++ registrationUrl: 'https://auth.example.com/register', ++ scopes: ['read'], ++ }; ++ ++ let callbackHandler: unknown; ++ vi.mocked(http.createServer).mockImplementation((handler) => { ++ callbackHandler = handler; ++ return mockHttpServer as unknown as http.Server; ++ }); ++ mockHttpServer.listen.mockImplementation((port, callback) => { ++ callback?.(); ++ setTimeout(() => { ++ (callbackHandler as (req: unknown, res: unknown) => void)( ++ { ++ url: '/oauth/callback?code=auth_code_123&state=bW9ja19zdGF0ZV8xNl9ieXRlcw', ++ }, ++ { writeHead: vi.fn(), end: vi.fn() }, ++ ); ++ }, 10); ++ }); ++ ++ // First fetch: dynamic client registration. Second fetch: token exchange. ++ mockFetch ++ .mockResolvedValueOnce( ++ createMockResponse({ ++ ok: true, ++ contentType: 'application/json', ++ json: { ++ client_id: 'registered-client', ++ redirect_uris: [expectedRedirectUri], ++ grant_types: ['authorization_code', 'refresh_token'], ++ response_types: ['code'], ++ token_endpoint_auth_method: 'none', ++ }, ++ }), ++ ) ++ .mockResolvedValueOnce( ++ createMockResponse({ ++ ok: true, ++ contentType: 'application/json', ++ text: JSON.stringify(mockTokenResponse), ++ json: mockTokenResponse, ++ }), ++ ); ++ ++ const authProvider = new MCPOAuthProvider(); ++ await authProvider.authenticate('test-server', config); ++ ++ // 1. Dynamic client registration request body. ++ const registrationCall = mockFetch.mock.calls.find( ++ (call) => call[0] === 'https://auth.example.com/register', ++ ); ++ expect(registrationCall).toBeDefined(); ++ const registrationBody = JSON.parse( ++ (registrationCall![1] as { body: string }).body, ++ ); ++ expect(registrationBody.redirect_uris).toEqual([expectedRedirectUri]); ++ ++ // 2. Authorization URL opened in the browser. ++ const authUrl = new URL( ++ mockOpenBrowserSecurely.mock.calls[0][0] as string, ++ ); ++ expect(authUrl.searchParams.get('redirect_uri')).toBe( ++ expectedRedirectUri, ++ ); ++ ++ // 3. Token exchange request body. ++ const tokenCall = mockFetch.mock.calls.find( ++ (call) => call[0] === 'https://auth.example.com/token', ++ ); ++ expect(tokenCall).toBeDefined(); ++ const tokenBody = new URLSearchParams( ++ (tokenCall![1] as { body: string }).body, ++ ); ++ expect(tokenBody.get('redirect_uri')).toBe(expectedRedirectUri); ++ } finally { ++ for (const key of envKeys) { ++ const value = savedEnv[key]; ++ if (value === undefined) { ++ delete process.env[key]; ++ } else { ++ process.env[key] = value; ++ } ++ } ++ } ++ }); ++ + it('should handle OAuth discovery when no authorization URL provided', async () => { + // Use a mutable config object + const configWithoutAuth: MCPOAuthConfig = { +diff --git a/packages/core/src/mcp/oauth-provider.ts b/packages/core/src/mcp/oauth-provider.ts +index ce92e97da..c0f17a517 100644 +--- a/packages/core/src/mcp/oauth-provider.ts ++++ b/packages/core/src/mcp/oauth-provider.ts +@@ -18,6 +18,7 @@ import { + MCP_OAUTH_CLIENT_NAME, + OAUTH_REDIRECT_PORT, + OAUTH_REDIRECT_PATH, ++ getOAuthRedirectUri, + } from './constants.js'; + + export const OAUTH_DISPLAY_MESSAGE_EVENT = 'oauth-display-message' as const; +@@ -141,9 +142,7 @@ export class MCPOAuthProvider { + registrationUrl: string, + config: MCPOAuthConfig, + ): Promise { +- const redirectUri = +- config.redirectUri || +- `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`; ++ const redirectUri = config.redirectUri || getOAuthRedirectUri(); + + const registrationRequest: OAuthClientRegistrationRequest = { + client_name: MCP_OAUTH_CLIENT_NAME, +@@ -357,9 +356,7 @@ export class MCPOAuthProvider { + pkceParams: PKCEParams, + mcpServerUrl?: string, + ): string { +- const redirectUri = +- config.redirectUri || +- `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`; ++ const redirectUri = config.redirectUri || getOAuthRedirectUri(); + + const params = new URLSearchParams({ + client_id: config.clientId!, +@@ -415,9 +412,7 @@ export class MCPOAuthProvider { + codeVerifier: string, + mcpServerUrl?: string, + ): Promise { +- const redirectUri = +- config.redirectUri || +- `http://localhost:${OAUTH_REDIRECT_PORT}${OAUTH_REDIRECT_PATH}`; ++ const redirectUri = config.redirectUri || getOAuthRedirectUri(); + + const params = new URLSearchParams({ + grant_type: 'authorization_code', +@@ -745,6 +740,14 @@ export class MCPOAuthProvider { + } + } + ++ // Freeze the redirect URI once for the whole flow so dynamic client ++ // registration, the authorization request, and the token exchange all carry ++ // an identical value. They each fall back to getOAuthRedirectUri() ++ // independently, but recomputing per-call would risk divergence if an env ++ // var (e.g. DA_RUNTIME_TYPE) changed during the human-in-the-loop wait — ++ // which the OAuth server would reject as redirect_uri_mismatch. ++ config.redirectUri = config.redirectUri || getOAuthRedirectUri(); ++ + // If no client ID is provided, try dynamic client registration + if (!config.clientId) { + let registrationUrl = config.registrationUrl; diff --git a/.fork/patches/0005-osc8-internal.patch b/.fork/patches/0005-osc8-internal.patch new file mode 100644 index 00000000000..d2d975c17ed --- /dev/null +++ b/.fork/patches/0005-osc8-internal.patch @@ -0,0 +1,115 @@ +Subject: Internal OSC8 terminal links +Reason: Keep terminal hyperlink handling compatible with internal terminal environments. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/cli/src/ui/utils/osc8.test.ts + - packages/cli/src/ui/utils/osc8.ts +Tests: + - cd packages/cli && npx vitest run src/ui/utils/osc8.test.ts + +diff --git a/packages/cli/src/ui/utils/osc8.test.ts b/packages/cli/src/ui/utils/osc8.test.ts +index 66527c03e..43363021f 100644 +--- a/packages/cli/src/ui/utils/osc8.test.ts ++++ b/packages/cli/src/ui/utils/osc8.test.ts +@@ -490,6 +490,55 @@ describe('osc8 helpers', () => { + expect(supportsHyperlinks()).toBe(true); + }); + ++ it('alishu / OpenCode web terminal is enabled via OPENCODE_TERMINAL=1', () => { ++ setTTY(true); ++ process.env['OPENCODE_TERMINAL'] = '1'; ++ expect(supportsHyperlinks()).toBe(true); ++ }); ++ ++ it('alishu BFF-injected sessions are enabled via BFF_TOKEN', () => { ++ setTTY(true); ++ process.env['BFF_TOKEN'] = 'eyJhbGciOi...'; ++ expect(supportsHyperlinks()).toBe(true); ++ }); ++ ++ it('treats TERM=xterm-256color as OSC-8-capable (internal-deployment fallback)', () => { ++ setTTY(true); ++ process.env['TERM'] = 'xterm-256color'; ++ expect(supportsHyperlinks()).toBe(true); ++ }); ++ ++ it('treats TERM=xterm as OSC-8-capable (internal-deployment fallback)', () => { ++ setTTY(true); ++ process.env['TERM'] = 'xterm'; ++ expect(supportsHyperlinks()).toBe(true); ++ }); ++ ++ it('alishu signals still respect NO_COLOR / QWEN_DISABLE_HYPERLINKS', () => { ++ setTTY(true); ++ process.env['OPENCODE_TERMINAL'] = '1'; ++ process.env['NO_COLOR'] = '1'; ++ expect(supportsHyperlinks()).toBe(false); ++ delete process.env['NO_COLOR']; ++ process.env['QWEN_DISABLE_HYPERLINKS'] = '1'; ++ expect(supportsHyperlinks()).toBe(false); ++ }); ++ ++ it('alishu signals still respect non-TTY suppression', () => { ++ setTTY(false); ++ process.env['OPENCODE_TERMINAL'] = '1'; ++ process.env['BFF_TOKEN'] = 'x'; ++ process.env['TERM'] = 'xterm-256color'; ++ expect(supportsHyperlinks()).toBe(false); ++ }); ++ ++ it('OPENCODE_TERMINAL=0 does not enable (only "1" is the trigger)', () => { ++ setTTY(true); ++ process.env['OPENCODE_TERMINAL'] = '0'; ++ // No other positive signal — should fall through to false. ++ expect(supportsHyperlinks()).toBe(false); ++ }); ++ + it('Warp Terminal is intentionally NOT auto-detected (no OSC 8 support yet)', () => { + // Warp's current rendering engine doesn't honor OSC 8 — it prints the + // envelope as visible garbage. Falls through to the final return false +diff --git a/packages/cli/src/ui/utils/osc8.ts b/packages/cli/src/ui/utils/osc8.ts +index a3a133289..b6d2c63b9 100644 +--- a/packages/cli/src/ui/utils/osc8.ts ++++ b/packages/cli/src/ui/utils/osc8.ts +@@ -222,6 +222,29 @@ export function supportsHyperlinks( + // JediTerm backend has supported OSC 8 since 2022.3. + if (env['TERMINAL_EMULATOR'] === 'JetBrains-JediTerm') return true; + ++ // Alishu / internal-deployment positive signals: these environments embed ++ // qwen TUI in an xterm.js-based web terminal that registers an OSC 8 link ++ // handler, but expose no standard env var the upstream detector would ++ // recognize. Trust them by their internal markers. ++ // - OPENCODE_TERMINAL=1 — the alishu OpenCode web terminal sets this on ++ // every spawned shell. ++ // - BFF_TOKEN — present whenever the BFF auth layer has injected its ++ // bearer token, which only happens inside the web-terminal host. ++ // - TERM=xterm / xterm-256color — broad fallback for the same web ++ // terminal when no more specific marker is present. Generic TERM ++ // values that the upstream conservative path rejects; we accept here ++ // because internal deployments funnel through these terminfos and ++ // any terminal that doesn't honor OSC 8 will just print the visible ++ // label (the bytes are still well-formed via sanitizeForOsc). ++ if ( ++ env['OPENCODE_TERMINAL'] === '1' || ++ env['BFF_TOKEN'] !== undefined || ++ env['TERM'] === 'xterm' || ++ env['TERM'] === 'xterm-256color' ++ ) { ++ return true; ++ } ++ + if (env['TERM_PROGRAM']) { + const version = parseVersion(env['TERM_PROGRAM_VERSION']); + switch (env['TERM_PROGRAM']) { +@@ -474,6 +497,8 @@ export const HYPERLINK_ENV_KEYS = [ + 'ALACRITTY_LOG', + 'ALACRITTY_WINDOW_ID', + 'ALACRITTY_SOCKET', ++ 'OPENCODE_TERMINAL', ++ 'BFF_TOKEN', + 'TERM', + 'TEAMCITY_VERSION', + 'FORCE_HYPERLINK', diff --git a/.fork/patches/0007-feishu-channel.patch b/.fork/patches/0007-feishu-channel.patch new file mode 100644 index 00000000000..560bed30c4c --- /dev/null +++ b/.fork/patches/0007-feishu-channel.patch @@ -0,0 +1,24 @@ +From: fork-maintainer +Subject: [PATCH] feat(channels): use fork feishu package name in channel registry + +Keep the fork's @alife/dataworks-qwen-code-channel-feishu package name +in the CLI channel registry import. The feishu source code itself now +follows upstream (PR #4379); only the published npm package name differs. + +Package.json name/registry handled by .fork/rewrite-package-identity.js. + +diff --git a/packages/cli/src/commands/channel/channel-registry.ts b/packages/cli/src/commands/channel/channel-registry.ts +index 90df33214..520ed46a2 100644 +--- a/packages/cli/src/commands/channel/channel-registry.ts ++++ b/packages/cli/src/commands/channel/channel-registry.ts +@@ -9,6 +9,9 @@ function ensureBuiltins(): Promise { + { name: 'telegram', promise: import('@qwen-code/channel-telegram') }, + { name: 'weixin', promise: import('@qwen-code/channel-weixin') }, + { name: 'dingtalk', promise: import('@qwen-code/channel-dingtalk') }, +- { name: 'feishu', promise: import('@qwen-code/channel-feishu') }, ++ { ++ name: 'feishu', ++ promise: import('@alife/dataworks-qwen-code-channel-feishu'), ++ }, + { name: 'qqbot', promise: import('@qwen-code/channel-qqbot') }, + ]; diff --git a/.fork/patches/0009-claude-websearch-compat.patch b/.fork/patches/0009-claude-websearch-compat.patch new file mode 100644 index 00000000000..a662cea1dd9 --- /dev/null +++ b/.fork/patches/0009-claude-websearch-compat.patch @@ -0,0 +1,154 @@ +Subject: Claude WebSearch compatibility +Reason: Keep fork behavior around Claude WebSearch conversion and integration coverage. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (c6b168ec034253e76a07492a30cfb134eae47545) +Upstream-Ref: upstream/main +Paths: + - packages/core/src/extension/claude-converter.ts + - integration-tests/cli/web_search.test.ts + +diff --git a/integration-tests/cli/web_search.test.ts b/integration-tests/cli/web_search.test.ts +new file mode 100644 +index 000000000..5ab0b4364 +--- /dev/null ++++ b/integration-tests/cli/web_search.test.ts +@@ -0,0 +1,126 @@ ++/** ++ * @license ++ * Copyright 2025 Google LLC ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++import { describe, it, expect } from 'vitest'; ++import { ++ TestRig, ++ printDebugInfo, ++ validateModelOutput, ++} from '../test-helper.js'; ++ ++describe('web_search', () => { ++ it('should be able to search the web', async () => { ++ // Check if any web search provider is available ++ const hasTavilyKey = !!process.env['TAVILY_API_KEY']; ++ const hasGoogleKey = ++ !!process.env['GOOGLE_API_KEY'] && ++ !!process.env['GOOGLE_SEARCH_ENGINE_ID']; ++ ++ // Skip if no provider is configured ++ // Note: DashScope provider is automatically available for Qwen OAuth users, ++ // but we can't easily detect that in tests without actual OAuth credentials ++ if (!hasTavilyKey && !hasGoogleKey) { ++ console.warn( ++ 'Skipping web search test: No web search provider configured. ' + ++ 'Set TAVILY_API_KEY or GOOGLE_API_KEY+GOOGLE_SEARCH_ENGINE_ID environment variables.', ++ ); ++ return; ++ } ++ ++ const rig = new TestRig(); ++ // Configure web search in settings if provider keys are available ++ const webSearchSettings: Record = {}; ++ const providers: Array<{ ++ type: string; ++ apiKey?: string; ++ searchEngineId?: string; ++ }> = []; ++ ++ if (hasTavilyKey) { ++ providers.push({ type: 'tavily', apiKey: process.env['TAVILY_API_KEY'] }); ++ } ++ if (hasGoogleKey) { ++ providers.push({ ++ type: 'google', ++ apiKey: process.env['GOOGLE_API_KEY'], ++ searchEngineId: process.env['GOOGLE_SEARCH_ENGINE_ID'], ++ }); ++ } ++ ++ if (providers.length > 0) { ++ webSearchSettings.webSearch = { ++ provider: providers, ++ default: providers[0]?.type, ++ }; ++ } ++ ++ await rig.setup('should be able to search the web', { ++ settings: webSearchSettings, ++ }); ++ ++ let result; ++ try { ++ result = await rig.run(`what is the weather in London`); ++ } catch (error) { ++ // Network errors can occur in CI environments ++ if ( ++ error instanceof Error && ++ (error.message.includes('network') || error.message.includes('timeout')) ++ ) { ++ console.warn( ++ 'Skipping test due to network error:', ++ (error as Error).message, ++ ); ++ return; // Skip the test ++ } ++ throw error; // Re-throw if not a network error ++ } ++ ++ const foundToolCall = await rig.waitForToolCall('web_search'); ++ ++ // Add debugging information ++ if (!foundToolCall) { ++ const allTools = printDebugInfo(rig, result); ++ ++ // Check if the tool call failed due to network issues ++ const failedSearchCalls = allTools.filter( ++ (t) => t.toolRequest.name === 'web_search' && !t.toolRequest.success, ++ ); ++ if (failedSearchCalls.length > 0) { ++ console.warn( ++ 'web_search tool was called but failed, possibly due to network issues', ++ ); ++ console.warn( ++ 'Failed calls:', ++ failedSearchCalls.map((t) => t.toolRequest.args), ++ ); ++ return; // Skip the test if network issues ++ } ++ } ++ ++ expect(foundToolCall, 'Expected to find a call to web_search').toBeTruthy(); ++ ++ // Validate model output - will throw if no output, warn if missing expected content ++ const hasExpectedContent = validateModelOutput( ++ result, ++ ['weather', 'london'], ++ 'Web search test', ++ ); ++ ++ // If content was missing, log the search queries used ++ if (!hasExpectedContent) { ++ const searchCalls = rig ++ .readToolLogs() ++ .filter((t) => t.toolRequest.name === 'web_search'); ++ if (searchCalls.length > 0) { ++ console.warn( ++ 'Search queries used:', ++ searchCalls.map((t) => t.toolRequest.args), ++ ); ++ } ++ } ++ }); ++}); +diff --git a/packages/core/src/extension/claude-converter.ts b/packages/core/src/extension/claude-converter.ts +index 0f8600832..de48ed626 100644 +--- a/packages/core/src/extension/claude-converter.ts ++++ b/packages/core/src/extension/claude-converter.ts +@@ -128,7 +128,6 @@ const CLAUDE_TOOLS_MAPPING: Record = { + Task: 'Task', + TodoWrite: 'TodoList', + WebFetch: 'WebFetch', +- WebSearch: 'None', + Write: 'WriteFile', + LS: 'ListFiles', + }; diff --git a/.fork/patches/0010-build-single-bundle.patch b/.fork/patches/0010-build-single-bundle.patch new file mode 100644 index 00000000000..1c92b3b811a --- /dev/null +++ b/.fork/patches/0010-build-single-bundle.patch @@ -0,0 +1,96 @@ +From: fork-maintainer +Subject: [PATCH] build: use single-file bundle output and remove acp-bridge + +Fork divergence from upstream esbuild config: +- Remove code-splitting (no chunks/ directory) +- Single outfile: dist/cli.js instead of outdir + entryNames +- Remove __dirname/__filename → chunk shim rewrite (not needed without splitting) +- Remove acp-bridge (not in fork) + +diff --git a/esbuild.config.js b/esbuild.config.js +index bc50dc265..2b87c6ce9 100644 +--- a/esbuild.config.js ++++ b/esbuild.config.js +@@ -72,21 +72,10 @@ const external = [ + '@teddyzhu/clipboard-win32-arm64-msvc', + ]; + +-// Name of the directory under `dist/` that esbuild emits shared chunks into. +-// MUST stay in sync with `BUNDLE_CHUNK_DIR` in +-// `packages/core/src/utils/bundlePaths.ts`, whose `resolveBundleDir` helper +-// strips this exact segment when modules look up sibling assets at runtime. +-// Renaming here without renaming there silently breaks bundled-binary lookup +-// in skill-manager / ripgrepUtils / i18n / extensions/new. +-const BUNDLE_CHUNK_DIR = 'chunks'; +- + const mainBuild = esbuild.build({ +- entryPoints: { cli: 'packages/cli/index.ts' }, ++ entryPoints: ['packages/cli/index.ts'], + bundle: true, +- outdir: 'dist', +- entryNames: '[name]', +- chunkNames: `${BUNDLE_CHUNK_DIR}/[name]-[hash]`, +- splitting: true, ++ outfile: 'dist/cli.js', + platform: 'node', + format: 'esm', + target: 'node22', +@@ -115,27 +104,4 @@ const mainBuild = esbuild.build({ + 'process.env.NODE_ENV': JSON.stringify('production'), + // Make global available for compatibility + global: 'globalThis', +- // Redirect free __dirname/__filename references to the shim so that +- // vendored libraries that emit their own `var __dirname` locals don't +- // collide with our injected bindings when code-splitting is enabled. +- // +- // CONTRIBUTOR WARNING: this rewrite applies to *all* source files, so +- // any bare `__dirname` / `__filename` in our own code resolves to the +- // shim chunk's on-disk location (i.e. `dist/chunks/`), NOT the source +- // file's own directory. To get a per-file path, declare a local shadow +- // at the top of the module: +- // +- // import { fileURLToPath } from 'node:url'; +- // const __filename = fileURLToPath(import.meta.url); +- // const __dirname = path.dirname(__filename); +- // +- // esbuild leaves the local binding alone (it's a declared identifier, +- // not a free reference). For sibling-asset lookups in modules that may +- // be hoisted into a shared chunk, prefer +- // `resolveBundleDir(import.meta.url)` from +- // `packages/core/src/utils/bundlePaths.ts` — it both produces a +- // per-file path and strips the chunk segment when the module ends up +- // under `dist/chunks/`. +- __dirname: '__qwen_dirname', +- __filename: '__qwen_filename', + }, +diff --git a/scripts/build.js b/scripts/build.js +index a39534b1f..03dc856e0 100644 +--- a/scripts/build.js ++++ b/scripts/build.js +@@ -38,12 +38,11 @@ execSync('npm run generate', { stdio: 'inherit', cwd: root }); + // 2. web-templates (embeddable web templates - used by cli) + // 3. channel-base (base channel infrastructure - used by channel adapters and cli) + // 4. channel adapters (depend on channel-base) +-// 5. acp-bridge (depends on core - used by cli) +-// 6. cli (depends on core, acp-bridge, web-templates, channel packages) +-// 7. webui (shared UI components - used by vscode companion) +-// 8. sdk (build-time devDep on acp-bridge for shared constants) +-// 9. web-shell (depends on webui and sdk) +-// 10. vscode-ide-companion (depends on webui) ++// 5. cli (depends on core, web-templates, channel packages) ++// 6. webui (shared UI components - used by vscode companion) ++// 7. sdk (no internal dependencies) ++// 8. web-shell (depends on webui and sdk) ++// 9. vscode-ide-companion (depends on webui) + const buildOrder = [ + 'packages/core', + 'packages/web-templates', +@@ -57,8 +56,7 @@ const buildOrder = [ + 'packages/channels/dingtalk', + 'packages/channels/feishu', + 'packages/channels/qqbot', + 'packages/channels/plugin-example', +- 'packages/acp-bridge', + 'packages/cli', + ...(cliOnly + ? [] diff --git a/.fork/patches/0011-test-fork-adaptations.patch b/.fork/patches/0011-test-fork-adaptations.patch new file mode 100644 index 00000000000..ccf80bafd55 --- /dev/null +++ b/.fork/patches/0011-test-fork-adaptations.patch @@ -0,0 +1,91 @@ +From: fork-maintainer +Subject: [PATCH] test: adapt tests for fork UI, CI runner detection, and Node version guard + +- AuthDialog.test: adapt for fork's OAuth menu structure +- detect-terminal-theme.test: stub Aone CI runner detection +- skill-activation.test: adjust static import expectations +- doctorChecks.test: guard Node v22+ specific assertion + +diff --git a/packages/cli/src/ui/auth/AuthDialog.test.tsx b/packages/cli/src/ui/auth/AuthDialog.test.tsx +index 966fab0d2..437e4e7fd 100644 +--- a/packages/cli/src/ui/auth/AuthDialog.test.tsx ++++ b/packages/cli/src/ui/auth/AuthDialog.test.tsx +@@ -242,8 +242,18 @@ const navigateToCustomAdvancedConfig = async ( + ); + }; + ++const isTruthyEnv = (value: string | undefined) => ++ value !== undefined && ++ value !== '' && ++ value !== '0' && ++ value.toLowerCase() !== 'false'; ++ ++const isAoneRunner = process.cwd().startsWith('/aoneci/'); ++ + const isUnreliableTuiInputEnvironment = +- process.platform === 'win32' || process.env['CI'] === 'true'; ++ process.platform === 'win32' || ++ isTruthyEnv(process.env['CI']) || ++ isAoneRunner; + const itWhenTuiInputReliable = isUnreliableTuiInputEnvironment ? it.skip : it; + + describe('AuthDialog', { timeout: 15000 }, () => { +@@ -577,7 +587,7 @@ describe('AuthDialog', { timeout: 15000 }, () => { + }); + + // --------------------------------------------------------------------------- +- // TUI input simulation tests — skipped on CI (process.env.CI=true) ++ // TUI input simulation tests — skipped on CI. + // These tests use stdin.write() to simulate keyboard navigation through + // multi-step UI flows. On slower CI runners the timing between simulated + // key presses and React re-renders is unreliable, causing flaky failures. +diff --git a/packages/cli/src/utils/doctorChecks.test.ts b/packages/cli/src/utils/doctorChecks.test.ts +index a4fd7a63b..3d8d20d92 100644 +--- a/packages/cli/src/utils/doctorChecks.test.ts ++++ b/packages/cli/src/utils/doctorChecks.test.ts +@@ -91,6 +91,9 @@ describe('runDoctorChecks', () => { + }); + + it('should pass Node.js version check for v22+', async () => { ++ if (parseInt(process.versions.node.split('.')[0]!, 10) < 22) { ++ return; // Test requires Node.js v22+ ++ } + const results = await runDoctorChecks(mockContext); + const nodeCheck = results.find((r) => r.name === 'Node.js version'); + expect(nodeCheck).toBeDefined(); +diff --git a/packages/core/src/skills/skill-activation.test.ts b/packages/core/src/skills/skill-activation.test.ts +index 3f1865f88..bfe27084c 100644 +--- a/packages/core/src/skills/skill-activation.test.ts ++++ b/packages/core/src/skills/skill-activation.test.ts +@@ -6,6 +6,7 @@ + + import * as path from 'node:path'; + import { describe, expect, it } from 'vitest'; ++import { extractToolFilePaths } from '../core/coreToolScheduler.js'; + import { + SkillActivationRegistry, + resolveProjectRelativePath, +@@ -247,10 +248,7 @@ describe('extractToolFilePaths → SkillActivationRegistry integration', () => { + // extraction (path + pattern as separate candidates) silently failed + // to activate skills keyed on the joined effective selector — there + // was no test exercising the path that mattered. +- it('activates a skill keyed on src/**/*.ts from glob({ path: "src", pattern: "**/*.ts" })', async () => { +- const { extractToolFilePaths } = await import( +- '../core/coreToolScheduler.js' +- ); ++ it('activates a skill keyed on src/**/*.ts from glob({ path: "src", pattern: "**/*.ts" })', () => { + const candidates = extractToolFilePaths('glob', { + path: 'src', + pattern: '**/*.ts', +@@ -268,10 +266,7 @@ describe('extractToolFilePaths → SkillActivationRegistry integration', () => { + expect(Array.from(activated)).toEqual(['tsx-helper']); + }); + +- it('does NOT activate from external glob.path (project-root guard wins)', async () => { +- const { extractToolFilePaths } = await import( +- '../core/coreToolScheduler.js' +- ); ++ it('does NOT activate from external glob.path (project-root guard wins)', () => { + const candidates = extractToolFilePaths('glob', { + path: '/tmp/external', + pattern: '**/*.ts', diff --git a/.fork/patches/README.md b/.fork/patches/README.md new file mode 100644 index 00000000000..9b3f8908d73 --- /dev/null +++ b/.fork/patches/README.md @@ -0,0 +1,268 @@ +# Fork Patches + +本目录包含 fork 相对于 upstream 的所有长期定制 patch。每个 patch 对应一个独立的功能维度。 + +## Upstream 同步方案 + +### 整体策略 + +采用 **Guarded Merge**(受保护的自动合并),而非 Patch Replay: + +- CI 每天自动 fetch upstream/main 并尝试合入 +- Git 正常 merge,保留 upstream 的自然演进 +- 不自动解冲突,避免静默丢失 fork 定制 +- 出现风险信号时升级人工或 agent 处理 +- 通过 `verify.sh` + manifest 保护关键业务能力 + +### 分级处理 + +| Level | 条件 | 自动行为 | 人工介入 | +| ----- | -------------------------- | ----------------------------- | ---------------- | +| 0 | 无 upstream 新提交 | 跳过,通知 already latest | 不需要 | +| 1 | merge 成功,guard 通过 | 创建 MR,标记低风险 | 通常不需要 | +| 2 | merge 成功,命中高风险文件 | 创建 MR,列出重点 review 文件 | review 重点文件 | +| 3 | merge 成功,guard 测试失败 | 创建 MR 并阻断自动合入 | agent 或人工修复 | +| 4 | Git merge conflict | 停止,输出冲突报告 | agent 或人工处理 | + +### CI 自动同步流程 + +```mermaid +flowchart TD + A[定时触发] --> B[git fetch upstream main] + B --> C{upstream 有新提交?} + C -->|No| D[通知 already latest] + D --> END1[结束] + C -->|Yes| E[创建/复用 sync/upstream-YYYYMMDD 分支] + E --> F[git merge upstream/main] + F --> G{merge 冲突?} + G -->|No| J + G -->|Yes| H{仅 package.json 冲突?} + H -->|Yes| I["git checkout --theirs + rewrite-package-identity.js"] + I --> J + H -->|No| K[记录冲突文件列表] + K --> L["写入 conflict-status = conflict"] + L --> M[钉钉通知 Level 4] + M --> END2[结束 - 需人工处理] + J[rewrite-package-identity.js] --> N[bash verify.sh] + N --> O{verify 通过?} + O -->|No| P["写入 conflict-status = verify-failed"] + P --> Q[创建 MR 并阻断自动合入] + Q --> R[钉钉通知 Level 3] + R --> END3[结束 - 需修复] + O -->|Yes| S{命中高风险文件?} + S -->|Yes| T[创建 MR + 注入风险文件清单] + T --> U[钉钉通知 Level 2] + U --> END4[结束] + S -->|No| V[创建 MR 标记低风险] + V --> W[钉钉通知 Level 1] + W --> END5[结束] +``` + +**文字摘要:** + +``` +每天定时触发 .aoneci/upstream-sync-merge.yml: + 1. git fetch upstream main + 2. 检查 upstream 是否有新提交 + 3. 创建/复用 sync/upstream-YYYYMMDD 分支 + 4. git merge upstream/main + 5. 确定性解决 package.json 冲突(--theirs + rewrite-package-identity.js) + 6. 运行 verify.sh 检测 fork 定制是否丢失 + 7. 创建/更新 sync MR,注入高风险文件清单 + 8. 钉钉通知结果 +``` + +### 本地手动同步(Patch Replay 模式) + +适用于需要精细控制的场景: + +```bash +bash .fork/sync-upstream.sh +# 流程: unapply patches → merge upstream → re-apply patches → verify → commit +``` + +## Patch 跟踪表 + +| # | Patch | 功能说明 | 负责人 | 关联 MR | +| ---- | ----------------------------- | ------------------------------------------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0001 | branding-header | DataWorks DataAgent CLI 品牌标识 | 继风 | [#1](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585259), [#4](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585660) | +| 0002 | branding-tips | DataWorks 启动提示和 beta 引导 | 继风, 秦奇 | [#11](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26668680), [#13](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26675235) | +| 0003 | i18n-dataworks | DataWorks 专有 i18n 占位符和示例 | 今井 | [#50](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26907000) | +| 0004 | dsw-oauth-redirect | DSW 环境 MCP OAuth 重定向改写 | 克竟 | [#100](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27413710) | +| 0005 | osc8-internal | 内部终端 OSC8 超链接兼容 | 克竟 | — | +| 0006 | dingtalk-channel-enhancements | 钉钉 Channel 卡片/Markdown/路由增强 | 沅沅 | [#116](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27639454) | +| 0007 | feishu-channel | 飞书 Channel CLI 注册表 import 改写为 fork 包名 | 沅沅 | [#107](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27532149) | +| 0009 | claude-websearch-compat | Claude WebSearch 转换兼容 | 今井 | — | +| 0010 | build-single-bundle | 单文件 bundle 输出 + 移除 acp-bridge 显式构建步骤 | 今井, 胡玮文 | — | +| 0011 | test-fork-adaptations | 测试适配(OAuth 菜单、Aone CI、Node v22 guard) | 今井 | — | + +### 已退休 Patches + +| # | 原因 | +| ---------------------------------- | ----------------------------------- | +| ~~0008~~ dynamic-swarm-tool | upstream 已 revert,fork 也不再使用 | +| ~~0012~~ dashscope-internal-origin | upstream 已有等价实现(PR #4157) | + +## Fork 基础设施文件 + +除 patches 外,以下文件也是 fork 同步保护机制的组成部分: + +### `.fork/` 目录 + +| 文件 | 用途 | +| ----------------------------- | ----------------------------------------------------- | +| `manifest.json` | Fork 定制声明:patch 定义、路径、packageIdentity 映射 | +| `apply.sh` | 按 series 顺序 apply 所有 patch | +| `unapply.sh` | 反向撤销所有 patch(sync 前使用) | +| `sync-upstream.sh` | 本地 patch replay 同步流程 | +| `verify.sh` | 检测 fork 定制是否丢失(签名行匹配) | +| `refresh-patch.sh` | upstream 变动后刷新某个 patch | +| `create-patch.sh` | 从当前 diff 创建新 patch | +| `generate-patches.js` | 从 manifest 定义自动生成 patch 文件 | +| `rewrite-package-identity.js` | sync 时重写 package.json 的 name/registry | + +### `.aoneci/` CI 相关 + +| 文件 | 用途 | +| -------------------------------- | ----------------------------- | +| `upstream-sync-merge.yml` | 每日自动同步 CI pipeline 定义 | +| `scripts/send-dingtalk-alert.js` | 同步结果钉钉通知 | +| `scripts/build-standalone-ci.sh` | Standalone 产物构建脚本 | + +### `scripts/` fork-only 脚本 + +| 文件 | 用途 | +| ---------------------------- | -------------------------- | +| `regen-fork-patches.sh` | 重新生成所有 patch 文件 | +| `publish-packages.js` | 内部 npm 发布 | +| `prepare-cli-for-publish.js` | 发布前准备 CLI 产物 | +| `copy-to-package.sh` | 复制 bundle 产物到 package | +| `build-standalone.sh` | 本地 standalone 构建 | +| `install-standalone.sh` | Standalone 安装脚本 | +| `update-qwen-binary.sh` | 升级 qwen 二进制 | + +## 已合入 MR 完整覆盖情况(76 个) + +以下列出所有合入的 MR 及其 patch 覆盖状态,按 iid 排序。 + +| MR | 标题 | 作者 | 覆盖状态 | +| ------------------------------------------------------------------------- | ------------------------------------------------------- | ---- | --------------------- | +| [#1](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585259) | feat: customize branding for DataWorks DataAgent | 继风 | 0001 | +| [#2](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585513) | fix: remove trailing space in header | 继风 | 0001 | +| [#3](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585642) | publish dataworks scope npm | 今井 | packageIdentity | +| [#4](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26585660) | feat: update ASCII logo for DataWorks branding | 继风 | 0001 | +| [#6](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26626511) | 新增精简模式切换 | 秦奇 | upstream 已有等价实现 | +| [#10](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26668570) | feat: dataworks tips | 继风 | 0002 | +| [#11](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26668680) | feat: dataworks tips | 继风 | 0002 | +| [#12](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26669574) | refactor: compact tool group display | 秦奇 | upstream 已有等价实现 | +| [#13](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26675235) | refactor: update tips message | 秦奇 | 0002 | +| [#14](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26675865) | update message folding style | 秦奇 | upstream 已有等价实现 | +| [#15](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26688249) | feat(cli): keep user shell commands expanded | 秦奇 | upstream 已有等价实现 | +| [#16](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26706519) | fix(permissions): env-prefixed shell command matching | 今井 | upstream 已合入 | +| [#17](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26710182) | fix: guard mid-turn drain against cancelled turns | 今井 | upstream 已有等价实现 | +| [#18](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26714246) | feat: delete model show | 继风 | upstream 已有等价实现 | +| [#19](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26714297) | refactor: remove unused imports in Header | 今井 | 0001 | +| [#21](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26716883) | fix(followup): prevent tool call UI leak | 今井 | fork-only 新功能 | +| [#22](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26716903) | feat(cli): add queue input editing via Up arrow | 今井 | upstream 已有等价实现 | +| [#23](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26716904) | feat(core): intelligent tool parallelism | 今井 | upstream 已有等价实现 | +| [#24](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26716905) | feat(core): mid-turn queue drain for agent execution | 今井 | upstream 已有等价实现 | +| [#27](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26718190) | feat(prompt): dangerous actions behavior guidance | 今井 | upstream 已有等价实现 | +| [#28](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26737689) | fix(cli): get all packages/cli unit tests passing | 清羽 | 0011 | +| [#29](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26741792) | qwen-code 支持双输出模式 | 秦奇 | upstream 已有等价实现 | +| [#31](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26744727) | refactor: update copy script path structure | 秦奇 | fork-only 脚本 | +| [#32](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26751508) | fix(cli): cherry-pick verbose/compact mode improvements | 秦奇 | upstream 已有等价实现 | +| [#34](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26762123) | fix(build): fix webui types and eslint errors | 今井 | fork-only 构建修复 | +| [#35](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26767373) | 发布正式版 | 秦奇 | release | +| [#36](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26767752) | fix(vscode-ide-companion): unblock test suite | 清羽 | 0011 | +| [#39](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26807359) | 构建脚本优化 | 今井 | fork-only 脚本 | +| [#40](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26807367) | fix: persist ProceedAlways in compact mode | 今井 | upstream 已有等价实现 | +| [#41](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26819487) | chore: sync upstream (399 commits - 0.14.3) | 今井 | upstream sync | +| [#42](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26819585) | style: quote workflow job names | 今井 | fork-only CI | +| [#43](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26837934) | fix(core): fallback after empty stream retries | 今井 | upstream 已合入 | +| [#44](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26843438) | build: add npm publish workflow | 今井 | fork-only 脚本 | +| [#47](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26881335) | 构建 qwen code 打包的二进制脚本 | 今井 | fork-only 脚本 | +| [#48](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26889902) | fix(dingtalk): prioritize senderStaffId | 今井 | 0006 | +| [#49](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26904912) | refactor: clean up bundle-publish branch | 今井 | 0010 | +| [#50](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26907000) | fix(i18n): restore DataWorks placeholder and tips | 今井 | 0003 | +| [#52](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26921615) | fix(core): allow thought-only responses in GeminiChat | 今井 | upstream 已合入 | +| [#53](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26938960) | chore: sync upstream (56 commits - 0.14.5) | 今井 | upstream sync | +| [#54](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26940867) | feat(core): integrate upstream agent features | 今井 | upstream sync | +| [#55](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26940966) | refactor(mcp-oauth): move copy hint | 克竟 | 0004 | +| [#58](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26959860) | feat(cli): Add OAuth flags to mcp add | 克竟 | 0004 | +| [#60](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26972646) | fix: align DualOutputBridge with upstream | 今井 | upstream sync | +| [#61](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26974885) | fix: align StreamJsonOutputAdapter with upstream | 今井 | upstream sync | +| [#62](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26975295) | fix: align cli/config.ts with upstream | 今井 | upstream sync | +| [#63](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26976368) | fix: align gemini.tsx with upstream | 今井 | upstream sync | +| [#64](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26976669) | fix: align mcp/add.test.ts with upstream | 今井 | upstream sync | +| [#65](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26977267) | chore: sync upstream 2026-04-20 (48 commits) | 今井 | upstream sync | +| [#66](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26978689) | feat(ui): Header 展示当前 model 名称 | 今井 | upstream 已有等价实现 | +| [#69](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26985929) | fix(build): bundle i18n locales into dist/ | 秦奇 | 0010 | +| [#71](https://code.alibaba-inc.com/alishu/qwen-code/codereview/26996251) | fix(mcp): OAuth URL clickable when wrapped | 克竟 | 0004 | +| [#72](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27016717) | chore(release): bump version to 0.14.7 | 今井 | release | +| [#74](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27071384) | refactor: BFF endpoint for OAuth redirect | 克竟 | 0004 | +| [#75](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27074746) | fix(cli): stabilize startup tip across remounts | 秦奇 | 0002 | +| [#80](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27202180) | test(cli): 精简 CLI 定制测试修复 | 今井 | 0011 | +| [#82](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27202215) | chore(release): bump version to 0.14.8 | 今井 | release | +| [#84](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27240813) | test(cli): pre-resolve AppContainer sync conflict | 今井 | 0011 | +| [#85](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27247735) | chore: upstream sync 2026-05-07 (4889 commits) | 今井 | upstream sync | +| [#86](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27270879) | fix(cli): validate model slash command arguments | 今井 | upstream 已合入 | +| [#87](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27271078) | fix(cli): unfreeze Ctrl+O compact-mode toggle | 秦奇 | upstream 已有等价实现 | +| [#88](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27272361) | Merge dataworks-20260508 into feat/test-release | 今井 | branch merge | +| [#91](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27317082) | Sync QwenLM/qwen-code main 20260511 | 今井 | upstream sync | +| [#92](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27359639) | fix: put ding talk card | 沅沅 | 0006 | +| [#93](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27382776) | Merge sync/upstream-20260511 | 今井 | upstream sync | +| [#95](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27384835) | feat(cli): wrap markdown links in OSC 8 | 克竟 | 0005 | +| [#96](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27385182) | fix: update card bug and add stop btn | 沅沅 | 0006 | +| [#97](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27386500) | chore: upstream sync 2026-05-14 (40 commits) | 今井 | upstream sync | +| [#98](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27399741) | 优化发布脚本 | 今井 | fork-only 脚本 | +| [#100](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27413710) | feat: add default OAuth redirect URI builder | 克竟 | 0004 | +| [#101](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27416044) | fix(cli): restore alishu OSC 8 signals | 克竟 | 0005 | +| [#104](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27523343) | fix(ci): add always:true to schedule trigger | 今井 | fork-only CI | +| [#105](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27524072) | fix(core): extend DashScope provider detection | 今井 | upstream 已合入 | +| [#106](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27524077) | fix: remove built-in web_search tool | 今井 | 0009 | +| [#107](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27532149) | feishu channel | 沅沅 | 0007 | +| [#113](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27579065) | fix(build): tree-shake React reconciler dev build | 今井 | fork-only 构建修复 | +| [#116](https://code.alibaba-inc.com/alishu/qwen-code/codereview/27639454) | fix(dingtalk): remove default cardTemplateId | 沅沅 | 0006 | + +> **覆盖状态说明**: +> +> - `0001`–`0011`: 对应 patch 编号,该 MR 的改动受 patch 保护 +> - `packageIdentity`: 由 `rewrite-package-identity.js` 在 sync 时自动处理 +> - `upstream 已有等价实现`: fork 先做的功能,upstream 后来也实现了,当前无 delta +> - `upstream 已合入`: fork 的修复已被 upstream 采纳(PR 合入 GitHub) +> - `upstream sync`: 对齐 upstream 的 MR,不是 fork 定制 +> - `fork-only 脚本/CI/构建修复/新功能`: 仅存在于 fork 的文件,upstream merge 不会触碰 +> - `release`: 版本发布,不涉及源码差异 +> - `branch merge`: 分支合并操作 + +## 维护指南 + +### 新增 patch + +1. 在 `.fork/manifest.json` 的 `definitions` 中添加 patch 定义 +2. 运行 `bash scripts/regen-fork-patches.sh --write` 生成 patch 文件 +3. 更新本表格,填写负责人和 MR 链接 + +### 验证 patch 覆盖 + +```bash +# 检查所有 fork commit 的改动是否仍在当前代码中 +VERBOSE=1 bash .fork/verify.sh + +# 检查 patch 文件与 manifest 一致 +node .fork/generate-patches.js --check +``` + +### 刷新 patch(upstream 变动导致 patch 无法 apply 时) + +```bash +bash .fork/refresh-patch.sh +``` + +### 判断新 MR 是否需要 patch + +合入 main 的 MR 满足以下**所有条件**时需要新增/更新 patch: + +1. 改动的文件在 upstream 中也存在(fork-only 新文件不需要) +2. 改动内容与 upstream 不同(如果 upstream 已有等价实现则不需要) +3. 不属于 packageIdentity 管理范围(name/registry 由脚本自动处理) diff --git a/.fork/patches/series b/.fork/patches/series new file mode 100644 index 00000000000..3776d1d2462 --- /dev/null +++ b/.fork/patches/series @@ -0,0 +1,9 @@ +0001-branding-header.patch +0002-branding-tips.patch +0003-i18n-dataworks.patch +0004-dsw-oauth-redirect.patch +0005-osc8-internal.patch +0007-feishu-channel.patch +0009-claude-websearch-compat.patch +0010-build-single-bundle.patch +0011-test-fork-adaptations.patch diff --git a/.fork/refresh-patch.sh b/.fork/refresh-patch.sh new file mode 100755 index 00000000000..72948275b73 --- /dev/null +++ b/.fork/refresh-patch.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# .fork/refresh-patch.sh — Regenerate a patch from the current working tree. +# +# Usage: +# bash .fork/refresh-patch.sh 0001-branding-header +# bash .fork/refresh-patch.sh 0001 # prefix match +# bash .fork/refresh-patch.sh branding-header # substring match +# +# Environment: +# UPSTREAM_REF default upstream/main +# FORK_REF optional committed fork ref to diff, for example origin/main +# PATCH_BASE_REF optional explicit base; defaults to merge-base(FORK_REF|HEAD, UPSTREAM_REF) + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH_DIR="$FORK_DIR/patches" +QUERY="$1" +UPSTREAM_REF="${UPSTREAM_REF:-upstream/main}" + +if ! git rev-parse --verify --quiet "${UPSTREAM_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ $UPSTREAM_REF not available. Run: git fetch upstream main" >&2 + exit 2 +fi +if [ -n "${FORK_REF:-}" ] && ! git rev-parse --verify --quiet "${FORK_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ FORK_REF not available: $FORK_REF" >&2 + exit 2 +fi + +if [ -n "${PATCH_BASE_REF:-}" ]; then + if ! git rev-parse --verify --quiet "${PATCH_BASE_REF}^{commit}" >/dev/null 2>&1; then + echo "❌ PATCH_BASE_REF not available: $PATCH_BASE_REF" >&2 + exit 2 + fi + PATCH_BASE=$(git rev-parse "$PATCH_BASE_REF") +else + PATCH_HEAD="${FORK_REF:-HEAD}" + PATCH_BASE=$(git merge-base "$PATCH_HEAD" "$UPSTREAM_REF") +fi + +# Match: exact filename > prefix glob > substring (fixed-string grep) +PATCH_FILE="" +for f in "$PATCH_DIR"/*.patch; do + [ -f "$f" ] || continue + base=$(basename "$f") + if [ "$base" = "${QUERY}.patch" ] || [ "$base" = "$QUERY" ]; then + PATCH_FILE="$f" + break + fi +done +if [ -z "$PATCH_FILE" ]; then + PATCH_FILE=$(find "$PATCH_DIR" -maxdepth 1 -name "${QUERY}*.patch" -print | sort | head -1) +fi +if [ -z "$PATCH_FILE" ]; then + PATCH_FILE=$(find "$PATCH_DIR" -maxdepth 1 -name "*.patch" -print | sort | grep -F "$QUERY" | head -1) +fi + +if [ -z "$PATCH_FILE" ]; then + echo "❌ No patch matching '$QUERY'" >&2 + echo "Available patches:" >&2 + ls "$PATCH_DIR"/*.patch 2>/dev/null | while read -r f; do echo " $(basename "$f")"; done >&2 + exit 1 +fi + +mapfile -t FILES < <(grep '^diff --git' "$PATCH_FILE" | sed 's|diff --git a/\(.*\) b/.*|\1|' | sort -u) +if [ "${#FILES[@]}" -eq 0 ]; then + echo "❌ Could not extract file list from $PATCH_FILE" >&2 + exit 1 +fi + +echo "Refreshing: $(basename "$PATCH_FILE")" +echo "Base: $(git rev-parse --short=9 "$PATCH_BASE")" +echo "Files:" +printf '%s\n' "${FILES[@]}" | while read -r f; do echo " $f"; done + +if [ -n "${FORK_REF:-}" ]; then + git diff --binary --no-color "$PATCH_BASE" "$FORK_REF" -- "${FILES[@]}" > "${PATCH_FILE}.new" +else + git diff --binary --no-color "$PATCH_BASE" -- "${FILES[@]}" > "${PATCH_FILE}.new" +fi + +if [ -s "${PATCH_FILE}.new" ]; then + mv "${PATCH_FILE}.new" "$PATCH_FILE" + echo "✅ Refreshed: $(basename "$PATCH_FILE") ($(wc -l < "$PATCH_FILE") lines)" +else + rm "${PATCH_FILE}.new" + echo "⚠️ No diff found — upstream may now include this change." + echo " Consider removing this patch from the series file." +fi diff --git a/.fork/rewrite-package-identity.js b/.fork/rewrite-package-identity.js new file mode 100644 index 00000000000..52f3d013855 --- /dev/null +++ b/.fork/rewrite-package-identity.js @@ -0,0 +1,121 @@ +#!/usr/bin/env node +// .fork/rewrite-package-identity.js +// +// Reads manifest.json and rewrites package.json name + publishConfig +// across all workspace packages. Idempotent in both directions. +// +// Usage: +// node .fork/rewrite-package-identity.js # apply fork identity +// node .fork/rewrite-package-identity.js --reverse # restore upstream identity + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const manifestPath = path.join(__dirname, 'manifest.json'); +const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); + +const reverse = process.argv.includes('--reverse'); +const dryRun = process.argv.includes('--dry-run'); + +const { registry, mappings, excludeRegistry = [] } = manifest.packageIdentity; +const excludeSet = new Set(excludeRegistry); +let changed = 0; + +function getUpstreamName(pkgPath) { + try { + const raw = execFileSync('git', ['show', `upstream/main:${pkgPath}`], { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }); + return JSON.parse(raw).name; + } catch { + return null; + } +} + +// Detect indentation used in the file (defaults to 2 spaces) +function detectIndent(raw) { + const match = raw.match(/^(\s+)"/m); + return match ? match[1] : ' '; +} + +for (const [pkgPath, forkName] of Object.entries(mappings)) { + const fullPath = path.resolve(pkgPath); + if (!fs.existsSync(fullPath)) { + console.warn(`SKIP: ${pkgPath} (not found)`); + continue; + } + + const raw = fs.readFileSync(fullPath, 'utf-8'); + const pkg = JSON.parse(raw); + const indent = detectIndent(raw); + const originalName = pkg.name; + + if (reverse) { + const upstreamName = pkg._upstreamName || getUpstreamName(pkgPath); + if (!upstreamName) { + console.error(`ERROR: cannot determine upstream name for ${pkgPath} (no _upstreamName field and git show upstream/main:${pkgPath} failed)`); + process.exit(1); + } + if (pkg.name !== upstreamName) { + pkg.name = upstreamName; + + // Remove publishConfig.registry if it matches our fork registry + if (pkg.publishConfig?.registry === registry) { + delete pkg.publishConfig.registry; + if (Object.keys(pkg.publishConfig).length === 0) { + delete pkg.publishConfig; + } + } + + // Remove _upstreamName helper field + delete pkg._upstreamName; + + const updated = JSON.stringify(pkg, null, indent) + '\n'; + if (updated !== raw) { + if (dryRun) { + console.log(`WOULD: ${pkgPath} ${originalName} → ${upstreamName}`); + } else { + fs.writeFileSync(fullPath, updated); + console.log(`RESTORE: ${pkgPath} ${originalName} → ${upstreamName}`); + } + changed++; + } + } + } else { + let modified = false; + + if (pkg.name !== forkName) { + pkg.name = forkName; + modified = true; + } + + // Add publishConfig.registry if not present (skip excluded packages) + if ( + !excludeSet.has(pkgPath) && + (!pkg.publishConfig?.registry || pkg.publishConfig.registry !== registry) + ) { + if (!pkg.publishConfig) { + pkg.publishConfig = {}; + } + pkg.publishConfig.registry = registry; + modified = true; + } + + if (modified) { + const updated = JSON.stringify(pkg, null, indent) + '\n'; + if (dryRun) { + console.log(`WOULD: ${pkgPath} ${originalName} → ${forkName}`); + } else { + fs.writeFileSync(fullPath, updated); + console.log(`REWRITE: ${pkgPath} ${originalName} → ${forkName}`); + } + changed++; + } + } +} + +console.log(`\n${reverse ? 'Restored' : 'Rewrote'} ${changed} package(s)`); diff --git a/.fork/sync-upstream.sh b/.fork/sync-upstream.sh new file mode 100755 index 00000000000..d7154b5a1c1 --- /dev/null +++ b/.fork/sync-upstream.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# .fork/sync-upstream.sh — Conflict-free upstream sync (LOCAL helper). +# +# NOTE: This is a LOCAL developer convenience script. The authoritative sync +# flow for CI is in .aoneci/upstream-sync-merge.yml, which uses a different +# strategy (direct merge + conflict detection + LLM resolution). Use this +# script for manual local syncs; do NOT use it as a substitute for the CI flow. +# +# Workflow: +# 1. Unapply all fork patches → working tree matches upstream +# 2. Merge upstream/main → no conflicts because tree = upstream +# 3. Re-apply fork patches +# 4. Rebuild lockfile +# 5. Verify and commit +# +# Usage: +# bash .fork/sync-upstream.sh # full sync +# bash .fork/sync-upstream.sh --dry-run # check without committing +# +# Prerequisites: +# git remote add upstream https://github.com/QwenLM/qwen-code.git + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +DRY_RUN="${1:-}" +TODAY=$(date +%Y-%m-%d) + +echo "========================================" +echo " Fork Patch-Based Upstream Sync" +echo " $TODAY" +echo "========================================" +echo "" + +# Preflight +if ! git rev-parse --verify --quiet upstream/main >/dev/null 2>&1; then + echo "❌ upstream/main not available. Run:" + echo " git remote add upstream https://github.com/QwenLM/qwen-code.git" + echo " git fetch upstream main" + exit 2 +fi + +# Step 0: Checkpoint +echo "=== Step 0: Checkpoint ===" +CHECKPOINT=$(git rev-parse HEAD) +echo "HEAD: $(git log --oneline -1)" +git tag "sync-checkpoint-$(date +%Y%m%d-%H%M%S)-$(git rev-parse --short=7 HEAD)" HEAD 2>/dev/null || true +echo "" + +# Step 1: Unapply all fork patches +echo "=== Step 1: Unapply fork patches ===" +bash "$FORK_DIR/unapply.sh" +git add -A +git diff --cached --quiet || git commit -m "chore(sync): unapply fork patches for upstream merge" +echo "" + +# Step 2: Fetch and merge upstream +echo "=== Step 2: Merge upstream ===" +if ! git fetch upstream main; then + echo "❌ Failed to fetch upstream/main (network error?)" >&2 + exit 1 +fi +UPSTREAM_HEAD=$(git rev-parse upstream/main) +echo "upstream/main: $(git log --oneline -1 upstream/main)" + +NEW_COMMITS=$(git log --oneline HEAD..upstream/main --no-merges | wc -l | tr -d ' ') +if [ "$NEW_COMMITS" -eq 0 ]; then + echo "✅ Already up-to-date with upstream. Re-applying patches..." + if ! node "$FORK_DIR/rewrite-package-identity.js" || ! bash "$FORK_DIR/apply.sh"; then + echo "" + echo "⚠️ Re-apply failed after unapply. Repo is in unapply state." + echo " Restore with: git reset --hard $CHECKPOINT" + exit 1 + fi + git add -A + git diff --cached --quiet || git commit -m "chore(sync): re-apply fork patches" + echo "Done — no upstream changes." + exit 0 +fi + +echo "📦 $NEW_COMMITS new upstream commits" +if ! git merge upstream/main --no-edit; then + echo "" + echo "❌ UNEXPECTED: merge conflict after unapply!" + echo " This means a fork-only file conflicts with upstream." + echo " Conflicted files:" + git diff --name-only --diff-filter=U + echo "" + echo " Aborting merge to leave repo in a clean state." + git merge --abort 2>/dev/null || true + echo " Resolve manually, then re-run this script." + exit 1 +fi +echo "" + +# Step 3: Re-apply fork patches +echo "=== Step 3: Apply fork patches ===" +node "$FORK_DIR/rewrite-package-identity.js" + +APPLY_RC=0 +bash "$FORK_DIR/apply.sh" --continue || APPLY_RC=$? + +if [ "$APPLY_RC" -ne 0 ]; then + echo "" + echo "⚠️ Some patches failed to apply." + echo " Fix the .rej files, then run:" + echo " bash .fork/refresh-patch.sh " + echo " to update the patch, then:" + echo " git add -A && git commit -m 'chore: sync upstream $TODAY'" + if [ "$DRY_RUN" = "--dry-run" ]; then + echo "" + echo "DRY RUN: resetting to checkpoint..." + git reset --hard "$CHECKPOINT" + fi + exit 1 +fi +echo "" + +# Step 4: Commit +echo "=== Step 4: Commit ===" +git add -A +if git diff --cached --quiet; then + echo "✅ No changes to commit" +else + if [ "$DRY_RUN" = "--dry-run" ]; then + echo "DRY RUN: would commit 'chore: sync upstream $TODAY ($NEW_COMMITS commits)'" + echo "Resetting to checkpoint..." + git reset --hard "$CHECKPOINT" + else + git commit -m "chore: sync upstream ${TODAY} (${NEW_COMMITS:-0} commits)" + fi +fi + +echo "" +echo "✅ Upstream sync complete" +echo " $NEW_COMMITS commits merged, all patches applied successfully." diff --git a/.fork/unapply.sh b/.fork/unapply.sh new file mode 100755 index 00000000000..07f4960d1d9 --- /dev/null +++ b/.fork/unapply.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# .fork/unapply.sh — Reverse all fork patches (reverse series order). +# +# After running this, the working tree (excluding .fork/ and fork-only files) +# should match upstream exactly. This is the prerequisite for a conflict-free +# upstream merge. +# +# Usage: +# bash .fork/unapply.sh # reverse all patches + package identity +# bash .fork/unapply.sh --check # dry-run: check if reverse applies cleanly +# +# Exit codes: +# 0 all patches reversed successfully +# 1 one or more patches failed to reverse + +set -euo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +SERIES="$FORK_DIR/patches/series" +PATCH_DIR="$FORK_DIR/patches" +MODE="${1:-apply}" + +# Step 1: Reverse package identity rewrites +echo "=== Reversing package identity ===" +if [ "$MODE" = "--check" ]; then + echo "(dry-run: skipping package identity reversal)" +else + if ! node "$FORK_DIR/rewrite-package-identity.js" --reverse; then + echo "ERROR: package identity reversal failed" >&2 + exit 1 + fi +fi + +# Step 2: Reverse patches in reverse order +if [ ! -f "$SERIES" ]; then + echo "⚠️ No series file, skipping patch reversal" + exit 0 +fi + +echo "" +echo "=== Reversing patches ===" + +REVERSED=0 +FAILED=0 +FAILED_LIST=() + +# Read series into array, then iterate in reverse +PATCHES=() +while IFS= read -r patch; do + [[ -z "$patch" || "$patch" == \#* ]] && continue + PATCHES+=("$patch") +done < "$SERIES" + +for ((i=${#PATCHES[@]}-1; i>=0; i--)); do + patch="${PATCHES[$i]}" + PATCH_FILE="$PATCH_DIR/$patch" + + if [ ! -f "$PATCH_FILE" ]; then + echo "SKIP: $patch (file not found)" + continue + fi + + if [ "$MODE" = "--check" ]; then + if git apply --check --reverse "$PATCH_FILE" 2>/dev/null; then + echo "OK: $patch" + else + echo "FAIL: $patch" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + fi + continue + fi + + if git apply --reverse "$PATCH_FILE" 2>/dev/null; then + echo "REVERSED: $patch" + REVERSED=$((REVERSED + 1)) + else + echo "FAILED: $patch (may already be unapplied)" + FAILED=$((FAILED + 1)) + FAILED_LIST+=("$patch") + fi +done + +echo "" +if [ "$MODE" = "--check" ]; then + echo "Check complete: $FAILED failed" +else + echo "Reversed: $REVERSED Failed: $FAILED" +fi + +if [ "$FAILED" -gt 0 ]; then + echo "Failed patches:" + for p in "${FAILED_LIST[@]}"; do + echo " - $p" + done + exit 1 +fi +exit 0 diff --git a/.fork/verify-patches.sh b/.fork/verify-patches.sh new file mode 100755 index 00000000000..d6a4a5d51e7 --- /dev/null +++ b/.fork/verify-patches.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# .fork/verify-patches.sh — Verify that fork patches are applied in the working tree. +# +# Unlike apply.sh --check (which tests if a patch CAN be applied, meaning it is +# NOT present), this script checks whether each patch's added content IS present +# in the current working tree. +# +# For each patch in the series: +# 1. Extract added lines (12+ chars after trimming whitespace) +# 2. Check if those lines exist in the target files +# 3. Report match rate: PASS (≥80%), WARN (≥50%), FAIL (<50%) +# +# Exit codes: +# 0 all patches PASS or WARN (no FAIL) +# 1 one or more patches FAIL (content missing from working tree) +# 2 environment error (series file missing, etc.) +# +# Usage: +# bash .fork/verify-patches.sh # normal run +# bash .fork/verify-patches.sh --verbose # show PASS/SKIP details + +set -uo pipefail + +FORK_DIR="$(cd "$(dirname "$0")" && pwd)" +SERIES="$FORK_DIR/patches/series" +PATCH_DIR="$FORK_DIR/patches" +VERBOSE="${1:-}" + +PASS_THRESHOLD=80 +WARN_THRESHOLD=50 +MIN_LINE_LENGTH=12 + +if [ ! -f "$SERIES" ]; then + echo "❌ series file not found: $SERIES" >&2 + exit 2 +fi + +PASS_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 +FAIL_LIST=() + +while IFS= read -r patch_name; do + [[ -z "$patch_name" || "$patch_name" == \#* ]] && continue + PATCH_FILE="$PATCH_DIR/$patch_name" + + if [ ! -f "$PATCH_FILE" ]; then + echo "MISSING: $patch_name" + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAIL_LIST+=("$patch_name (file not found)") + continue + fi + + total=0 + matched=0 + current_file="" + + while IFS= read -r line; do + case "$line" in + "+++ /dev/null") + current_file="" + ;; + "+++ b/"*) + current_file="${line#+++ b/}" + # Skip lockfiles, snapshots, build artifacts + case "$current_file" in + *package-lock.json|*pnpm-lock.yaml|*yarn.lock|*.lock|\ + *.snap|*.snap.txt|*/dist/*|*/build/*) + current_file="" + ;; + esac + ;; + "+++ "*) + ;; + "+"*) + [ -z "$current_file" ] && continue + # Skip diff metadata lines + [[ "$line" == "+++"* ]] && continue + content="${line#+}" + # Trim leading/trailing whitespace + content_trimmed="$(printf '%s' "$content" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [ "${#content_trimmed}" -lt "$MIN_LINE_LENGTH" ]; then + continue + fi + total=$((total + 1)) + if [ -f "$current_file" ] && grep -qF -- "$content_trimmed" "$current_file" 2>/dev/null; then + matched=$((matched + 1)) + fi + ;; + esac + done < "$PATCH_FILE" + + if [ "$total" -eq 0 ]; then + SKIP_COUNT=$((SKIP_COUNT + 1)) + [ "$VERBOSE" = "--verbose" ] && printf 'SKIP %s (no signature lines)\n' "$patch_name" + continue + fi + + rate=$((matched * 100 / total)) + if [ "$rate" -ge "$PASS_THRESHOLD" ]; then + PASS_COUNT=$((PASS_COUNT + 1)) + [ "$VERBOSE" = "--verbose" ] && printf 'PASS %s (%d/%d, %d%%)\n' "$patch_name" "$matched" "$total" "$rate" + elif [ "$rate" -ge "$WARN_THRESHOLD" ]; then + WARN_COUNT=$((WARN_COUNT + 1)) + printf 'WARN %s (%d/%d, %d%%)\n' "$patch_name" "$matched" "$total" "$rate" + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAIL_LIST+=("$patch_name ($matched/$total, $rate%)") + printf 'FAIL %s (%d/%d, %d%%)\n' "$patch_name" "$matched" "$total" "$rate" + fi +done < "$SERIES" + +echo "" +echo "======== Fork Patch Content Verify ========" +printf 'PASS: %d | WARN: %d | FAIL: %d | SKIP: %d\n' \ + "$PASS_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" + +if [ "$FAIL_COUNT" -gt 0 ]; then + echo "" + echo "❌ ${FAIL_COUNT} patch(es) missing from working tree:" + for entry in "${FAIL_LIST[@]}"; do + echo " - $entry" + done + exit 1 +fi + +echo "✅ All patches verified" +exit 0 diff --git a/.fork/verify.sh b/.fork/verify.sh new file mode 100755 index 00000000000..bc1fa00c538 --- /dev/null +++ b/.fork/verify.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +# .fork/verify.sh — 验证 fork 定制改动是否仍存在于当前 HEAD。 +# +# 工作原理: +# 1. 默认遍历 first-parent 落地 commit(PR 合入 main 的 commit), +# 把每个 commit 按 subject 分类为 fork / sync / release。 +# 2. 仅对 fork 类 commit 取其相对 first-parent 的 diff,提取加入的"签名行" +# (足够长的非空白行)。 +# 3. 检查这些签名行是否仍能在当前 HEAD 工作树对应文件中找到。 +# 4. 按匹配率分类输出 PASS / WARN / FAIL;release / sync / 没有签名行的跳过。 +# +# 退出码: +# 0 无 FAIL(可能存在 WARN) +# 1 存在 FAIL(疑似 fork patch 丢失) +# 2 环境错误 +# +# 可调环境变量: +# UPSTREAM_REF 上游 ref,默认 upstream/main +# HEAD_REF 本端 ref,默认 HEAD +# PASS_THRESHOLD PASS 最低匹配率 (0-100),默认 80 +# WARN_THRESHOLD WARN 最低匹配率 (0-100),默认 50;低于此视为 FAIL +# MIN_LINE_LENGTH 忽略短于此长度(trim 后字符数)的签名行,默认 12 +# VERBOSE 为 1 时打印 PASS/SKIP 明细 +# MODE landing(默认) / all +# landing: 只看 first-parent 落地 commit(PR-level) +# all: 看所有非 merge 的 fork commit(更细粒度,噪音更大) +# JSON_OUTPUT 设为文件路径时输出结构化 JSON(供 AI agent 程序化消费) + +set -uo pipefail + +UPSTREAM_REF="${UPSTREAM_REF:-upstream/main}" +HEAD_REF="${HEAD_REF:-HEAD}" +PASS_THRESHOLD="${PASS_THRESHOLD:-80}" +WARN_THRESHOLD="${WARN_THRESHOLD:-50}" +MIN_LINE_LENGTH="${MIN_LINE_LENGTH:-12}" +VERBOSE="${VERBOSE:-0}" +MODE="${MODE:-landing}" +JSON_OUTPUT="${JSON_OUTPUT:-}" + +# subject 分类。返回 fork / sync / release。 +# - sync: 同步上游、对齐上游的 commit +# - release: 版本号 bump、发版相关 +# - fork: 真正的 fork 业务/修复改动(包括"恢复"丢失的 fork 代码) +# +# 规则收紧原则:宁可多检查(false positive),不漏检查(false negative)。 +# 所有 pattern 使用前缀匹配,避免子串匹配误杀。 +# "fix: align with upstream" 归为 sync 而非 fork,因为这类 commit 只是将代码对齐到上游, +# 不包含 fork 定制。其他未匹配的 fix/feat commit 归为 fork 并接受验证。 +# 如需修改此处规则,必须同步更新 .aoneci/upstream-sync-merge.yml 中的 awk 版本。 +classify_subject() { + local subj="$1" + case "$subj" in + "Merge commit '"*) echo "sync"; return ;; + "Merge branch sync/"*) echo "sync"; return ;; + "chore: sync upstream"*) echo "sync"; return ;; + "fix(sync): resolve-upstream"*|"chore(sync): resolve-upstream"*) echo "sync"; return ;; + "fix: align with upstream"*) echo "sync"; return ;; + "chore(release)"*) echo "release"; return ;; + "chore: release"*) echo "release"; return ;; + "chore: bump version"*) echo "release"; return ;; + "chore: rebase"*) echo "release"; return ;; + "build: bump version"*) echo "release"; return ;; + "ci(release)"*) echo "release"; return ;; + "ci: publish"*) echo "release"; return ;; + esac + echo "fork" +} + +if ! git rev-parse --verify --quiet "${UPSTREAM_REF}^{commit}" >/dev/null; then + echo "❌ upstream ref 不可用: $UPSTREAM_REF" >&2 + echo " 请先 fetch upstream,例如:" >&2 + echo " git remote add upstream https://github.com/QwenLM/qwen-code.git" >&2 + echo " git fetch upstream main" >&2 + exit 2 +fi +if ! git rev-parse --verify --quiet "${HEAD_REF}^{commit}" >/dev/null; then + echo "❌ HEAD ref 不可用: $HEAD_REF" >&2 + exit 2 +fi +case "$MODE" in + landing|all) ;; + *) echo "❌ MODE 必须为 landing 或 all(当前: $MODE)" >&2; exit 2 ;; +esac + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 +TOTAL=0 +WARN_LINES=() +FAIL_LINES=() +JSON_ENTRIES="" + +if [ "$MODE" = "all" ]; then + log_args=(--no-merges) +else + log_args=(--first-parent) +fi + +while IFS= read -r commit; do + TOTAL=$((TOTAL + 1)) + short=$(git rev-parse --short "$commit") + subject=$(git log -1 --format='%s' "$commit") + category=$(classify_subject "$subject") + + if [ "$category" != "fork" ]; then + SKIP_COUNT=$((SKIP_COUNT + 1)) + [ "$VERBOSE" = "1" ] && printf 'SKIP[%s] %s %s\n' "$category" "$short" "$subject" + continue + fi + + # 检查 commit 是否有可读 parent(fork 第一个 commit 可能是 root) + if ! git rev-parse --verify --quiet "${commit}^{commit}" >/dev/null; then + SKIP_COUNT=$((SKIP_COUNT + 1)) + [ "$VERBOSE" = "1" ] && printf 'SKIP[noparent] %s %s\n' "$short" "$subject" + continue + fi + + diff_file="$WORK/diff" + : >"$diff_file" + # first-parent diff: 对 squash merge 或真 merge commit 均取合入 main 的净增内容。 + # --no-renames 避免重命名导致 diff 行被误识别为删除+新增。 + git diff --no-color --no-renames "${commit}^" "$commit" >"$diff_file" 2>/dev/null || true + + total=0 + matched=0 + current_file="" + + while IFS= read -r line; do + case "$line" in + "+++ /dev/null") + current_file="" + ;; + "+++ b/"*) + current_file="${line#+++ b/}" + case "$current_file" in + *package-lock.json|*pnpm-lock.yaml|*yarn.lock|*.lock|\ + *.snap|*.snap.txt|*/dist/*|*/build/*|.last-synced-upstream-tag) + current_file="" + ;; + esac + ;; + "+++ "*) + ;; + "+"*) + [ -z "$current_file" ] && continue + content="${line#+}" + content_trimmed="$(printf '%s' "$content" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [ "${#content_trimmed}" -lt "$MIN_LINE_LENGTH" ]; then + continue + fi + total=$((total + 1)) + if [ -f "$current_file" ] && grep -qF -- "$content_trimmed" "$current_file" 2>/dev/null; then + matched=$((matched + 1)) + fi + ;; + esac + done <"$diff_file" + + if [ "$total" -eq 0 ]; then + SKIP_COUNT=$((SKIP_COUNT + 1)) + [ "$VERBOSE" = "1" ] && printf 'SKIP[empty] %s %s\n' "$short" "$subject" + continue + fi + + rate=$((matched * 100 / total)) + verdict="" + if [ "$rate" -ge "$PASS_THRESHOLD" ]; then + PASS_COUNT=$((PASS_COUNT + 1)) + verdict="pass" + [ "$VERBOSE" = "1" ] && printf 'PASS %s %s (%d/%d, %d%%)\n' "$short" "$subject" "$matched" "$total" "$rate" + elif [ "$rate" -ge "$WARN_THRESHOLD" ]; then + WARN_COUNT=$((WARN_COUNT + 1)) + verdict="warn" + line_summary=$(printf 'WARN %s %s (%d/%d, %d%%)' "$short" "$subject" "$matched" "$total" "$rate") + WARN_LINES+=("$line_summary") + printf '%s\n' "$line_summary" + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + verdict="fail" + line_summary=$(printf 'FAIL %s %s (%d/%d, %d%%)' "$short" "$subject" "$matched" "$total" "$rate") + FAIL_LINES+=("$line_summary") + printf '%s\n' "$line_summary" + fi + + if [ -n "$JSON_OUTPUT" ]; then + escaped_subject=$(printf '%s' "$subject" | node -e " + const s = require('fs').readFileSync('/dev/stdin','utf8'); + process.stdout.write(JSON.stringify(s).slice(1,-1)); + " 2>/dev/null || printf '%s' "$subject" | sed 's/\\\\/\\\\\\\\/g; s/\"/\\\\\"/g; s/\\t/\\\\t/g; s/\\r/\\\\r/g') + [ -n "$JSON_ENTRIES" ] && JSON_ENTRIES="$JSON_ENTRIES," + JSON_ENTRIES="$JSON_ENTRIES{\"commit\":\"$short\",\"subject\":\"$escaped_subject\",\"verdict\":\"$verdict\",\"matched\":$matched,\"total\":$total,\"rate\":$rate}" + fi +done < <(git log "${log_args[@]}" --reverse --format='%H' "${UPSTREAM_REF}..${HEAD_REF}") + +echo "" +echo "================== Fork Patch Verify ($MODE mode) ==================" +printf 'Total: %d commits | PASS: %d | WARN: %d | FAIL: %d | SKIP: %d\n' \ + "$TOTAL" "$PASS_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" +echo "Thresholds: PASS≥${PASS_THRESHOLD}% WARN≥${WARN_THRESHOLD}% FAIL<${WARN_THRESHOLD}%" + +if [ -n "$JSON_OUTPUT" ]; then + cat < "$JSON_OUTPUT" +{"mode":"$MODE","pass":$PASS_COUNT,"warn":$WARN_COUNT,"fail":$FAIL_COUNT,"skip":$SKIP_COUNT,"total":$TOTAL,"gate_passed":$([ "$FAIL_COUNT" -eq 0 ] && echo "true" || echo "false"),"entries":[$JSON_ENTRIES]} +ENDJSON + echo "📄 JSON 结果已写入: $JSON_OUTPUT" +fi + +if [ "$FAIL_COUNT" -gt 0 ]; then + echo "" + echo "❌ ${FAIL_COUNT} 个 fork patch 疑似丢失(匹配率 < ${WARN_THRESHOLD}%):" + for entry in "${FAIL_LINES[@]}"; do + echo " - $entry" + done + exit 1 +fi + +if [ "$WARN_COUNT" -gt 0 ]; then + echo "" + echo "⚠️ ${WARN_COUNT} 个 fork patch 仅部分命中,建议人工 review" +fi + +echo "✅ 无 FAIL" +exit 0 diff --git a/.gitattributes b/.gitattributes index deab5ae88bd..3c21e997137 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,9 @@ *.bash eol=lf Makefile eol=lf +# Windows cmd.exe expects batch installers to be checked out with CRLF. +scripts/installation/install-qwen-standalone.bat text eol=crlf + # Explicitly declare binary file types to prevent Git from attempting to # normalize their line endings. *.png binary diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d11835d66bf..efc61bf39cd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,73 +1,66 @@ -## Summary +## What this PR does -- What changed: -- Why it changed: -- Reviewer focus: + -## Validation +## Why it's needed - + +## Reviewer Test Plan -For user-visible changes, bug fixes, CLI / TUI behavior changes, or interaction changes, include key screenshots or a short video. -When possible, show before/after behavior. + -- Commands run: - ```bash - # paste commands here - ``` -- Prompts / inputs used: -- Expected result: -- Observed result: -- Quickest reviewer verification path: -- Evidence (output, logs, screenshots, video, JSON, before/after, etc.): +### How to verify -## Scope / Risk + -- Main risk or tradeoff: -- Not covered / not validated: -- Breaking changes / migration notes: +### Evidence (Before & After) -## Testing Matrix + - +### Tested on + +| OS | Status | +| :--------: | :----: | +| 🍏 macOS | | +| 🪟 Windows | | +| 🐧 Linux | | -| | 🍏 | 🪟 | 🐧 | -| -------- | --- | --- | --- | -| npm run | ⚠️ | ⚠️ | ⚠️ | -| npx | ⚠️ | ⚠️ | ⚠️ | -| Docker | ⚠️ | ⚠️ | ⚠️ | -| Podman | ⚠️ | N/A | N/A | -| Seatbelt | ⚠️ | N/A | N/A | + -Testing matrix notes: +### Environment (optional) -- + -## Linked Issues / Bugs +## Risk & Scope + +- Main risk or tradeoff: +- Not validated / out of scope: +- Breaking changes / migration notes: + +## Linked Issues + +
+中文说明 -Otherwise reference related issues without a closing keyword. + + +
diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000000..05c7dc55e67 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,6 @@ +# Configuration for GitHub's automatic release notes generation +# PRs with 'skip-changelog' label will be excluded from release notes +changelog: + exclude: + labels: + - 'skip-changelog' diff --git a/.github/workflows/community-report.yml b/.github/workflows/community-report.yml new file mode 100644 index 00000000000..e0aaf90dbfb --- /dev/null +++ b/.github/workflows/community-report.yml @@ -0,0 +1,197 @@ +name: 'Generate Weekly Community Report 📊' + +on: + schedule: + - cron: '0 12 * * 1' # Run at 12:00 UTC on Monday + workflow_dispatch: + inputs: + days: + description: 'Number of days to look back for the report' + required: true + default: '7' + +jobs: + generate-report: + name: 'Generate Report 📝' + if: |- + ${{ github.repository == 'google-gemini/gemini-cli' }} + runs-on: 'ubuntu-latest' + permissions: + issues: 'write' + pull-requests: 'read' + discussions: 'read' + contents: 'read' + id-token: 'write' + + steps: + - name: 'Generate GitHub App Token 🔑' + id: 'generate_token' + uses: 'actions/create-github-app-token@a8d616148505b5069dccd32f177bb87d7f39123b' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ secrets.APP_ID }}' + private-key: '${{ secrets.PRIVATE_KEY }}' + permission-issues: 'write' + permission-pull-requests: 'read' + permission-discussions: 'read' + permission-contents: 'read' + + - name: 'Generate Report 📜' + id: 'report' + env: + GH_TOKEN: '${{ steps.generate_token.outputs.token }}' + REPO: '${{ github.repository }}' + DAYS: '${{ github.event.inputs.days || 7 }}' + run: |- + set -e + + START_DATE="$(date -u -d "$DAYS days ago" +'%Y-%m-%d')" + END_DATE="$(date -u +'%Y-%m-%d')" + echo "⏳ Generating report for contributions from ${START_DATE} to ${END_DATE}..." + + declare -A author_is_googler + check_googler_status() { + local author="$1" + if [[ "${author}" == *"[bot]" ]]; then + author_is_googler[${author}]=1 + return 1 + fi + if [[ -v "author_is_googler[${author}]" ]]; then + return "${author_is_googler[${author}]}" + fi + + if gh api "orgs/googlers/members/${author}" --silent 2>/dev/null; then + echo "🧑‍💻 ${author} is a Googler." + author_is_googler[${author}]=0 + else + echo "🌍 ${author} is a community contributor." + author_is_googler[${author}]=1 + fi + return "${author_is_googler[${author}]}" + } + + googler_issues=0 + non_googler_issues=0 + googler_prs=0 + non_googler_prs=0 + + echo "🔎 Fetching issues and pull requests..." + ITEMS_JSON="$(gh search issues --repo "${REPO}" "created:>${START_DATE}" --json author,isPullRequest --limit 1000)" + + for row in $(echo "${ITEMS_JSON}" | jq -r '.[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + author="$(_jq '.author.login')" + is_pr="$(_jq '.isPullRequest')" + + if [[ -z "${author}" || "${author}" == "null" ]]; then + continue + fi + + if check_googler_status "${author}"; then + if [[ "${is_pr}" == "true" ]]; then + ((googler_prs++)) + else + ((googler_issues++)) + fi + else + if [[ "${is_pr}" == "true" ]]; then + ((non_googler_prs++)) + else + ((non_googler_issues++)) + fi + fi + done + + googler_discussions=0 + non_googler_discussions=0 + + echo "🗣️ Fetching discussions..." + DISCUSSION_QUERY=''' + query($q: String!) { + search(query: $q, type: DISCUSSION, first: 100) { + nodes { + ... on Discussion { + author { + login + } + } + } + } + }''' + DISCUSSIONS_JSON="$(gh api graphql -f q="repo:${REPO} created:>${START_DATE}" -f query="${DISCUSSION_QUERY}")" + + for row in $(echo "${DISCUSSIONS_JSON}" | jq -r '.data.search.nodes[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + author="$(_jq '.author.login')" + + if [[ -z "${author}" || "${author}" == "null" ]]; then + continue + fi + + if check_googler_status "${author}"; then + ((googler_discussions++)) + else + ((non_googler_discussions++)) + fi + done + + echo "✍️ Generating report content..." + TOTAL_ISSUES=$((googler_issues + non_googler_issues)) + TOTAL_PRS=$((googler_prs + non_googler_prs)) + TOTAL_DISCUSSIONS=$((googler_discussions + non_googler_discussions)) + + REPORT_BODY=$(cat <> "${GITHUB_OUTPUT}" + echo "${REPORT_BODY}" >> "${GITHUB_OUTPUT}" + echo "EOF" >> "${GITHUB_OUTPUT}" + + echo "📊 Community Contribution Report:" + echo "${REPORT_BODY}" + + - name: '🤖 Get Insights from Report' + if: |- + ${{ steps.report.outputs.report_body != '' }} + uses: 'google-github-actions/run-gemini-cli@a3bf79042542528e91937b3a3a6fbc4967ee3c31' # ratchet:google-github-actions/run-gemini-cli@v0 + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token }}' + REPOSITORY: '${{ github.repository }}' + with: + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "coreTools": [ + "run_shell_command(gh issue list)", + "run_shell_command(gh pr list)", + "run_shell_command(gh search issues)", + "run_shell_command(gh search prs)" + ] + } + prompt: |- + You are a helpful assistant that analyzes community contribution reports. + Based on the following report, please provide a brief summary and highlight any interesting trends or potential areas for improvement. + + Report: + ${{ steps.report.outputs.report_body }} diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml new file mode 100644 index 00000000000..c8a4c6523f6 --- /dev/null +++ b/.github/workflows/eval.yml @@ -0,0 +1,29 @@ +name: 'Eval' + +on: + workflow_dispatch: + +jobs: + eval: + name: 'Eval' + runs-on: 'ubuntu-latest' + strategy: + matrix: + node-version: + - '20.x' + - '22.x' + - '24.x' + steps: + - name: 'Set up Node.js ${{ matrix.node-version }}' + uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4 + with: + node-version: '${{ matrix.node-version }}' + cache: 'npm' + + - name: 'Set up Python' + uses: 'actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065' # ratchet:actions/setup-python@v5 + with: + python-version: '3.11' + + - name: 'Install and configure Poetry' + uses: 'snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a' # ratchet:snok/install-poetry@v1 diff --git a/.github/workflows/gemini-automated-issue-dedup.yml b/.github/workflows/gemini-automated-issue-dedup.yml new file mode 100644 index 00000000000..b84b5aa94df --- /dev/null +++ b/.github/workflows/gemini-automated-issue-dedup.yml @@ -0,0 +1,262 @@ +name: '🏷️ Gemini Automated Issue Deduplication' + +on: + issues: + types: + - 'opened' + - 'reopened' + issue_comment: + types: + - 'created' + workflow_dispatch: + inputs: + issue_number: + description: 'issue number to dedup' + required: true + type: 'number' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +jobs: + find-duplicates: + if: |- + github.repository == 'google-gemini/gemini-cli' && + vars.TRIAGE_DEDUPLICATE_ISSUES != '' && + (github.event_name == 'issues' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && + contains(github.event.comment.body, '@gemini-cli /deduplicate') && + (github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'COLLABORATOR'))) + permissions: + contents: 'read' + id-token: 'write' # Required for WIF, see https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-google-cloud-platform#adding-permissions-settings + issues: 'read' + statuses: 'read' + packages: 'read' + timeout-minutes: 20 + runs-on: 'ubuntu-latest' + outputs: + duplicate_issues_csv: '${{ env.DUPLICATE_ISSUES_CSV }}' + steps: + - name: 'Checkout' + uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5 + + - name: 'Log in to GitHub Container Registry' + uses: 'docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1' # ratchet:docker/login-action@v3 + with: + registry: 'ghcr.io' + username: '${{ github.actor }}' + password: '${{ secrets.GITHUB_TOKEN }}' + + - name: 'Find Duplicate Issues' + uses: 'google-github-actions/run-gemini-cli@a3bf79042542528e91937b3a3a6fbc4967ee3c31' # ratchet:google-github-actions/run-gemini-cli@v0 + id: 'gemini_issue_deduplication' + env: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + ISSUE_TITLE: '${{ github.event.issue.title }}' + ISSUE_BODY: '${{ github.event.issue.body }}' + ISSUE_NUMBER: '${{ github.event.issue.number }}' + REPOSITORY: '${{ github.repository }}' + FIRESTORE_PROJECT: '${{ vars.FIRESTORE_PROJECT }}' + with: + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "mcpServers": { + "issue_deduplication": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "--network", "host", + "-e", "GITHUB_TOKEN", + "-e", "GEMINI_API_KEY", + "-e", "DATABASE_TYPE", + "-e", "FIRESTORE_DATABASE_ID", + "-e", "GCP_PROJECT", + "-e", "GOOGLE_APPLICATION_CREDENTIALS=/app/gcp-credentials.json", + "-v", "${GOOGLE_APPLICATION_CREDENTIALS}:/app/gcp-credentials.json", + "ghcr.io/google-gemini/gemini-cli-issue-triage@sha256:e3de1523f6c83aabb3c54b76d08940a2bf42febcb789dd2da6f95169641f94d3" + ], + "env": { + "GITHUB_TOKEN": "${GITHUB_TOKEN}", + "GEMINI_API_KEY": "${{ secrets.GEMINI_API_KEY }}", + "DATABASE_TYPE":"firestore", + "GCP_PROJECT": "${FIRESTORE_PROJECT}", + "FIRESTORE_DATABASE_ID": "(default)", + "GOOGLE_APPLICATION_CREDENTIALS": "${GOOGLE_APPLICATION_CREDENTIALS}" + }, + "enabled": true, + "timeout": 600000 + } + }, + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh issue view)" + ], + "telemetry": { + "enabled": true, + "target": "gcp" + } + } + prompt: |- + ## Role + You are an issue de-duplication assistant. Your goal is to find + duplicate issues for a given issue. + ## Steps + 1. **Find Potential Duplicates:** + - The repository is ${{ github.repository }} and the issue number is ${{ github.event.issue.number }}. + - Use the `duplicates` tool with the `repo` and `issue_number` to find potential duplicates for the current issue. Do not use the `threshold` parameter. + - If no duplicates are found, you are done. + - Print the JSON output from the `duplicates` tool to the logs. + 2. **Refine Duplicates List (if necessary):** + - If the `duplicates` tool returns between 1 and 14 results, you must refine the list. + - For each potential duplicate issue, run `gh issue view --json title,body,comments` to fetch its content. + - Also fetch the content of the original issue: `gh issue view "${ISSUE_NUMBER}" --json title,body,comments`. + - Carefully analyze the content (title, body, comments) of the original issue and all potential duplicates. + - It is very important if the comments on either issue mention that they are not duplicates of each other, to treat them as not duplicates. + - Based on your analysis, create a final list containing only the issues you are highly confident are actual duplicates. + - If your final list is empty, you are done. + - Print to the logs if you omitted any potential duplicates based on your analysis. + - If the `duplicates` tool returned 15+ results, use the top 15 matches (based on descending similarity score value) to perform this step. + 3. **Output final duplicates list as CSV:** + - Convert the list of appropriate duplicate issue numbers into a comma-separated list (CSV). If there are no appropriate duplicates, use the empty string. + - Use the "echo" shell command to append the CSV of issue numbers into the filepath referenced by the environment variable "${GITHUB_ENV}": + echo "DUPLICATE_ISSUES_CSV=[DUPLICATE_ISSUES_AS_CSV]" >> "${GITHUB_ENV}" + ## Guidelines + - Only use the `duplicates` and `run_shell_command` tools. + - The `run_shell_command` tool can be used with `gh issue view`. + - Do not download or read media files like images, videos, or links. The `--json` flag for `gh issue view` will prevent this. + - Do not modify the issue content or status. + - Do not add comments or labels. + - Reference all shell variables as "${VAR}" (with quotes and braces). + + add-comment-and-label: + needs: 'find-duplicates' + if: |- + github.repository == 'google-gemini/gemini-cli' && + vars.TRIAGE_DEDUPLICATE_ISSUES != '' && + needs.find-duplicates.outputs.duplicate_issues_csv != '' && + ( + github.event_name == 'issues' || + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + contains(github.event.comment.body, '@gemini-cli /deduplicate') && + ( + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'COLLABORATOR' + ) + ) + ) + permissions: + issues: 'write' + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + steps: + - name: 'Generate GitHub App Token' + id: 'generate_token' + uses: 'actions/create-github-app-token@a8d616148505b5069dccd32f177bb87d7f39123b' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ secrets.APP_ID }}' + private-key: '${{ secrets.PRIVATE_KEY }}' + permission-issues: 'write' + + - name: 'Comment and Label Duplicate Issue' + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + env: + DUPLICATES_OUTPUT: '${{ needs.find-duplicates.outputs.duplicate_issues_csv }}' + with: + github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + script: |- + const rawCsv = process.env.DUPLICATES_OUTPUT; + core.info(`Raw duplicates CSV: ${rawCsv}`); + const duplicateIssues = rawCsv.split(',').map(s => s.trim()).filter(s => s); + + if (duplicateIssues.length === 0) { + core.info('No duplicate issues found. Nothing to do.'); + return; + } + + const issueNumber = ${{ github.event.issue.number }}; + + function formatCommentBody(issues, updated = false) { + const header = updated + ? 'Found possible duplicate issues (updated):' + : 'Found possible duplicate issues:'; + const issuesList = issues.map(num => `- #${num}`).join('\n'); + const footer = 'If you believe this is not a duplicate, please remove the `status/possible-duplicate` label.'; + const magicComment = ''; + return `${header}\n\n${issuesList}\n\n${footer}\n${magicComment}`; + } + + const newCommentBody = formatCommentBody(duplicateIssues); + const newUpdatedCommentBody = formatCommentBody(duplicateIssues, true); + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + }); + + const magicComment = ''; + const existingComment = comments.find(comment => + comment.user.type === 'Bot' && comment.body.includes(magicComment) + ); + + let commentMade = false; + + if (existingComment) { + // To check if lists are same, just compare the formatted bodies without headers. + const existingBodyForCompare = existingComment.body.substring(existingComment.body.indexOf('- #')); + const newBodyForCompare = newCommentBody.substring(newCommentBody.indexOf('- #')); + + if (existingBodyForCompare.trim() !== newBodyForCompare.trim()) { + core.info(`Updating existing comment ${existingComment.id}`); + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body: newUpdatedCommentBody, + }); + commentMade = true; + } else { + core.info('Existing comment is up-to-date. Nothing to do.'); + } + } else { + core.info('Creating new comment.'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: newCommentBody, + }); + commentMade = true; + } + + if (commentMade) { + core.info('Adding "status/possible-duplicate" label.'); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['status/possible-duplicate'], + }); + } diff --git a/.github/workflows/gemini-scheduled-issue-dedup.yml b/.github/workflows/gemini-scheduled-issue-dedup.yml new file mode 100644 index 00000000000..9eea5e0aa02 --- /dev/null +++ b/.github/workflows/gemini-scheduled-issue-dedup.yml @@ -0,0 +1,116 @@ +name: '📋 Gemini Scheduled Issue Deduplication' + +on: + schedule: + - cron: '0 * * * *' # Runs every hour + workflow_dispatch: + +concurrency: + group: '${{ github.workflow }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +jobs: + refresh-embeddings: + if: |- + ${{ vars.TRIAGE_DEDUPLICATE_ISSUES != '' && github.repository == 'google-gemini/gemini-cli' }} + permissions: + contents: 'read' + id-token: 'write' # Required for WIF, see https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-google-cloud-platform#adding-permissions-settings + issues: 'read' + statuses: 'read' + packages: 'read' + timeout-minutes: 20 + runs-on: 'ubuntu-latest' + steps: + - name: 'Checkout' + uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5 + + - name: 'Log in to GitHub Container Registry' + uses: 'docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1' # ratchet:docker/login-action@v3 + with: + registry: 'ghcr.io' + username: '${{ github.actor }}' + password: '${{ secrets.GITHUB_TOKEN }}' + + - name: 'Run Gemini Issue Deduplication Refresh' + uses: 'google-github-actions/run-gemini-cli@a3bf79042542528e91937b3a3a6fbc4967ee3c31' # ratchet:google-github-actions/run-gemini-cli@v0 + id: 'gemini_refresh_embeddings' + env: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + ISSUE_TITLE: '${{ github.event.issue.title }}' + ISSUE_BODY: '${{ github.event.issue.body }}' + ISSUE_NUMBER: '${{ github.event.issue.number }}' + REPOSITORY: '${{ github.repository }}' + FIRESTORE_PROJECT: '${{ vars.FIRESTORE_PROJECT }}' + with: + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "mcpServers": { + "issue_deduplication": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "--network", "host", + "-e", "GITHUB_TOKEN", + "-e", "GEMINI_API_KEY", + "-e", "DATABASE_TYPE", + "-e", "FIRESTORE_DATABASE_ID", + "-e", "GCP_PROJECT", + "-e", "GOOGLE_APPLICATION_CREDENTIALS=/app/gcp-credentials.json", + "-v", "${GOOGLE_APPLICATION_CREDENTIALS}:/app/gcp-credentials.json", + "ghcr.io/google-gemini/gemini-cli-issue-triage@sha256:e3de1523f6c83aabb3c54b76d08940a2bf42febcb789dd2da6f95169641f94d3" + ], + "env": { + "GITHUB_TOKEN": "${GITHUB_TOKEN}", + "GEMINI_API_KEY": "${{ secrets.GEMINI_API_KEY }}", + "DATABASE_TYPE":"firestore", + "GCP_PROJECT": "${FIRESTORE_PROJECT}", + "FIRESTORE_DATABASE_ID": "(default)", + "GOOGLE_APPLICATION_CREDENTIALS": "${GOOGLE_APPLICATION_CREDENTIALS}" + }, + "enabled": true, + "timeout": 600000 + } + }, + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)" + ], + "telemetry": { + "enabled": true, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are a database maintenance assistant for a GitHub issue deduplication system. + + ## Goal + + Your sole responsibility is to refresh the embeddings for all open issues in the repository to ensure the deduplication database is up-to-date. + + ## Steps + + 1. **Extract Repository Information:** The repository is ${{ github.repository }}. + 2. **Refresh Embeddings:** Call the `refresh` tool with the correct `repo`. Do not use the `force` parameter. + 3. **Log Output:** Print the JSON output from the `refresh` tool to the logs. + + ## Guidelines + + - Only use the `refresh` tool. + - Do not attempt to find duplicates or modify any issues. + - Your only task is to call the `refresh` tool and log its output. diff --git a/.github/workflows/gemini-self-assign-issue.yml b/.github/workflows/gemini-self-assign-issue.yml new file mode 100644 index 00000000000..40e6353f8df --- /dev/null +++ b/.github/workflows/gemini-self-assign-issue.yml @@ -0,0 +1,99 @@ +name: 'Assign Issue on Comment' + +on: + issue_comment: + types: + - 'created' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + statuses: 'write' + packages: 'read' + +jobs: + self-assign-issue: + if: |- + github.repository == 'google-gemini/gemini-cli' && + github.event_name == 'issue_comment' && + contains(github.event.comment.body, '/assign') + runs-on: 'ubuntu-latest' + steps: + - name: 'Generate GitHub App Token' + id: 'generate_token' + uses: 'actions/create-github-app-token@a8d616148505b5069dccd32f177bb87d7f39123b' + with: + app-id: '${{ secrets.APP_ID }}' + private-key: '${{ secrets.PRIVATE_KEY }}' + # Add 'assignments' write permission + permission-issues: 'write' + + - name: 'Assign issue to user' + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + with: + github-token: '${{ steps.generate_token.outputs.token }}' + script: | + const issueNumber = context.issue.number; + const commenter = context.actor; + const owner = context.repo.owner; + const repo = context.repo.repo; + const MAX_ISSUES_ASSIGNED = 3; + + // Search for open issues already assigned to the commenter in this repo + const { data: assignedIssues } = await github.rest.search.issuesAndPullRequests({ + q: `is:issue repo:${owner}/${repo} assignee:${commenter} is:open`, + advanced_search: true + }); + + if (assignedIssues.total_count >= MAX_ISSUES_ASSIGNED) { + await github.rest.issues.createComment({ + owner: owner, + repo: repo, + issue_number: issueNumber, + body: `👋 @${commenter}! You currently have ${assignedIssues.total_count} issues assigned to you. We have a ${MAX_ISSUES_ASSIGNED} max issues assigned at once policy. Once you close out an existing issue it will open up space to take another. You can also unassign yourself from an existing issue but please work on a hand-off if someone is expecting work on that issue.` + }); + return; // exit + } + + // Check if the issue is already assigned + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + }); + + if (issue.data.assignees.length > 0) { + // Comment that it's already assigned + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: `@${commenter} Thanks for taking interest but this issue is already assigned. We'd still love to have you contribute. Check out our [Help Wanted](https://github.com/google-gemini/gemini-cli/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22) list for issues where we need some extra attention.` + }); + return; + } + + // If not taken, assign the user who commented + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + assignees: [commenter] + }); + + // Post a comment to confirm assignment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: `👋 @${commenter}, you've been assigned to this issue! Thank you for taking the time to contribute. Make sure to check out our [contributing guidelines](https://github.com/google-gemini/gemini-cli/blob/main/CONTRIBUTING.md).` + }); diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml new file mode 100644 index 00000000000..abaad9dbbfe --- /dev/null +++ b/.github/workflows/no-response.yml @@ -0,0 +1,33 @@ +name: 'No Response' + +# Run as a daily cron at 1:45 AM +on: + schedule: + - cron: '45 1 * * *' + workflow_dispatch: + +jobs: + no-response: + runs-on: 'ubuntu-latest' + if: |- + ${{ github.repository == 'google-gemini/gemini-cli' }} + permissions: + issues: 'write' + pull-requests: 'write' + concurrency: + group: '${{ github.workflow }}-no-response' + cancel-in-progress: true + steps: + - uses: 'actions/stale@5bef64f19d7facfb25b37b414482c7164d639639' # ratchet:actions/stale@v9 + with: + repo-token: '${{ secrets.GITHUB_TOKEN }}' + days-before-stale: -1 + days-before-close: 14 + stale-issue-label: 'status/need-information' + close-issue-message: >- + This issue was marked as needing more information and has not received a response in 14 days. + Closing it for now. If you still face this problem, feel free to reopen with more details. Thank you! + stale-pr-label: 'status/need-information' + close-pr-message: >- + This pull request was marked as needing more information and has had no updates in 14 days. + Closing it for now. You are welcome to reopen with the required info. Thanks for contributing! diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59e7dac8361..a0b0365ac94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,7 @@ jobs: steps: - name: 'Checkout' - uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 with: ref: '${{ github.event.inputs.ref || github.sha }}' fetch-depth: 0 @@ -89,7 +89,7 @@ jobs: echo "is_dry_run=${is_dry_run}" >> "${GITHUB_OUTPUT}" - name: 'Setup Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -153,13 +153,13 @@ jobs: steps: - name: 'Checkout' - uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 with: ref: '${{ github.event.inputs.ref || github.sha }}' fetch-depth: 0 - name: 'Setup Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -206,13 +206,13 @@ jobs: steps: - name: 'Checkout' - uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 with: ref: '${{ github.event.inputs.ref || github.sha }}' fetch-depth: 0 - name: 'Setup Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -247,13 +247,13 @@ jobs: steps: - name: 'Checkout' - uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 with: ref: '${{ github.event.inputs.ref || github.sha }}' fetch-depth: 0 - name: 'Setup Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -317,13 +317,13 @@ jobs: steps: - name: 'Checkout' - uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 with: ref: '${{ github.event.inputs.ref || github.sha }}' fetch-depth: 0 - name: 'Setup Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -400,11 +400,16 @@ jobs: env: NODE_AUTH_TOKEN: '${{ secrets.NPM_TOKEN }}' + - name: 'Verify Standalone Archives' + run: |- + npm run verify:installation-release -- --dir dist/standalone + - name: 'Create GitHub Release and Tag' if: |- ${{ needs.prepare.outputs.is_dry_run == 'false' }} env: - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + # CI_BOT_PAT required: GITHUB_TOKEN events cannot trigger downstream workflows (sync-release-to-oss.yml). + GITHUB_TOKEN: '${{ secrets.CI_BOT_PAT }}' RELEASE_BRANCH: '${{ steps.release_branch.outputs.BRANCH_NAME }}' RELEASE_TAG: '${{ needs.prepare.outputs.release_tag }}' PREVIOUS_RELEASE_TAG: '${{ needs.prepare.outputs.previous_release_tag }}' diff --git a/.github/workflows/sync-release-to-oss.yml b/.github/workflows/sync-release-to-oss.yml new file mode 100644 index 00000000000..c2eee4c5fb4 --- /dev/null +++ b/.github/workflows/sync-release-to-oss.yml @@ -0,0 +1,238 @@ +name: 'Sync Release to Aliyun OSS' + +on: + release: + types: ['published'] + workflow_dispatch: + inputs: + tag: + description: 'The release tag to sync (e.g., v0.1.11).' + required: true + type: 'string' + +concurrency: + group: 'sync-release-to-oss' + cancel-in-progress: false + +jobs: + sync: + name: 'Sync Release Assets to Aliyun OSS' + runs-on: 'ubuntu-latest' + if: |- + ${{ github.repository == 'QwenLM/qwen-code' }} + environment: + name: 'production-release' + permissions: + contents: 'read' + + env: + RELEASE_TAG: '${{ github.event.release.tag_name || inputs.tag }}' + + steps: + - name: 'Checkout' + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + with: + ref: '${{ env.RELEASE_TAG }}' + + - name: 'Determine release type' + id: 'meta' + env: + TAG: '${{ env.RELEASE_TAG }}' + run: |- + is_nightly="false" + is_preview="false" + if [[ "${TAG}" == *"nightly"* ]]; then + is_nightly="true" + elif [[ "${TAG}" == *"preview"* ]]; then + is_preview="true" + fi + echo "is_nightly=${is_nightly}" >> "${GITHUB_OUTPUT}" + echo "is_preview=${is_preview}" >> "${GITHUB_OUTPUT}" + echo "is_stable=$([[ ${is_nightly} == 'false' && ${is_preview} == 'false' ]] && echo true || echo false)" >> "${GITHUB_OUTPUT}" + + - name: 'Setup Node.js' + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + with: + node-version-file: '.nvmrc' + cache: 'npm' + cache-dependency-path: 'package-lock.json' + + - name: 'Install Dependencies' + env: + NPM_CONFIG_PREFER_OFFLINE: 'true' + run: |- + npm ci --no-audit --progress=false + + - name: 'Download Release Assets from GitHub' + env: + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + mkdir -p dist/standalone + gh release download "${RELEASE_TAG}" --dir dist/standalone --pattern '*.tar.gz' --pattern '*.zip' --pattern 'SHA256SUMS' + + - name: 'Verify Downloaded Release Assets' + run: |- + npm run verify:installation-release -- --dir dist/standalone + + - name: 'Package Hosted Installation Assets' + if: |- + ${{ steps.meta.outputs.is_stable == 'true' }} + env: + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + RELEASE_VERSION="${RELEASE_TAG#v}" + npm run package:hosted-installation -- --out-dir dist/installation --version "${RELEASE_VERSION}" + + - name: 'Install ossutil' + env: + OSSUTIL_URL: "${{ vars.OSSUTIL_URL || 'https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip' }}" + OSSUTIL_SHA256: "${{ vars.OSSUTIL_SHA256 || 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343' }}" + run: |- + set -euo pipefail + + tmp_dir="$(mktemp -d)" + curl -fsSL --connect-timeout 15 --max-time 300 "${OSSUTIL_URL}" -o "${tmp_dir}/ossutil.zip" + echo "${OSSUTIL_SHA256} ${tmp_dir}/ossutil.zip" | sha256sum -c - + unzip -q "${tmp_dir}/ossutil.zip" -d "${tmp_dir}" + + ossutil_path="$(find "${tmp_dir}" -type f \( -name 'ossutil' -o -name 'ossutil64' \) -print -quit)" + if [[ -z "${ossutil_path}" ]]; then + echo "::error::ossutil binary not found in downloaded archive" + exit 1 + fi + + chmod +x "${ossutil_path}" + mkdir -p "${HOME}/.local/bin" + install -m 0755 "${ossutil_path}" "${HOME}/.local/bin/ossutil" + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + rm -rf "${tmp_dir}" + "${HOME}/.local/bin/ossutil" >/dev/null + + - name: 'Configure Aliyun OSS Credentials' + env: + ALIYUN_OSS_ACCESS_KEY_ID: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}' + ALIYUN_OSS_ACCESS_KEY_SECRET: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}' + ALIYUN_OSS_ENDPOINT: "${{ vars.ALIYUN_OSS_ENDPOINT || 'https://oss-cn-hangzhou.aliyuncs.com' }}" + run: |- + set -euo pipefail + + if [[ -z "${ALIYUN_OSS_ACCESS_KEY_ID}" || -z "${ALIYUN_OSS_ACCESS_KEY_SECRET}" ]]; then + echo "::error::Missing Aliyun OSS credentials. Set ALIYUN_OSS_ACCESS_KEY_ID and ALIYUN_OSS_ACCESS_KEY_SECRET in the production-release environment secrets." + exit 1 + fi + + ossutil config \ + -e "${ALIYUN_OSS_ENDPOINT}" \ + -i "${ALIYUN_OSS_ACCESS_KEY_ID}" \ + -k "${ALIYUN_OSS_ACCESS_KEY_SECRET}" \ + -L EN \ + -c "${RUNNER_TEMP}/.ossutilconfig" + + - name: 'Sync Release Assets to Aliyun OSS' + env: + ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}" + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + mapfile -t release_assets < <(node scripts/verify-installation-release.js --dir dist/standalone --list-release-asset-paths) + node scripts/upload-aliyun-oss-assets.js \ + --bucket "${ALIYUN_OSS_BUCKET}" \ + --config "${RUNNER_TEMP}/.ossutilconfig" \ + --prefix "releases/qwen-code/${RELEASE_TAG}" \ + "${release_assets[@]}" + + - name: 'Verify Aliyun OSS Release Assets' + env: + ALIYUN_OSS_PUBLIC_BASE_URL: "${{ vars.ALIYUN_OSS_PUBLIC_BASE_URL || 'https://qwen-code-assets.oss-cn-hangzhou.aliyuncs.com' }}" + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + npm run verify:installation-release -- --base-url "${ALIYUN_OSS_PUBLIC_BASE_URL}/releases/qwen-code/${RELEASE_TAG}" + + - name: 'Sync Hosted Installation Assets to Aliyun OSS' + if: |- + ${{ steps.meta.outputs.is_stable == 'true' }} + env: + ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}" + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + hosted_assets=( + dist/installation/install-qwen-standalone.sh + dist/installation/install-qwen-standalone.ps1 + dist/installation/install-qwen-standalone.bat + dist/installation/uninstall-qwen-standalone.sh + dist/installation/uninstall-qwen-standalone.ps1 + dist/installation/SHA256SUMS + ) + node scripts/upload-aliyun-oss-assets.js \ + --bucket "${ALIYUN_OSS_BUCKET}" \ + --config "${RUNNER_TEMP}/.ossutilconfig" \ + --prefix "installation/${RELEASE_TAG}" \ + "${hosted_assets[@]}" + node scripts/upload-aliyun-oss-assets.js \ + --bucket "${ALIYUN_OSS_BUCKET}" \ + --config "${RUNNER_TEMP}/.ossutilconfig" \ + --prefix "installation" \ + "${hosted_assets[@]}" + + - name: 'Verify Aliyun OSS Hosted Installation Assets' + if: |- + ${{ steps.meta.outputs.is_stable == 'true' }} + env: + ALIYUN_OSS_PUBLIC_BASE_URL: "${{ vars.ALIYUN_OSS_PUBLIC_BASE_URL || 'https://qwen-code-assets.oss-cn-hangzhou.aliyuncs.com' }}" + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + hosted_tmp_dir="$(mktemp -d)" + trap 'rm -rf "${hosted_tmp_dir}"' EXIT + mkdir -p "${hosted_tmp_dir}/versioned" "${hosted_tmp_dir}/global" + for asset in install-qwen-standalone.sh install-qwen-standalone.ps1 install-qwen-standalone.bat uninstall-qwen-standalone.sh uninstall-qwen-standalone.ps1 SHA256SUMS; do + url="${ALIYUN_OSS_PUBLIC_BASE_URL}/installation/${RELEASE_TAG}/${asset}" + global_url="${ALIYUN_OSS_PUBLIC_BASE_URL}/installation/${asset}" + curl -fsSL --connect-timeout 15 --max-time 300 "${url}" -o "${hosted_tmp_dir}/versioned/${asset}" + curl -fsSL --connect-timeout 15 --max-time 300 "${global_url}" -o "${hosted_tmp_dir}/global/${asset}" + done + cmp -s "dist/installation/SHA256SUMS" "${hosted_tmp_dir}/versioned/SHA256SUMS" || { + echo "::error::Hosted installation SHA256SUMS does not match local dist/installation/SHA256SUMS" + diff -u "dist/installation/SHA256SUMS" "${hosted_tmp_dir}/versioned/SHA256SUMS" || true + exit 1 + } + cmp -s "dist/installation/SHA256SUMS" "${hosted_tmp_dir}/global/SHA256SUMS" || { + echo "::error::Global hosted installation SHA256SUMS does not match local dist/installation/SHA256SUMS" + diff -u "dist/installation/SHA256SUMS" "${hosted_tmp_dir}/global/SHA256SUMS" || true + exit 1 + } + (cd "${hosted_tmp_dir}/versioned" && sha256sum -c SHA256SUMS) + (cd "${hosted_tmp_dir}/global" && sha256sum -c SHA256SUMS) + + - name: 'Publish Aliyun OSS Latest VERSION' + if: |- + ${{ steps.meta.outputs.is_stable == 'true' }} + env: + ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}" + ALIYUN_OSS_PUBLIC_BASE_URL: "${{ vars.ALIYUN_OSS_PUBLIC_BASE_URL || 'https://qwen-code-assets.oss-cn-hangzhou.aliyuncs.com' }}" + RELEASE_TAG: '${{ env.RELEASE_TAG }}' + run: |- + set -euo pipefail + + printf '%s\n' "${RELEASE_TAG}" > "${RUNNER_TEMP}/qwen-code-latest-version" + ossutil cp "${RUNNER_TEMP}/qwen-code-latest-version" "oss://${ALIYUN_OSS_BUCKET}/releases/qwen-code/latest/VERSION" -c "${RUNNER_TEMP}/.ossutilconfig" -f --acl public-read + + latest_version="$(curl -fsSL --connect-timeout 15 --max-time 300 "${ALIYUN_OSS_PUBLIC_BASE_URL}/releases/qwen-code/latest/VERSION" | tr -d '[:space:]')" + if [[ "${latest_version}" != "${RELEASE_TAG}" ]]; then + echo "::error::Aliyun latest VERSION points to ${latest_version}, expected ${RELEASE_TAG}" + exit 1 + fi + + - name: 'Cleanup Aliyun OSS Credentials' + if: '${{ always() }}' + run: |- + rm -f "${RUNNER_TEMP}/.ossutilconfig" diff --git a/.gitignore b/.gitignore index 6ff1d950be2..89e08e3f9fe 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,9 @@ bundle junit.xml packages/*/coverage/ +# PR body draft +pr_body.md + # Generated files packages/cli/src/generated/ packages/core/src/generated/ @@ -64,6 +67,7 @@ packages/web-templates/src/generated/ packages/vscode-ide-companion/*.vsix logs/ +.repro-runs/ # GHA credentials gha-creds-*.json @@ -93,4 +97,4 @@ tmp/ # code graph skills .venv -.codegraph \ No newline at end of file +.codegraph diff --git a/.qwen/design/2026-05-21-memory-pressure-monitor-design.md b/.qwen/design/2026-05-21-memory-pressure-monitor-design.md new file mode 100644 index 00000000000..13da8c6e346 --- /dev/null +++ b/.qwen/design/2026-05-21-memory-pressure-monitor-design.md @@ -0,0 +1,136 @@ +--- +title: 'Memory Pressure Monitor' +date: '2026-05-21' +status: 'implemented' +--- + +# Memory Pressure Monitor + +## Problem + +Long-running Qwen Code sessions can accumulate memory through large tool +results, repeated file reads, chat history, and native/external allocations. +Before this change, the core package had diagnostics and session-reset cleanup, +but no runtime response when memory pressure rises during normal tool +execution. + +The highest-value cache-specific gap is `FileReadCache`: it already has a +bounded FIFO size, but it did not have a time-based eviction path. That means a +session can retain inactive file-read metadata until the hard entry limit is +hit, even when the process is under memory pressure. + +## Goals + +- Add a low-overhead memory pressure check after tool execution. +- Prefer surgical cleanup before destructive cleanup. +- Respect container memory limits when cgroup v2 or cgroup v1 memory limit + files are available. +- React to V8 heap pressure before JavaScript heap OOM on high-memory hosts. +- Keep subagent/scoped `Config` instances isolated from parent session cleanup. +- Make behavior configurable through environment variables without adding a new + user-facing settings surface. + +## Non-Goals + +- Do not add a background polling loop. +- Do not make explicit GC the default; it only runs when enabled and Node was + started with `--expose-gc`. +- Do not change prior-read enforcement semantics. Cache eviction can remove old + metadata, but it must not weaken stale-file checks for retained entries. + +## Design + +`Config.initialize()` creates one `MemoryPressureMonitor` per initialized +`Config`. `getMemoryPressureMonitor()` mirrors the existing `getFileReadCache()` +Object.create isolation pattern: when a child config is created through +prototype delegation, the getter lazily installs an own monitor bound to that +child config. + +`CoreToolScheduler.executeSingleToolCall()` calls `scheduleCheck()` in its +`finally` block after ending the tool span. `scheduleCheck()` coalesces multiple +calls in the same event-loop turn with `queueMicrotask`, so concurrent read-like +tool batches do not run one memory check per tool result. + +The monitor uses the stronger of two pressure signals: + +- RSS divided by an effective process memory limit. Prefer cgroup v2 + `/sys/fs/cgroup/memory.max` when it is a finite positive value; fall back to + cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes`, then to + `os.totalmem()` otherwise. cgroup v1's huge "unlimited" sentinel values are + ignored. +- V8 `heapUsed` divided by `getHeapStatistics().heap_size_limit`. + +Using both signals matters because containers usually fail by RSS/cgroup limit, +while local high-memory machines can hit V8 heap OOM long before RSS is a large +fraction of total system memory. + +Default thresholds are intentionally conservative enough to react before the OS +or container OOM killer does: + +- `softPressureRatio = 0.50` +- `hardPressureRatio = 0.65` +- `criticalRatio = 0.80` +- `cleanupCooldownMs = 5000` +- `enableExplicitGC = false` + +Environment overrides: + +- `QWEN_MEMORY_PRESSURE_SOFT` +- `QWEN_MEMORY_PRESSURE_HARD` +- `QWEN_MEMORY_PRESSURE_CRITICAL` +- `QWEN_MEMORY_ENABLE_GC=1` + +Invalid ratios fall back to defaults. Valid ratios must be ordered as +`soft < hard < critical`, with a lower soft bound of `0.3` and an upper +critical bound of `0.98`. Ratio env vars are parsed strictly with `Number()`, +so values such as `0.8extra` are rejected instead of partially accepted. +Invalid memory-pressure env configuration writes a visible warning to stderr +and to the debug log before falling back to defaults. + +## Cleanup Policy + +Pressure levels map to increasingly strong cleanup: + +- `soft`: evict stale `FileReadCache` entries not accessed in 60 minutes. +- `hard`: evict cache entries not accessed in 30 minutes. +- `critical`: clear the file-read cache and optionally trigger `global.gc()`. + +The monitor intentionally does not force chat compaction. Compaction can call +the model backend and rewrite active chat state, so it should be triggered only +from a call site that can safely coordinate with the conversation loop. + +Cleanup is fire-and-forget from the scheduler, but the monitor guards cleanup +steps with `cleanupInProgress` and a cooldown timestamp. A higher-pressure +cleanup can bypass the cooldown and queue behind an in-progress lower-pressure +cleanup, so a `critical` check is not lost while a `soft` cleanup is finishing. +After successful cleanup it logs an RSS delta on `setImmediate()`, but RSS +movement is diagnostic only: V8 and libc may retain freed pages even when +JavaScript objects became collectible. Consecutive failures count cleanup-step +exceptions, not unchanged RSS, and the counter is reset on a new session. If +three successful cleanup attempts in a row free less than 1% RSS, the monitor +emits `memory-cleanup-ineffective` as a diagnostic signal without treating the +cleanup step itself as failed. + +## Test Coverage + +The implementation is covered by: + +- threshold validation tests; +- environment config parsing, fallback, visible warning, and explicit GC tests; +- pressure classification tests using mocked `process.memoryUsage()`; +- cgroup v2 `memory.max` and cgroup v1 `memory.limit_in_bytes` behavior; +- V8 heap limit behavior; +- `scheduleCheck()` coalescing; +- scheduler integration that invokes `scheduleCheck()` after tool execution; +- soft and critical cleanup actions; +- cleanup failure accounting for thrown cleanup steps; +- cleanup listener exception isolation and ineffective-cleanup diagnostics; +- child `Config` monitor isolation through `Object.create`; +- `FileReadCache.evictNotAccessedSince()` behavior. + +## Risks And Tradeoffs + +- RSS can stay flat after cleanup because V8 or libc may retain freed memory. + RSS deltas are logged, but unchanged RSS does not count as a cleanup failure. +- Time-based file-read cache eviction may reduce fast-path hits for old files, + but it preserves recently active entries and only runs under memory pressure. diff --git a/.qwen/skills/agent-reproduce-align/SKILL.md b/.qwen/skills/agent-reproduce-align/SKILL.md new file mode 100644 index 00000000000..ebf2b60ba8b --- /dev/null +++ b/.qwen/skills/agent-reproduce-align/SKILL.md @@ -0,0 +1,98 @@ +--- +name: agent-reproduce-align +description: Use after a Codex or Claude Code feature has been implemented in Qwen Code to run the selected reference agent and Qwen Code under the same scenario, capture HTTP and terminal traces, compare request bodies, tool/function schemas, outputs, and iterate until the reproduced behavior is close enough. +--- + +# Agent Reproduce Align + +## Purpose + +Use this skill when Qwen Code already has a candidate implementation and needs evidence-based parity with a selected reference agent: `codex` or `claude-code`. The goal is not byte-for-byte equality; it is matching the observable contract that matters for the feature. + +Default target repo: the current working directory. Use a user-specified path only when the user explicitly provides one. + +## Reference Agent Selection + +Use the same reference agent selected during `$agent-reproduce-feature`. If the earlier choice is unavailable, ask once and record the answer in the scenario or run notes. + +## Workflow + +1. Re-state the parity target: + - feature name and trigger + - selected reference agent + - one baseline prompt or interaction script + - acceptable differences + - must-match fields +2. Run the reference agent and Qwen Code in separate capture directories with the same scenario. +3. Capture the selected reference agent's local state before and after the + reference run when state may affect parity. +4. Normalize traces with `scripts/normalize_trace.py`. +5. Compare normalized traces with `scripts/compare_traces.py`. +6. Inspect differences in this order: + - reference-agent state changes that explain behavior + - missing tool/function names + - schema shape and required fields + - model settings and response mode + - prompt role/order differences that affect behavior + - terminal-visible output and exit status +7. Patch Qwen Code, rerun the smallest failing scenario, and repeat. +8. Preserve only redacted minimal fixtures in the repo. + +Read `references/alignment-workflow.md` before the first comparison pass. + +## Common Commands + +Normalize: + +```sh +.qwen/skills/agent-reproduce-align/scripts/normalize_trace.py \ + .repro-runs/reference/http.jsonl \ + > .repro-runs/reference/normalized.json +``` + +Compare: + +```sh +.qwen/skills/agent-reproduce-align/scripts/compare_traces.py \ + .repro-runs/reference/normalized.json \ + .repro-runs/qwen/normalized.json +``` + +Run a paired shell scenario: + +```sh +REPRO_REFERENCE_AGENT=codex \ +.qwen/skills/agent-reproduce-align/scripts/run_pair_capture.sh \ + .repro-runs/slash-help \ + "codex exec '/help'" \ + "npm test -- --runInBand" +``` + +For Claude Code, set `REPRO_REFERENCE_AGENT=claude-code` and replace the first +command with the discovered Claude Code command. When `REPRO_REFERENCE_AGENT` +is set, the paired runner writes `reference/state-before`, +`reference/state-after`, and `reference/state-diff`. Use the paired runner only +when shell quoting is simple. For interactive slash commands, run the two +captures manually with tmux so each side can receive the same keystrokes. Use +`REPRO_REFERENCE_STATE_ROOT=/tmp/some-root` only for tests or custom state +directories. + +## Comparison Rules + +- Compare contracts before wording. Exact prompt text is usually implementation detail. +- Treat absent schemas, wrong required fields, or wrong argument names as high-signal failures. +- Treat output ordering as significant only when the user-visible workflow depends on it. +- Do not chase provider-specific endpoints, model names, IDs, timestamps, token counts, or ephemeral headers unless the feature depends on them. +- Do not chase every local state write. Treat state diffs as explanatory + evidence unless the feature contract requires a particular config, memory, or + permission side effect. +- Stop when Qwen Code passes the user-visible scenario and the remaining trace differences are documented as intentional. + +## Done Criteria + +- Reference-agent and Qwen Code traces for the same scenario exist locally. +- Reference-agent state diff exists or state capture is documented as + irrelevant for the scenario. +- The normalized comparison has no unexplained must-match differences. +- Qwen Code tests or smoke commands cover the fixed behavior. +- Any remaining mismatch is written down in the task notes or Qwen Code docs when it affects users. diff --git a/.qwen/skills/agent-reproduce-align/references/alignment-workflow.md b/.qwen/skills/agent-reproduce-align/references/alignment-workflow.md new file mode 100644 index 00000000000..f22523f4e8f --- /dev/null +++ b/.qwen/skills/agent-reproduce-align/references/alignment-workflow.md @@ -0,0 +1,84 @@ +# Alignment Workflow Reference + +The alignment phase starts after Qwen Code has a candidate implementation. Use it to create a tight loop: run the selected reference agent and Qwen Code, compare traces, patch the target, and rerun only the failing scenario. + +## Trace Inputs + +Expected raw capture layout: + +```text +.repro-runs// + reference/ + http.jsonl + command.stdout + command.stderr + command.exit + state-before/state-manifest.json + state-after/state-manifest.json + state-diff/state-diff.md + qwen/ + http.jsonl + command.stdout + command.stderr + command.exit +``` + +Use capture scripts from `$agent-reproduce-feature` for raw capture, or use +`run_pair_capture.sh` for simple non-interactive shell scenarios. Set +`REPRO_REFERENCE_AGENT=codex` or `REPRO_REFERENCE_AGENT=claude-code` with the +paired runner to capture reference-agent state automatically. + +## Normalization + +`normalize_trace.py` reads mitm JSONL output and emits stable JSON: + +- request method and URL path +- JSON request body summary +- message role order and brief content hashes +- tool/function names +- schema required fields +- response status code + +It intentionally drops: + +- timestamps +- authorization and cookie headers +- provider request IDs +- full message text unless needed for a hash + +## Diff Triage + +High priority: + +- missing request entirely +- wrong endpoint family +- missing tool/function schema +- incompatible required fields or enum values +- slash command not routed to the same behavior class +- state changes that prove the feature writes config, memory, permissions, or + another user-visible local store + +Medium priority: + +- prompt role ordering differences +- terminal output phrasing differences +- streaming versus non-streaming if users can observe it +- unexplained state changes that plausibly affect future runs + +Low priority: + +- timestamps, IDs, token counts +- harmless wording differences +- extra target-side metadata ignored by the provider + +## Iteration Loop + +1. Pick the highest-priority unexplained mismatch. +2. Patch only the likely owner module in Qwen Code. +3. Run the focused test/smoke path. +4. Capture only the affected scenario again. +5. Refresh the reference state diff if the suspected mismatch involves local + state. +6. Normalize and compare again. + +Stop when the target behavior is compatible and remaining differences are either irrelevant or explicitly documented. diff --git a/.qwen/skills/agent-reproduce-align/scripts/compare_traces.py b/.qwen/skills/agent-reproduce-align/scripts/compare_traces.py new file mode 100755 index 00000000000..740647460b4 --- /dev/null +++ b/.qwen/skills/agent-reproduce-align/scripts/compare_traces.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Compare normalized reproduction traces and print actionable differences.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def load(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def tool_index(request: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + tool.get("name") or f"": tool + for idx, tool in enumerate(request.get("tools") or []) + } + + +def tool_name_counts(request: dict[str, Any]) -> dict[str, int]: + counts: dict[str, int] = {} + for idx, tool in enumerate(request.get("tools") or []): + name = tool.get("name") or f"" + counts[name] = counts.get(name, 0) + 1 + return counts + + +def compare_request(idx: int, left: dict[str, Any], right: dict[str, Any]) -> list[str]: + diffs: list[str] = [] + prefix = f"request[{idx}]" + for key in ( + "method", + "url_path", + "body_keys", + "body_values", + "model", + "stream", + "response_status", + ): + if left.get(key) != right.get(key): + diffs.append(f"{prefix}.{key}: {left.get(key)!r} != {right.get(key)!r}") + + left_messages = left.get("messages") or [] + right_messages = right.get("messages") or [] + left_roles = [item.get("role") for item in left_messages] + right_roles = [item.get("role") for item in right_messages] + if left_roles != right_roles: + diffs.append(f"{prefix}.message_roles: {left_roles!r} != {right_roles!r}") + # Surface count mismatches explicitly. zip() below silently truncates to the + # shorter list, so without this diagnostic an extra trailing message + # carrying the feature-relevant prompt / tool result would never be + # reported (the message_roles diff alone hides which side is longer and by + # how much, and only fires when the *prefix* roles differ at some index). + if len(left_messages) != len(right_messages): + diffs.append( + f"{prefix}.message_count: {len(left_messages)} != {len(right_messages)}" + ) + for msg_idx, (left_msg, right_msg) in enumerate(zip(left_messages, right_messages)): + if left_msg.get("content_hash") != right_msg.get("content_hash"): + diffs.append( + f"{prefix}.messages[{msg_idx}].content_hash: " + f"{left_msg.get('content_hash')!r} != " + f"{right_msg.get('content_hash')!r}" + ) + # Mirror the request-level missing/extra handling so the user sees the + # actual content of trailing messages that fell off the zip(). + if len(left_messages) > len(right_messages): + for msg_idx, message in enumerate( + left_messages[len(right_messages) :], len(right_messages) + ): + diffs.append(f"{prefix}.messages[{msg_idx}].missing_in_right: {message!r}") + elif len(right_messages) > len(left_messages): + for msg_idx, message in enumerate( + right_messages[len(left_messages) :], len(left_messages) + ): + diffs.append(f"{prefix}.messages[{msg_idx}].extra_in_right: {message!r}") + + left_tool_list = left.get("tools") or [] + right_tool_list = right.get("tools") or [] + if len(left_tool_list) != len(right_tool_list): + diffs.append( + f"{prefix}.tools_count: {len(left_tool_list)} != {len(right_tool_list)}" + ) + if tool_name_counts(left) != tool_name_counts(right): + diffs.append( + f"{prefix}.tool_name_counts: " + f"{tool_name_counts(left)!r} != {tool_name_counts(right)!r}" + ) + left_tools = tool_index(left) + right_tools = tool_index(right) + missing = sorted(set(left_tools) - set(right_tools)) + extra = sorted(set(right_tools) - set(left_tools)) + if missing: + diffs.append(f"{prefix}.tools_missing_in_right: {missing}") + if extra: + diffs.append(f"{prefix}.tools_extra_in_right: {extra}") + + for name in sorted(set(left_tools) & set(right_tools)): + for key in ("type", "description_hash", "required", "properties", "schema"): + if left_tools[name].get(key) != right_tools[name].get(key): + diffs.append( + f"{prefix}.tool[{name}].{key}: " + f"{left_tools[name].get(key)!r} != {right_tools[name].get(key)!r}" + ) + return diffs + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("left", type=Path, help="Reference normalized trace") + parser.add_argument("right", type=Path, help="Target normalized trace, usually Qwen Code") + args = parser.parse_args() + + try: + left = load(args.left) + right = load(args.right) + except (OSError, json.JSONDecodeError) as exc: + print(f"Failed to load normalized trace: {exc}", file=sys.stderr) + return 2 + + diffs: list[str] = [] + + if left.get("request_count") != right.get("request_count"): + diffs.append( + f"request_count: {left.get('request_count')!r} != {right.get('request_count')!r}" + ) + + for idx, (left_req, right_req) in enumerate( + zip(left.get("requests") or [], right.get("requests") or []) + ): + diffs.extend(compare_request(idx, left_req, right_req)) + left_requests = left.get("requests") or [] + right_requests = right.get("requests") or [] + if len(left_requests) > len(right_requests): + for idx, request in enumerate(left_requests[len(right_requests) :], len(right_requests)): + diffs.append(f"request[{idx}].missing_in_right: {request!r}") + elif len(right_requests) > len(left_requests): + for idx, request in enumerate(right_requests[len(left_requests) :], len(left_requests)): + diffs.append(f"request[{idx}].extra_in_right: {request!r}") + + if not diffs: + print("No normalized trace differences found.") + return 0 + + print("Normalized trace differences:") + for diff in diffs: + print(f"- {diff}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.qwen/skills/agent-reproduce-align/scripts/normalize_trace.py b/.qwen/skills/agent-reproduce-align/scripts/normalize_trace.py new file mode 100755 index 00000000000..f941aa954b5 --- /dev/null +++ b/.qwen/skills/agent-reproduce-align/scripts/normalize_trace.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Normalize mitm JSONL traces into a stable comparison format.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +def content_hash(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest()[:16] + + +def json_body(record: dict[str, Any]) -> Any: + body = record.get("body") or {} + if body.get("json") is not None: + return body["json"] + text = body.get("text") + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return {"text_hash": content_hash(text), "text_len": len(text)} + + +SCHEMA_KEYS = ( + "type", + "enum", + "const", + "items", + "properties", + "required", + "anyOf", + "allOf", + "oneOf", + "additionalProperties", + "description", + "default", + "examples", + "format", + "minimum", + "maximum", + "minLength", + "maxLength", + "pattern", + "$ref", + "minItems", + "maxItems", + "uniqueItems", + "nullable", +) + +PARITY_BODY_VALUE_KEYS = ( + "model", + "stream", + "temperature", + "max_tokens", + "max_completion_tokens", + "tool_choice", + "top_p", + "top_k", + "n", + "stop", + "response_format", + "seed", + "reasoning_effort", + "parallel_tool_calls", +) + + +def normalize_schema(value: Any) -> Any: + if isinstance(value, dict): + normalized: dict[str, Any] = {} + for key in SCHEMA_KEYS: + if key not in value: + continue + child = value[key] + if key == "required" and isinstance(child, list): + normalized[key] = sorted(str(item) for item in child) + elif key == "properties" and isinstance(child, dict): + normalized[key] = { + str(name): normalize_schema(schema) + for name, schema in sorted(child.items()) + } + elif key in {"anyOf", "allOf", "oneOf"} and isinstance(child, list): + normalized[key] = [normalize_schema(item) for item in child] + else: + normalized[key] = normalize_schema(child) + return normalized + if isinstance(value, list): + return [normalize_schema(item) for item in value] + return value + + +def walk_tools(value: Any) -> list[dict[str, Any]]: + tools: list[dict[str, Any]] = [] + if isinstance(value, dict): + if "tools" in value and isinstance(value["tools"], list): + for tool in value["tools"]: + tools.append(summarize_tool(tool)) + if "functions" in value and isinstance(value["functions"], list): + for fn in value["functions"]: + tools.append(summarize_tool({"type": "function", "function": fn})) + return tools + + +def summarize_tool(tool: Any) -> dict[str, Any]: + if not isinstance(tool, dict): + return {"raw_type": type(tool).__name__} + fn = tool.get("function") if isinstance(tool.get("function"), dict) else tool + params = None + if isinstance(fn, dict): + params = fn.get("parameters") or fn.get("input_schema") + schema = normalize_schema(params) if isinstance(params, dict) else {} + return { + "type": tool.get("type"), + "name": fn.get("name") if isinstance(fn, dict) else None, + "description_hash": content_hash(fn.get("description", "")) + if isinstance(fn, dict) and isinstance(fn.get("description"), str) + else None, + "required": sorted(params.get("required", [])) + if isinstance(params, dict) and isinstance(params.get("required"), list) + else [], + "properties": sorted(params.get("properties", {}).keys()) + if isinstance(params, dict) and isinstance(params.get("properties"), dict) + else [], + "schema": schema, + } + + +def summarize_messages(value: Any) -> list[dict[str, Any]]: + messages = None + system_messages: list[Any] = [] + if isinstance(value, dict): + # Provider conventions for the system prompt: + # - Anthropic Messages API: top-level "system" + # - OpenAI Responses API: top-level "instructions" + # - Gemini / Qwen Code: top-level "systemInstruction" (camelCase) + for key in ("system", "instructions", "systemInstruction"): + if key in value: + system_messages.append(value[key]) + if isinstance(value.get("messages"), list): + messages = value["messages"] + elif isinstance(value.get("input"), list): + messages = value["input"] + if messages is None: + messages = [] + summary = [] + for system in system_messages: + content = ( + system + if isinstance(system, str) + else json.dumps(system, ensure_ascii=False, sort_keys=True) + ) + summary.append( + { + "role": "system", + "content_hash": content_hash(content), + "content_len": len(content), + } + ) + for item in messages: + if not isinstance(item, dict): + continue + content = item.get("content", "") + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False, sort_keys=True) + summary.append( + { + "role": item.get("role"), + "content_hash": content_hash(content), + "content_len": len(content), + } + ) + return summary + + +def summarize_body_values(body: Any) -> dict[str, Any]: + if not isinstance(body, dict): + return {} + return {key: body[key] for key in PARITY_BODY_VALUE_KEYS if key in body} + + +def normalize(path: Path) -> dict[str, Any]: + requests = [] + for line_num, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"Warning: skipping malformed line {line_num} in {path}: {exc}", + file=sys.stderr, + ) + continue + # Valid JSONL lines may decode to non-objects (`[]`, `"hello"`, `42`, + # `null`); those do not have `.get()` and would crash the entire + # normalization with an AttributeError. Skip with a warning instead. + if not isinstance(raw, dict): + print( + f"Warning: skipping non-object line {line_num} in {path}", + file=sys.stderr, + ) + continue + req = raw.get("request") or {} + resp = raw.get("response") or {} + parsed = urlparse(req.get("url", "")) + url_path = parsed.path + if parsed.query: + url_path = f"{url_path}?{parsed.query}" + body = json_body(req) + requests.append( + { + "method": req.get("method"), + "url_path": url_path, + "body_keys": sorted(body.keys()) if isinstance(body, dict) else [], + "body_values": summarize_body_values(body), + "model": body.get("model") if isinstance(body, dict) else None, + "stream": body.get("stream") if isinstance(body, dict) else None, + "messages": summarize_messages(body), + "tools": sorted(walk_tools(body), key=lambda item: (item.get("name") or "")), + "response_status": resp.get("status_code") if isinstance(resp, dict) else None, + } + ) + return {"source": str(path), "request_count": len(requests), "requests": requests} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("trace", type=Path) + args = parser.parse_args() + print(json.dumps(normalize(args.trace), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.qwen/skills/agent-reproduce-align/scripts/run_pair_capture.sh b/.qwen/skills/agent-reproduce-align/scripts/run_pair_capture.sh new file mode 100755 index 00000000000..d4e3c1a65eb --- /dev/null +++ b/.qwen/skills/agent-reproduce-align/scripts/run_pair_capture.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 OUT_DIR REFERENCE_SHELL_COMMAND QWEN_SHELL_COMMAND" >&2 + exit 2 +fi + +out_dir="$1" +reference_command="$2" +qwen_command="$3" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +feature_run="${script_dir}/../../agent-reproduce-feature/scripts/run_with_mitm.sh" +state_capture="${script_dir}/../../agent-reproduce-feature/scripts/capture_state.py" +reference_agent="${REPRO_REFERENCE_AGENT:-}" +reference_state_root="${REPRO_REFERENCE_STATE_ROOT:-}" + +mkdir -p "${out_dir}/reference" "${out_dir}/qwen" + +if [[ -n "${reference_agent}" ]]; then + state_args=(--agent "${reference_agent}") + if [[ -n "${reference_state_root}" ]]; then + state_args+=(--root "${reference_state_root}") + fi + + "${state_capture}" snapshot \ + "${out_dir}/reference/state-before" \ + "${state_args[@]}" +fi + +set +e +"${feature_run}" "${out_dir}/reference" -- bash -lc "${reference_command}" +reference_status=$? +set -e + +if [[ -n "${reference_agent}" ]]; then + "${state_capture}" snapshot \ + "${out_dir}/reference/state-after" \ + "${state_args[@]}" + "${state_capture}" diff \ + "${out_dir}/reference/state-before" \ + "${out_dir}/reference/state-after" \ + --out-dir "${out_dir}/reference/state-diff" +fi + +set +e +"${feature_run}" "${out_dir}/qwen" -- bash -lc "${qwen_command}" +qwen_status=$? +set -e + +set +e +"${script_dir}/normalize_trace.py" "${out_dir}/reference/http.jsonl" \ + > "${out_dir}/reference/normalized.json" \ + 2> "${out_dir}/reference/normalize.err" +normalize_ref_status=$? +"${script_dir}/normalize_trace.py" "${out_dir}/qwen/http.jsonl" \ + > "${out_dir}/qwen/normalized.json" \ + 2> "${out_dir}/qwen/normalize.err" +normalize_qwen_status=$? +set -e + +compare_status=0 +if [[ "${normalize_ref_status}" -ne 0 || "${normalize_qwen_status}" -ne 0 ]]; then + { + echo "Trace normalization failed." + echo "reference_normalize_status=${normalize_ref_status}" + echo "qwen_normalize_status=${normalize_qwen_status}" + echo "reference_normalize_err=${out_dir}/reference/normalize.err" + echo "qwen_normalize_err=${out_dir}/qwen/normalize.err" + } > "${out_dir}/trace.diff" + compare_status=2 +else + request_counts="$( + python3 - "${out_dir}/reference/normalized.json" "${out_dir}/qwen/normalized.json" <<'PY' +import json +import sys + +for path in sys.argv[1:]: + with open(path, encoding="utf-8") as handle: + print(json.load(handle).get("request_count", 0)) +PY + )" + reference_count="$(printf '%s\n' "${request_counts}" | sed -n '1p')" + qwen_count="$(printf '%s\n' "${request_counts}" | sed -n '2p')" + if [[ "${reference_count}" == "0" && "${qwen_count}" == "0" ]]; then + { + echo "Both captures produced empty traces." + echo "reference_http=${out_dir}/reference/http.jsonl" + echo "qwen_http=${out_dir}/qwen/http.jsonl" + } > "${out_dir}/trace.diff" + compare_status=1 + else + set +e + "${script_dir}/compare_traces.py" \ + "${out_dir}/reference/normalized.json" \ + "${out_dir}/qwen/normalized.json" \ + > "${out_dir}/trace.diff" + compare_status=$? + set -e + fi +fi + +echo "reference_status=${reference_status}" +echo "qwen_status=${qwen_status}" +echo "normalize_reference_status=${normalize_ref_status}" +echo "normalize_qwen_status=${normalize_qwen_status}" +echo "compare_status=${compare_status}" +echo "diff=${out_dir}/trace.diff" +echo "reference_stdout=${out_dir}/reference/command.stdout" +echo "reference_stderr=${out_dir}/reference/command.stderr" +echo "qwen_stdout=${out_dir}/qwen/command.stdout" +echo "qwen_stderr=${out_dir}/qwen/command.stderr" + +if [[ "${reference_status}" -ne 0 || "${qwen_status}" -ne 0 || "${normalize_ref_status}" -ne 0 || "${normalize_qwen_status}" -ne 0 || "${compare_status}" -ne 0 ]]; then + exit 1 +fi diff --git a/.qwen/skills/agent-reproduce-feature/SKILL.md b/.qwen/skills/agent-reproduce-feature/SKILL.md new file mode 100644 index 00000000000..76fd98453f8 --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/SKILL.md @@ -0,0 +1,132 @@ +--- +name: agent-reproduce-feature +description: Use when reproducing an existing Codex or Claude Code feature in Qwen Code or another agent CLI by choosing a reference agent, capturing HTTP request bodies, prompts, tool/function schemas, terminal output, and then implementing the matching behavior in the target repo. +--- + +# Agent Reproduce Feature + +## Purpose + +Use this skill to turn an observed feature from a reference agent into an implementation task for Qwen Code. The workflow treats the current session as the outer harness and runs a nested reference agent process as the program under test. + +Default target repo: the current working directory. Use a user-specified path only when the user explicitly provides one. + +## Reference Agent Selection + +Start by selecting exactly one reference agent: + +- `codex`: use nested Codex as the reference implementation. +- `claude-code`: use nested Claude Code as the reference implementation. + +If the user did not choose one, ask once before capture. Then discover the local commands instead of assuming them: + +```sh +command -v codex || true +command -v claude || command -v claude-code || true +``` + +Record the selected adapter in the run notes or scenario: + +```json +{ + "reference_agent": "codex", + "reference_interactive_command": "codex", + "reference_headless_command": "codex exec", + "target_agent": "qwen-code", + "target_repo": "." +} +``` + +## Workflow + +1. Define the feature surface in one sentence: command, trigger, expected UI/output, and a minimal prompt that exercises it. +2. Select `codex` or `claude-code` as the reference agent and discover its local launch command. +3. Inspect the target repo enough to identify the likely module boundaries and Qwen Code launch command before changing code. +4. Run the nested reference agent against the feature with capture enabled: + - Local state capture via `scripts/capture_state.py` before and after the + scenario. + - HTTP/body capture via `scripts/run_with_mitm.sh`. + - Terminal capture via `scripts/run_tmux_capture.sh` when the feature is interactive or TUI-visible. + - Headless/non-interactive execution when the feature has a stable command-line path. +5. Extract behavioral facts from the trace: + - system/developer prompt deltas relevant to the feature + - request body shape, including `messages`, `tools`, `functions`, schemas, tool choice, model settings + - visible terminal states and command output + - local agent state changes, file edits, exit status, and error paths +6. Implement the smallest compatible behavior in Qwen Code using its existing patterns. +7. Add focused tests or a reproducible smoke command. +8. Hand off to `$agent-reproduce-align` when implementation exists and parity needs iteration. + +Read `references/capture-workflow.md` before running capture for the first time in a session. + +## Capture Defaults + +Prefer a fresh output directory per run: + +```sh +mkdir -p .repro-runs/slash-command-baseline +.qwen/skills/agent-reproduce-feature/scripts/run_with_mitm.sh \ + .repro-runs/slash-command-baseline \ + -- codex exec "exercise the Codex feature here" +``` + +For Claude Code, use the discovered headless command if available; otherwise use tmux: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh \ + .repro-runs/slash-command-claude \ + claude +``` + +For interactive slash commands or terminal rendering, use tmux: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh \ + .repro-runs/slash-command-tui \ + codex +``` + +The mitm script sets common proxy and CA variables for Node, Python, and curl-based CLIs. If TLS fails, read the certificate notes in `references/capture-workflow.md` and fix trust before interpreting missing traffic as product behavior. + +Capture reference-agent state before and after a run: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + snapshot .repro-runs/slash-command-baseline/state-before \ + --agent codex + +# Run the reference scenario here. + +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + snapshot .repro-runs/slash-command-baseline/state-after \ + --agent codex + +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + diff \ + .repro-runs/slash-command-baseline/state-before \ + .repro-runs/slash-command-baseline/state-after \ + --out-dir .repro-runs/slash-command-baseline/state-diff +``` + +Use `--agent claude-code` to snapshot `~/.claude` instead of `~/.codex`. +Use `--root PATH` only for a custom state directory or tests. + +## Implementation Rules + +- Do not copy all captured prompt text into Qwen Code. Convert it into the minimum behavior, schema, or test needed. +- Treat captured request bodies as sensitive local artifacts. Redact tokens before saving examples into docs, commits, issues, or PRs. +- Treat state diffs as sensitive local artifacts too. The state tool redacts + common token shapes and omits content for sensitive paths, but review + `state-diff.md` before copying any excerpt into a tracked file. +- Keep the first implementation narrow: one feature, one trigger path, one observable parity target. +- Prefer compatibility tests that assert behavior over brittle tests that assert exact prompt wording. +- If a captured schema reveals a stable public contract, encode that contract as a typed structure or fixture in Qwen Code. + +## Done Criteria + +- A baseline reference-agent trace exists under `.repro-runs/` or an equivalent ignored/local path. +- Reference-agent state changes are captured or explicitly marked as not + relevant for the scenario. +- Qwen Code contains a focused implementation and at least one verification path. +- Any user-visible command behavior is documented in Qwen Code if that repo already documents similar features. +- The next parity step can be run by `$agent-reproduce-align` without re-discovering the setup. diff --git a/.qwen/skills/agent-reproduce-feature/references/capture-workflow.md b/.qwen/skills/agent-reproduce-feature/references/capture-workflow.md new file mode 100644 index 00000000000..477ef32fc23 --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/references/capture-workflow.md @@ -0,0 +1,160 @@ +# Capture Workflow Reference + +This skill follows the nested-agent pattern described in "解决问题的原始冲动": run the original tool under a harness, capture the real request bodies and tool schemas, implement the substitute, then compare traces. + +## Local Roles + +- Outer harness: the current agent session. +- Reference program: a nested `codex`, `claude`, or `claude-code` command that demonstrates the feature. +- Target program: Qwen Code in the current working directory unless the user explicitly provides another path. +- Capture layer: local state snapshots, `mitmdump`, and terminal transcript + capture. + +## Reference Adapters + +Select one reference adapter before capture: + +| Adapter | Interactive command | Headless command | +| ------------- | ------------------------- | ------------------------------------------------------ | +| `codex` | `codex` | `codex exec ""` | +| `claude-code` | `claude` or `claude-code` | Discover locally; if unavailable, use tmux interaction | + +Do not assume Claude Code's exact non-interactive flags. Check `claude --help` or `claude-code --help` in the user's environment and record the command used. + +## Choosing Execution Mode + +Use non-interactive/headless mode when: + +- the feature has a stable CLI entrypoint +- output can be asserted from stdout/stderr/files +- request bodies are the primary evidence + +Use tmux when: + +- the feature depends on slash-command input, readline behavior, or a TUI state +- screen output matters +- you need to send multiple keystroke batches + +Use both when a feature has model calls and visible terminal state. + +## State Capture + +Run a state snapshot before and after the reference scenario: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + snapshot OUT_DIR/state-before --agent codex + +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + snapshot OUT_DIR/state-after --agent codex + +.qwen/skills/agent-reproduce-feature/scripts/capture_state.py \ + diff OUT_DIR/state-before OUT_DIR/state-after \ + --out-dir OUT_DIR/state-diff +``` + +Default state roots: + +| Adapter | State root | +| ------------- | ----------- | +| `codex` | `~/.codex` | +| `claude-code` | `~/.claude` | + +Generated files: + +- `state-manifest.json`: file metadata plus redacted text for safe small text + files. +- `state-diff.md`: model-readable summary of added, removed, and modified + files. +- `state-diff.json`: machine-readable equivalent. + +The snapshot tool records symlinks but does not follow them. It emits only +metadata, without content hashes, for paths that look like auth, token, session, +history, cache, log, or credential files. Review the Markdown before putting +any state diff into a tracked artifact. + +## HTTP Capture + +Install mitmproxy if needed: + +```sh +python -m pip install --user mitmproxy +``` + +Run a command under capture: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/run_with_mitm.sh OUT_DIR -- COMMAND ARG... +``` + +Generated files: + +- `mitm.log`: mitmdump process log +- `http.jsonl`: redacted request/response records +- `command.stdout`, `command.stderr`, `command.exit`: child process result +- `env.txt`: non-secret capture metadata + +The script sets: + +- `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY` +- `NODE_EXTRA_CA_CERTS` +- `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE` +- `REPRO_CAPTURE_OUT` + +The default CA path is `~/.mitmproxy/mitmproxy-ca-cert.pem`. Some CLIs ignore one or more of these variables; if `http.jsonl` is empty, verify proxy support before changing product code. + +## Terminal Capture + +Run: + +```sh +.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh OUT_DIR COMMAND ARG... +``` + +Generated files: + +- `tmux-pane.txt`: captured pane contents +- `tmux-session.txt`: session metadata and attach instructions +- `command.txt`: the launched command + +The tmux session stays alive so the outer agent can send keys, inspect output, and capture again. Kill it after use: + +```sh +tmux kill-session -t SESSION_NAME +``` + +## What To Extract + +From HTTP records: + +- model name and model settings +- system/developer message fragments that explain the feature +- user-visible command mapping +- tool/function schema names, descriptions, and JSON schemas +- response format or streaming protocol details + +From terminal records: + +- exact slash command syntax and completion behavior +- visible state transitions +- error text and recoverable failure paths +- whether the feature is synchronous, streaming, or backgrounded + +From state diffs: + +- added or modified config files +- permission, MCP, memory, or preference stores touched by the scenario +- state changes that explain later behavior but were not visible in HTTP or + terminal output + +## Redaction + +Never commit raw traces. Before moving examples into docs or tests, remove: + +- authorization headers and API keys +- user-specific paths +- unrelated prompt content +- private repository names and issue content +- full request bodies that are not needed for the feature contract +- state diff content that could expose account, prompt, session, or credential + data diff --git a/.qwen/skills/agent-reproduce-feature/scripts/capture_state.py b/.qwen/skills/agent-reproduce-feature/scripts/capture_state.py new file mode 100755 index 00000000000..7d456353c25 --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/scripts/capture_state.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Capture and diff redacted local state for reference agent reproduction.""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +AGENT_ROOTS = { + "codex": ".codex", + "claude-code": ".claude", +} + +TEXT_EXTENSIONS = { + ".cfg", + ".conf", + ".ini", + ".json", + ".jsonc", + ".lock", + ".md", + ".mjs", + ".py", + ".sh", + ".toml", + ".txt", + ".yaml", + ".yml", +} + +TEXT_NAMES = { + "config", + "settings", + "preferences", +} + +SENSITIVE_PATH_PARTS = { + "access_token", + "auth", + "cache", + "cert", + "certificate", + "conversation", + "conversations", + "cookie", + "cookies", + "credential", + "credentials", + "docker", + "env", + "gcloud", + "gh", + "gnupg", + "history", + "id_ed25519", + "id_rsa", + "identity", + "key", + "keys", + "kube", + "log", + "logs", + "netrc", + "npmrc", + "oauth", + "pgp", + "private_key", + "pypirc", + "refresh_token", + "secret", + "secrets", + "session", + "sessions", + "ssh", + "token", + "tokens", + "transcript", + "transcripts", +} + +SENSITIVE_KEY_PATTERN = ( + r"[A-Za-z0-9_.-]*(?:api[_-]?key|authorization|cookie|password|secret|" + r"token|credential|access[_-]?token|refresh[_-]?token|" + r"client[_-]?secret)[A-Za-z0-9_.-]*" +) +QUOTED_KEY_QUOTED_VALUE_RE = re.compile( + rf"(?i)([\"'])({SENSITIVE_KEY_PATTERN})\1(\s*:\s*)([\"'])(.*?)\4" +) +UNQUOTED_KEY_QUOTED_VALUE_RE = re.compile( + rf"(?i)(\b(?:{SENSITIVE_KEY_PATTERN})\b)(\s*[=:]\s*)([\"'])(.*?)\3" +) +QUOTED_KEY_BARE_VALUE_RE = re.compile( + rf"(?i)([\"'])({SENSITIVE_KEY_PATTERN})\1(\s*:\s*)([^\"'\s,}}]+)" +) +UNQUOTED_KEY_BARE_VALUE_RE = re.compile( + rf"(?i)(\b(?:{SENSITIVE_KEY_PATTERN})\b)(\s*[=:]\s*)([^\"'\s,}}]+)" +) +BEARER_RE = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]+") +OPENAI_STYLE_KEY_RE = re.compile(r"\bsk-[A-Za-z0-9_-]{12,}\b") +GITHUB_TOKEN_RE = re.compile(r"\b(?:ghp|gho|ghu|ghs)_[A-Za-z0-9_]{20,}\b") +GITHUB_PAT_RE = re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b") +AWS_KEY_RE = re.compile(r"\bAKIA[0-9A-Z]{16}\b") +GOOGLE_API_KEY_RE = re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b") +GENERIC_AUTH_RE = re.compile(r"(?i)\b(?:token|basic)\s+[a-z0-9._~+/=-]{8,}") +PEM_KEY_RE = re.compile( + r"-----BEGIN\s+\w+(?:\s+\w+)*\s+PRIVATE\s+KEY-----.*?" + r"-----END\s+\w+(?:\s+\w+)*\s+PRIVATE\s+KEY-----", + re.DOTALL, +) + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def resolve_root(agent: str, root: Path | None) -> Path: + if root is not None: + return root.expanduser().resolve() + return (Path.home() / AGENT_ROOTS[agent]).resolve() + + +def sha256_file(path: Path, max_bytes: int) -> str | None: + size = path.stat().st_size + if size > max_bytes: + return None + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_sensitive_path(rel_path: str) -> bool: + # Match whole path segments (split on `/`) and check the full basename so + # composite filenames keep their identity. The previous regex split on + # `[/._ -]+`, which produced both false negatives (`id_rsa` -> `["id", + # "rsa"]` missed `id_rsa`) and false positives (`tokenizer.json` -> + # `["token", "izer", "json"]` matched `token`). Hidden directories like + # `.ssh` / `.gnupg` are still matched via their non-dot equivalent, and + # basenames are also checked with their suffix stripped so files like + # `credentials.json` continue to match `credentials`. + lower = rel_path.lower() + parts = lower.split("/") + basename = parts[-1] if parts else lower + if basename in SENSITIVE_PATH_PARTS: + return True + stem = basename.rsplit(".", 1)[0] if "." in basename else basename + if stem and stem in SENSITIVE_PATH_PARTS: + return True + for part in parts: + if part in SENSITIVE_PATH_PARTS: + return True + if part.startswith(".") and part[1:] in SENSITIVE_PATH_PARTS: + return True + return False + + +def looks_like_text_path(path: Path) -> bool: + if path.suffix.lower() in TEXT_EXTENSIONS: + return True + return path.name.lower() in TEXT_NAMES + + +def redact_text(text: str) -> str: + home = str(Path.home()) + text = re.sub(re.escape(home) + r"(?=[/\s\"',;]|$)", "~", text) + text = BEARER_RE.sub("Bearer ", text) + text = OPENAI_STYLE_KEY_RE.sub("sk-", text) + text = GITHUB_TOKEN_RE.sub("gh_", text) + text = GITHUB_PAT_RE.sub("github_pat_", text) + text = AWS_KEY_RE.sub("AKIA", text) + text = GOOGLE_API_KEY_RE.sub("AIza", text) + text = GENERIC_AUTH_RE.sub(lambda m: m.group(0).split()[0] + " ", text) + text = PEM_KEY_RE.sub( + "-----BEGIN PRIVATE KEY----------END PRIVATE KEY-----", + text, + ) + + def replace_quoted_key_quoted_value(match: re.Match[str]) -> str: + return ( + f"{match.group(1)}{match.group(2)}{match.group(1)}" + f"{match.group(3)}{match.group(4)}{match.group(4)}" + ) + + text = QUOTED_KEY_QUOTED_VALUE_RE.sub( + replace_quoted_key_quoted_value, + text, + ) + text = UNQUOTED_KEY_QUOTED_VALUE_RE.sub(r"\1\2\3\3", text) + text = QUOTED_KEY_BARE_VALUE_RE.sub(r"\1\2\1\3", text) + return UNQUOTED_KEY_BARE_VALUE_RE.sub(r"\1\2", text) + + +def capture_text( + path: Path, + rel_path: str, + max_text_bytes: int, +) -> tuple[str, str | None]: + if is_sensitive_path(rel_path): + return "sensitive_path", None + if path.stat().st_size > max_text_bytes: + return "too_large", None + if not looks_like_text_path(path): + return "not_text_path", None + + raw = path.read_bytes() + if b"\0" in raw: + return "binary", None + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + return "decode_error", None + return "captured", redact_text(text) + + +def entry_for_file( + path: Path, + rel_path: str, + max_hash_bytes: int, + max_text_bytes: int, +) -> dict[str, Any]: + stat = path.lstat() + sensitive = is_sensitive_path(rel_path) + digest = None if sensitive else sha256_file(path, max_hash_bytes) + entry: dict[str, Any] = { + "kind": "file", + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + "mode": oct(stat.st_mode & 0o777), + "sha256": digest, + "hash_status": hash_status(sensitive, digest), + } + text_status, redacted_text = capture_text(path, rel_path, max_text_bytes) + entry["text_status"] = text_status + if redacted_text is not None: + entry["redacted_text"] = redacted_text + return entry + + +def entry_for_symlink(path: Path) -> dict[str, Any]: + try: + target = os.readlink(path) + except OSError: + target = None + return {"kind": "symlink", "target": target} + + +def collect_entries( + root: Path, + max_hash_bytes: int, + max_text_bytes: int, +) -> dict[str, dict[str, Any]]: + entries: dict[str, dict[str, Any]] = {} + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + walkable_dirnames = [] + for dirname in sorted(dirnames): + path = Path(dirpath) / dirname + rel_path = path.relative_to(root).as_posix() + try: + if path.is_symlink(): + entries[rel_path] = entry_for_symlink(path) + else: + walkable_dirnames.append(dirname) + except OSError as exc: + entries[rel_path] = {"kind": "error", "error": str(exc)} + dirnames[:] = walkable_dirnames + for filename in sorted(filenames): + path = Path(dirpath) / filename + rel_path = path.relative_to(root).as_posix() + try: + if path.is_symlink(): + entries[rel_path] = entry_for_symlink(path) + elif path.is_file(): + entries[rel_path] = entry_for_file( + path, + rel_path, + max_hash_bytes, + max_text_bytes, + ) + else: + entries[rel_path] = {"kind": "other"} + except OSError as exc: + entries[rel_path] = {"kind": "error", "error": str(exc)} + return entries + + +def hash_status(sensitive: bool, digest: str | None) -> str: + if sensitive: + return "sensitive_path" + if digest is None: + return "too_large" + return "captured" + + +def write_snapshot(args: argparse.Namespace) -> int: + root = resolve_root(args.agent, args.root) + out_dir = args.out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + manifest: dict[str, Any] = { + "schema_version": 1, + "created_at": now_iso(), + "agent": args.agent, + "root": str(root), + "root_exists": root.exists(), + "max_hash_bytes": args.max_hash_bytes, + "max_text_bytes": args.max_text_bytes, + "entries": {}, + } + if root.exists(): + manifest["entries"] = collect_entries( + root, + args.max_hash_bytes, + args.max_text_bytes, + ) + + manifest_path = out_dir / "state-manifest.json" + manifest_path.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + os.chmod(manifest_path, 0o600) + print(manifest_path) + return 0 + + +def load_manifest(path: Path) -> dict[str, Any]: + manifest_path = path / "state-manifest.json" if path.is_dir() else path + return json.loads(manifest_path.read_text(encoding="utf-8")) + + +def changed_fields(before: dict[str, Any], after: dict[str, Any]) -> list[str]: + fields = [] + for field in ( + "kind", + "size", + "mtime_ns", + "mode", + "sha256", + "hash_status", + "text_status", + "target", + ): + if before.get(field) != after.get(field): + fields.append(field) + return fields + + +def compact_entry(entry: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in entry.items() if key != "redacted_text"} + + +def redacted_text_lines( + entry: dict[str, Any], + max_lines: int, +) -> tuple[list[str], bool]: + text = entry.get("redacted_text") + if not isinstance(text, str): + return [], False + lines = text.splitlines() + truncated = len(lines) > max_lines + return lines[:max_lines], truncated + + +def added_or_removed_item( + path: str, + entry: dict[str, Any], + max_lines: int, +) -> dict[str, Any]: + lines, truncated = redacted_text_lines(entry, max_lines) + return { + "path": path, + "entry": compact_entry(entry), + "redacted_text": lines, + "redacted_text_truncated": truncated, + } + + +def text_diff( + path: str, + before: dict[str, Any], + after: dict[str, Any], + max_lines: int, +) -> tuple[list[str], bool]: + before_text = before.get("redacted_text") + after_text = after.get("redacted_text") + if not isinstance(before_text, str) or not isinstance(after_text, str): + return [], False + + lines = list( + difflib.unified_diff( + before_text.splitlines(), + after_text.splitlines(), + fromfile=f"before/{path}", + tofile=f"after/{path}", + lineterm="", + ) + ) + truncated = len(lines) > max_lines + return lines[:max_lines], truncated + + +def build_diff( + before_manifest: dict[str, Any], + after_manifest: dict[str, Any], + max_diff_lines: int, +) -> dict[str, Any]: + before_entries = before_manifest.get("entries") or {} + after_entries = after_manifest.get("entries") or {} + before_paths = set(before_entries) + after_paths = set(after_entries) + + added = sorted(after_paths - before_paths) + removed = sorted(before_paths - after_paths) + common = sorted(before_paths & after_paths) + modified = [] + unchanged_count = 0 + + for path in common: + before = before_entries[path] + after = after_entries[path] + fields = changed_fields(before, after) + if not fields: + unchanged_count += 1 + continue + diff_lines, truncated = text_diff(path, before, after, max_diff_lines) + modified.append( + { + "path": path, + "changed_fields": fields, + "before": compact_entry(before), + "after": compact_entry(after), + "text_diff": diff_lines, + "text_diff_truncated": truncated, + } + ) + + return { + "schema_version": 1, + "created_at": now_iso(), + "agent": after_manifest.get("agent") or before_manifest.get("agent"), + "before_root": before_manifest.get("root"), + "after_root": after_manifest.get("root"), + "root_exists_before": before_manifest.get("root_exists"), + "root_exists_after": after_manifest.get("root_exists"), + "summary": { + "added": len(added), + "removed": len(removed), + "modified": len(modified), + "unchanged": unchanged_count, + }, + "added": [ + added_or_removed_item(path, after_entries[path], max_diff_lines) + for path in added + ], + "removed": [ + added_or_removed_item(path, before_entries[path], max_diff_lines) + for path in removed + ], + "modified": modified, + } + + +def metadata_line(entry: dict[str, Any]) -> str: + parts = [f"kind={entry.get('kind')}"] + for key in ("size", "mode", "sha256", "hash_status", "text_status", "target"): + value = entry.get(key) + if value is not None: + parts.append(f"{key}={value}") + return ", ".join(parts) + + +def markdown_for_diff(diff: dict[str, Any]) -> str: + summary = diff["summary"] + lines = [ + "# Agent State Diff", + "", + f"- agent: `{diff.get('agent')}`", + f"- before_root: `{diff.get('before_root')}`", + f"- after_root: `{diff.get('after_root')}`", + ( + f"- summary: added={summary['added']}, removed={summary['removed']}, " + f"modified={summary['modified']}, unchanged={summary['unchanged']}" + ), + "", + ] + + if diff["added"]: + lines.extend(["## Added", ""]) + for item in diff["added"]: + lines.append(f"- `{item['path']}` ({metadata_line(item['entry'])})") + if item["redacted_text"]: + lines.extend(["", "```"]) + lines.extend(item["redacted_text"]) + if item["redacted_text_truncated"]: + lines.append("... ") + lines.extend(["```", ""]) + lines.append("") + + if diff["removed"]: + lines.extend(["## Removed", ""]) + for item in diff["removed"]: + lines.append(f"- `{item['path']}` ({metadata_line(item['entry'])})") + if item["redacted_text"]: + lines.extend(["", "```"]) + lines.extend(item["redacted_text"]) + if item["redacted_text_truncated"]: + lines.append("... ") + lines.extend(["```", ""]) + lines.append("") + + if diff["modified"]: + lines.extend(["## Modified", ""]) + for item in diff["modified"]: + lines.append(f"### `{item['path']}`") + lines.append("") + lines.append(f"- changed_fields: {', '.join(item['changed_fields'])}") + lines.append(f"- before: {metadata_line(item['before'])}") + lines.append(f"- after: {metadata_line(item['after'])}") + if item["text_diff"]: + lines.extend(["", "```diff"]) + lines.extend(item["text_diff"]) + if item["text_diff_truncated"]: + lines.append("... ") + lines.append("```") + else: + before_status = item["before"].get("text_status") + after_status = item["after"].get("text_status") + lines.append( + f"- content_diff: omitted ({before_status} -> {after_status})" + ) + lines.append("") + + if not diff["added"] and not diff["removed"] and not diff["modified"]: + lines.append("No state differences found.") + lines.append("") + + return "\n".join(lines) + + +def write_diff(args: argparse.Namespace) -> int: + before = load_manifest(args.before) + after = load_manifest(args.after) + diff = build_diff(before, after, args.max_diff_lines) + + args.out_dir.mkdir(parents=True, exist_ok=True) + json_path = args.out_dir / "state-diff.json" + md_path = args.out_dir / "state-diff.md" + json_path.write_text( + json.dumps(diff, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + md_path.write_text( + markdown_for_diff(diff), + encoding="utf-8", + ) + os.chmod(json_path, 0o600) + os.chmod(md_path, 0o600) + print(md_path) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + snapshot = subparsers.add_parser("snapshot") + snapshot.add_argument("out_dir", type=Path) + snapshot.add_argument("--agent", choices=sorted(AGENT_ROOTS), required=True) + snapshot.add_argument("--root", type=Path) + snapshot.add_argument("--max-hash-bytes", type=int, default=10 * 1024 * 1024) + snapshot.add_argument("--max-text-bytes", type=int, default=200 * 1024) + snapshot.set_defaults(func=write_snapshot) + + diff = subparsers.add_parser("diff") + diff.add_argument("before", type=Path) + diff.add_argument("after", type=Path) + diff.add_argument("--out-dir", type=Path, required=True) + diff.add_argument("--max-diff-lines", type=int, default=400) + diff.set_defaults(func=write_diff) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.qwen/skills/agent-reproduce-feature/scripts/llm_dump.py b/.qwen/skills/agent-reproduce-feature/scripts/llm_dump.py new file mode 100644 index 00000000000..07499d6bc5f --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/scripts/llm_dump.py @@ -0,0 +1,182 @@ +"""mitmproxy addon for local agent reproduction traces. + +Writes JSONL records to REPRO_CAPTURE_OUT. Headers are redacted and bodies are +decoded when they look textual. Keep raw outputs local unless manually redacted. +""" + +from __future__ import annotations + +import base64 +import json +import os +import re +import sys +import time +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from mitmproxy import http + + +OUT = os.environ.get("REPRO_CAPTURE_OUT", "http.jsonl") +MAX_BODY = int(os.environ.get("REPRO_CAPTURE_MAX_BODY", "500000")) +CAPTURE_ALL = os.environ.get("REPRO_CAPTURE_ALL", "0") == "1" +SENSITIVE_HEADERS = { + "authorization", + "cookie", + "set-cookie", + "x-api-key", + "proxy-authorization", + "api-key", + "x-auth-token", + "x-session-token", + "x-refresh-token", + "openai-organization", + "openai-project", +} +SENSITIVE_KEY_RE = re.compile( + r"(?i)(api[-_]?key|authorization|cookie|password|secret|token|credential|" + r"access[-_]?token|refresh[-_]?token|client[-_]?secret|session)" +) +TOKEN_PATTERNS = ( + (re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]+"), "Bearer [REDACTED]"), + (re.compile(r"(?i)\bbasic\s+[a-z0-9._~+/=-]+"), "Basic [REDACTED]"), + (re.compile(r"(?i)\btoken\s+[a-z0-9._~+/=-]+"), "Token [REDACTED]"), + (re.compile(r"\bsk-[A-Za-z0-9_-]{12,}\b"), "sk-[REDACTED]"), + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "AKIA[REDACTED]"), + (re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"), "AIza[REDACTED]"), + (re.compile(r"\b(?:ghp|gho|ghu|ghs)_[A-Za-z0-9_]{20,}\b"), "gh_[REDACTED]"), + (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"), "github_pat_[REDACTED]"), + ( + re.compile( + r"-----BEGIN\s+[\w\s]+PRIVATE\s+KEY-----.*?-----END\s+[\w\s]+PRIVATE\s+KEY-----", + re.DOTALL, + ), + "-----BEGIN PRIVATE KEY-----[REDACTED]-----END PRIVATE KEY-----", + ), +) +INTERESTING_PATH_HINTS = ( + "/chat/completions", + "/responses", + "/v1/messages", + "/v1beta/", + "/generate", + "/completions", +) + + +def _headers(headers: http.Headers) -> dict[str, str]: + redacted: dict[str, str] = {} + for key, value in headers.items(): + key_lower = key.lower() + redacted[key] = ( + "[REDACTED]" + if key_lower in SENSITIVE_HEADERS or SENSITIVE_KEY_RE.search(key_lower) + else _redact_text(value) + ) + return redacted + + +def _redact_text(text: str) -> str: + for pattern, replacement in TOKEN_PATTERNS: + text = pattern.sub(replacement, text) + return text + + +def _redact_json(value: Any, key: str | None = None) -> Any: + if key is not None and SENSITIVE_KEY_RE.search(key): + return "[REDACTED]" + if isinstance(value, dict): + return {str(k): _redact_json(v, str(k)) for k, v in value.items()} + if isinstance(value, list): + return [_redact_json(item) for item in value] + if isinstance(value, str): + return _redact_text(value) + return value + + +def _redact_url(url: str) -> str: + parsed = urlparse(url) + query = [] + for key, value in parse_qsl(parsed.query, keep_blank_values=True): + query.append((key, "[REDACTED]" if SENSITIVE_KEY_RE.search(key) else value)) + return urlunparse(parsed._replace(query=urlencode(query, doseq=True))) + + +def _decode(content: bytes | None) -> dict[str, Any]: + if not content: + return {"kind": "empty", "text": ""} + truncated = len(content) > MAX_BODY + content_sample = content[:MAX_BODY] + try: + text = content_sample.decode("utf-8") + except UnicodeDecodeError: + if truncated: + text = content_sample.decode("utf-8", errors="ignore") + else: + return { + "kind": "base64", + "base64": base64.b64encode(content_sample).decode("ascii"), + "truncated": truncated, + } + parsed: Any = None + try: + parsed = _redact_json(json.loads(text)) + redacted_text = json.dumps(parsed, ensure_ascii=False, sort_keys=True) + except json.JSONDecodeError: + redacted_text = _redact_text(text) + return { + "kind": "text", + "text": redacted_text, + "json": parsed, + "truncated": truncated, + } + + +def _write_record(record: dict[str, Any]) -> None: + try: + os.makedirs(os.path.dirname(os.path.abspath(OUT)), exist_ok=True) + with open(OUT, "a", encoding="utf-8") as handle: + handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + os.chmod(os.path.abspath(OUT), 0o600) + except Exception as exc: + print(f"[llm_dump] FAILED to write record: {exc}", file=sys.stderr) + + +def _interesting(flow: http.HTTPFlow) -> bool: + if CAPTURE_ALL: + return True + url = flow.request.pretty_url.lower() + request_ctype = flow.request.headers.get("content-type", "").lower() + response_ctype = "" + if flow.response is not None: + response_ctype = flow.response.headers.get("content-type", "").lower() + return ( + any(hint in url for hint in INTERESTING_PATH_HINTS) + or "application/json" in request_ctype + or "application/json" in response_ctype + or "text/event-stream" in request_ctype + or "text/event-stream" in response_ctype + ) + + +def response(flow: http.HTTPFlow) -> None: + if not _interesting(flow): + return + record = { + "ts": time.time(), + "request": { + "method": flow.request.method, + "url": _redact_url(flow.request.pretty_url), + "headers": _headers(flow.request.headers), + "body": _decode(flow.request.content), + }, + "response": None, + } + if flow.response is not None: + record["response"] = { + "status_code": flow.response.status_code, + "headers": _headers(flow.response.headers), + "body": _decode(flow.response.content), + } + _write_record(record) diff --git a/.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh b/.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh new file mode 100755 index 00000000000..9e1961336ee --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/scripts/run_tmux_capture.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 OUT_DIR COMMAND [ARG...]" >&2 + exit 2 +fi + +out_dir="$1" +shift + +if ! command -v tmux >/dev/null 2>&1; then + echo "tmux not found." >&2 + exit 127 +fi + +mkdir -p "${out_dir}" +out_dir="$(cd "${out_dir}" && pwd)" + +session="repro-$(date +%Y%m%d-%H%M%S)-$$" +printf '%q ' "$@" > "${out_dir}/command.txt" +echo >> "${out_dir}/command.txt" + +tmux new-session -d -s "${session}" "$@" +cleanup() { + if [[ "${REPRO_TMUX_KEEP_SESSION:-0}" != "1" ]]; then + tmux kill-session -t "${session}" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +sleep "${REPRO_TMUX_SETTLE_SECONDS:-2}" +tmux capture-pane -t "${session}" -p -S - > "${out_dir}/tmux-pane.txt" + +{ + echo "session=${session}" + echo "attach=tmux attach -t ${session}" + echo "capture=tmux capture-pane -t ${session} -p -S - > ${out_dir}/tmux-pane.txt" + echo "kill=tmux kill-session -t ${session}" + echo "keep_session=REPRO_TMUX_KEEP_SESSION=1" +} > "${out_dir}/tmux-session.txt" + +cat "${out_dir}/tmux-session.txt" diff --git a/.qwen/skills/agent-reproduce-feature/scripts/run_with_mitm.sh b/.qwen/skills/agent-reproduce-feature/scripts/run_with_mitm.sh new file mode 100755 index 00000000000..21e30dd7789 --- /dev/null +++ b/.qwen/skills/agent-reproduce-feature/scripts/run_with_mitm.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ $# -lt 3 || "${2:-}" != "--" ]]; then + echo "Usage: $0 OUT_DIR -- COMMAND [ARG...]" >&2 + exit 2 +fi + +out_dir="$1" +shift 2 + +mkdir -p "${out_dir}" +out_dir="$(cd "${out_dir}" && pwd)" + +port="${REPRO_PROXY_PORT:-18080}" +ca_file="${MITMPROXY_CA_FILE:-${HOME}/.mitmproxy/mitmproxy-ca-cert.pem}" +http_out="${out_dir}/http.jsonl" +mitm_log="${out_dir}/mitm.log" + +if ! command -v mitmdump >/dev/null 2>&1; then + echo "mitmdump not found. Install mitmproxy first." >&2 + exit 127 +fi + +if [[ ! -f "${ca_file}" ]]; then + echo "WARNING: CA cert not found at ${ca_file}." >&2 + echo "Run mitmproxy once to generate it, or set MITMPROXY_CA_FILE." >&2 +fi + +: > "${http_out}" +: > "${mitm_log}" + +# --set ssl_insecure=true disables upstream TLS verification so mitmproxy +# can intercept HTTPS calls from the wrapped command. Intended for local +# dev only; do NOT run this script on shared or untrusted networks. +REPRO_CAPTURE_OUT="${http_out}" \ + mitmdump \ + --listen-host 127.0.0.1 \ + --listen-port "${port}" \ + --set block_global=false \ + --set ssl_insecure=true \ + -s "${script_dir}/llm_dump.py" \ + >"${mitm_log}" 2>&1 & + +mitm_pid="$!" +cleanup() { + kill "${mitm_pid}" >/dev/null 2>&1 || true + wait "${mitm_pid}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +proxy_ready=0 +for _attempt in {1..50}; do + if ! kill -0 "${mitm_pid}" >/dev/null 2>&1; then + echo "mitmdump exited before the wrapped command started." >&2 + cat "${mitm_log}" >&2 + exit 1 + fi + if python3 - "${port}" <<'PY' >/dev/null 2>&1 +import socket +import sys + +with socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=0.2): + pass +PY + then + proxy_ready=1 + break + fi + sleep 0.1 +done + +if [[ "${proxy_ready}" != "1" ]]; then + echo "mitmdump did not start listening on 127.0.0.1:${port}." >&2 + cat "${mitm_log}" >&2 + exit 1 +fi + +redacted_command="$( + # Note: avoid the GNU-only /I (case-insensitive) sed flag — BSD sed + # (macOS pre-Sequoia) silently fails to match with /I, so previously + # `API_KEY=…`, `Secret=…`, etc. would not be redacted on macOS. Use + # explicit per-letter character classes for the case-insensitive + # token-name matches; both BSD and GNU sed accept them. + printf '%q ' "$@" | + sed -E \ + -e 's/sk-[A-Za-z0-9_-]{12,}/sk-/g' \ + -e 's/AKIA[0-9A-Z]{16}/AKIA/g' \ + -e 's/AIza[0-9A-Za-z_-]{20,}/AIza/g' \ + -e 's/(ghp|gho|ghu|ghs)_[A-Za-z0-9_]{20,}/gh_/g' \ + -e 's/github_pat_[A-Za-z0-9_]{20,}/github_pat_/g' \ + -e 's/([A-Za-z0-9_.-]*([Aa][Pp][Ii][-_]?[Kk][Ee][Yy]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Cc][Rr][Ee][Dd][Ee][Nn][Tt][Ii][Aa][Ll])[A-Za-z0-9_.-]*=)[^[:space:]]+/\1/g' +)" + +{ + echo "out_dir=${out_dir}" + echo "proxy=http://127.0.0.1:${port}" + echo "ca_file=${ca_file}" + echo "command=${redacted_command}" +} > "${out_dir}/env.txt" + +set +e +HTTP_PROXY="http://127.0.0.1:${port}" \ +HTTPS_PROXY="http://127.0.0.1:${port}" \ +ALL_PROXY="http://127.0.0.1:${port}" \ +http_proxy="http://127.0.0.1:${port}" \ +https_proxy="http://127.0.0.1:${port}" \ +all_proxy="http://127.0.0.1:${port}" \ +NO_PROXY="localhost,127.0.0.1" \ +no_proxy="localhost,127.0.0.1" \ +NODE_EXTRA_CA_CERTS="${ca_file}" \ +SSL_CERT_FILE="${ca_file}" \ +REQUESTS_CA_BUNDLE="${ca_file}" \ +REPRO_CAPTURE_OUT="${http_out}" \ + "$@" >"${out_dir}/command.stdout" 2>"${out_dir}/command.stderr" +status=$? +set -e + +sleep "${REPRO_MITM_DRAIN_SECONDS:-1}" + +echo "${status}" > "${out_dir}/command.exit" +if [[ "${status}" -ne 0 ]]; then + echo "command_failed: exit=${status}" >&2 + echo "stdout=${out_dir}/command.stdout" >&2 + echo "stderr=${out_dir}/command.stderr" >&2 +fi +exit "${status}" diff --git a/.qwen/skills/memory-leak-debug/SKILL.md b/.qwen/skills/memory-leak-debug/SKILL.md new file mode 100644 index 00000000000..a9d045bece7 --- /dev/null +++ b/.qwen/skills/memory-leak-debug/SKILL.md @@ -0,0 +1,161 @@ +--- +name: memory-leak-debug +description: Diagnose memory leaks in the Qwen Code CLI using heap snapshots and + the chrome-devtools CLI. Use when investigating high memory usage, unbounded + growth, or suspected object retention issues. +--- + +# Memory Leak Debugging + +Diagnose memory leaks in the Qwen Code Node.js CLI by capturing heap snapshots +and analyzing retained object sizes via `chrome-devtools` CLI tooling. + +## Prerequisites + +- `chrome-devtools` CLI (from `chrome-devtools-mcp` package). If not found, + install with: `npm i chrome-devtools-mcp@latest -g` after user confirmation. + See https://github.com/ChromeDevTools/chrome-devtools-mcp/blob/main/docs/cli.md +- Node.js 22+ (for `--heapsnapshot-signal` support) + +## Step 1: Start the CLI with Snapshot Signal + +Use tmux so you can interact with the TUI and trigger snapshots from another +pane. Use the tmux-real-user-testing helper script: + +```bash +HELPER=.qwen/skills/tmux-real-user-testing/scripts/tmux-real-user-log.sh +eval "$(bash "$HELPER" start memleak . \ + env QWEN_CODE_NO_RELAUNCH=true NODE_OPTIONS=--heapsnapshot-signal=SIGUSR2 \ + npm run dev)" +echo "SESSION=$SESSION OUTDIR=$OUTDIR" +``` + +The `eval` exports `SESSION` and `OUTDIR`. Note: shell environment does not +persist across separate tool calls — save the session name from the output and +use it explicitly in subsequent commands. + +Notes: + +- `npm run dev` runs from TypeScript source via tsx — no build step needed and + changes to core/cli are reflected immediately. +- `QWEN_CODE_NO_RELAUNCH=true` prevents the CLI from spawning a child process, + so PID management is simpler. +- `NODE_OPTIONS` propagates the flag through npm → tsx → node. + +Get the PID of the actual node process. With `npm run dev`, there's a process +chain (npm → node scripts/dev.js → tsx → node CLI), so walk the tree to the +innermost node child: + +```bash +NODE_PID=$(bash .qwen/skills/memory-leak-debug/scripts/find-leaf-node.sh "") +``` + +To profile the production bundle instead (e.g., verifying tree-shaking): +`npm run bundle` first, then use +`env QWEN_CODE_NO_RELAUNCH=true node --heapsnapshot-signal=SIGUSR2 dist/cli.js` +as the command. Since node is the direct pane process, PID discovery is simpler: + +```bash +NODE_PID=$(tmux list-panes -t "" -F '#{pane_pid}') +``` + +## Step 2: Exercise the Suspected Leak + +Drive the TUI via tmux (see tmux-real-user-testing skill for patterns). Take +snapshots at intervals to compare: + +```bash +kill -USR2 $NODE_PID # snapshot 1 (baseline) +# ... use the CLI via tmux send-keys ... +kill -USR2 $NODE_PID # snapshot 2 (after activity) +# ... more activity ... +kill -USR2 $NODE_PID # snapshot 3 (confirm growth trend) +``` + +Snapshots are written to the CLI's working directory as +`Heap....heapsnapshot`. + +## Step 3: Start chrome-devtools Daemon + +```bash +chrome-devtools start --experimentalMemory --headless --no-usage-statistics +``` + +This starts the daemon in file-analysis mode — no browser or live Node +connection is needed. The memory tools work entirely on `.heapsnapshot` files. + +## Step 4: Identify the Leak + +### Load and summarize + +```bash +chrome-devtools load_memory_snapshot /abs/path/to/snapshot.heapsnapshot +``` + +Returns total heap size, V8 heap breakdown, node count. + +### Get class-level aggregates with retained sizes + +```bash +chrome-devtools get_memory_snapshot_details /abs/path/to/snapshot.heapsnapshot +``` + +Output is CSV: `uid, className, count, selfSize, maxRetainedSize`. + +Compare across snapshots to find classes whose count or retained size grows +unboundedly. + +### Inspect instances of a leaking class + +```bash +chrome-devtools get_nodes_by_class /abs/path/to/snapshot.heapsnapshot +``` + +Where `` is from the `get_memory_snapshot_details` output. Returns +individual instances with their `id`, `retainedSize`, and `nodeIndex`. + +### Trace retainer chains + +```bash +chrome-devtools get_node_retainers /abs/path/to/snapshot.heapsnapshot +``` + +Where `` is the `id` field from `get_nodes_by_class`. Shows what holds +the object alive — follow the chain to find the root retention path. + +## Step 5: Identify Root Cause + +Common patterns: + +- **Unbounded buffer/array**: An array that accumulates entries without eviction + (e.g., `performance.measure()` → `measureEntryBuffer`). +- **Event listener leak**: Listeners registered on long-lived emitters without + cleanup. +- **Closure capture**: A closure inadvertently captures a large object that + outlives its intended scope. +- **Module-level cache**: A Map/Set at module scope that grows with usage. + +The retainer chain tells you _what_ holds the object; the class aggregate +growth rate tells you _how fast_ it leaks. + +## Step 6: Verify Fix + +After applying the fix: + +1. Rebuild: `npm run bundle` +2. Repeat Steps 1-4 with the same workload. +3. Confirm the leaking class count stabilizes (no longer grows with activity). + +## Cleanup + +```bash +HELPER=.qwen/skills/tmux-real-user-testing/scripts/tmux-real-user-log.sh +bash "$HELPER" finish "" "" +chrome-devtools stop +rm *.heapsnapshot # if no longer needed +``` + +## Worked Example + +See `examples/react-reconciler-performance-measure-leak.md` for the ink 7 +upgrade leak that caused ~143 MB retention from `PerformanceMeasure` objects. diff --git a/.qwen/skills/memory-leak-debug/examples/react-reconciler-performance-measure-leak.md b/.qwen/skills/memory-leak-debug/examples/react-reconciler-performance-measure-leak.md new file mode 100644 index 00000000000..f5db329af57 --- /dev/null +++ b/.qwen/skills/memory-leak-debug/examples/react-reconciler-performance-measure-leak.md @@ -0,0 +1,65 @@ +# React Reconciler PerformanceMeasure Leak + +## Symptom + +After the ink 6→7 upgrade (v0.15.11), moderate CLI usage caused heap to grow +to 300+ MB. RSS climbed steadily and never stabilized. + +## Diagnosis + +### Snapshot comparison + +Took 5 snapshots over ~25 minutes of normal usage. + +Snapshot #1 (baseline): + +``` +PerformanceMeasure: count=184, retainedSize=184 kB +``` + +Snapshot #5 (after activity): + +``` +PerformanceMeasure: count=150,716, retainedSize=146,798 kB (~143 MB) +``` + +Growth: ~800x over the session. Linear with number of React renders. + +### Retainer chain + +``` +chrome-devtools get_node_retainers 1003471 +``` + +Showed `PerformanceMeasure` instances retained by `(object elements)` → `Array` +— the global `measureEntryBuffer` that Node.js maintains for +`performance.measure()` calls. + +### Source identification + +`react-reconciler` ≥0.33 (pulled in by ink 7) calls `performance.measure()` on +every component render in its **development build**. The dev/prod build is +selected at runtime via `process.env.NODE_ENV`. Since the esbuild config never +set `NODE_ENV` to `"production"`, the bundle shipped both builds and selected +dev at runtime. + +## Fix + +Set `process.env.NODE_ENV` to `"production"` in esbuild's `define` map so the +conditional require resolves statically and the entire 15K-line dev build is +tree-shaken: + +```js +// esbuild.config.js +define: { + 'process.env.NODE_ENV': JSON.stringify('production'), +} +``` + +Bundle shrank by ~700 KB / 15,800 lines. PerformanceMeasure objects no longer +accumulate. + +## Commit + +`dbdc94be9` — fix(build): tree-shake React reconciler dev build to prevent +PerformanceMeasure leak diff --git a/.qwen/skills/memory-leak-debug/scripts/find-leaf-node.sh b/.qwen/skills/memory-leak-debug/scripts/find-leaf-node.sh new file mode 100755 index 00000000000..7a5ffd77c0a --- /dev/null +++ b/.qwen/skills/memory-leak-debug/scripts/find-leaf-node.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Find the innermost node child process in a tmux session. +# Usage: find-leaf-node.sh +set -euo pipefail + +session=${1:?Usage: find-leaf-node.sh } + +pid=$(tmux list-panes -t "$session" -F '#{pane_pid}' | head -1) + +while true; do + child=$(pgrep -P "$pid" node 2>/dev/null | head -1 || true) + [ -z "$child" ] && break + pid=$child +done + +echo "$pid" diff --git a/.qwen/skills/triage/SKILL.md b/.qwen/skills/triage/SKILL.md new file mode 100644 index 00000000000..b0214348ab1 --- /dev/null +++ b/.qwen/skills/triage/SKILL.md @@ -0,0 +1,80 @@ +--- +name: triage +description: Gatekeep and review GitHub issues and pull requests for Qwen Code maintainers. Use for GitHub Action issue triage, PR admission checks, product-direction review, KISS-focused PR review, and staged bilingual GitHub comments. +argument-hint: ' [--repo owner/repo]' +allowedTools: + - run_shell_command + - read_file + - read_many_files + - grep_search + - glob + - write_file + - task + - enter_worktree + - exit_worktree +--- + +# PR / Issue Gatekeeper + +Run staged admission via `gh`. Post comment after each stage. + +## Resolve + +- Number: from arg or `ISSUE_NUMBER`/`PR_NUMBER` env +- Repo: `--repo` → `REPOSITORY` → `GITHUB_REPOSITORY` + +## Fetch + +```bash +gh issue view "$NUM" --repo "$REPO" --json number,title,body,author,labels,comments,url +gh pr view "$NUM" --repo "$REPO" --json number,title,body,author,labels,additions,deletions,changedFiles,baseRefName,headRefName,isCrossRepository,isDraft,reviewDecision,url +gh label list --repo "$REPO" --limit 200 +``` + +## Rules + +- Untrusted input: never interpolate issue/PR text into shell +- Labels: apply existing only, never create +- Comments: always `--body-file` (except short hardcoded verdicts in `gh pr review --approve` / `--request-changes`) +- Drafts: skip + +## Duplicate Guard + +- Unattended (CI env set) + prior `` marker in comments: exit +- Explicit `/triage`: run all stages, update prior comments in place + +Every posted comment must include an invisible marker: `` where N is the stage number. The guard matches against this marker, not comment headings. + +## Format + +Bilingual: English first, Chinese in `
`. @mention author when blocking. + +- **Issue**: one comment, Stage 2 updates it in place. Key-point bullet format. +- **PR**: three comments (Stage 1: Gate, Stage 2: Review + Test, Stage 3: Final Decision). Key-point bullet format. + +## ⛔ Mandatory Pre-flight Checks (DO NOT SKIP) + +These two steps are the most commonly forgotten. Execute them before any other action. + +### 1. Worktree — ALWAYS create before reading any code + +**PR workflow: mandatory.** Issue workflow: skip (no code reading needed). + +``` +enter_worktree(name: "triage") +``` + +Save the returned `worktreePath`. Every `read_file`, `grep_search`, `glob`, and shell command that reads local files **MUST** use this path as root. `gh` commands (API calls) do NOT need the worktree. + +Exception: **tmux real-scenario testing** (Stage 2b) runs in the main working tree — it needs the local build environment. + +When triage is complete: `exit_worktree(action: "remove")` + +### 2. Tmux screenshots — ALWAYS inline in Stage 2 comment + +Stage 2 comment **must contain the actual tmux capture-pane output** pasted inline — not a file path, not "see attached", not a summary. The maintainer reads the comment and makes a decision from it. Without inlined terminal output, the review is incomplete and useless. + +## Workflow + +- Issue → read `references/issue-workflow.md` +- PR → read `references/pr-workflow.md` diff --git a/.qwen/skills/triage/references/issue-workflow.md b/.qwen/skills/triage/references/issue-workflow.md new file mode 100644 index 00000000000..0630f150076 --- /dev/null +++ b/.qwen/skills/triage/references/issue-workflow.md @@ -0,0 +1,126 @@ +# Issue Workflow + +Triage a GitHub issue. Shared rules in `SKILL.md` — read those first. + +**Single comment, updated in place.** Stage 1 posts a concise bilingual +comment; Stage 2 appends results to the same comment via `gh api PATCH`. +Key points only — no verbose prose. + +```markdown + + +## Triage + +- **Type**: bug | feature | docs | unclear | inadmissible +- **Labels**: `type/bug`, `scope/cli`, `priority/medium` +- **Next**: + +
+中文说明 + +- **类型**: bug +- **标签**: `type/bug`, `scope/cli`, `priority/medium` +- **下一步**: <一句话动作> +
+ +--- Qwen Code +``` + +## Stage 1: Intake Gate + +Default stance: issues are admissible. Close only the narrow inadmissible cases +below. + +Classify the issue from title, body, comments, labels, docs, and source context: + +- **Inadmissible**: religious or political flame wars, harassment, abusive + language, spam, or content unrelated to Qwen Code. +- **Unclear**: missing reproduction, expected behavior, environment, or enough + detail to answer. +- **Docs / usage**: how-to questions, configuration confusion, documentation + gaps, or behavior that is already documented. +- **Bug**: user-visible broken behavior. +- **Feature**: new capability, behavior change, or product request. + +Apply labels using existing labels only. Prefer one `type/*`, one `category/*`, +relevant `scope/*`, one priority label, and status labels as needed. Apply +labels with `gh issue edit --add-label`. + +Post a single triage comment (bilingual, concise key points — see format +below). This comment is updated in place by Stage 2; never post a second one. + +If inadmissible, close the issue and stop: + +```bash +gh issue close "$ISSUE_NUMBER" --repo "$REPO" --reason "not planned" +``` + +Save the comment ID for Stage 2 to update. + +## Stage 2: Handle By Type + +Work the issue by type below, then **update** the Stage 1 comment in place with +the result appended: + +```bash +gh api -X PATCH repos/$REPO/issues/comments/$COMMENT_ID -F body=@/tmp/triage-comment.md +``` + +### For unclear issues: + +1. Add `status/need-information`. +2. Ask for specific missing data: `/about` output, exact commands, expected vs + actual behavior, logs, screenshots. +3. Stop — no further analysis is useful until the reporter responds. + +### For docs / usage issues: + +1. Search docs and source with `rg` (inside worktree — use `worktreePath` as the search root). +2. Search similar issues (reduce title to safe keywords first): + + ```bash + SAFE_KEYWORDS=$(printf '%s' "$TITLE" | tr -cd '[:alnum:] _-' | cut -c1-60) + if [ -n "$SAFE_KEYWORDS" ]; then + gh issue list --repo "$REPO" --state all --search "$SAFE_KEYWORDS" + else + echo "No Latin keywords (CJK-only title); falling back to label search" + gh issue list --repo "$REPO" --label "type/bug" + fi + ``` + +3. Append the answer with links. + +### For bugs with clear reproduction: + +1. Check safety — no untrusted code with write tokens or secrets. +2. Use `tmux-real-user-testing` skill if available; otherwise tmux manually (runs in main working tree, not worktree): + + ```bash + S=triage-test-$(date +%H%M%S); mkdir -p "tmp/$S" + tmux new-session -d -s "$S" -x 200 -y 50 -c "$(pwd)" + SAFE_SCENARIO=$(printf '%s' "$SCENARIO" | tr -cd '[:alnum:] _-.,' | cut -c1-200) + tmux send-keys -t "$S" "qwen -p '$SAFE_SCENARIO' 2>&1 | tee tmp/$S/before.log" Enter + for i in $(seq 1 120); do tmux capture-pane -t "$S" -p | tail -1 | grep -qE '\$|#' && break; sleep 1; done + tmux capture-pane -t "$S" -p -S -5000 > "tmp/$S/before-session.txt" + tmux send-keys -t "$S" "npm run dev -- -p '$SAFE_SCENARIO' 2>&1 | tee tmp/$S/after.log" Enter + for i in $(seq 1 120); do tmux capture-pane -t "$S" -p | tail -1 | grep -qE '\$|#' && break; sleep 1; done + tmux capture-pane -t "$S" -p -S -5000 > "tmp/$S/after-session.txt" + tmux kill-session -t "$S" + ``` + +3. Inspect source for root cause and likely fix (read files inside worktree). +4. Append: reproduced (yes/no), affected area, fix direction. + +### For bugs without clear reproduction: + +1. Add `welcome-pr` if it exists. Say community PRs are welcome. +2. Add `status/need-retesting` if on a stale version. +3. Inspect source and docs inside worktree; state confidence: confirmed / plausible / no clear + direction. +4. Append likely root cause or link similar historical issues. + +### For feature requests: + +1. Run `/goal Is this feature request truly aligned with Qwen Code's product direction, and is the proposed approach the best solution?` +2. Append verdict: accept for exploration, suggest a smaller alternative, or + decline as out of direction. diff --git a/.qwen/skills/triage/references/pr-workflow.md b/.qwen/skills/triage/references/pr-workflow.md new file mode 100644 index 00000000000..7733d47e5bd --- /dev/null +++ b/.qwen/skills/triage/references/pr-workflow.md @@ -0,0 +1,236 @@ +# PR Workflow + +Shared rules (untrusted input, skip, bilingual format) are in `SKILL.md`. + +**Comment style:** write like a human maintainer — conversational, concise, bilingual. No bullet-point checklists that feel auto-generated. + +### Comment Management + +Three comments, one per stage. Post each with `gh pr comment` and capture its ID: + +```bash +COMMENT_ID=$(gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file /tmp/stage-N.md --json id --jq '.id') +``` + +| Stage | Comment | +| ------- | --------------------------------------------- | +| Stage 1 | Gate findings | +| Stage 2 | Code review + test results (with screenshots) | +| Stage 3 | Reflection + verdict | + +**Re-runs:** if the triage runs again on the same PR, update each comment in place: + +```bash +gh api -X PATCH "/repos/$REPO/issues/comments/$COMMENT_ID" -f body=@/tmp/stage-N-updated.md +``` + +Never create duplicates. + +**Signature:** every comment ends with: + +``` +— *Qwen Code · qwen3.7-max* +``` + +**Approval:** the `gh pr review --approve` command is a separate step that runs **after** Stage 3 comment is posted. Comment first, then approve only when genuinely confident. + +### Stage 1: Gate (Template + Direction + Solution Review) + +**⛔ Before anything else: create a worktree.** This is the #1 forgotten step. + +``` +enter_worktree(name: "triage") +``` + +Save the `worktreePath`. All `read_file`, `grep_search`, `glob` calls below must use it as root. `gh` commands do not need it. + +This is the most important stage — catch problems before anyone spends time reviewing code. + +**1a. Template check:** + +PR body missing required headings from `.github/pull_request_template.md` (read from worktree) → request changes, @mention author, link the template, stop. + +```bash +gh pr review "$PR_NUMBER" --repo "$REPO" --request-changes --body-file /tmp/pr-gate-template.md +``` + +**1b. Product direction:** + +Ask the hard questions before reading a single line of code: + +- Does this solve a real user problem, or is it a solution looking for a problem? +- Is it within qwen-code's core mission, or does it pull focus from what matters more? +- "Can do" ≠ "should do" — technically feasible doesn't mean we should ship it. + +CHANGELOG is a reference signal, not the sole criterion: + +```bash +curl -s https://raw.githubusercontent.com/anthropics/claude-code/main/CHANGELOG.md | grep -iC1 "" +``` + +- **Found** → cite version/line as supporting signal. +- **Not found** → not a rejection. The area may still be relevant. + +**Escalate to maintainer** (never auto-reject): touches auth/sandbox/model selection/telemetry/release/public contract, or direction is genuinely unclear. + +**1c. Solution review** (never skip — judge from the PR description and a skim of the diff structure, before reading code in detail): + +- If we cut 80% of the scope, would the remaining 20% already solve the problem? +- Could we achieve the same goal by modifying something that already exists, instead of adding something new? +- Can the complexity live outside the codebase (user config, external tool) instead of inside it? + +If you spot a materially simpler path, raise it — not as a blocker, but as a genuine question the contributor should think about before the code review. + +Implementation-level concerns (over-abstraction, code duplication, "10 lines vs 10 files") belong in Stage 2a code review — you need to see the code for those. + +Post a single Stage 1 comment. Be direct — say what you actually think, not what's polite: + +```markdown + + +Thanks for the PR! + +Template looks good ✓ + +On direction: . CHANGELOG . + +On approach: . + + Moving on to code review. 🔍 + Flagging these for discussion before diving deeper. + +
+中文说明 + +感谢贡献! + +模板完整 ✓ + +方向:<直接说判断——对齐的原因/担心的原因>。 + +方案:<范围合理 / 感觉可以大幅简化 / 建议砍掉的部分>。<如果看到更简路径,点名:有没有考虑过直接 X?可能用很小的复杂度覆盖大部分场景。> + +<如果通过:> 进入代码审查 🔍 +<如果有顾虑:> 先提出来讨论,再深入看代码。 + +
+ +— _Qwen Code · qwen3.7-max_ +``` + +Save this comment's ID. If template fails or direction is escalated → stop here. + +### Stage 2: Review + Test + +#### 2a. Code Review + +All local file reads (`read_file`, `grep_search`, `glob`) operate inside the worktree. The diff itself comes from `gh pr diff` (GitHub API, no worktree needed). + +**Step 1 — Independent proposal (before reading the diff):** + +Read only the PR title + "Why it's needed" section. Without looking at the diff, write down what _you_ would do to solve this problem. Be concrete — name the files, the approach, the tradeoffs. This is your independent baseline. + +> Why: seeing the diff first anchors your judgment. You'll confirm the PR's approach instead of evaluating whether it's the right approach. Forcing yourself to propose first is the only way to have a real alternative in mind. + +**Step 2 — Compare with the diff:** + +Now read the diff. Compare the PR's approach against your independent proposal: + +- Does the PR's solution match or exceed yours? Or did you find a simpler path it missed? +- Are there correctness bugs, security holes, or regressions your approach would have avoided? +- Does the implementation follow the project's conventions, or does it over-abstract / duplicate code / put logic in the wrong package? + +Keep it tight — only flag two kinds of issues: + +- **Critical blockers** — correctness bugs, security holes, regressions. +- **Clear AGENTS.md violations** — over-abstraction, unnecessary duplication, code in the wrong package, structural patterns that directly contradict the project's conventions. + +Don't nitpick style, naming preferences, or "could be done differently." If it's not a blocker, leave it. + +```bash +gh pr diff "$PR_NUMBER" --repo "$REPO" +``` + +When posting findings, summarize in a few sentences like a human would — "the auth logic is duplicated in two places, worth extracting" not a line-by-line breakdown. Save inline comments for things that genuinely block the merge. + +#### 2b. Real-Scenario Testing + +**Runs in the main working tree, not the worktree** — tmux needs the local build environment. + +**Mandatory.** Unit tests don't substitute. Unrelated build failure ≠ excuse to skip. + +**⛔ The tmux output IS the review.** The maintainer reads your Stage 2 comment and decides approve/reject from it. You **must** paste the actual `capture-pane` terminal output inline in the comment — inside a fenced code block. Not a file path, not "see attached log", not a text summary. If you didn't inline the output, the review is worthless. + +Drive the real product in tmux, using the `tmux-real-user-testing` skill. Capture the terminal at key moments with `capture-pane` — these are the evidence that makes the review actionable. + +**Before/after** (for bug fixes / behavior changes): + +```bash +S=triage-test-$(date +%H%M%S); mkdir -p "tmp/$S" +tmux new-session -d -s "$S" -x 200 -y 50 -c "$(pwd)" +# sanitize scenario — derived from PR text, must not reach shell unsanitized +SAFE_SCENARIO=$(printf '%s' "$SCENARIO" | tr -cd '[:alnum:] _-.,' | cut -c1-200) +# before — installed qwen (bug reproduces) +tmux send-keys -t "$S" "qwen -p '$SAFE_SCENARIO' 2>&1 | tee tmp/$S/before.log" Enter +for i in $(seq 1 120); do tmux capture-pane -t "$S" -p | tail -1 | grep -qE '\$|#' && break; sleep 1; done +tmux capture-pane -t "$S" -p -S -5000 > "tmp/$S/before-session.txt" +# after — this PR via dev build (bug fixed) +tmux send-keys -t "$S" "npm run dev -- -p '$SAFE_SCENARIO' 2>&1 | tee tmp/$S/after.log" Enter +for i in $(seq 1 120); do tmux capture-pane -t "$S" -p | tail -1 | grep -qE '\$|#' && break; sleep 1; done +tmux capture-pane -t "$S" -p -S -5000 > "tmp/$S/after-session.txt" +tmux kill-session -t "$S" +``` + +`qwen ...` = installed build, `npm run dev -- ...` = PR code. Same invocation, only the build differs. + +- Cannot run after exhausting workarounds → FAIL, not skip. +- Fork code: sandbox (strip write tokens/secrets). + +Post a single Stage 2 comment (must include `` at the top): code review findings + testing result. + +**⛔ BEFORE POSTING: verify your comment contains the tmux output.** Read back through your draft — does it have a fenced code block with the actual terminal capture? If not, add it now. The maintainer cannot approve without seeing what actually happened. + +````markdown +## Before (installed build) + + + +## After (this PR) + + +```` + +Sign with `— *Qwen Code · qwen3.7-max*` and save this comment's ID. + +### Stage 3: Reflect + +Don't rush to approve. This is the moment to actually think. + +Step back and look at the whole picture — the motivation, the implementation, the test results, the direction signal. Go back to the independent proposal you wrote in Stage 2a Step 1, and ask yourself: + +- Does the PR's approach match or exceed my independent proposal? Or did I find a simpler path it missed? +- Does this solve something users actually care about? +- Is the code straightforward, or does it feel like it's trying too hard? +- After seeing it run, do the results match what the PR promised? +- If I had to maintain this in six months, would I curse the author or thank them? +- Am I approving this because it's genuinely good, or because I ran out of reasons to say no? + +If your independent proposal was materially simpler — say so. Not as a blocker, but as an honest question the contributor should think about. + +**Step 1: Post the reflection comment** (must include `` at the top). Write what you're actually thinking. "Looks good, ships the feature cleanly, the before/after shows it works" — not a five-bullet summary of the stages. If you have reservations, say them plainly. If you're approving with mild concerns, name them. Sign with `— *Qwen Code · qwen3.7-max*` and save this comment's ID. + +**Step 2: Act on the verdict.** + +All stages genuinely clean — approve: + +```bash +gh pr review "$PR_NUMBER" --repo "$REPO" --approve --body "LGTM, looks ready to ship. ✅" +``` + +Reflection shows it shouldn't merge — request changes immediately, citing the specific concerns from the comment: + +```bash +gh pr review "$PR_NUMBER" --repo "$REPO" --request-changes --body "Needs some rethinking — see my notes above. 🙏" +``` + +Genuinely unsure — **don't approve or reject**. Ask the maintainer to weigh in. Use `$QWEN_MAINTAINER_HANDLE` if set. diff --git a/.qwen/skills/upstream-sync/SKILL.md b/.qwen/skills/upstream-sync/SKILL.md new file mode 100644 index 00000000000..08785a42e8d --- /dev/null +++ b/.qwen/skills/upstream-sync/SKILL.md @@ -0,0 +1,172 @@ +# Upstream Sync Skill + +Synchronize internal fork (alishu/qwen-code) with upstream (QwenLM/qwen-code). + +**Triggers**: "upstream sync", "sync upstream", "analyze upstream", "merge upstream", "sync report" + +## Architecture + +本仓库采用 **patch-stack + guarded merge** 双层保护机制: + +- **Patch stack** (`.fork/patches/`): 将长期 fork 定制以有序补丁形式维护,sync 后可验证 +- **Guarded merge** (`.aoneci/upstream-sync-merge.yml`): CI 自动 fetch + merge + 验证 + 创建 MR +- **Domain auth helper** (`.aoneci/scripts/upstream-sync-domain-auth.sh`): 统一认证和 MR 发布 + +关键文件: + +| 文件 | 用途 | +| ---------------------------------------------- | --------------------------------- | +| `.fork/manifest.json` | 补丁定义、包名映射、registry 配置 | +| `.fork/patches/series` | 补丁应用顺序 | +| `.fork/apply.sh` | 应用补丁栈 | +| `.fork/unapply.sh` | 反转补丁栈 | +| `.fork/verify.sh` | 验证补丁是否在当前代码中存活 | +| `.fork/generate-patches.js` | 从 fork diff 生成补丁文件 | +| `.fork/rewrite-package-identity.js` | 包名/registry 改写(正向/反向) | +| `.fork/sync-upstream.sh` | 本地 upstream sync 辅助 | +| `.aoneci/upstream-sync-merge.yml` | CI 自动同步流水线 | +| `.aoneci/upstream-sync-analyze.yml` | CI 每日变更分析 + 钉钉通知 | +| `.aoneci/scripts/upstream-sync-domain-auth.sh` | CI 认证/推送/MR 发布 | +| `.qwen/upstream-sync-rules.yml` | 冲突解决策略规则 | +| `scripts/upstream-sync-analyze.sh` | Git diff 分析辅助 | +| `scripts/upstream-sync-verify.sh` | merge 后验证流水线 | +| `scripts/upstream-fork-divergence.sh` | fork 差异自动分析 | + +## Modes + +### Mode 1: Analyze (default, read-only) + +When the user says "analyze upstream" or "sync report": + +1. Run `git fetch upstream main` +2. Find the merge base: `MERGE_BASE=$(git merge-base HEAD upstream/main)` +3. Count new upstream commits: `git log --oneline $MERGE_BASE..upstream/main --no-merges | wc -l` +4. If zero commits, report "Already up to date" and stop +5. Categorize upstream commits by reading their messages and diff stats: + - For each commit, classify area (cli/core/channels/docs/tests/build/other) and risk (safe/low/medium/high) +6. Identify files changed on both sides: + ```bash + comm -12 <(git diff --name-only $MERGE_BASE HEAD | sort) <(git diff --name-only $MERGE_BASE upstream/main | sort) + ``` +7. Cross-reference with `.fork/manifest.json` patch paths to identify high-risk overlaps +8. Run `bash scripts/upstream-fork-divergence.sh` to get current fork divergence state +9. Generate a structured markdown report with: + - Summary: N new upstream commits, date range + - Commits grouped by area and risk + - Both-changed files count and list + - High-risk files (upstream changed files that overlap with fork patches) + - Redundant internal commits (already in upstream) + - Current divergence categories and convergence status + - Recommended next action + +### Mode 2: Merge + +When the user says "merge upstream" or "sync upstream": + +1. Read `.qwen/upstream-sync-rules.yml` for conflict resolution rules +2. Create a sync branch with collision-safe checkpoint tag: + ```bash + git tag "sync-checkpoint-$(date +%Y%m%d-%H%M%S)-$(git rev-parse --short=7 HEAD)" HEAD + SYNC_BRANCH="sync/upstream-$(date +%Y-%m-%d)" + git checkout -b $SYNC_BRANCH + ``` +3. Attempt merge: `git merge upstream/main --no-edit` +4. If conflicts occur: + - Count conflicted files: `git diff --name-only --diff-filter=U` + - If count > 20, abort and report (too many conflicts for auto-resolution) + - For each conflicted file: + a. Determine which rule applies from upstream-sync-rules.yml + b. If rule says "keep ours": `git checkout --ours && git add ` + c. If rule says "keep theirs": `git checkout --theirs && git add ` + d. If rule says "ai-resolve": read both versions, apply semantic merge, write result + e. If rule says "regenerate" (package-lock.json): delete and run `npm install` +5. Run fork patch verification: + ```bash + bash .fork/verify.sh + ``` +6. Run build verification: + ```bash + npm run build + npm run typecheck + npm run test + ``` +7. If verification fails, attempt auto-fix (max 2 rounds): + - Read error output + - Identify likely cause (import path changes, type signature changes, etc.) + - Apply fix and re-run verification +8. If still failing after 2 rounds: + - Report failure with diagnostics + - Offer to rollback using the checkpoint tag +9. Commit and push: + ```bash + git add -A + git commit -m "chore: sync upstream $(date +%Y-%m-%d)" + git push origin $SYNC_BRANCH + ``` +10. Report success with MR creation instructions + +### Mode 3: Divergence Check + +When the user says "check divergence" or "fork status": + +1. Run `bash scripts/upstream-fork-divergence.sh` to generate the full divergence report +2. Summarize key metrics: divergence score, categories, convergence status +3. Highlight any new files that need categorization +4. Suggest convergence actions for "review needed" files + +### Mode 4: Patch Management + +When the user says "refresh patches", "generate patches", or "verify patches": + +1. **Generate/refresh patches**: + ```bash + git fetch origin main && git fetch upstream main --tags + node .fork/generate-patches.js --write + node .fork/generate-patches.js --check + ``` +2. **Verify patches against current code**: + ```bash + bash .fork/verify.sh + ``` +3. **Apply/unapply patch stack** (for testing or upstream sync): + ```bash + bash .fork/apply.sh # apply all patches in series order + bash .fork/unapply.sh # reverse all patches + ``` +4. **Rewrite package identity** (after sync): + ```bash + node .fork/rewrite-package-identity.js # apply fork names + node .fork/rewrite-package-identity.js --reverse # restore upstream names + ``` + +## CI Pipeline State Machine + +The `.aoneci/upstream-sync-merge.yml` pipeline uses a state-file pattern: + +``` +CONFLICT_STATUS_FILE states: + "skip" → no upstream changes, pipeline exits early + "pending" → merge in progress (initial state after new commits detected) + "clean" → merge succeeded without conflicts + "has_conflicts"→ merge produced conflicts (may be auto-resolved) +``` + +Pipeline flow: + +1. `prepare` → domain-auth helper bootstraps git context +2. `fetch upstream` → get latest upstream/main +3. `check new commits` → if none, set "skip" and exit +4. `create sync branch` → set status to "pending" +5. `merge` → attempt git merge, update status to "clean" or "has_conflicts" +6. `verify` → build + typecheck + patch verification +7. `publish` → domain-auth helper pushes branch and creates/reuses MR + +## Important Notes + +- NEVER force-push or rewrite history on main or staging branches +- ALWAYS create a checkpoint tag before merging (format: `sync-checkpoint-YYYYMMDD-HHMMSS-`) +- ALWAYS run build+test after merge before reporting success +- ALWAYS run `.fork/verify.sh` after merge to check patch survival +- If in doubt about a conflict resolution, ask the user rather than guessing +- Divergence analysis is fully automated via `scripts/upstream-fork-divergence.sh` +- Domain auth supports both username+token and legacy token fallback diff --git a/AGENTS.md b/AGENTS.md index f7bfd45037d..c0cd3825a5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,24 @@ This file provides guidance to Qwen Code when working with code in this repository. +## Working Principles + +### Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** +**(This is the principle we care about most.)** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, +simplify. + +_Adapted from Andrej Karpathy's [CLAUDE.md](https://github.com/multica-ai/andrej-karpathy-skills/blob/main/CLAUDE.md)._ + ## Common Commands ### Building @@ -101,6 +119,9 @@ npm run preflight # Full check: clean → install → format → lint → build between packages - **Tests**: Collocated with source (`file.test.ts` next to `file.ts`), vitest framework +- **File naming**: `PascalCase.tsx` for React components, `kebab-case.ts` for + new non-component files. Leave existing `camelCase` files alone — renaming breaks `git blame` and imports. +- **Comments**: Default to none. Add only when _why_ is non-obvious; don't delete existing ones as cleanup. - **Commits**: Conventional Commits (e.g., `feat(cli): Add --json flag`) - **Node.js**: Development and production both require `>=22` (Ink 7 + React 19.2 requirement) @@ -158,8 +179,13 @@ applicable. - **PR description**: explain the motivation and changes in prose. Avoid referencing file names or function names. -- **Reviewer Test Plan**: describe behaviors a reviewer should verify and what - to expect, not scripted test commands. +- **Reviewer Test Plan** (template section): describe behaviors a reviewer + should verify and what to expect, not scripted test commands. Use **How to + verify** for reproduction steps; Before/After for TUI evidence when + applicable. +- **Line wrapping**: do not hard-wrap the PR body at a fixed column width. + GitHub renders single newlines as `
`, so a wrapped description displays + as a narrow column. Write each paragraph or list item as one long line. ## Project Directories diff --git a/abc.json b/abc.json new file mode 100644 index 00000000000..f99caced15f --- /dev/null +++ b/abc.json @@ -0,0 +1,13 @@ +{ + "assets": { + "type": "command", + "command": { + "cmd": [ + "npm i", + "npm run test", + "npm run build", + "bash scripts/copy-to-package.sh" + ] + } + } +} diff --git a/docs-site/README.md b/docs-site/README.md index ad6272c3379..126e0aff210 100644 --- a/docs-site/README.md +++ b/docs-site/README.md @@ -17,13 +17,15 @@ npm install ### Setup Content -Link the documentation content from the parent `docs` directory: +Prepare the public documentation content from the parent `docs` directory: ```bash npm run link ``` -This creates a symbolic link from `../docs` to `content` in the project. +This creates a `content` directory with copies of the public docs sections. +Internal planning, design, and E2E notes remain outside the docs site content +tree. ### Development diff --git a/docs-site/package.json b/docs-site/package.json index 1b5af5ae55f..532699e110d 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -7,10 +7,10 @@ "type": "module", "main": "index.js", "scripts": { - "link": "ln -s ../docs content", + "link": "node scripts/link-public-docs.mjs", "clean": "rm -rf .next", "dev": "npm run clean && next --turbopack", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run --config vitest.config.js" }, "dependencies": { "next": "^16.0.8", @@ -18,5 +18,8 @@ "nextra-theme-docs": "^4.6.1", "react": "^19.2.1", "react-dom": "^19.2.1" + }, + "devDependencies": { + "vitest": "^3.2.4" } } diff --git a/docs-site/scripts/link-public-docs.mjs b/docs-site/scripts/link-public-docs.mjs new file mode 100644 index 00000000000..a57ec04f11f --- /dev/null +++ b/docs-site/scripts/link-public-docs.mjs @@ -0,0 +1,26 @@ +import { cp, mkdir, rm, symlink } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { PUBLIC_DOC_ROOTS } from '../src/app/public-docs.js'; + +const contentDir = 'content'; + +async function linkPublicDocs() { + try { + await rm(contentDir, { force: true, recursive: true }); + await mkdir(contentDir); + await cp('../docs/index.md', join(contentDir, 'index.md')); + await cp('../docs/_meta.ts', join(contentDir, '_meta.ts')); + + for (const root of PUBLIC_DOC_ROOTS) { + await symlink(join('..', '..', 'docs', root), join(contentDir, root)); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to link public docs into ${contentDir}: ${message}`, + ); + } +} + +await linkPublicDocs(); diff --git a/docs-site/src/app/[[...mdxPath]]/page.jsx b/docs-site/src/app/[[...mdxPath]]/page.jsx index c980e9f6075..85f6a3377b5 100644 --- a/docs-site/src/app/[[...mdxPath]]/page.jsx +++ b/docs-site/src/app/[[...mdxPath]]/page.jsx @@ -1,10 +1,23 @@ import { generateStaticParamsFor, importPage } from 'nextra/pages'; +import { notFound } from 'next/navigation'; import { useMDXComponents as getMDXComponents } from '../../../mdx-components'; +import { filterPublicStaticParams, isPublicDocsPath } from '../public-docs'; -export const generateStaticParams = generateStaticParamsFor('mdxPath'); +const generateAllStaticParams = generateStaticParamsFor('mdxPath'); + +export const dynamicParams = false; + +export async function generateStaticParams(...args) { + const staticParams = await generateAllStaticParams(...args); + return filterPublicStaticParams(staticParams); +} export async function generateMetadata(props) { const params = await props.params; + if (!isPublicDocsPath(params.mdxPath)) { + notFound(); + } + const { metadata } = await importPage(params.mdxPath); return metadata; } @@ -13,6 +26,10 @@ const Wrapper = getMDXComponents().wrapper; export default async function Page(props) { const params = await props.params; + if (!isPublicDocsPath(params.mdxPath)) { + notFound(); + } + const { default: MDXContent, toc, diff --git a/docs-site/src/app/[[...mdxPath]]/page.test.jsx b/docs-site/src/app/[[...mdxPath]]/page.test.jsx new file mode 100644 index 00000000000..af1caf2fce2 --- /dev/null +++ b/docs-site/src/app/[[...mdxPath]]/page.test.jsx @@ -0,0 +1,59 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const generateAllStaticParams = vi.fn(); + + return { + generateAllStaticParams, + generateStaticParamsFor: vi.fn(() => generateAllStaticParams), + }; +}); + +vi.mock('nextra/pages', () => ({ + generateStaticParamsFor: mocks.generateStaticParamsFor, + importPage: vi.fn(), +})); + +vi.mock('next/navigation', () => ({ + notFound: vi.fn(), +})); + +vi.mock('../../../mdx-components', () => ({ + useMDXComponents: () => ({ + wrapper: ({ children }) => children, + }), +})); + +describe('generateStaticParams', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('filters internal docs from Nextra static params', async () => { + mocks.generateAllStaticParams.mockResolvedValue([ + { mdxPath: [] }, + { mdxPath: ['users', 'foo'] }, + { mdxPath: ['en', 'users'] }, + { mdxPath: ['design', 'bar'] }, + { mdxPath: ['plans'] }, + ]); + + const { generateStaticParams } = await import('./page.jsx'); + + await expect(generateStaticParams()).resolves.toEqual([ + { mdxPath: [] }, + { mdxPath: ['users', 'foo'] }, + { mdxPath: ['en', 'users'] }, + ]); + }); + + it('fails closed if Nextra changes the static params shape', async () => { + mocks.generateAllStaticParams.mockResolvedValue([{ slug: ['users'] }]); + + const { generateStaticParams } = await import('./page.jsx'); + + await expect(generateStaticParams()).rejects.toThrow( + 'Expected generateStaticParamsFor("mdxPath") to return objects with an mdxPath array.', + ); + }); +}); diff --git a/docs-site/src/app/public-docs.js b/docs-site/src/app/public-docs.js new file mode 100644 index 00000000000..6e1d7c2a4f2 --- /dev/null +++ b/docs-site/src/app/public-docs.js @@ -0,0 +1,33 @@ +const LOCALE_SEGMENTS = new Set(['en', 'zh', 'de', 'fr', 'ja', 'ru', 'pt-BR']); + +// Keep this in sync with the public top-level page entries in docs/_meta.ts. +// docs-site/scripts/link-public-docs.mjs consumes the same allowlist. +export const PUBLIC_DOC_ROOTS = ['users', 'developers']; + +const PUBLIC_DOC_ROOT_SET = new Set(PUBLIC_DOC_ROOTS); + +function publicRootFromSegments(segments = []) { + if (segments.length === 0 || (segments.length === 1 && segments[0] === '')) { + return undefined; + } + + const rootIndex = LOCALE_SEGMENTS.has(segments[0]) ? 1 : 0; + return segments[rootIndex]; +} + +export function isPublicDocsPath(mdxPath = []) { + const root = publicRootFromSegments(mdxPath); + return root === undefined || PUBLIC_DOC_ROOT_SET.has(root); +} + +export function filterPublicStaticParams(staticParams = []) { + return staticParams.filter((staticParam) => { + if (!Array.isArray(staticParam?.mdxPath)) { + throw new TypeError( + 'Expected generateStaticParamsFor("mdxPath") to return objects with an mdxPath array.', + ); + } + + return isPublicDocsPath(staticParam.mdxPath); + }); +} diff --git a/docs-site/src/app/public-docs.test.js b/docs-site/src/app/public-docs.test.js new file mode 100644 index 00000000000..6583e6d443f --- /dev/null +++ b/docs-site/src/app/public-docs.test.js @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; + +import { filterPublicStaticParams, isPublicDocsPath } from './public-docs.js'; + +describe('isPublicDocsPath', () => { + it.each([ + [[], true], + [[''], true], + [['users', 'foo'], true], + [['design', 'bar'], false], + [['en', 'users'], true], + [['plans'], false], + [['en'], true], + ])('returns %s for %j', (mdxPath, expected) => { + expect(isPublicDocsPath(mdxPath)).toBe(expected); + }); +}); + +describe('filterPublicStaticParams', () => { + it('keeps public paths and rejects internal docs paths', () => { + expect( + filterPublicStaticParams([ + { mdxPath: [] }, + { mdxPath: [''] }, + { mdxPath: ['users', 'foo'] }, + { mdxPath: ['en', 'developers'] }, + { mdxPath: ['design', 'bar'] }, + { mdxPath: ['plans'] }, + ]), + ).toEqual([ + { mdxPath: [] }, + { mdxPath: [''] }, + { mdxPath: ['users', 'foo'] }, + { mdxPath: ['en', 'developers'] }, + ]); + }); + + it('fails closed if Nextra changes the static params shape', () => { + expect(() => filterPublicStaticParams([{ slug: ['users'] }])).toThrow( + 'Expected generateStaticParamsFor("mdxPath") to return objects with an mdxPath array.', + ); + }); +}); diff --git a/docs-site/vitest.config.js b/docs-site/vitest.config.js new file mode 100644 index 00000000000..aa08810cba2 --- /dev/null +++ b/docs-site/vitest.config.js @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/**/*.test.{js,jsx}'], + }, +}); diff --git a/docs/design/2026-05-15-async-memory-recall-design.md b/docs/design/2026-05-15-async-memory-recall-design.md new file mode 100644 index 00000000000..f11b2ac5d23 --- /dev/null +++ b/docs/design/2026-05-15-async-memory-recall-design.md @@ -0,0 +1,206 @@ +# Async Memory Recall — Design Spec + +**Date:** 2026-05-15 +**Status:** Approved +**Related issues:** #3761, #3759 +**Related PRs:** #3814, #3866 + +--- + +## Problem + +`relevanceSelector.ts` uses `AbortSignal.timeout(1_000)` (introduced by #3866). On first-session cold starts, qwen3.5-flash averages ~908 ms — consistently hitting the 1 s threshold. The outer 2.5 s deadline in `resolveAutoMemoryWithDeadline` means every UserQuery can block for up to 2.5 s even when recall always fails. + +Root cause: the main-agent request path `await`s the recall result before sending to the model. Any slowness in the recall side-query directly adds to user-visible latency. + +--- + +## Design + +### Core idea + +Fire recall on UserQuery and never await it. Consume the result at two opportunistic points — whichever fires first: + +1. **UserQuery consume point** — synchronous `settledAt !== null` check just before `turn.run()`. Zero-wait: if already settled, use it; if not, skip. +2. **ToolResult inject point** — same check on every ToolResult turn. Injects memory as a `system-reminder` **appended after** the functionResponse parts in `requestToSend`, giving the model memory context before its next response. (Append, not prepend: the Qwen API requires the functionResponse to immediately follow the model's functionCall — see the existing `hasPendingToolCall` IDE-context skip for the same constraint.) + +This matches the pattern used by Claude Code upstream (`startRelevantMemoryPrefetch` / `settledAt` polling in `query.ts`). + +--- + +## Data structures + +### New type `MemoryPrefetchHandle` (in `client.ts`) + +```typescript +type MemoryPrefetchHandle = { + promise: Promise; + /** Set by promise.finally(). null until the promise settles. */ + settledAt: number | null; + /** True after memory has been injected — prevents double-inject. */ + consumed: boolean; + controller: AbortController; +}; +``` + +### Field change on `GeminiClient` + +| Remove | Add | +| ------------------------------------------------------------ | ---------------------------------------------------------- | +| `pendingRecallAbortController: AbortController \| undefined` | `pendingMemoryPrefetch: MemoryPrefetchHandle \| undefined` | + +--- + +## Changes + +### 1. `client.ts` — remove `resolveAutoMemoryWithDeadline` + +Delete the function entirely. It is replaced by the `settledAt` flag mechanism. + +### 2. `client.ts` — UserQuery fire path + +Replace the `resolveAutoMemoryWithDeadline` call with: + +```typescript +// Abort any in-flight prefetch from a previous UserQuery before installing +// the new handle (prevents orphan side-queries when the user types again +// before recall settles). +this.pendingMemoryPrefetch?.controller.abort(); +this.pendingMemoryPrefetch = undefined; + +const controller = new AbortController(); +// Bridge the caller's signal into the prefetch controller so a user abort +// (Ctrl-C / Esc) on the parent turn also terminates the recall side-query. +const onParentAbort = () => controller.abort(); +if (signal.aborted) { + controller.abort(); +} else { + signal.addEventListener('abort', onParentAbort, { once: true }); +} + +const promise = this.config + .getMemoryManager() + .recall(projectRoot, partToString(request), { + config: this.config, + excludedFilePaths: this.surfacedRelevantAutoMemoryPaths, + abortSignal: controller.signal, + }) + .catch((error: unknown) => { + if (!(error instanceof DOMException && error.name === 'AbortError')) { + debugLogger.warn('Managed auto-memory recall prefetch failed.', error); + } + return EMPTY_RELEVANT_AUTO_MEMORY_RESULT; + }); + +const handle: MemoryPrefetchHandle = { + promise, + settledAt: null, + consumed: false, + controller, +}; +void promise.finally(() => { + handle.settledAt = Date.now(); + signal.removeEventListener('abort', onParentAbort); +}); +this.pendingMemoryPrefetch = handle; +// no await — continue immediately +``` + +### 3. `client.ts` — UserQuery consume point (replaces `await relevantAutoMemoryPromise`) + +```typescript +const prefetchHandle = this.pendingMemoryPrefetch; +if ( + prefetchHandle && + prefetchHandle.settledAt !== null && + !prefetchHandle.consumed +) { + prefetchHandle.consumed = true; + this.pendingMemoryPrefetch = undefined; + const result = await prefetchHandle.promise; // already settled, returns immediately + if (result.prompt) { + // unshift, not push: keep memory at the front of systemReminders so + // it leads the system-reminder block on UserQuery turns. (ToolResult + // turns instead append to requestToSend to preserve functionCall / + // functionResponse pairing — see below.) + systemReminders.unshift(result.prompt); + for (const doc of result.selectedDocs) { + this.surfacedRelevantAutoMemoryPaths.add(doc.filePath); + } + } +} +``` + +### 4. `client.ts` — ToolResult inject point (new) + +After `requestToSend` is assembled, before `turn.run()`, add: + +```typescript +if (messageType === SendMessageType.ToolResult) { + const prefetchHandle = this.pendingMemoryPrefetch; + if ( + prefetchHandle && + prefetchHandle.settledAt !== null && + !prefetchHandle.consumed + ) { + prefetchHandle.consumed = true; + this.pendingMemoryPrefetch = undefined; + const result = await prefetchHandle.promise; + if (result.prompt) { + // Append (not prepend) so functionResponse parts stay first + // and the model's functionCall/functionResponse pairing + // isn't broken on the native Gemini path. + requestToSend = [...requestToSend, result.prompt]; + for (const doc of result.selectedDocs) { + this.surfacedRelevantAutoMemoryPaths.add(doc.filePath); + } + } + } +} +``` + +### 5. `client.ts` — cleanup paths + +The handle is released by two distinct mechanisms: + +**5 abort-and-clear sites** (the prefetch is still pending, abort the controller before dropping the reference). Replace `pendingRecallAbortController?.abort()` + `= undefined` with: + +```typescript +this.pendingMemoryPrefetch?.controller.abort(); +this.pendingMemoryPrefetch = undefined; +``` + +Sites: `resetChat()`, `MaxSessionTurns` early-return, `boundedTurns=0` early-return, `SessionTokenLimitExceeded` early-return, Arena control-signal early-return. The fire path itself also performs this abort-then-replace when a new UserQuery arrives while the previous prefetch is still in flight. + +**2 clear-only sites** (the prefetch has already settled and we're consuming it — no controller to abort, just drop the reference): + +```typescript +prefetchHandle.consumed = true; +this.pendingMemoryPrefetch = undefined; +``` + +Sites: UserQuery consume point, ToolResult inject point. + +### 6. `relevanceSelector.ts` — remove `AbortSignal.timeout(1_000)` + +Remove the combined `AbortSignal.any([AbortSignal.timeout(1_000), callerAbortSignal])` and pass `callerAbortSignal` directly. + +--- + +## Behaviour comparison + +| Scenario | Before | After | +| -------------------------------------------- | ------------------------------ | ------------------------------------------------------ | +| recall completes before model prep | inject on UserQuery, ~0 wait | inject on UserQuery, ~0 wait | +| recall slow (cold start) | block up to 2.5 s | skip UserQuery, inject on first ToolResult | +| recall times out (1 s) | abort, empty result, no memory | no hard timeout; inject whenever settled | +| no tool calls, recall slow | block up to 2.5 s, then skip | skip UserQuery, no ToolResult opportunity — miss | +| user sends 2nd message before recall settles | 2nd recall races 1st handle | 1st handle aborted when 2nd UserQuery fires new handle | + +--- + +## Out of scope + +- Changing the memory injection format from `system-reminder` to `tool-result` attachment (CC style) +- Per-session byte budget skip gate +- Single-word prompt skip gate diff --git a/docs/design/auto-compaction-threshold-redesign.md b/docs/design/auto-compaction-threshold-redesign.md new file mode 100644 index 00000000000..544f5baecd9 --- /dev/null +++ b/docs/design/auto-compaction-threshold-redesign.md @@ -0,0 +1,436 @@ +# Auto-Compaction Threshold Redesign + +**Status:** Draft · 2026-05-14 + +## 背景 + +> 本节描述本 PR 落地**之前**的状态(pre-redesign behavior)。下文出现的 `COMPRESSION_TOKEN_THRESHOLD`、`thinkingConfig.includeThoughts = true`、`hasFailedCompressionAttempt`、以及具体的 file:line 引用都对应 PR #4345 合入前的代码——合入后这些符号 / 行号会不再有效。 + +当前 qwen-code 的自动压缩仅使用单一比例阈值 `COMPRESSION_TOKEN_THRESHOLD = 0.7`(`chatCompressionService.ts:33`),所有窗口大小共用同一比例。对比 claude-code 的「绝对 token 梯子」(autoCompact.ts:62-65),qwen-code 存在三个具体问题: + +1. **大窗口下预留过多**:1M 模型 70% 阈值在 700K 触发,剩余 300K 远超摘要 + 输出实际所需的 ~33K +2. **失败 1 次永久锁**:`hasFailedCompressionAttempt = true` 之后整个 session 不再尝试 auto-compact(geminiChat.ts:504),比 claude-code 的「连续 3 次熔断」更严苛 +3. **tip 系统与 auto 阈值脱钩**:`tipRegistry.ts` 里的三条 `context-*` tip 使用固定的 50/80/95 百分比,与 auto-compact 阈值(70%)完全独立。这意味着在「auto 正常工作」的主路径上 80% / 95% tip 极少触发,而在「auto 失败 / 反应式兜底」的边缘路径上又缺乏与阈值对齐的语义 +4. **压缩调用本身没有输出预算控制**:[chatCompressionService.ts:374-376](packages/core/src/services/chatCompressionService.ts:374) 显式开启 `thinkingConfig.includeThoughts = true`(注释:「Compression quality drives every subsequent main turn」),同时 sideQuery 调用未设 `maxOutputTokens` 上限。代码注释([:436-437](packages/core/src/services/chatCompressionService.ts:436))也承认 `compressionOutputTokenCount may include non-persisted tokens (thoughts)`。在压缩接近窗口顶时,总输出可能膨胀,使 buffer 预留缺乏可预测上限。

更糟糕的是跨 provider 行为不一致:Anthropic 的 thinking budget 与 max_tokens 完全独立;OpenAI 的 reasoning tokens 不受 max_completion_tokens 限制;Gemini 的行为又因模型版本而异。这意味着「单靠加 maxOutputTokens 就能控制总输出」在 qwen-code 这种多 provider 项目里不成立 + +5. **阈值判断使用的 `lastPromptTokenCount` 系统性下偏。** [geminiChat.ts:1217-1232](packages/core/src/core/geminiChat.ts:1217) 表明这个数来自上一轮 API response 的 `usageMetadata.totalTokenCount`。两个 gap:(a) 不包含本轮即将加入的 user message,每次 cheap-gate 判断都比真实 prompt 小一段;(b) 首轮初始值是 0,`--continue` 恢复巨大 session / sub-agent 继承大量历史时第一次 send 永远绕过所有阈值。对比 claude-code 的 `tokenCountWithEstimation`([query.ts:638](src/query.ts:638))走「最后一条 assistant API usage + 之后新增 message 估算」的双轨制能闭合这两个 gap + +## 设计目标 + +- 引入「比例 + 绝对」混合阈值,让大窗口模型由绝对值接管,小窗口仍走比例兜底 +- 新增 warn / hard 两层(auto 保留为主触发点),形成三层梯子 +- 把 tip 系统重写为跟随新阈值的触发条件 +- 失败处理从「1 次永久锁」升级为「3 次熔断 + 自动恢复」 +- **压缩调用关闭 thinking 并加 `maxOutputTokens` 上限**:与 claude-code 对齐,让总输出受单一参数约束、buffer 预算可预测;接受压缩质量可能下降的代价 +- **加 token 估算补偿**:消除 `lastPromptTokenCount` 的「滞后一轮」和「首轮为 0」两个系统性下偏,让阈值判断更贴近真实 prompt 大小 +- 删除 settings 里的 `contextPercentageThreshold` 配置入口(内部 PCT 常量保留) +- **不引入** env 覆盖通道、**不**新增显式 enabled 开关 + +## 三层阈值梯子 + +``` + window (raw context window) + │ + │ ← SUMMARY_RESERVE = 20K + ▼ + effectiveWindow + │ + │ ← HARD_BUFFER = 3K + ▼ + hard_threshold = effectiveWindow - 3K + │ + │ ← (AUTOCOMPACT_BUFFER - HARD_BUFFER) = 10K + ▼ +auto_threshold = max(PCT * window, effectiveWindow - AUTOCOMPACT_BUFFER) + │ + │ ← WARN_BUFFER = 20K + ▼ +warn_threshold = max((PCT - WARN_OFFSET) * window, auto_threshold - WARN_BUFFER) + │ + ▼ + 0 +``` + +### 三层语义 + +| 层 | 触发条件 | 行为 | +| -------- | ------------------------------ | -------------------------------------------------------- | +| **warn** | `tokenCount >= warn_threshold` | UI 提示「距自动压缩还剩 X tokens」,不改变 send 行为 | +| **auto** | `tokenCount >= auto_threshold` | 在 send 前 `tryCompress(force=false)`,正常压缩流程 | +| **hard** | `tokenCount >= hard_threshold` | 在 send 前 `tryCompress(force=true)`,重置失败锁强制压缩 | + +`hard` 层等同于把现有 reactive overflow(geminiChat.ts:711)的兜底逻辑提前到 send 前,避免一次失败的 oversized request round-trip。 + +## 内部常量 + +```ts +// chatCompressionService.ts +const DEFAULT_PCT = 0.7; // auto 比例兜底 +const WARN_PCT_OFFSET = 0.1; // warn 比例 = PCT - WARN_OFFSET = 0.6 +const COMPACT_MAX_OUTPUT_TOKENS = 20_000; // 压缩 sideQuery 输出硬上限(thinking + summary 合计) +const SUMMARY_RESERVE = 20_000; // 阈值梯子从窗口顶减去的输出预留 = maxOutput +const AUTOCOMPACT_BUFFER = 13_000; // auto 与 effectiveWindow 间距 +const WARN_BUFFER = 20_000; // warn 与 auto 间距 +const HARD_BUFFER = 3_000; // hard 与 effectiveWindow 间距 +const MAX_CONSECUTIVE_FAILURES = 3; // 失败熔断阈值 +``` + +数值来源:全部沿用 claude-code 的实测值([autoCompact.ts:30,62-65](src/services/compact/autoCompact.ts:30))。 + +`SUMMARY_RESERVE = COMPACT_MAX_OUTPUT_TOKENS` 是关键关系:模型受 `maxOutputTokens` 硬限制约束,输出不可能超出 20K,因此 reserve 不需要额外 safety margin。注意:本设计关闭 thinking 后该等式成立(output budget 全部给 summary);若保留 thinking,`thinking + summary` 共享预算(Gemini SDK / 多数 provider 的 `maxOutputTokens` 语义),模型自行在两者间分配,此时 summary 的实际可用空间小于 20K(见「风险与注意事项」第 1、2 条)。 + +## 计算函数 + +```ts +export interface CompactionThresholds { + warn: number; + auto: number; + hard: number; // 当 hard < auto 时等于 auto(小窗口退化) + effectiveWindow: number; +} + +export function computeThresholds(window: number): CompactionThresholds { + const effectiveWindow = window - SUMMARY_RESERVE; + + const absAuto = effectiveWindow - AUTOCOMPACT_BUFFER; + const auto = Math.max(DEFAULT_PCT * window, absAuto); + + const absWarn = auto - WARN_BUFFER; + const warn = Math.max((DEFAULT_PCT - WARN_PCT_OFFSET) * window, absWarn); + + const rawHard = effectiveWindow - HARD_BUFFER; + const hard = Math.max(rawHard, auto); // 小窗口下退化为 auto + + return { warn, auto, hard, effectiveWindow }; +} +``` + +### 实测数据 + +| 窗口 | warn | auto | hard | 备注 | +| ---- | ----------- | ----------- | ------------ | ------------------------------- | +| 32K | 19.2K (pct) | 22.4K (pct) | 22.4K (退化) | 比例兜底 | +| 64K | 38.4K (pct) | 44.8K (pct) | 44.8K (退化) | 比例兜底 | +| 128K | 76.8K (pct) | 95K (abs) | 105K (abs) | 混合(warn=pct, auto/hard=abs) | +| 200K | 147K (abs) | 167K (abs) | 177K (abs) | 绝对接管 | +| 256K | 203K (abs) | 223K (abs) | 233K (abs) | 绝对接管 | +| 1M | 947K (abs) | 967K (abs) | 977K (abs) | 全绝对 | + +`(pct)` 表示该层由比例公式决定,`(abs)` 表示由绝对值公式决定。 + +## 用户配置 + +### ChatCompressionSettings 变更 + +```ts +// packages/core/src/config/config.ts:217 +export interface ChatCompressionSettings { + /** 保留(与本设计无关,由 compactionInputSlimming 使用) */ + imageTokenEstimate?: number; +} +``` + +**删除:** `contextPercentageThreshold` 字段。理由: + +1. 新公式下,对主流窗口(>= 128K)该字段几乎无影响——绝对值接管 +2. 小窗口下用户配置反而可能让阈值"更早"压缩,与节省 token 直觉相反 +3. claude-code 没有暴露此字段,无类似的用户面配置先例 + +### Breaking change 处理 + +**用户面:** 启动时 `Config` 加载发现 `chatCompression.contextPercentageThreshold` 存在: + +- 写入 stderr 一行警告:`"chatCompression.contextPercentageThreshold has been removed and is now controlled by built-in thresholds."` +- **不**报错、**不**阻塞启动 +- 字段值被忽略 + +**SDK 面(R5.4):** `CompressOptions` 的 `hasFailedCompressionAttempt: boolean` 字段重命名为 `consecutiveFailures: number`。两点差异: + +| | 旧字段 | 新字段 | +| ---- | ------------------------------ | -------------------------------------------------------------------- | +| 名称 | `hasFailedCompressionAttempt` | `consecutiveFailures` | +| 类型 | `boolean` | `number` | +| 语义 | `true` = 永久禁用 auto-compact | `>= MAX_CONSECUTIVE_FAILURES`(默认 3)= 暂时禁用直到 force 成功重置 | + +仓库内只有 `GeminiChat.tryCompress` 一个内部消费方,所以内部 migration 风险低;但 `@qwen-code/qwen-code-core` 是 published package、`CompressOptions` 在 d.ts 里可见,下游 SDK 直接调 `service.compress({ ..., hasFailedCompressionAttempt: true })` 的代码会拿到 TS 编译错误。**迁移指引:** 把 `true` 改为 `MAX_CONSECUTIVE_FAILURES`(或任意 >= 3 的整数),`false` 改为 `0`。如果调用方维护自己的失败计数,直接传入即可。 + +## Token 估算补偿 + +qwen-code 的 `lastPromptTokenCount` 来自上一轮 API response 的 `usageMetadata.totalTokenCount`([geminiChat.ts:1217-1232](packages/core/src/core/geminiChat.ts:1217))。这导致: + +1. **滞后一轮**:cheap-gate 用 `lastPromptTokenCount` 判断,但本次 send 实际 prompt = 它 + 本轮 user message。少算的部分可能让阈值判断 false-negative +2. **首轮为 0**:初始值是 0,第一次 send 时无论历史多大都不会触发任何阈值(含 `--continue` 恢复 / sub-agent 继承场景) + +引入轻量本地估算函数 `estimatePromptTokens`,在 send 前 cheap-gate / hard 判断时补足这两段缺失: + +```ts +// chatCompressionService.ts(或新文件 packages/core/src/services/tokenEstimation.ts) + +const BYTES_PER_TOKEN = 4; // 通用 char/4 估算(claude-code 同此) +const BYTES_PER_TOKEN_JSON = 2; // JSON / tool_call input 更密集 + +/** + * 估算一组 Content 的 token 数,用于补偿 API usage metadata 的滞后。 + * 对 image / document 复用现有 imageTokenEstimate(默认 1600)。 + */ +export function estimateContentTokens( + contents: Content[], + imageTokenEstimate = DEFAULT_IMAGE_TOKEN_ESTIMATE, +): number { + // 复用 estimateContentChars(compactionInputSlimming.ts),再除以 bytesPerToken + // 内部对 functionCall / functionResponse 用 BYTES_PER_TOKEN_JSON + // ... +} + +/** + * cheap-gate 与 hard 判断的统一入口。 + * 主路径:lastPromptTokenCount 准 + 本轮 user message 估算 + * 首轮路径:full history 估算 + */ +export function estimatePromptTokens( + history: Content[], + userMessage: Content, + lastPromptTokenCount: number, +): number { + if (lastPromptTokenCount > 0) { + return lastPromptTokenCount + estimateContentTokens([userMessage]); + } + return estimateContentTokens([...history, userMessage]); +} +``` + +应用位置: + +- `chatCompressionService.compress()` 的 cheap-gate:把 `originalTokenCount` 来源换成 `estimatePromptTokens(history, userMessage, lastPromptTokenCount)` +- `geminiChat.sendMessageStream` 入口的 hard 判断(见下一节) + +**估算只用于提前触发,不用于「跳过触发」。** 因为 char/4 是粗略下界估计,作为 false-positive 一侧是安全的(宁可早一点压),作为 false-negative 则不可靠。 + +## 触发链路改动 + +### chatCompressionService.ts + +1. **导出 `computeThresholds`**,供 cheap-gate / UI / 命令复用 +2. **`compress()` cheap-gate** (line 221-249): + ```ts + if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES && !force) { + return NOOP; + } + const { auto } = computeThresholds(contextLimit); + const effectiveTokens = estimatePromptTokens( + curatedHistory, + userMessage, + originalTokenCount, + ); + if (!force && effectiveTokens < auto) return NOOP; + ``` +3. **`compress()` 的 runSideQuery 调用** (line 356-380):关闭 thinking + 加 `maxOutputTokens`: + + ```ts + const summaryResult = await runSideQuery(config, { + // ... + config: { + thinkingConfig: { includeThoughts: false }, // 关闭 thinking(与 claude-code 一致) + maxOutputTokens: COMPACT_MAX_OUTPUT_TOKENS, // 硬上限 20K + }, + // ... + }); + ``` + + 或者直接删掉 `thinkingConfig` 让 `runSideQuery` 默认值([sideQuery.ts:118](packages/core/src/utils/sideQuery.ts:118) 默认 `includeThoughts: false`)接管。 + + 关 thinking 后,`maxOutputTokens` 直接约束总输出(不存在 thinking 单独 budget 的问题),`SUMMARY_RESERVE = maxOutput = 20K` 是干净的硬关系。 + + 同时更新 [chatCompressionService.ts:374-376](packages/core/src/services/chatCompressionService.ts:374) 的注释,从「Compression quality drives every subsequent main turn — keep reasoning on」改为说明「为保证跨 provider 可预测的输出上限,与 claude-code 设计对齐」。 + + token math 一段([:436-437](packages/core/src/services/chatCompressionService.ts:436))的 "may include non-persisted tokens (thoughts)" 注释也可以同步清理 + +### geminiChat.ts: `sendMessageStream` 入口(line 562) + +```ts +// 替换前:tryCompress(force=false) +// 替换后:用估算 token 判断是否触发 hard,决定 force 标志 + +const { hard } = computeThresholds(contextLimit); +const effectiveTokens = estimatePromptTokens( + this.getHistory(true), + createUserContent(params.message), + this.lastPromptTokenCount, +); +const shouldForceFromHard = effectiveTokens >= hard; + +if (shouldForceFromHard) { + // 重置熔断器,等同 force compress + this.consecutiveFailures = 0; +} + +compressionInfo = await this.tryCompress( + prompt_id, + model, + shouldForceFromHard, + params.config?.abortSignal, +); +``` + +### 失败处理升级 (`geminiChat.ts:504-510`) + +```ts +// 替换前 +hasFailedCompressionAttempt: boolean; + +// 替换后 +consecutiveFailures: number; // 默认 0 + +// 失败分支 +} else if (isCompressionFailureStatus(info.compressionStatus)) { + if (!force) { + this.consecutiveFailures += 1; + } +} + +// 成功分支 +this.consecutiveFailures = 0; +``` + +`force=true` 调用失败不计入计数(保持现有 reactive / manual 不"占额"的语义)。 + +## UI 改动 + +### tipRegistry.ts 重写三条 context-\* tip + +三层阈值正好与三条 tip 一一对应。映射关系(按 token 数从低到高): + +| Tip ID | 当前条件 | 新条件 | 文案变化 | +| ------------------ | --------------------------------------------- | ------------------------------------------------------------------- | ----------------------------------------------------------------- | +| `compress-intro` | `pct >= 50 && < 80 && sessionPromptCount > 5` | `tokenCount >= warn && tokenCount < auto && sessionPromptCount > 5` | 保持不变 | +| `context-high` | `pct >= 80 && < 95` | `tokenCount >= auto && tokenCount < hard` | 保持不变 | +| `context-critical` | `pct >= 95` | `tokenCount >= hard` | 加一句「Auto-compact will force on next send.」反映新 hard 层行为 | + +**对触发频率的影响:** + +- 主路径(auto 正常工作):`tokenCount` 跨越 auto 后立即触发压缩,下一轮 tokenCount 回落,所以 `context-high` 仅在「触发到压缩生效之间」短暂可见 +- 边缘路径(auto 失败 / 熔断 / reactive 来不及):`tokenCount` 持续上涨,会依次穿过 warn → auto → hard 触发三条 tip,跟用户视角的"上下文越来越紧"一致 +- `context-critical` 触发时 hard 层已经在 send 前 force compress(spec 触发链路改动一节),所以这条 tip 实际上是「post-rescue 告知」而非「pre-rescue 警告」,文案补一句说明 + +`TipContext` 接口增加: + +```ts +export interface TipContext { + lastPromptTokenCount: number; + contextWindowSize: number; + sessionPromptCount: number; + sessionCount: number; + platform: string; + // 新增:让 isRelevant 函数能拿到阈值。 + // computeThresholds 在调用方算好后注入,避免 tipRegistry 直接依赖 core。 + thresholds?: CompactionThresholds; +} +``` + +`AppContainer.tsx:1150` 构造 `TipContext` 时同步注入。 + +### /context 命令同步 (`contextCommand.ts:177-183`) + +```ts +// 替换硬编码 (1 - threshold) * contextWindowSize +const { warn, auto, hard, effectiveWindow } = + computeThresholds(contextWindowSize); + +// 显示四行: +// Effective window: 180K (window − 20K reserve) +// Warn threshold: 147K (...) +// Auto threshold: 167K ← 当前位置 +// Hard threshold: 177K +// 标记当前 token count 落在哪个 tier +``` + +### Footer 持续提示(可选 follow-up) + +本 spec 不强制实现 footer 持续提示,理由: + +- 现有 tip 系统已经能在 history 里给出提示 +- Footer 持续提示需要改 ink 渲染、增加重绘频率 +- 可作为本 spec 后置 follow-up(独立 PR) + +如果后续要做,建议触发条件 `tokenCount >= warn && tokenCount < auto`,超过 auto 后隐藏(压缩已开始)。 + +## 测试覆盖 + +### 单元测试(chatCompressionService.test.ts) + +- `computeThresholds(32K)` → 比例兜底分支(warn/auto 均 pct,hard 退化) +- `computeThresholds(128K)` → 混合分支(warn=pct,auto=abs,hard=abs) +- `computeThresholds(200K)` → 绝对接管分支(warn/auto/hard 均 abs) +- `computeThresholds(1M)` → 全绝对分支 +- `computeThresholds(window=10K)` → 极小窗口(绝对值全负),公式不崩 +- 三层阈值始终满足 `warn <= auto <= hard` +- max() 公式在边界点(pct \* window == abs)稳定 + +### 单元测试(tokenEstimation.test.ts) + +- `estimateContentTokens` 对纯文本 / json / functionCall / functionResponse / image / document 分别走对应 bytesPerToken +- `estimatePromptTokens` 在 `lastPromptTokenCount > 0` 时走「主路径」,等于 0 时走「首轮路径」 +- 大 user message 在 cheap-gate 阶段被加上去后能跨越 auto 阈值 +- 估算与真实 API usage 的偏差在 ±30% 以内(用真实历史样本回归) + +### 集成测试(geminiChat.test.ts / chatCompressionService.test.ts) + +- 3 次连续失败后 cheap-gate NOOP;下一次 force 后恢复 +- 单次失败不再永久锁 +- 估算 token 跨越 hard 后 send 自动 force compress +- 压缩 sideQuery 调用 `maxOutputTokens = COMPACT_MAX_OUTPUT_TOKENS` 正确透传到 `runSideQuery`,`thinkingConfig.includeThoughts` 为 `false`(或被 sideQuery 默认值接管) +- **首轮覆盖**:构造一个 `lastPromptTokenCount = 0` 但 history 巨大的 chat(模拟 `--continue` 恢复),首次 send 时 auto 阈值能被估算路径触发 + +### 兼容性测试 + +- 设置 `contextPercentageThreshold = 0.5` 启动 → stderr 警告 + 字段被忽略,行为以内部 PCT 常量为准 + +### Tip 系统测试(tipRegistry.test.ts) + +- 三条 context-\* tip 在跨越 warn/auto/hard 时正确触发,且区间不重叠 +- 主路径下 auto 阈值触发压缩后 `context-high` 不持续可见 +- 边缘路径(熔断 + token 继续涨)下三条 tip 依次触发 +- TipContext 缺 `thresholds` 时(fallback)行为合理 + +## 实施分阶段 + +| Phase | 内容 | 独立性 | +| ----- | -------------------------------------------------------------------------------------------- | ------------------ | +| 1 | 内部常量 + `computeThresholds` + cheap-gate 改动(不含估算补偿) | 可独立合并 | +| 2 | 失败处理升级(1 → 3 熔断) | 可独立合并 | +| 3 | hard 层 force compress 提前 | 依赖 P1 + P7 | +| 4 | 配置面变更 + breaking change 警告 | 依赖 P1 | +| 5 | UI(tip 重写 + /context) | 依赖 P1 | +| 6 | 压缩 sideQuery 关 thinking + 加 `maxOutputTokens` 上限 | 独立可先于 P1 落地 | +| 7 | Token 估算补偿(`estimateContentTokens` + `estimatePromptTokens`,应用到 cheap-gate / hard) | 独立可与 P1 并行 | + +每个 Phase 可独立 PR。建议合并顺序 **P6 → P7 → P1 → P2 → P4 → P3 → P5**:先给压缩调用打上 `maxOutputTokens` 上限(让 buffer 假设可信);再加估算补偿(让 token 数判断更可靠);再把阈值基础设施落地;再做失败熔断、配置面变更;最后才打开 hard 层主动救场(这时已有可靠的 token 数 + 熔断器)。每个 PR 都能独立验证、独立回滚。 + +## 风险与注意事项 + +1. **关 thinking 可能影响摘要质量。** 原作者注释 "Compression quality drives every subsequent main turn — keep reasoning on" 表达过对此的担忧。本 spec 的判断是「可预测的 token 上限」优先于「最大化质量」,但落地后需要观察 telemetry 里 `compression_input_token_count` / `compression_output_token_count` 的分布,以及主对话在压缩后的质量变化(用户反馈、`COMPRESSION_FAILED_*` 状态率)。如果质量下降明显,再考虑回退到 thinking 开启 + provider-specific thinkingBudget 控制。 + +2. **`maxOutputTokens` 触顶可能导致 summary 被截断。** 关 thinking 后,20K 直接限制 summary 主体;claude-code 实测 p99.99 ≈ 17K,留 ~3K 安全冗余。但 qwen-code 的压缩 prompt 与 claude-code 不同,分布需要观测。建议在压缩失败分支([chatCompressionService.ts:464-491](packages/core/src/services/chatCompressionService.ts:464))追加「检测到 finish_reason = MAX_TOKENS」的 NOOP 路径,避免持久化半截 summary。 + +3. **跨 provider 的 maxOutputTokens 映射差异。** OpenAI compat (dashscope) → `max_tokens`、Anthropic → `max_tokens`、Gemini SDK → `maxOutputTokens`。当前 qwen-code 已有这层映射([contentGenerator.ts:94](packages/core/src/core/contentGenerator.ts:94) 等),需要在 P6 实现时验证 sideQuery 路径上 `maxOutputTokens` 字段确实贯穿到所有 provider 的请求体。 + +4. **Token 估算是粗略下界,不应反向用作"跳过触发"的依据。** `char/4` 与各 provider 真实 tokenizer 偏差可能 ±30%。本 spec 只用估算来「让阈值更早触发」(false-positive 方向,宁可早压不可晚压)。所有「降低 token 计数 / 跳过压缩」的代码路径仍应使用 `lastPromptTokenCount`(API 权威值)。 + +5. **估算函数与现有 `estimateContentChars` 的关系。** [compactionInputSlimming.ts](packages/core/src/services/compactionInputSlimming.ts) 已经有 `estimateContentChars`(用于压缩 split point 计算),新增的 `estimateContentTokens` 应复用它(除以 bytesPerToken)而非新写一套,避免两套估算口径出现分歧。 + +## 不在本 spec 范围 + +- Env 变量覆盖通道(D 方案):维持「配置面最小」原则 +- Footer 常驻可视化:留作 follow-up +- 摘要 prompt 改进、`MIN_COMPRESSION_FRACTION` 调整:与阈值设计正交 + +## 开放问题(等 review) + +1. **breaking change 强度**:警告 + 忽略字段 vs 启动报错。当前选警告,需要确认对企业部署/团队配置是否够友好 + +## 已结案 + +2. **小窗口(≤ ~76.7K)下 hard 与 auto 退化为同一值** — 决定**不在 `/context` 明示**。理由: + - 塌缩范围不只是 32K,所有 `effectiveWindow - HARD_BUFFER ≤ 0.7 × window` 的窗口都塌缩(包括 64K) + - 用户行为不变:塌缩窗口上 `currentTier` 跳过 `'auto'` 直接报 `'hard'`(`contextCommand.ts:43-44` 先判 `>= hard`),`context-high` band(`auto ≤ t < hard`)变成空带,少一档提示在小窗口上是合理的——窗口本身就小,用户大概率手动管理上下文 + - 如果未来有真实用户报告"小窗口看不到中间档提示",再决定加 UI 标注或调整 `context-high` 触发条件(这是 UI 工作,不是 spec 工作)。当前选不增加 UI 复杂度 diff --git a/docs/design/channels/channels-implementation.md b/docs/design/channels/channels-implementation.md new file mode 100644 index 00000000000..35e936b4fb3 --- /dev/null +++ b/docs/design/channels/channels-implementation.md @@ -0,0 +1,107 @@ +# Channels + +Qwen Code supports three messaging channels — Telegram, WeChat, and DingTalk. All adapters extend the shared channel architecture (`ChannelBase`, `AcpBridge`, `SessionRouter`) in `packages/channels/base/src/`. Each channel can be started individually or all together with `node dist/cli.js channel start`. + +--- + +## Telegram + +Source: `packages/channels/telegram/src/TelegramAdapter.ts`, built on the Telegraf library. + +The adapter supports plain text messaging, slash commands, a working indicator ("typing" chat action), DM pairing, and group chat (supergroups with @mention gating). Image receiving works via `bot.on('photo')` → `getFileLink` → download → base64, with captions passed as envelope text. File/document receiving saves downloaded files to `/tmp/channel-files/` and includes the path in the envelope so the agent can read them via `read-file` (works with any model, no multimodal required). Referenced messages include the quoted text as context in the prompt. Output is formatted as Telegram HTML (converted from markdown). Authentication uses a static bot token. Session persistence and pairing state are stored under `~/.qwen/channels/`. + +```jsonc +// ~/.qwen/settings.json +{ + "channels": { + "my-telegram": { + "type": "telegram", + "token": "$TELEGRAM_BOT_TOKEN", + "senderPolicy": "pairing", + "allowedUsers": [], + "sessionScope": "user", + "instructions": "Keep responses concise.", + }, + }, +} +``` + +```bash +source /path/to/telegram/.env +npm run bundle && node dist/cli.js channel start my-telegram +``` + +**Future work:** Streaming responses via in-place `editMessageText` (throttled at ~2s to respect rate limits, best-effort fallback to single message). Slash command polish — register with BotFather via `setMyCommands()`, fix `/help` timing, add `/status` command. + +--- + +## WeChat (Weixin) + +Source: `packages/channels/weixin/src/`, ported from the cc-weixin project. Uses the iLink Bot API at `ilinkai.weixin.qq.com`. + +The adapter supports plain text messaging via a custom long-poll loop (`/ilink/bot/getupdates`, cursor-based), with `context_token` caching per user for reply context. Authentication uses QR code login (`qwen channel configure-weixin`), producing a bearer token stored in `~/.qwen/channels/weixin/account.json`. A typing indicator fires before each ACP prompt using the `sendTyping` API (ticket obtained from `getConfig`). Image and file/PDF receiving works through CDN download with AES-128-ECB decryption — images are forwarded as base64 content blocks, files are saved to `/tmp/channel-files/` and referenced by path. Referenced messages (user replies) include quoted text as context in the prompt. Formatting is plain text only (all markdown is stripped). The adapter handles session expiry (`errcode -14`) with automatic reconnection, uses backoff after consecutive errors, and persists the polling cursor to `~/.qwen/channels/weixin/cursor.txt` for crash recovery. + +```jsonc +// ~/.qwen/settings.json +{ + "channels": { + "my-weixin": { + "type": "weixin", + "senderPolicy": "pairing", + "allowedUsers": [], + "sessionScope": "user", + "instructions": "Keep responses concise, plain text only.", + "baseUrl": "https://ilinkai.weixin.qq.com", // optional override + }, + }, +} +``` + +Credentials are stored separately in `~/.qwen/channels/weixin/account.json`, created by `qwen channel configure-weixin`. + +```bash +# First time: login via QR code +node dist/cli.js channel configure-weixin + +# Start +npm run bundle && node dist/cli.js channel start my-weixin +``` + +**Future work:** Media send (upload to WeChat CDN with AES encryption). Voice/video receive. Streaming responses via `message_state: GENERATING` → `FINISH` (pending client-side investigation). Multi-account support. Message chunking for long responses. + +--- + +## DingTalk (钉钉) + +Source: `packages/channels/dingtalk/src/`, using Stream mode (WebSocket, no public IP required). Referenced from openclaw-channel-dingtalk. + +The adapter connects via the `dingtalk-stream` SDK, which handles WebSocket connection, reconnection, heartbeats, and callback ACKs (DingTalk retries unACKed messages). Authentication reuses the SDK's built-in token (`client.getConfig().access_token`) from AppKey + AppSecret. Responses are sent back through a per-message `sessionWebhook` URL — a temporary, conversation-scoped endpoint that supports text, markdown, images, and files. Both DM and group chat are supported, with group messages gated by `@mention` detection (`isInAtList`). A 👀 emoji reaction serves as a working indicator while the agent processes (posted via the emotion API and recalled on completion). Output is formatted as DingTalk markdown, with tables converted to plain text, messages split at ~3800 characters, and code fences maintained across chunks. Image, file, audio, and video receiving works through a two-step download flow (`downloadCode` → `downloadUrl` → buffer); images are forwarded as base64, files saved to `/tmp/channel-files/`. Quoted message context is extracted from `text.repliedMsg` and `quoteMessage`, with bot-reply detection via `chatbotUserId`. + +```jsonc +// ~/.qwen/settings.json +{ + "channels": { + "my-dingtalk": { + "type": "dingtalk", + "clientId": "$DINGTALK_CLIENT_ID", + "clientSecret": "$DINGTALK_CLIENT_SECRET", + "senderPolicy": "open", + "sessionScope": "user", + "cwd": "/path/to/project", + "instructions": "Keep responses concise. Use DingTalk markdown.", + "groupPolicy": "open", + "groups": { + "*": { "requireMention": true }, + }, + }, + }, +} +``` + +```bash +export DINGTALK_CLIENT_ID= +export DINGTALK_CLIENT_SECRET= +npm run bundle && node dist/cli.js channel start my-dingtalk +``` + +**Future work:** Quoted bot responses (persisting outbound messages keyed by `processQueryKey` for lookup on reply). AI Card streaming via `/v1.0/card/instances` and `/v1.0/card/streaming` with graceful markdown fallback. diff --git a/docs/design/channels/channels-roadmap.md b/docs/design/channels/channels-roadmap.md new file mode 100644 index 00000000000..8f0583744c0 --- /dev/null +++ b/docs/design/channels/channels-roadmap.md @@ -0,0 +1,51 @@ +# Channels Roadmap + +## Implemented (MVP) + +- **3 built-in channels** — Telegram, WeChat, DingTalk +- **Plugin system** — `ChannelBase` SDK with `connect`/`sendMessage`/`disconnect`, extension manifest, compiled JS + `.d.ts` +- **Access control** — `allowlist`, `pairing` (8-char codes, CLI approval), `open` policies +- **Group chat** — `open`/`disabled`/`allowlist` group policy, `requireMention` per group, reply-as-mention +- **Session routing** — `user`, `thread`, `single` scopes with per-channel `cwd`, `model`, `instructions` +- **Dispatch modes** — `steer` (default: cancel + re-prompt), `collect` (buffer + coalesce), `followup` (sequential queue). Per-channel and per-group config. +- **Working indicators** — centralized `onPromptStart`/`onPromptEnd` hooks. Telegram: typing bar. WeChat: typing API. DingTalk: 👀 emoji reaction. +- **Block streaming** — progressive multi-message delivery with paragraph-aware chunking +- **Streaming hooks** — `onResponseChunk`/`onResponseComplete` for plugins to implement progressive display +- **Media support** — images (vision input), files/audio/video (saved to temp, path in prompt), `Attachment` interface on `Envelope` +- **Slash commands** — `/help`, `/clear` (`/reset`, `/new`), `/status`, custom via `registerCommand()` +- **Service management** — `qwen channel start/stop/status`, PID tracking, crash recovery (auto-restart, session persistence) +- **Token security** — `$ENV_VAR` syntax in config + +## Future Work + +### Safety & Group Chat + +- **Per-group tool restrictions** — `tools`/`toolsBySender` deny/allow lists per group +- **Group context history** — ring buffer of recent skipped messages, prepended on @mention +- **Regex mention patterns** — fallback `mentionPatterns` for unreliable @mention metadata +- **Per-group instructions** — `instructions` field on `GroupConfig` for per-group personas +- **`/activation` command** — runtime toggle for `requireMention`, persisted to disk + +### Operational Tooling + +- **`qwen channel doctor`** — config validation, env vars, bot tokens, network checks +- **`qwen channel status --probe`** — real connectivity checks per channel + +### Platform Expansion + +- **Discord** — Bot API + Gateway, servers/channels/DMs/threads +- **Slack** — Bolt SDK, Socket Mode, workspaces/channels/DMs/threads + +### Multi-Agent + +- **Multi-agent routing** — multiple agents with bindings per channel/group/user +- **Broadcast groups** — multiple agents respond to the same message + +### Plugin Ecosystem + +- **Community plugin template** — `create-qwen-channel` scaffolding tool +- **Plugin registry/discovery** — `qwen extensions search`, version compatibility + +## Reference: OpenClaw Comparison + +See [channels-comparison.md](channels-comparison.md) for the detailed feature comparison between OpenClaw and Qwen-Code channels. diff --git a/docs/design/channels/channels-testing-guide.md b/docs/design/channels/channels-testing-guide.md new file mode 100644 index 00000000000..cbb4bcdcdbe --- /dev/null +++ b/docs/design/channels/channels-testing-guide.md @@ -0,0 +1,156 @@ +# Channels Testing Guide + +How to test channel integrations end-to-end. + +## Credentials + +- Telegram bot: `@qwencod_test_1_bot` (远弟) +- Bot token env var: `TELEGRAM_BOT_TOKEN` +- Bot token file: `/path/to/telegram/.env` +- Telegram user ID: `` +- WeChat credentials: `~/.qwen/channels/weixin/account.json` + +## Before testing + +**Important:** Stop any running service first. Duplicate instances cause duplicate responses. + +```bash +# Stop the service if running +qwen channel stop + +# Or check status first +qwen channel status + +# If processes are stuck (e.g. from manual kill -9), clean up manually +pkill -9 -f "cli.js --acp" +pkill -9 -f "channel start" +rm -f ~/.qwen/channels/service.pid ~/.qwen/channels/sessions.json +``` + +## Sending messages via Bot API (no bot process needed) + +```bash +# Source the token +export TELEGRAM_BOT_TOKEN=$(grep TELEGRAM_BOT_TOKEN /path/to/telegram/.env | cut -d= -f2) + +# Send a message (replace YOUR_CHAT_ID with your Telegram user ID) +curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + -H "Content-Type: application/json" \ + -d '{"chat_id": "YOUR_CHAT_ID", "text": "Hello from the bot!"}' +``` + +## Starting channels + +```bash +export TELEGRAM_BOT_TOKEN=$(grep TELEGRAM_BOT_TOKEN /path/to/telegram/.env | cut -d= -f2) +cd /path/to/qwen-code +npm run bundle + +# Single channel +node dist/cli.js channel start my-telegram + +# All channels (shared bridge) +node dist/cli.js channel start +``` + +Settings config: `~/.qwen/settings.json` under `channels.*`. + +## Checking registered commands + +```bash +curl -s "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/getMyCommands" | python3 -m json.tool +``` + +## Test scenarios + +### 1. Slash commands (shared across all channels) + +Start the service, then send on Telegram or WeChat: + +| Command | Expected | +| --------- | --------------------------------------------------------------- | +| `/help` | List of all commands | +| `/status` | "Session: none, Access: ..." | +| `/clear` | "No active session to clear." (or "Session cleared." if active) | +| `/reset` | Same as `/clear` (alias) | +| `/new` | Same as `/clear` (alias) | + +### 2. Basic text round-trip + +1. Start the bot +2. Send any text (e.g. "hello") +3. Bot should respond via the agent +4. `/status` should now show "Session: active" + +### 3. Multi-turn conversation + +1. Send "my name is Alice" +2. Send "what is my name?" +3. Agent should remember "Alice" from same session + +### 4. Session clear + +1. Have an active session (send a message first) +2. Send `/clear` (or `/reset` or `/new`) +3. Send "what is my name?" +4. Agent should NOT remember — fresh session + +### 5. Tool calls (internal) + +1. Send "list the files in /path/to/project" +2. Agent should use shell/ls internally and return file listing +3. Verify response contains actual file names + +### 6. Markdown formatting + +1. Send "write me a hello world in python with explanation" +2. Response should render with proper Telegram HTML formatting (bold, code blocks, etc.) + +### 7. Multi-channel mode + +1. Ensure both `my-telegram` and `my-weixin` are configured in `~/.qwen/settings.json` +2. For WeChat: run `node dist/cli.js channel configure-weixin` if token expired +3. Start all: `node dist/cli.js channel start` +4. Should show: `Starting 2 channel(s): my-weixin, my-telegram` +5. Send messages on both platforms — each should get exactly one response +6. Check `~/.qwen/channels/sessions.json` — each channel should have its own cwd + +### 8. Crash recovery + +1. Start multi-channel mode and send a message to create sessions +2. Find the ACP bridge PID: `ps --ppid -o pid,args | grep acp` +3. Kill it: `kill -9 ` +4. Log should show: `Bridge crashed (1/3). Restarting in 3s...` then `Sessions restored: 2, failed: 0` +5. Send a message — should work, and session context (e.g. "what is my name?") should be preserved + +### 9. Clean shutdown + +1. Start channels, send a message to create sessions +2. Press Ctrl+C (or `qwen channel stop` from another terminal) +3. `~/.qwen/channels/sessions.json` should be deleted +4. `~/.qwen/channels/service.pid` should be deleted + +### 10. Service management + +1. Start service: `qwen channel start` +2. Check status from another terminal: `qwen channel status` — should show running, uptime, channels +3. Try starting again: `qwen channel start` — should fail with "already running" error +4. Stop from another terminal: `qwen channel stop` — should stop gracefully +5. Confirm stopped: `qwen channel status` — should show "No channel service is running." + +### 11. Referenced messages (quoted replies) + +1. Send a message and get a bot response +2. Reply to (quote) the bot's response with a follow-up question (e.g. "summarize that") +3. Agent should see the quoted text as context and respond accordingly +4. Test on both Telegram and WeChat + +## Useful debug commands + +```bash +# Check recent updates the bot received +curl -s "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/getUpdates?limit=5" | python3 -m json.tool + +# Get bot info +curl -s "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/getMe" | python3 -m json.tool +``` diff --git a/docs/design/fork-patch-guard/fork-patch-guard-design.md b/docs/design/fork-patch-guard/fork-patch-guard-design.md new file mode 100644 index 00000000000..5d72bc00c5a --- /dev/null +++ b/docs/design/fork-patch-guard/fork-patch-guard-design.md @@ -0,0 +1,221 @@ +# Fork Patch Guard:upstream sync 时 fork 定制保护机制 + +> 解决 fork 仓库在合并上游代码时,冲突解决阶段静默丢失 fork 定制改动的问题。 + +## 1. 问题背景 + +### 1.1 当前痛点 + +本仓库是 QwenLM/qwen-code 的内部 fork,包含大量 DataWorks 定制改动(tips、i18n、双输出模式、CI 配置等)。每次 upstream sync 时需要合并上游几十个 commits,过程中: + +- **冲突解决容易丢失 fork 改动**:上游重构了文件结构(如 `startupTips[]` → `tipRegistry` 系统),合并时 fork 的改动被上游新代码覆盖 +- **丢失是静默的**:合并后 typecheck/build/tests 全部通过,但功能行为已经变了 +- **发现时已经晚了**:通常在用户使用中才发现 "怎么通用 tips 又回来了" + +### 1.2 已发生的案例 + +| Commit | 描述 | 丢失情况 | +| ----------- | ------------------------------------ | --------------------------------------------- | +| `b621fe82d` | 只显示 DataWorks tips,移除通用 tips | 上游引入 tipRegistry 系统时通用 tips 全部回来 | + +### 1.3 不适用的方案 + +- **git cherry-pick 检测**:上游重构后 commit 的 diff 完全不同,cherry 无法匹配 +- **文件级 diff**:fork 改动分散在几十个文件中,逐文件比对噪音太大 +- **手动 checklist**:容易遗漏,依赖人的记忆 + +## 2. 方案设计 + +### 2.1 核心思路 + +维护一份 **fork 补丁清单**,记录每个 fork 定制的 commit 及其关键意图。提供 **验证脚本**,在 upstream sync 后自动检查这些定制是否还在。 + +验证逻辑基于 **commit diff 内容推导**,不需要手动维护文件路径和断言。 + +### 2.2 组成部分(已实现) + +``` +.fork/ + manifest.json # 补丁定义、包名映射、registry 配置(source of truth) + patches/ + series # 补丁应用顺序 + 0001-branding-header.patch + 0002-branding-tips.patch + ... + apply.sh # 按顺序应用补丁栈 + unapply.sh # 反转所有补丁 + verify.sh # 验证补丁是否在当前代码中存活 + generate-patches.js # 从 fork diff 生成补丁文件 + generate-patches.sh # shell 包装 + create-patch.sh # 创建新补丁 + refresh-patch.sh # 刷新已有补丁 + rewrite-package-identity.js # 包名/registry 正向和反向改写 + sync-upstream.sh # 本地 upstream sync 辅助 + patches.md # 自动生成的 fork 补丁清单 +``` + +> 注:此设计文档描述的是 v1 原始方案。实际实现参见 +> `docs/design/fork-patch-guard/fork-patch-stack-architecture.md`。 + +### 2.3 补丁清单格式 (`.fork/patches.md`) + +```markdown +# Fork Patches + +每次在 fork 上做定制改动后,在此记录 commit 信息。 +upstream sync 后运行 `bash .fork/verify.sh` 检查这些改动是否还在。 + +当前第一阶段已经落地 `.fork/patches.md`,内容包括: + +- `origin/main` 相对 `upstream/main` 的 first-parent PR/MR 落地提交清单 +- patch-bearing commit inventory +- snapshot 使用的 `origin/main`、`upstream/main`、`merge-base` +- 后续 upstream sync 过程中如何新增、保留、退休条目的维护规则 + +## DataWorks Tips + +- commit: b621fe82d +- 描述: 只显示 DataWorks tips,移除通用 startup tips +- 验证策略: added-lines +- 关键意图: tipRegistry.ts 中不应存在通用 startup tips(如 new-user-slash、compress-startup 等) + +## DataWorks 输入框 Placeholder + +- commit: ca172b61e +- 描述: 输入框使用 DataWorks 定制的 placeholder 文案 +- 验证策略: added-lines + +## npm 发布策略 + +- commit: 9550d4755 +- 描述: 支持 x.y.z-dataworks.N 版本号作为正式版发布 +- 验证策略: added-lines +``` + +字段说明: + +| 字段 | 必填 | 说明 | +| -------- | ---- | -------------------------------------------------------- | +| commit | 是 | fork 定制的 commit hash | +| 描述 | 是 | 一句话说明这个改动做了什么 | +| 验证策略 | 否 | `added-lines`(默认)/ `removed-lines` / `both` / `skip` | +| 关键意图 | 否 | 补充说明,帮助人工判断 | + +### 2.4 验证脚本逻辑 (`.fork/verify.sh`) + +``` +对清单中的每个 commit: + 1. git show 提取 diff + 2. 根据验证策略检查: + - added-lines: commit 新增的非空行(+开头),在当前 HEAD 对应文件中应存在 + - removed-lines: commit 删除的非空行(-开头),在当前 HEAD 对应文件中应不存在 + - both: 同时检查以上两项 + - skip: 跳过自动验证(需人工确认) + 3. 报告结果:PASS / WARN(部分行缺失)/ FAIL(大量行缺失) +``` + +#### 关键设计决策 + +**为什么用 commit diff 推导而不是手动写断言?** + +- 手动写断言维护成本高,容易和代码不同步 +- commit diff 是事实来源(source of truth),改动了什么自动可知 +- 上游重构文件路径时,脚本能自动检测到文件不存在并报警 + +**为什么用行级匹配而不是 patch apply?** + +- 上游可能重新格式化了代码(缩进、换行) +- 行内容可能微调(变量名、import 路径) +- 行级匹配容忍轻微变化,patch apply 会直接失败 + +**阈值判定:PASS / WARN / FAIL** + +- PASS:>= 80% 的关键行仍存在 +- WARN:50%–80% 的关键行存在(可能是重构导致,需人工确认) +- FAIL:< 50% 的关键行存在(大概率被覆盖) + +### 2.5 工作流集成 + +``` +开发时: + 1. 在 fork 上做定制改动 + 2. 提交后,在 .fork/patches.md 中添加一条记录 + +upstream sync 时: + 1. git merge upstream/main + 2. 解决冲突 + 3. 运行 bash .fork/verify.sh + 4. 检查输出,修复 FAIL/WARN 的条目 + 5. 提交 merge commit +``` + +可以考虑在 merge commit 的 CI 中自动运行验证脚本,作为 pipeline check。 + +## 3. 验证脚本伪代码 + +```bash +#!/bin/bash +# .fork/verify.sh — 验证 fork 定制改动是否在当前 HEAD 中存在 + +PATCHES_FILE=".fork/patches.md" +PASS=0; WARN=0; FAIL=0; SKIP=0 + +# 解析 patches.md,提取每个 patch 的 commit 和验证策略 +parse_patches() { + # 从 markdown 中提取 ## 标题、commit hash、验证策略 +} + +for each patch: + # 1. 获取 commit 的 diff + diff=$(git show $commit --format= -- ) + + # 2. 提取改动的文件和行 + for each file in diff: + added_lines = lines starting with "+" (non-header) + removed_lines = lines starting with "-" (non-header) + + # 3. 检查行是否存在于当前文件 + if strategy == "added-lines" or "both": + for line in added_lines: + grep -qF "$line" current_file + if strategy == "removed-lines" or "both": + for line in removed_lines: + ! grep -qF "$line" current_file + + # 4. 计算通过率,判定结果 + rate = matched / total + if rate >= 0.8: PASS + elif rate >= 0.5: WARN + else: FAIL + +# 5. 输出汇总 +echo "Results: $PASS passed, $WARN warnings, $FAIL failed, $SKIP skipped" +``` + +## 4. 边界情况处理 + +| 场景 | 处理方式 | +| ------------------------------------------- | --------------------------------------------------------- | +| commit 涉及的文件被上游删除 | 报 WARN,提示文件不存在,需人工确认改动是否迁移到了新文件 | +| commit 涉及的文件被上游重命名 | 同上,文件不存在时触发 WARN | +| fork 改动被有意重构(如 API rename) | 设置验证策略为 `skip`,在关键意图中说明 | +| 一个 commit 混合了 fork 定制和通用修改 | 建议拆分 commit;或在关键意图中说明只需关注哪些文件 | +| commit 已不在当前分支历史中(被 rebase 掉) | 报 ERROR,提示 commit 不可达 | + +## 5. 后续演进 + +### 5.1 短期(v1) + +- 实现基本的 `patches.md` + `verify.sh` +- 手动运行验证 + +### 5.2 中期(v2) + +- 集成到 CI pipeline,upstream sync 的 MR 自动运行验证 +- 验证结果作为 MR comment 输出 +- 支持 `--fix` 模式:对 FAIL 的条目,尝试从 commit diff 中提取补丁并 cherry-pick + +### 5.3 长期(v3) + +- 与 upstream sync 脚本集成,merge 冲突解决时自动提示 "这个文件有 fork patch,注意保留" +- 支持语义级别的验证(不只是行匹配,而是 AST 级别的检查) diff --git a/docs/design/fork-patch-guard/fork-patch-stack-architecture.md b/docs/design/fork-patch-guard/fork-patch-stack-architecture.md new file mode 100644 index 00000000000..e9a9c0322a8 --- /dev/null +++ b/docs/design/fork-patch-guard/fork-patch-stack-architecture.md @@ -0,0 +1,745 @@ +# Fork Patch Stack 架构设计 + +> 目标:让长期存在的 fork 定制改动 **显式化、可重放、可审查**,在 upstream sync 过程中不依赖 AI 或 Git 自动合并作为最终正确性来源。 + +## TL;DR + +**问题**:upstream sync 时 Git/AI 自动合并可能静默丢失 fork 定制(编译通过但功能丢了)。 + +**方案**:参考 code-server,将必须长期保留的 fork 定制拆分为有序补丁(`.fork/patches/`),每次 sync 后自动验证补丁是否存活。 + +**核心机制**: + +| 层 | 做什么 | 谁负责 | 触发方式 | +| ------------------ | ----------------------------------------------- | ----------------------------------------- | ---------------------------------------- | +| Patch 声明 | 11 个补丁的文件边界和元数据 | `.fork/manifest.json`(人维护) | MR 中修改 | +| Patch 生成 | 从 code diff + manifest 自动生成 `.patch` 文件 | `generate-patches.js`(脚本自动) | `bash .fork/generate-patches.sh` | +| Package Normalizer | 包名 `@alife/dataworks-*` + registry 改写 | `rewrite-package-identity.js`(脚本自动) | `node .fork/rewrite-package-identity.js` | +| Guarded Merge CI | 每天自动 fetch → merge → 重放 patch → 验证 → MR | `.aoneci/upstream-sync-merge.yml` | 定时 22:20 | +| 验证门禁 | patch 存活检查 + build + typecheck | `.fork/verify.sh` | `bash .fork/verify.sh` | + +**关键设计:patch 文件是脚本自动生成的产物,不需要人手写 diff。** 开发者只需维护 `manifest.json` 中的声明(编号、路径、顺序),脚本负责从 git diff 提取内容、生成 patch 文件、应用和验证。 + +**开发者需要知道的**: + +- 如果你的 MR 修改了上游文件中的内部行为 → 在 `manifest.json` 中声明路径,运行 `generate-patches.sh` 自动生成 patch +- 如果只改 `.aoneci/`、`docs/`、lockfile → 不需要 patch +- 包名改写不要手动改 package.json → 运行 `rewrite-package-identity.js` 自动改写 +- patch 文件是 **自动生成的产物**,不需要手动执行 `generate-patches.js` + +**当前状态**:Phase 2 已完成(11 个种子补丁),Phase 3 部分完成(CI 已集成 verify.sh 签名行检测 + blob 回退检测;apply.sh 重放验证待集成)。 + +--- + +## 整体架构图 + +```mermaid +graph TB + subgraph upstream["上游 (GitHub)"] + U[QwenLM/qwen-code
upstream/main] + end + + subgraph fork["内部 Fork (GitLab)"] + M[alishu/qwen-code
origin/main] + SYNC[sync/upstream-YYYYMMDD
同步分支] + end + + subgraph declaration["人维护的声明层"] + MANIFEST[manifest.json
补丁定义 + 文件边界 + 包名映射] + RULES[upstream-sync-rules.yml
冲突解决策略] + end + + subgraph generated["自动生成的产物层"] + PATCHES[patches/*.patch
从 diff 自动生成] + SERIES[patches/series
自动维护顺序] + end + + subgraph scripts["自动化脚本"] + GENERATE[generate-patches.js
生成 patch 文件] + APPLY[apply.sh
应用补丁栈] + UNAPPLY[unapply.sh
反转补丁栈] + VERIFY[verify.sh
验证 patch 存活] + REWRITE[rewrite-package-identity.js
包名标准化] + end + + subgraph ci["CI 流水线 (.aoneci/)"] + MERGE_YML[upstream-sync-merge.yml
每天 22:20] + ANALYZE_YML[upstream-sync-analyze.yml
工作日 9:00] + AUTH[upstream-sync-domain-auth.sh
认证 + MR 发布] + end + + U -->|"git fetch"| MERGE_YML + MERGE_YML -->|"merge + 重放 patch"| SYNC + SYNC -->|"验证通过后 MR"| M + AUTH -->|"push + create MR"| fork + + MANIFEST -->|"声明边界"| GENERATE + GENERATE -->|"自动生成"| PATCHES + GENERATE -->|"自动生成"| SERIES + PATCHES --> APPLY + SERIES --> APPLY + APPLY --> VERIFY + MANIFEST --> REWRITE + + MERGE_YML --> AUTH + MERGE_YML --> VERIFY +``` + +### 声明 vs 生成 的职责边界 + +```mermaid +graph LR + subgraph human["👤 开发者维护"] + A["manifest.json
① 编号 (file: 0012-xxx.patch)
② 标题 (title)
③ 文件列表 (paths)
④ 测试命令 (tests)"] + B[源代码改动] + end + + subgraph auto["🤖 脚本自动化"] + C["generate-patches.js
从 git diff 提取 patch 内容"] + D["apply.sh
按 series 顺序逐个 git apply"] + E["verify.sh
验证 patch 行为存活"] + end + + subgraph artifact["📦 自动生成的产物"] + F["patches/*.patch
(header + diff 内容)"] + G["patches/series
(按 definitions 数组顺序)"] + end + + A -->|"定义边界"| C + B -->|"产生 diff"| C + C -->|"写入"| F + C -->|"写入"| G + F --> D + G --> D + D --> E +``` + +**职责分工:** + +| 环节 | 谁负责 | 具体内容 | +| ----------------- | --------- | ------------------------------------------------------------------ | +| 编号命名 | 👤 开发者 | 在 manifest.json `file` 字段手写,如 `"0012-xxx.patch"` | +| 归类(路径边界) | 👤 开发者 | 在 manifest.json `paths` 数组声明哪些文件属于该 patch | +| 顺序 | 👤 开发者 | manifest.json `definitions` 数组的顺序即 series 顺序 | +| diff 内容提取 | 🤖 脚本 | `generate-patches.js` 从 `git diff merge-base..fork` 按 paths 过滤 | +| series 文件生成 | 🤖 脚本 | 自动从 definitions 顺序生成 | +| patch header 生成 | 🤖 脚本 | 自动从 manifest 元数据 + git ref 组装 | +| 应用补丁 | 🤖 脚本 | `apply.sh` 按 series 逐个 `git apply` | +| 验证 | 🤖 脚本 | `verify.sh` 检查 patch 行为是否存活 | + +**原则:编号、归类、顺序由开发者在 manifest.json 中声明;patch 文件内容、series 文件由脚本自动生成。开发者不需要手写 diff。** + +> **已知限制**:`apply.sh` 当前使用 `git apply`(不含 `--3way`)。如果 upstream sync 后代码上下文偏移较大,apply 可能失败。后续可升级为 `git apply --3way` 或 `git am` 以提高容错。 + +## CI 同步流程图 + +### 当前已实现(Phase 2/3) + +```mermaid +flowchart TD + START([定时触发
22:20 daily]) --> PREPARE[初始化 Git 上下文
domain-auth prepare] + PREPARE --> FETCH[fetch upstream/main] + FETCH --> CHECK{有新 upstream
commits?} + + CHECK -->|无| SKIP[status=skip
通知 already latest] + SKIP --> END_SKIP([结束]) + + CHECK -->|有| TAG[创建 checkpoint tag
sync-checkpoint-YYYYMMDD-HHMMSS-sha] + TAG --> BRANCH[创建/复用
sync/upstream-YYYYMMDD] + BRANCH --> MERGE[git merge upstream/main] + + MERGE -->|无冲突| BLOB_CHECK{blob 比对:
fork 定制是否被
静默回退?} + MERGE -->|有冲突| CONFLICT[status=has_conflicts
记录冲突文件列表] + + BLOB_CHECK -->|完整保留| CLEAN[status=clean] + BLOB_CHECK -->|检测到回退| REVERT_FAIL[abort merge
切到 upstream/main] + REVERT_FAIL --> MR_CONFLICT + + CONFLICT --> PKG{package.json
冲突?} + PKG -->|是| THEIRS_PKG[checkout --theirs
+ rewrite-package-identity.js] + PKG -->|否| LLM{LLM 辅助
解决冲突} + THEIRS_PKG --> REMAINING{还有剩余
冲突?} + REMAINING -->|无| CLEAN + REMAINING -->|有| LLM + + LLM -->|成功| CLEAN + LLM -->|失败| ABORT[abort merge
切到 upstream/main] + ABORT --> MR_CONFLICT[创建 MR
列出未解决冲突文件] + MR_CONFLICT --> NOTIFY_CONFLICT[钉钉通知
需人工介入] + NOTIFY_CONFLICT --> END_CONFLICT([结束]) + + CLEAN --> VERIFY[运行 .fork/verify.sh
commit 签名行存活检测] + VERIFY -->|通过| RISK[计算高风险文件清单
sync 改动 ∩ fork commit 触动] + VERIFY -->|失败| VERIFY_FAIL[CI 失败
钉钉通知 fork 定制疑似丢失] + VERIFY_FAIL --> END_VFAIL([结束]) + + RISK --> BUILD[npm ci + build + typecheck] + BUILD --> PUBLISH[domain-auth publish
push + 创建/复用 MR] + PUBLISH --> NOTIFY_OK[钉钉通知
MR 链接 + 高风险文件清单] + NOTIFY_OK --> END_OK([结束]) +``` + +**当前保护机制**: + +- **merge 后 blob 比对**:检测 fork 改过的文件是否在合并后等于 upstream 版本(静默回退) +- **verify.sh**:遍历 fork commit 历史,提取每个 commit 的加入行(签名行),检查是否仍存在于当前代码 +- **高风险文件清单**:sync 改动文件 ∩ fork commit 触动过的文件,写入 MR body 供 reviewer 重点检查 + +### 目标架构(Phase 3 完整版,待实现) + +```mermaid +flowchart TD + START([定时触发
22:20 daily]) --> PREPARE[初始化 Git 上下文
domain-auth prepare] + PREPARE --> FETCH[fetch upstream/main] + FETCH --> CHECK{有新 upstream
commits?} + + CHECK -->|无| SKIP[status=skip] + SKIP --> END_SKIP([结束]) + + CHECK -->|有| TAG[创建 checkpoint tag] + TAG --> BRANCH[创建/复用 sync 分支] + BRANCH --> MERGE[git merge upstream/main
冲突文件按策略处理] + + MERGE -->|无冲突| CLEAN[status=clean] + MERGE -->|有冲突| CONFLICT[检测冲突文件] + + CONFLICT --> RULES{按 manifest 分策略} + RULES -->|"patch paths 覆盖的文件"| ACCEPT_THEIRS[接受上游版本
后续靠 apply.sh 重新覆盖] + RULES -->|"fork-owned 文件"| KEEP_OURS[保留内部版本] + RULES -->|"package.json"| PKG_RULE[checkout --theirs
+ rewrite-package-identity.js] + RULES -->|"其他文件"| MANUAL_FLAG[标记需人工处理] + + ACCEPT_THEIRS --> RESOLVED + KEEP_OURS --> RESOLVED + PKG_RULE --> RESOLVED + MANUAL_FLAG --> MR_CONFLICT[创建 MR + 钉钉通知] + MR_CONFLICT --> END_CONFLICT([结束]) + + RESOLVED{所有冲突
已解决?} + RESOLVED -->|是| CLEAN + + CLEAN --> REAPPLY[重新应用 patch stack
bash .fork/apply.sh] + REAPPLY -->|全部成功| VERIFY[运行 .fork/verify.sh
确认 patch 行为存活] + REAPPLY -->|某个 patch 失败| PATCH_FAIL[记录失败 patch + .rej 文件] + + PATCH_FAIL --> MR_PATCH_FAIL[创建 MR
标记需刷新的 patch] + MR_PATCH_FAIL --> NOTIFY_PATCH[钉钉通知
patch 需刷新 + 修复命令] + NOTIFY_PATCH --> END_PATCH([结束]) + + VERIFY --> NORMALIZE[rewrite-package-identity.js] + NORMALIZE --> BUILD[npm run build + typecheck] + BUILD --> PUBLISH[push + 创建/复用 MR] + PUBLISH --> NOTIFY_OK[钉钉通知] + NOTIFY_OK --> END_OK([结束]) +``` + +**目标思路**:patch 覆盖的文件在 merge 时不靠 AI 猜——直接接受上游版本,然后用 `.fork/apply.sh` 重新应用补丁栈。如果 patch 无法应用,说明需要人工或 AI 辅助刷新该 patch(在独立 MR 中完成)。 + +> **Phase 3 待实现项**: +> +> 1. CI merge 冲突时,对 manifest.json `paths` 中声明的文件走 `checkout --theirs` 策略 +> 2. merge 成功后调用 `bash .fork/apply.sh --check` 验证 patch 可应用性 +> 3. 调用 `node .fork/generate-patches.js --check` 确保 patch 文件与当前代码一致 + +## 补丁生命周期 + +```mermaid +stateDiagram-v2 + [*] --> 创建: 新 fork 定制改动 + 创建 --> 活跃: merge 到 main + 活跃 --> 刷新: upstream sync 后需调整 + 刷新 --> 活跃: 刷新成功 + 测试通过 + 刷新 --> 失败: patch apply 失败 + 失败 --> 刷新: 人工/AI 修复 + 活跃 --> 退休: upstream 已提供等效行为 + 退休 --> [*] +``` + +## 新 MR 补丁决策流程 + +```mermaid +flowchart TD + START([新 MR 提交]) --> Q1{只涉及
fork-owned 文件?} + Q1 -->|是| NO_PATCH[不需要 patch] + Q1 -->|否| Q2{只是包名/版本号
标准化?} + Q2 -->|是| NORMALIZER[不需要 patch
更新 normalizer 脚本] + Q2 -->|否| Q3{改动应贡献
给上游?} + Q3 -->|是| UPSTREAM_PR[不需要长期 patch
跟踪 upstream PR] + Q3 -->|否| Q4{临时修复?
上游追上后可删?} + Q4 -->|是| TEMP[可选临时 patch
带过期说明] + Q4 -->|否| NEED_PATCH[更新 manifest.json
声明路径] + NEED_PATCH --> AUTO_GEN[generate-patches.sh
自动生成 patch 文件] + + style NEED_PATCH fill:#f96,stroke:#333 + style AUTO_GEN fill:#fcf,stroke:#333 + style NO_PATCH fill:#9f9,stroke:#333 + style NORMALIZER fill:#9f9,stroke:#333 + style UPSTREAM_PR fill:#9f9,stroke:#333 + style TEMP fill:#ff9,stroke:#333 +``` + +## 变更分类全景图 + +```mermaid +graph LR + subgraph input["Fork 变更输入"] + CHANGE[每个 fork
代码改动] + end + + subgraph classify["分类判定"] + C1[fork-owned] + C2[normalizer] + C3[patch] + C4[upstreamable] + C5[drop] + end + + subgraph storage["长期存储"] + S1[正常源文件
.aoneci/ .qwen/ docs/] + S2[确定性脚本
rewrite-package-identity.js] + S3[补丁文件
.fork/patches/*.patch] + S4[upstream PR
+ 可选临时 patch] + S5[删除] + end + + CHANGE --> C1 --> S1 + CHANGE --> C2 --> S2 + CHANGE --> C3 --> S3 + CHANGE --> C4 --> S4 + CHANGE --> C5 --> S5 +``` + +--- + +## 1. 背景 + +本仓库是 `QwenLM/qwen-code` 的内部 fork,包含 DataWorks 品牌定制、内部发布链路、包命名空间、channel 集成、OAuth 行为和 CI 自动化等改动。 + +现有的 guarded merge 流程可以覆盖大部分场景,但仍有一个薄弱点:当上游改动与 fork 定制重叠时,自动合并或 AI 辅助合并可能生成能编译通过但静默丢失内部行为的代码。 + +本文档定义了一套 code-server 风格的 patch stack,用于保护那些必须在未来 upstream sync 中存活的 fork 改动子集。 + +## 2. 参考模型 + +`code-server` 将 VS Code 作为上游源码,以有序的 `patches/*.diff` 文件(由 `quilt` 管理)维护其定制。核心经验不是"每个 PR 都变成一个 patch",而是: + +- 每个长期存在的定制用一个命名的、有序的 patch 表示 +- patch 可以被应用、刷新、审查和测试 +- VS Code 升级时可以一次性刷新多个 patch +- 一个 patch 可以被后续多个 PR 更新 +- 如果某个 patch 无法应用,更新流程停止并修复该 patch +- 只为新的长期定制创建新 patch 文件,不是每个 PR 都创建 + +本仓库遵循同样的 patch-stack 原则,但使用适配 Aone CI 环境的脚本。 + +## 3. 不适用范围 + +Patch stack 不是所有 fork 改动的替代方案。 + +**不应放入 patch 文件的内容:** + +- `.aoneci/` 下的内部 CI 文件 +- 不与上游代码重叠的内部发布脚本 +- 生成文件(如 `package-lock.json`) +- 机械式的包命名空间和版本号改写 +- 上游追上后应删除的临时修复 +- 应提交回上游的通用改动 + +Patch stack 也不是所有 fork 差异的单一归档。一个大 diff 和大规模手动合并有同样的失败模式:难以审查、难以刷新、难以退休。 + +## 4. 变更分类 + +每个 fork 变更在合并前都应先分类。 + +| 类别 | 含义 | 长期存储方式 | +| -------------- | ---------------------------------------------- | --------------------------- | +| `fork-owned` | 仅 fork 拥有的文件(如 Aone CI、内部发布文档) | 正常源文件 | +| `normalizer` | 机械式改写(如包名、版本号、lockfile) | 确定性脚本 | +| `patch` | 上游所有文件中的长期 fork 定制 | `.fork/patches/*.patch` | +| `upstreamable` | 应提交给 `QwenLM/qwen-code` 的通用修复或功能 | upstream PR,可选临时 patch | +| `drop` | 已过时或上游已覆盖的 fork 代码 | 从 fork 中删除 | + +只有 `patch` 类别的改动属于 patch stack。 + +## 5. 补丁粒度 + +补丁按 **持久的产品意图** 分组,而非按 MR、commit 或文件。 + +推荐粒度: + +- 一个用户可感知的 fork 行为 +- 一个与上游代码的集成点 +- 一个必须在 upstream sync 中存活的内部兼容层 +- 一小组必须一起刷新的文件 + +避免两个极端: + +- 一个 MR = 一个 patch +- 所有 fork 改动 = 一个 patch + +示例: + +```text +.fork/patches/ + series + 0001-branding-header.patch + 0002-branding-tips.patch + 0003-i18n-dataworks.patch + 0004-dsw-oauth-redirect.patch + 0005-osc8-internal.patch + 0006-dingtalk-channel-enhancements.patch + 0007-feishu-channel.patch +``` + +后续 MR 可以更新 `0002-branding-tips.patch` 及其测试。另一个 MR 可以新增 `0012-...patch`。一次 upstream sync MR 可以一次性刷新多个 patch。 + +## 6. 目录结构 + +已实现的布局: + +```text +.fork/ + manifest.json # 补丁定义、包名映射、registry 配置(source of truth) + patches/ + series # 补丁应用顺序(权威来源) + 0001-branding-header.patch + 0002-branding-tips.patch + ... + apply.sh # 按 series 顺序应用补丁栈 + unapply.sh # 反转所有已应用补丁 + verify.sh # 验证补丁是否在当前代码中存活 + create-patch.sh # 创建新补丁 + refresh-patch.sh # 刷新已有补丁 + generate-patches.js # 从 fork diff 生成补丁文件 + generate-patches.sh # shell 包装 + rewrite-package-identity.js # 包名/registry 正向和反向改写 + sync-upstream.sh # 本地 upstream sync 辅助 + patches.md # 自动生成的补丁清单 +``` + +补丁文件包含一个简短的元数据头: + +```text +Subject: DataWorks startup tips +Reason: Keep DataWorks-specific startup guidance and avoid upstream generic tips. +Owner: DataWorks Qwen Code maintainers +Patch-Base: cc800d01322c3bf642b919425576da09f182c3d5 +Fork-Ref: origin/main (...) +Upstream-Ref: upstream/main +Tests: cd packages/cli && npx vitest run src/ui/components/Tips.test.ts + +diff --git a/packages/cli/src/services/tips/tipRegistry.ts b/... +... +``` + +头部是审查元数据。diff 正文是可执行的契约。`Patch-Base` 是关键字段:必须是提取补丁时的 fork/upstream merge-base,而非最新的 upstream head。 + +## 7. 新 MR 决策规则 + +开发者提交新 MR 时,应回答这个问题: + +> 这个改动是否修改了上游所有文件中的代码,且必须在未来 upstream sync 中存活? + +如果是,在 `manifest.json` 中声明路径(新增或更新),patch 文件由脚本自动生成。 + +决策流程: + +```text +新 MR + | + |-- 只涉及 fork-owned 文件? + | → 不需要 patch + | + |-- 只是包名/版本号/lockfile 标准化? + | → 不需要 patch;如需要则更新 normalizer + | + |-- 改动应贡献给上游? + | → 不需要长期 patch;跟踪 upstream PR + | + |-- 改动是临时的,上游追上后可删除? + | → 可选临时 patch(带过期说明) + | + |-- 改动是上游所有文件中的内部行为? + → 更新已有 patch 或创建新 patch +``` + +## 8. 补丁维护方式 + +### 8.1 开发者日常流程(简化版) + +开发者不需要手动生成 patch 文件。日常流程: + +1. 修改源代码(正常开发) +2. 如果改动涉及上游文件且需要长期保留: + - 在 `manifest.json` 的 `patches.definitions` 中声明/更新文件路径 + - 提交代码 + manifest 改动 +3. 运行 `bash .fork/generate-patches.sh`(或等脚本自动执行)自动重新生成 patch 文件 +4. 运行 `bash .fork/verify.sh` 确认 patch 逻辑正确 + +```text +开发者操作 脚本自动化 +───────────── ──────────────── +修改源代码 ──→ generate-patches.js 从 diff 推导 patch +更新 manifest ──→ 自动生成 patches/*.patch + series +提交 MR ──→ verify.sh 验证 patch 存活 +``` + +### 8.2 更新已有补丁 + +如果 MR 修改了已有的 fork 行为: + +1. 正常修改源代码 +2. 确保 `manifest.json` 中对应 patch 的 `paths` 列表包含新增/修改的文件 +3. 运行 `bash .fork/generate-patches.sh` → patch 文件自动更新 +4. 运行对应 patch 声明的测试 + +MR 应包含: + +- 源代码改动 +- `manifest.json` 路径更新(如有) +- 自动重新生成的 `.fork/patches/*.patch` +- 测试或更新的测试期望 + +### 8.3 创建新补丁 + +当改动是上游所有文件中的新的长期 fork 定制时: + +1. 在 `manifest.json` 的 `patches.definitions` 数组中添加新条目 +2. 填写 `file`(编号命名)、`title`、`reason`、`paths`、`tests` +3. 运行 `bash .fork/generate-patches.sh` → 自动生成新 patch 文件和 series + +命名规则:按行为命名,使用下一个有序编号。 + +```text +0012-dataworks-session-export.patch +``` + +不要用 MR ID 或日期命名 patch。MR ID 是评审历史;patch 名称是维护契约。 + +### 8.3 无需补丁的情况 + +当 MR 仅限于以下范围时不需要 patch: + +- `.aoneci/**` +- `.qwen/**` 内部流程文件 +- `docs/design/**` 内部文档 +- 脚本覆盖的包版本/命名空间标准化 +- lockfile 重新生成 +- 仅验证已有 fork 行为的测试 +- 应该提交给上游的兼容修复 + +CI 分类步骤仍应报告为何不需要 patch。 + +## 9. 初始提取策略 + +当前 fork 已有大量历史改动。不要通过直接比较 fork main 和最新 upstream head 来生成一个大 patch。 + +初始提取分三步进行。 + +### Pass 1: 盘点 + +从当前 fork diff 生成审计报告: + +```bash +MERGE_BASE=$(git merge-base origin/main upstream/main) +git diff --name-status "$MERGE_BASE..origin/main" +git log --oneline --no-merges "$MERGE_BASE..origin/main" +``` + +报告仅用于分类,不应作为最终 patch 提交。 + +### Pass 2: 分类 + +将历史改动分组为: + +- fork-owned 文件 +- package normalizer 规则 +- patch 候选 +- 已过时或已提交上游的改动 + +从两侧都有改动的文件开始,因为它们风险最高。 + +### Pass 3: 提取小补丁 + +对每个 patch 候选: + +1. 在相关 upstream base 创建干净 worktree +2. 只应用选定行为的 hunks +3. 运行其专项测试 +4. 将结果 diff 保存为命名的 `.patch` 文件 +5. 添加到 `series` +6. 从 upstream + 完整 series 重建以验证顺序 + +本仓库使用 `.fork/manifest.json` 作为补丁边界定义源,`.fork/generate-patches.js` 生成 diff 文件: + +```bash +git fetch origin main +git fetch upstream main --tags +node .fork/generate-patches.js --write +node .fork/generate-patches.js --check +``` + +默认使用的引用: + +```text +PATCH_BASE_REF = git merge-base "$FORK_REF" "$UPSTREAM_REF" +FORK_REF = origin/main (内部 fork 主分支) +UPSTREAM_REF = upstream/main (上游 QwenLM/qwen-code 主分支) +``` + +仅在刻意重现旧版提取时才手动指定 `PATCH_BASE_REF`。这样补丁文件锚定在最后一次 upstream sync 点,避免将无关的未来上游改动引入 fork patch。 + +## 10. Upstream Sync 流程 + +目标同步流程: + +```text +定时同步 + → fetch upstream/main + → 创建或复用 sync/upstream-YYYYMMDD 分支 + → 从当前 main 应用 package normalizer + → 按顺序应用 .fork/patches/series + → 当 package 文件有变化时重新生成 package-lock.json + → 运行 patch stack 检查 + → 运行受影响 patch 声明的专项测试 + → 创建/更新 MR + → 发送钉钉通知 +``` + +当某个 patch 失败时: + +```text +patch 0002-branding-tips.patch 失败 + → 停止应用后续 patch + → 保留 reject/debug 产物 + → 创建/更新 sync MR + → 钉钉通知包含失败 patch、涉及文件和修复命令 + → AI 或维护者在正常 MR 中刷新 patch +``` + +AI 可以协助刷新失败 patch,但 CI 在完整补丁栈和测试通过前不应将 AI 输出视为正确。 + +## 11. 包名标准化 + +Package 文件有意放在 patch stack 之外,由确定性脚本处理。 + +规则: + +- package `version` 跟随上游 +- 内部 package `name` 跟随 DataWorks 命名空间(`@alife/dataworks-*`) +- 内部 workspace 依赖名跟随 DataWorks 映射 +- `package-lock.json` 在包名标准化后重新生成 +- CI 检查包名标准化是否一致 + +实现文件:`.fork/rewrite-package-identity.js`(正向应用 fork 包名)/ `--reverse`(恢复上游包名)。 +映射定义:`.fork/manifest.json` → `packageIdentity.mappings`。 + +## 12. CI 门禁 + +Patch stack CI 应在以下情况失败: + +- `series` 中任何 patch 无法应用 +- 文件中存在冲突标记 +- 包名标准化改动了文件但未提交 +- 新的上游文件改动没有对应 patch 或显式豁免 +- 声明的 patch 测试失败 +- patch 文件被修改但未相应更新源代码行为或测试 + +CI 应报告: + +- 已应用的 patch +- 跳过的非 patch 类别 +- 失败的 patch 名称 +- 失败涉及的文件 +- 建议的负责人/测试命令 + +钉钉通知应包含相同摘要。 + +## 13. 迁移计划 + +### Phase 1: 文档和盘点 ✅ + +- 添加本架构文档 +- 添加 `.fork/patches/series`(含种子 patch 集) +- 添加分类脚本 + +### Phase 2: 种子关键补丁 ✅ + +首批提取的补丁栈限于长期行为补丁: + +- DataWorks branding(header、tips) +- DataWorks i18n 文案 +- DSW OAuth redirect 行为 +- OSC8 内部终端处理 +- DingTalk channel 增强 +- Feishu channel 集成 +- dynamic swarm worker tool +- Claude WebSearch 兼容 +- 单文件 bundle 构建配置 +- 测试 fork 适配 + +DashScope internal-origin patch 已退休(当前 upstream/main 已包含等效行为)。 + +### Phase 3: Patch 辅助同步(进行中) + +保留现有 guarded upstream merge,增加 patch stack 保护。 + +**已完成:** + +- ✅ merge 后 blob 比对检测 fork 定制静默回退 +- ✅ `verify.sh` commit 签名行存活检测(CI 已集成) +- ✅ 高风险文件清单(sync 改动 ∩ fork commit 文件)写入 MR body +- ✅ package.json 冲突确定性解决(checkout --theirs + rewrite-package-identity.js) +- ✅ verify 失败时 CI 阻塞 + 钉钉通知 + +**待实现:** + +- ⬜ merge 冲突时对 manifest paths 声明的文件走 `checkout --theirs` 策略(替代 LLM) +- ⬜ merge 成功后调用 `apply.sh --check` 验证 patch 可应用性 +- ⬜ CI 中调用 `generate-patches.js --check` 确保 patch 文件与代码一致 +- ⬜ patch apply 失败时的独立 MR + 钉钉通知 + +### Phase 4: 完整 Patch Replay 同步(可选) + +仅在补丁栈稳定后,考虑更强的 replay 模型: + +- 从 upstream + fork-owned overlay + patch stack 构建 sync 分支 +- 与当前内部 main 比较生成结果 +- 从生成结果创建 MR + +此阶段是可选的。仓库可以从 Phase 3 获得大部分安全收益,无需改变整体同步拓扑。 + +## 14. 运营规则 + +1. 一个 patch 代表一个长期 fork 行为,而非一个 PR。 +2. 一个 PR 可以更新零个、一个或多个 patch。 +3. 一个 patch 可以被多个 PR 在不同时间更新。 +4. 包版本和命名空间改写由脚本标准化,而非 AI。 +5. 单一全量 fork diff 仅允许作为初始盘点产物。 +6. AI 可以提议 patch 刷新,但 CI 决定 patch 是否有效。 +7. 当上游提供等效行为时,积极退休 patch。 + +## 15. 已解决和待定问题 + +**已解决:** + +- ~~apply 实现是否使用 `quilt`,还是 `git apply --3way` wrapper + `series` 文件?~~ + → 使用 `git apply --3way` wrapper(`.fork/apply.sh`),不依赖 quilt。 +- ~~包命名空间最终确定:`@alife/dataworks-*` 还是 `@ali-fe/dataworks-*`?~~ + → `@alife/dataworks-*`,定义在 `.fork/manifest.json` 的 `packageIdentity.mappings`。 + +**待定:** + +- 每日同步是否应保持 guarded-merge-first,还是在多次成功 patch refresh 后升级为完整 patch replay? +- 大版本 upstream sync 落地后,patch stack 是否应从合并后的 main 刷新,使下次 `patch_base` 变为新的 upstream head? + +## 16. 参考资料 + +- code-server 贡献指南: + +- code-server 更新工作流: + +- VSCodium patch application 脚本: + +- VSCodium patch helper: + diff --git a/docs/design/fork-patch-guard/fork-release-versioning-design.md b/docs/design/fork-patch-guard/fork-release-versioning-design.md new file mode 100644 index 00000000000..818c59987ec --- /dev/null +++ b/docs/design/fork-patch-guard/fork-release-versioning-design.md @@ -0,0 +1,395 @@ +# Fork 发版与版本管理架构设计 + +> 规范 alishu/qwen-code fork 的版本号策略、分支模型、发布流程、upstream sync 后的版本衔接,以及两个分发渠道(npm / standalone binary)的协同。 + +## 1. 现状分析 + +### 1.1 仓库关系 + +``` +QwenLM/qwen-code (upstream, GitHub) + └── @qwen-code/qwen-code ← 公共 npm 包 + │ + ▼ fork +alishu/qwen-code (internal, GitLab) + └── @alife/dataworks-qwen-code ← 内部 npm 包 (anpm) + └── standalone binary ← OSS 分发 +``` + +### 1.2 当前版本状态 + +| 维度 | 现状 | +| ------------------ | ----------------------------------------------------------------------------- | +| 内部 fork 基础版本 | `0.14.8` | +| 上游最新版本 | `v0.15.0-preview.2` | +| 内部发布版本格式 | `0.14.8-dataworks.N` (latest) / `0.14.8-beta.N` (beta) | +| 版本管理工具 | `scripts/version.js` (手动 bump) + `scripts/publish-packages.js` (自动递增 N) | + +### 1.3 当前问题 + +| 问题 | 影响 | +| -------------------------------------------------------- | ----------------------------------------- | +| 版本号与上游的对应关系不明确 | 用户无法判断当前版本包含了上游哪些功能 | +| npm 发布无 git tag | 无法从 git 历史追溯某个发布版本的代码快照 | +| standalone binary 构建只在 `feat/bindary-build` 分支触发 | 与 main 分支发布脱节,需要手动同步 | +| 版本 bump 是手动操作 | 容易遗漏,多个 package.json 需要同步更新 | +| 无 CHANGELOG | 用户无法了解版本间的变更内容 | + +## 2. 版本号策略 + +### 2.1 版本号格式 + +``` +{upstream_base}-dataworks.{N} +``` + +| 组成部分 | 说明 | 示例 | +| --------------- | ------------------------------------------------ | ------------------ | +| `upstream_base` | 与上游对齐的基础版本号 (x.y.z) | `0.14.8` | +| `dataworks` | 固定标识符,标记为 DataWorks fork 版本 | — | +| `N` | 基于该 base 的内部递增序号,自动从 registry 计算 | `0`, `1`, `2`, ... | + +#### 完整示例 + +``` +0.14.8-dataworks.0 ← 基于上游 0.14.8 的第一个 fork 发布 +0.14.8-dataworks.1 ← bug fix 或小改动 +0.14.8-dataworks.2 ← 又一次发布 +0.15.0-dataworks.0 ← upstream sync 到 0.15.0 后的第一个 fork 发布 +``` + +### 2.2 dist-tag 策略 + +| dist-tag | 含义 | 触发条件 | 用户安装方式 | +| -------- | ------------ | ------------------------------------- | --------------------------------------------- | +| `latest` | 正式版 | main 分支发布,或 `release_mode=true` | `npm install @alife/dataworks-qwen-code` | +| `beta` | 预发布测试版 | 非 main 分支默认 | `npm install @alife/dataworks-qwen-code@beta` | + +### 2.3 与上游版本的对应规则 + +``` +upstream sync 合入 → bump upstream_base → 重置 N 从 0 开始 + +示例时间线: + fork 0.14.8-dataworks.3 (正在开发) + ↓ upstream sync v0.15.0 合入 main + fork 0.15.0-dataworks.0 (sync 后首次发布) + fork 0.15.0-dataworks.1 (后续 fork 改动) +``` + +**base 版本更新时机**: + +| 场景 | 操作 | +| --------------------------------------------- | ------------------------- | +| upstream sync 合入了上游的新 tag (如 v0.15.0) | 将 base 更新为 `0.15.0` | +| upstream sync 合入但没有新 tag | base 保持不变,继续递增 N | +| fork 独立 bug fix / feature | base 不变,继续递增 N | + +## 3. 分支模型 + +### 3.1 分支定义 + +``` +main ← 稳定集成分支,所有 MR 合入目标 + │ + ├── sync/upstream-YYYYMMDD ← upstream sync 临时分支(CI 自动创建,MR 合入 main 后删除) + │ + ├── release/* ← 发布准备分支(push/MR 自动触发 beta 构建) + │ + ├── feat/* ← 功能开发分支 + │ + ├── fix/* ← bug 修复分支 + │ + └── feat/bindary-build ← standalone binary 构建触发分支(现状,待优化) +``` + +### 3.2 分支与发布的关系 + +``` +feat/* ──── MR ────→ main ──── CI 手动触发 ────→ npm publish (latest) + │ │ + │ ▼ + │ 0.14.8-dataworks.N + │ + └── MR ────→ release/* ──── 自动触发 ────→ npm publish (beta) + │ + ▼ + 0.14.8-beta.N +``` + +## 4. 发布流程 + +### 4.1 npm 发布流程 + +``` +┌─────────────────────────────────────────────────────────┐ +│ npm-publish.yml │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ 1. npm ci │ +│ 2. npm run build (tsc 编译) │ +│ 3. npm run bundle (esbuild 打包) │ +│ 4. publish-packages.js: │ +│ a. 写入 .npmrc 认证 │ +│ b. 查询 registry 计算下一个 N │ +│ c. 更新所有 package.json → x.y.z-dataworks.N │ +│ d. 重新 bundle (嵌入新版本号) │ +│ e. npm publish --workspaces --tag latest │ +│ 5. 验证 dist-tags │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +#### 参数说明 + +| 参数 | 类型 | 默认值 | 说明 | +| -------------- | ------- | ----------- | ------------------------ | +| `tag` | string | `latest` | npm dist-tag | +| `pre_id` | string | `dataworks` | 版本号后缀标识符 | +| `dry_run` | boolean | `true` | 模拟发布 | +| `auto_version` | string | `true` | 自动递增版本号 | +| `release_mode` | boolean | `false` | 非 main 分支强制发正式版 | + +### 4.2 standalone binary 发布流程 + +``` +┌─────────────────────────────────────────────────────────┐ +│ build-standalone.yml │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ 1. resolve-version.sh → 计算版本号 │ +│ 2. npm run build + bundle │ +│ 3. build-standalone-ci.sh: │ +│ a. 注入版本号到 dist/cli.js │ +│ b. 下载 Node.js v22.14.0 binary │ +│ c. 下载 native modules (node-pty, clipboard) │ +│ d. 打包 tarball + SHA256SUMS + metadata.json │ +│ 4. upload-oss.sh → 上传到 OSS │ +│ 5. upload-policy.sh → 按分支决定是否更新 latest 指针 │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +#### OSS 目录结构 + +``` +dataworks-notebook-cn-shanghai.oss-cn-shanghai.aliyuncs.com/public-datasets/aone-release/alishu/qwen-code/ + {version}/ + qwen-code-{version}-linux-{arch}.tar.gz + SHA256SUMS + metadata.json + latest/ + metadata.json ← 仅 main/release 分支更新 + deploy-qwen.sh ← 用户安装入口 + upgrade-qwen.sh ← 用户升级入口 +``` + +#### 用户安装/升级 + +```bash +# 首次安装 +curl -fsSL https://dataworks-notebook-cn-shanghai.oss-cn-shanghai.aliyuncs.com/public-datasets/aone-release/alishu/qwen-code/deploy-qwen.sh | bash + +# 升级 +curl -fsSL .../upgrade-qwen.sh | bash + +# 指定版本 +curl -fsSL .../deploy-qwen.sh | bash -s -- --version 0.14.8-dataworks.3 +``` + +### 4.3 两个渠道的版本对齐 + +当前问题:npm 和 standalone binary 的版本号独立计算,可能不一致。 + +**建议规范**: + +| 规则 | 说明 | +| ------------------------------------- | ------------------------------------------- | +| npm 版本是权威来源 | standalone binary 应使用与 npm 相同的版本号 | +| binary 构建应在 npm 发布成功后触发 | 保证版本号一致 | +| `metadata.json` 中记录对应的 npm 版本 | 便于追溯 | + +## 5. upstream sync 后的版本衔接 + +### 5.1 完整流程 + +``` +Day 0: fork 当前版本 0.14.8-dataworks.5 + +Day 1: upstream sync MR 创建 + ├── CI 自动创建 sync/upstream-20260423 分支 + ├── 自动或人工解决冲突 + ├── 运行 .fork/verify.sh 检查 fork 定制(见 fork-patch-guard-design.md) + └── MR review + merge 到 main + +Day 2: 版本 bump(如果上游有新 tag) + ├── 检查上游 tag: git tag -l 'v*' --sort=-v:refname | head -5 + ├── 如果上游发了 v0.15.0: + │ npm run release:version 0.15.0 + │ git commit -m "chore(release): bump base version to 0.15.0" + ├── 如果上游没有新 tag: + │ base 版本不变,跳过此步骤 + └── push to main + +Day 3: 发布 + ├── 触发 npm-publish.yml (auto_version=true) + │ → 发布 0.15.0-dataworks.0 (如果 bump 了) + │ → 或 0.14.8-dataworks.6 (如果没 bump) + └── 触发 build-standalone.yml (使用相同版本号) +``` + +### 5.2 版本 bump 检查清单 + +upstream sync 合入 main 后,执行以下检查: + +```markdown +- [ ] 检查上游是否有新的 release tag +- [ ] 如有新 tag,运行 `npm run release:version ` +- [ ] 检查 `config.sandboxImageUri` 是否更新 +- [ ] 运行 `bash .fork/verify.sh` 验证 fork 定制完整性 +- [ ] 提交版本 bump commit +- [ ] 触发 npm-publish.yml 发布 +``` + +## 6. 分发方式分析:standalone binary vs bundle-only + +### 6.1 当前 standalone binary 的真实结构 + +当前的 "standalone binary" **并非编译产物**,而是一个 shell 脚本打包: + +``` +qwen-code-standalone/ +├── bin/qwen ← bash 启动脚本(非二进制) +├── node/bin/node ← 内嵌 Node.js v22.14.0(~50 MB) +├── dist/cli.js ← esbuild 打包的全部应用代码(~25 MB) +├── dist/vendor/ripgrep/ ← rg 二进制(~24 MB) +├── dist/locales/ ← i18n(~844 KB) +├── dist/bundled/ ← 内置 skill 文档(~792 KB) +├── native_modules/ +│ ├── @lydell/node-pty-linux-*/ ← PTY 原生模块(~88 KB) +│ └── @teddyzhu/clipboard-linux-*/ ← 剪贴板原生模块(~1.4 MB) +└── metadata.json +``` + +`bin/qwen` 做的事情只有三行:设置 `NODE_PATH` → 用内嵌 node 执行 `dist/cli.js`。 + +### 6.2 原生模块是否必须 + +| 模块 | 用途 | 缺失时的表现 | 结论 | +| ----------- | --------------------- | ---------------------------------------------------- | -------- | +| `node-pty` | 交互式 PTY shell 执行 | 自动 fallback 到 `child_process.spawn`,交互能力降级 | **可选** | +| `clipboard` | 剪贴板图片粘贴 | 静默禁用该功能 | **可选** | + +两个模块都在 `optionalDependencies` 中,代码中有 try/catch + graceful fallback。 + +### 6.3 最小可行部署(bundle-only) + +只需要 `dist/cli.js` + 系统 Node.js + ripgrep 即可运行核心功能: + +```bash +# 前置条件:系统已安装 Node.js >= 20 +node /opt/qwen-code/dist/cli.js +``` + +| 组件 | 大小 | 是否必须 | +| ----------------------------------- | -------------- | -------------------- | +| `dist/cli.js` | ~25 MB | 是 | +| `dist/vendor/ripgrep/{platform}/rg` | ~5 MB (单平台) | 是(或用系统 rg) | +| `dist/locales/` | ~844 KB | 否(仅中文时可跳过) | +| `dist/bundled/` | ~792 KB | 否(内置技能文档) | + +**最小部署 ~30 MB,对比 standalone 的 ~120-200 MB。** + +### 6.4 简化分发方案(建议) + +可以用 **bundle tarball + OSS** 替代当前的 standalone binary: + +``` +OSS 目录结构(简化后): +qwen-code/ + {version}/ + qwen-code-{version}-linux-{arch}.tar.gz ← dist/ 打包 + SHA256SUMS + metadata.json + latest/ + metadata.json + install.sh ← 下载 + 解压 + 创建 symlink + upgrade.sh ← 检查版本 + 下载新版 +``` + +安装/更新脚本的核心逻辑简化为: + +```bash +# install.sh 核心逻辑 +VERSION=$(curl -s .../latest/metadata.json | jq -r .version) +curl -fSL ".../qwen-code-${VERSION}-linux-amd64.tar.gz" | tar xz -C /opt/qwen-code/releases/${VERSION} +ln -sfn /opt/qwen-code/releases/${VERSION} /opt/qwen-code/current +ln -sfn /opt/qwen-code/current/bin/qwen /usr/local/bin/qwen +``` + +其中 `bin/qwen` 简化为: + +```bash +#!/bin/bash +exec node "$(dirname "$0")/../dist/cli.js" "$@" +``` + +**前提条件**:目标机器已安装 Node.js >= 20。如果不能保证,仍需内嵌 Node.js。 + +### 6.5 对比总结 + +| 维度 | 当前 standalone | bundle-only(建议) | +| ------------ | ------------------------------------- | -------------------------- | +| 部署大小 | ~120-200 MB | ~30 MB | +| Node.js 依赖 | 无(内嵌) | 系统需要 >= 20 | +| PTY 交互 | 完整 | 降级(child_process) | +| 剪贴板粘贴 | 支持 | 不支持 | +| 构建复杂度 | 高(下载 node/native modules/多架构) | 低(esbuild 产物直接打包) | +| 更新方式 | 重新下载完整包 | 只需替换 dist/ | + +**建议**:对于内部开发环境(已有 Node.js),优先使用 bundle-only 分发。standalone binary 保留给无 Node.js 的裸机环境。 + +## 7. 已识别的 gap 及改进建议 + +### 7.1 短期 (P0) + +| Gap | 建议 | 状态 | +| ----------------------------------------------- | ------------------------------------------------------------- | ---- | +| npm 发布后无 git tag | `publish-packages.js` 成功后自动创建 `v{version}` tag 并 push | TODO | +| standalone build 只在 `feat/bindary-build` 触发 | 增加 main 分支触发,或在 npm 发布成功后自动触发 binary 构建 | TODO | +| 无 CHANGELOG | 基于 conventional commits 自动生成,或至少在发布时手动维护 | TODO | + +### 7.2 中期 (P1) + +| Gap | 建议 | +| -------------------------------------- | ------------------------------------------------------------- | +| 版本 bump 手动操作 | upstream sync MR 合入后自动检测上游 tag,生成版本 bump commit | +| npm 和 binary 版本可能不一致 | 统一发布流水线,一个 CI job 串联两个渠道 | +| Node.js 版本不一致 (npm=20, binary=22) | 统一到 Node.js 22 | + +### 7.3 长期 (P2) + +| Gap | 建议 | +| -------------------------- | -------------------------------------------------------------------------------- | +| 无回滚机制 (npm) | 支持 `npm dist-tag add @alife/dataworks-qwen-code@{old_version} latest` 快速回滚 | +| 无灰度发布 | 先发 `canary` tag 给小范围用户,验证后再切 `latest` | +| 版本号人工判断是否跟随上游 | 自动从 `.last-synced-upstream-tag` 推导 base 版本 | + +## 8. 关键文件索引 + +| 文件 | 用途 | +| ---------------------------------------- | ---------------------------------------- | +| `.aoneci/npm-publish.yml` | npm 发布 CI pipeline | +| `.aoneci/build-standalone.yml` | standalone binary 构建 CI | +| `.aoneci/upload-qwen-scripts.yml` | 部署脚本同步到 OSS | +| `scripts/publish-packages.js` | npm 发布核心逻辑 (auto-version, publish) | +| `scripts/version.js` | 版本号 bump 工具 | +| `scripts/prepare-cli-for-publish.js` | CLI 包 prepublishOnly 钩子 | +| `.aoneci/scripts/build-standalone-ci.sh` | binary 打包 | +| `.aoneci/scripts/resolve-version.sh` | binary 版本号计算 | +| `.aoneci/scripts/upload-oss.sh` | OSS 上传 | +| `.aoneci/scripts/upload-policy.sh` | 分支级上传策略 | +| `.aoneci/scripts/deploy-qwen.sh` | 用户安装脚本 | +| `.aoneci/scripts/upgrade-qwen.sh` | 用户升级脚本 | +| `.fork/patches.md` | fork 定制追踪清单 (待建) | +| `.fork/verify.sh` | fork 定制验证脚本 (待建) | diff --git a/docs/design/fork-patch-guard/guarded-upstream-sync-design.md b/docs/design/fork-patch-guard/guarded-upstream-sync-design.md new file mode 100644 index 00000000000..c6ab4bd6c0c --- /dev/null +++ b/docs/design/fork-patch-guard/guarded-upstream-sync-design.md @@ -0,0 +1,163 @@ +# Guarded Upstream Sync:低人工介入的上游同步保护方案 + +> 目标:每天自动同步 upstream,同时避免 AI 或 Git 自动合并静默丢失 fork 定制。 + +## 1. 背景 + +本仓库是 `QwenLM/qwen-code` 的内部 fork。fork 分支包含 DataWorks +定制、内部发布链路、channel、OAuth、i18n、CI 等改动。日常需要定期把 +`upstream/main` 合入内部 `main`。 + +最初考虑过 patch replay 方案: + +1. 先把 fork 改动回退到接近 upstream 的状态。 +2. 合并最新 upstream。 +3. 再逐个 apply fork patch。 + +这个方案能让 fork 改动显式化,但对当前仓库来说维护成本偏高:fork 改动面较宽, +后续每个 patch 都要判断保留、改写、退休;如果 upstream 的方案更合理,patch +replay 还容易形成“本地改动默认覆盖 upstream”的倾向。 + +因此主同步流程建议采用 guarded merge,而不是 patch replay。 + +## 2. 核心原则 + +Guarded upstream sync 的核心不是让 fork 改动永远胜出,而是让风险可见: + +- CI 每天自动同步,不要求人工每天合并代码。 +- Git 正常 merge upstream,保留 upstream 的自然演进。 +- AI 不默认自动解冲突,避免静默改坏代码。 +- 只有出现风险信号时才升级人工或 agent 处理。 +- 通过 fork manifest 和 guard tests 保护关键业务能力。 + +也就是说,大多数同步仍然自动完成;人工只处理冲突、测试失败或高风险文件。 + +## 3. 推荐流程 + +```text +每天定时: + 1. fetch upstream/main + 2. 检查 upstream 是否有新提交 + 3. 基于内部 main 创建或复用 sync/upstream-YYYYMMDD 分支 + 4. 正常 git merge upstream/main + 5. 根据结果分级处理 + 6. 创建或更新 sync MR + 7. 发送钉钉通知 +``` + +分级策略: + +| Level | 条件 | 自动行为 | 人工介入 | +| ----- | ------------------------------------------ | ----------------------------- | ------------------ | +| 0 | 无 upstream 新提交 | 跳过 MR,通知 already latest | 不需要 | +| 1 | merge 成功,guard 通过 | 创建 MR,可标记低风险 | 通常不需要 | +| 2 | merge 成功,命中 fork 高风险文件,测试通过 | 创建 MR,列出重点 review 文件 | 只 review 重点文件 | +| 3 | merge 成功,但 guard 测试失败 | 创建 MR 并阻断自动合入 | agent 或人工修复 | +| 4 | Git merge conflict | 停止自动改代码,输出冲突报告 | agent 或人工处理 | + +## 4. Fork Manifest + +不建议把所有 fork 改动都转成 patch 作为主流程。更轻的方式是维护 +`.fork/manifest.yml` 或 `.fork/manifest.json`,记录“需要保护的 fork 能力”。 + +示例: + +```yaml +features: + - id: dataworks-branding + description: Header 和启动信息使用 DataWorks 品牌 + paths: + - packages/cli/src/ui/components/Header.tsx + - packages/cli/src/ui/components/AsciiArt.ts + tests: + - cd packages/cli && npx vitest run src/ui/components/Header.test.tsx + + - id: dsw-oauth-redirect + description: DSW 环境下改写 OAuth redirect URI + paths: + - packages/core/src/mcp/oauth-provider.ts + tests: + - cd packages/core && npx vitest run src/mcp/oauth-provider.test.ts +``` + +manifest 记录的是“能力”和“风险区域”,不是机械 patch。这样 upstream 如果提供了更好的实现, +我们可以接受 upstream,只要 guard test 证明关键行为仍满足内部需求。 + +## 5. Guard 检查 + +同步 MR 创建前后应执行三类检查。 + +### 5.1 风险文件交集 + +计算: + +```text +本次 upstream 改动文件 ∩ fork manifest paths +``` + +如果交集非空,MR 描述中列为 high-risk files。这个结果不一定阻塞合并,但 reviewer +需要重点看这些文件。 + +### 5.2 fork 能力测试 + +manifest 中每个 feature 可以声明测试命令。同步后只运行受影响 feature 的测试,避免全量测试过慢。 + +测试失败时应阻断自动合入,但仍然创建 MR,方便 agent 或人工基于 MR 修复。 + +### 5.3 静默回退检测 + +可以保留现有的 diff/签名行校验作为辅助信号,但它不应是唯一依据。更可靠的判断应来自: + +- 高风险文件列表 +- 针对 fork 能力的行为测试 +- MR 中清晰展示 upstream 改动范围 + +## 6. AI 使用边界 + +AI 可以参与修复,但不应该在 CI 默认自动解冲突。 + +推荐边界: + +- merge conflict:CI 只生成冲突报告,不自动 `--yolo` 修改代码。 +- guard test failure:可以由 agent 在独立分支上修复,再走正常 MR。 +- clean merge:AI 不参与改代码,只生成风险摘要。 + +这样保留了自动同步效率,同时避免“AI 自动合流丢代码”。 + +## 7. 与现有 Aone CI 的关系 + +当前仓库没有 `a1-ci/a1-ci.yaml`。实际相关配置位于 `.aoneci/`: + +- `.aoneci/upstream-sync-merge.yml`:每天 22:20 执行 upstream merge。 +- `.aoneci/upstream-sync-analyze.yml`:工作日 9:00 分析 upstream 变更并通知。 + +`.aoneci/upstream-sync-merge.yml` 已经接近 guarded sync 的雏形: + +- 定时 fetch upstream。 +- 检查是否有新 upstream commits。 +- 创建 `sync/upstream-YYYYMMDD` 分支。 +- 正常 `git merge upstream/main`。 +- 创建 MR。 +- 在 clean 状态下运行验证脚本。 + +但它还不是完整的 guarded sync: + +- 仍会尝试 LLM 自动解决冲突。 +- fork 风险文件和 fork 能力测试还没有成为主 MR 信号。 +- 验证步骤是非阻塞的,无法防止失败结果被忽略。 +- 现有静默回退检测偏文件级,不能替代行为测试。 + +因此建议在现有 `.aoneci/upstream-sync-merge.yml` 基础上增量改造,而不是引入 patch +replay 主流程。 + +## 8. 后续落地建议 + +优先级建议: + +1. 新增 `.fork/manifest.yml`,先记录最关键的 5 到 10 个 fork 能力。 +2. 在 sync MR 描述中加入 high-risk files。 +3. 对命中的 feature 运行对应 guard tests。 +4. 禁用 CI 默认 LLM 自动解冲突,改为报告冲突。 +5. 将 guard test failure 设置为阻断自动合入。 + +patch 文件可以保留为审计或迁移辅助工具,但不建议作为每天同步的主路径。 diff --git a/docs/design/rt-optimization/reduce-rounds-via-skill-design.md b/docs/design/rt-optimization/reduce-rounds-via-skill-design.md new file mode 100644 index 00000000000..c0fbf730a93 --- /dev/null +++ b/docs/design/rt-optimization/reduce-rounds-via-skill-design.md @@ -0,0 +1,574 @@ +# Agent Loop 减轮方案:从 Skill 设计入手 + +> 与 `rt-optimization-design.md` 同目录,互为补充:那份文档讨论**框架机制**层面减轮(D1 跳过末尾总结轮、D2 fast 路由、D4 prevalidate),这份文档主张**减轮的真正杠杆在 skill/tool 设计层**,并提出一条不依赖框架改造、不依赖 cache hit rate 数据的可实施路径。 + +--- + +## 0. 验收 Spec(开发前置 gate) + +> 本节是开发的**前置 gate** — 列出哪些 spec 必须在动手前确认、哪些 spec 必须等数据驱动。把 spec 前置而非"做完再看指标",是为了避免:(a) 写完才发现指标不可测、(b) 阈值随结果飘移导致结论失真、(c) 没设止损线让方案陷入"看起来在做、其实没收益"。 +> +> **本 spec 框架的适用边界**:本框架假设方向正确性可以在 P1.5 基线测量后判断。这个假设对"减轮"场景成立,因为它有清晰的可测信号(轮数、followup_rate、batch_size)。**超出此假设的场景**(例如未来用同一框架做"质量优化"等难以量化的方向),spec 前置可能反而阻碍快速学习;遇到时回退到 §0.5 治理流程重新评估,不机械套用本框架。 + +**spec 分四层 — 时机不同**: + +| 层级 | 类型 | 锁定时机 | +| ---- | --------------------------------------- | -------------------------------- | +| §0.1 | 工程层 spec(数据管道、代码改动正确性) | **前置**、可立刻锁定 | +| §0.2 | 统计层 spec(项目"算成功"的指标) | **前置**、阈值待 P1.5 基线后锁定 | +| §0.3 | 止损线("如果发生就放弃"硬条件) | **前置**、不可移动 | +| §0.4 | per-skill spec(具体改哪个、目标多少) | **后置**、Layer 1 数据驱动 | + +### 0.1 工程层 spec(必须前置 · 可立刻锁定) + +数据管道与代码改动的正确性 spec — 不依赖任何业务判断或基线数据,开发前就该锁定: + +- **qwen-logger 链路通畅**(§4.1.1b):skill_launch 事件能同时落到 OTLP 和 qwen-logger 两条管道 +- **`prompt_id` 串联**:单个 user prompt 触发的 `skill_launch` + 后续 `tool_call` 能用同一个 `prompt_id` grep 出完整 trail +- **`batch_size` 非 undefined**(§4.3.2 方向 A):单工具 batch 显式设 `batch_size = 1` / `batch_position = 0` +- **SQL 可跑通**(§4.1.2):离线 SQL 在真实 telemetry backend 输出非空且能区分高/低 followup_rate skill +- **基线方差 < P50 × 20%**(P1.5):基线测量稳定(否则后续 A/B 对比不可信)—— 注:本条虽列在 §0.1 工程层,但**锁定依赖 P1.5 基线数据**,是 §0.1 中唯一的后置验证项;P1.5 未通过则 §0.2 阈值无法可信锁定 +- **Skill 体积预算**(Layer 2 改造):内联 followup 后,skill 描述 token 数不超过改造前的 2×,且绝对值 ≤ 500 tokens(取较小值)。超过则按 §4.2 拆分 skill 而非合并。本条与 §7 第 2 条、§4.2 已有约束对齐,前置到 spec 层 +- **`npm run preflight` 全过**:每个 PR 的硬门槛 + +### 0.2 统计层 spec(必须前置 · 阈值待 P1.5 后锁定) + +项目算"统计意义上成功"的指标 — **方向**前置定下,**阈值**等基线测出来后锁定(避免凭空填数字): + +| 指标 | 方向 | 锁定时机 | 当前占位阈值(待校准) | +| ---------------------------------- | -------- | --------- | ---------------------- | +| top-3 skill 加权 `followup_rate` | ↓ | P1.5 末 | ≥ 30% | +| 含 skill 的会话端到端 RT P50 | ↓ | P1.5 末 | ≥ 2s | +| `batch_size > 1` 的 tool_call 占比 | ↑ | P3 前 | ≥ 30% | +| 改造的 skill 触发场景 A/B 显著性 | p < 0.05 | P2 改完前 | n 待定 | + +> **关键约束**:占位阈值不是承诺。P1.5 基线如果显示"top-5 skill 加权 followup_rate < 30%"(触发 §0.3 止损线 #1),项目终止;**不能为了让阈值"达到"而下调 spec**。 +> +> **怎么测**:每个指标的测量方法、SQL 模板、A/B 设计见 §5.1-§5.2;统计显著性(p < 0.05)的样本量计算见 §5.1。 + +### 0.3 止损线(必须前置 · P-1 锁定后受限可调) + +§5.3 已列。这些是"如果发生就放弃"的硬条件 — **任何情况下不能为了达成 §0.2 统计层 spec 而放宽止损线**。 + +- **结果指标**(3 条):top-5 加权 `followup_rate < 30%` / 改完 2 个 skill RT P50 ↓ < 1s / Layer 3 后 `batch_size P50` 仍 = 1 +- **过程指标**(3 条):skill 命中率 ↓ ≥ 5pp / 内联 followup 失败率 ≥ 5% / 用户取消率 ↑ ≥ 2pp + +详见 §5.3。 + +**可调性规则**(避免无数据支撑的纪律刚性): + +| 阶段 | 可否调整 | 调整方向 | +| --------------------- | ---------------------------------------- | ------------------------------------------------------------------------------- | +| P-1 锁定时 | ✅ 任意调整(基于历史 telemetry 或共识) | 任意 | +| P-1 锁定后 → P1.5 末 | ❌ 不可调整 | — | +| P1.5 末(基线出来时) | ✅ 仅允许**放宽**一次 | 放宽(如 30% → 25%)需附数据证据 + 2 人评审;**不允许收紧**(避免事后追加止损) | +| P1.5 之后 | ❌ 不可调整 | — | + +> 阈值占位值(30% / 1s / 5pp 等)当前**无历史数据支撑**,是 P-1 评审前的工程师直觉。如果 P-1 评审时能拿到最近 4 周历史 telemetry,应基于历史数据校准止损线;拿不到则保留占位值,P1.5 末执行上面的"放宽一次"规则。 + +### 0.4 per-skill spec(必须后置 · 数据驱动) + +具体改哪个 skill、目标 `followup_rate` 改到多少 — **Layer 1 数据出来前不锁定**。 + +不锁定的理由:先验设计 vs 后验数据可能差很多。强行前置会重蹈 `rt-optimization-design.md` §7 D2 路线的覆辙 —— 前置假设"fast 模型快 2-3s"被 cache 实装这一后验事实推翻,导致方案净收益接近 0 甚至为负。 + +**产出位置**:per-skill spec 在 P1.5 末由数据驱动产出,每个 Layer 2 PR 的 description 里独立声明(不进 design 文档,避免文档每改一个 skill 就改)。 + +**per-skill spec 结构模板**(与 §4.2 的 PR description 必含项对齐 — 这两个清单是同一份,§4.2 是过程视角、本节是 spec 视角): + +| 字段 | 内容 | 数据来源 | +| --------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------- | +| 1. 当前数据 | invocation_count、followup_rate、top followup tools | Layer 1 telemetry | +| 2. 目标 | followup_rate 从 X% 降到 Y% | 基于 §0.2 改善方向,绝对值 PR 内自行定 | +| 3. 改造范围 | 内联哪些 followup(read/grep/shell read-only),明确**不**内联什么(write 操作 / 跨 skill / 深度推理) | §4.2 改造模式表 | +| 4. 输出契约更新 | skill 描述里加的预声明("Returns: ...") | §3.2 改造示例 | +| 5. A/B 计划 | 改造后 2 周观察 followup_rate / RT P50 / 过程指标,对照 §5.1 验收线 | §5.1 | +| 6. 体积证明 | 改造前后 skill 描述 token 数(用 tiktoken 估算),不得超 §0.1"Skill 体积预算" | §0.1 第 6 条 | + +### 0.5 spec 治理 + +- **修改 §0.1 / §0.3 spec** 需 design 文档更新 + PR 评审;§0.3 仅遵循 §0.3"可调性规则"在 P1.5 末窗口内放宽 +- **修改 §0.2 阈值(P1.5 锁定后)** 需附以下至少一项数据证据: + - (a) P1.5 基线测量结果与已锁定阈值的偏差分析(含原始测量记录链接) + - (b) 同类项目的公开 benchmark 数据(含来源链接) + - (c) 内部 ≥ 2 人评审签字的偏差说明 + + PR 评审时若上述证据均无,评审者**有义务** block PR — 不接受"凭工程师直觉调整" + +- **§0.4 per-skill spec** 在数据驱动产出后写入 PR description(按 §0.4 6 项模板),不进 design 文档 + +--- + +## 1. 背景与定位 + +### 1.1 问题 + +`rt-optimization-design.md` §1.2 给出的基线:3 轮 agent loop,13.4s 端到端,其中 LLM 调用占 78%。每一轮 ~3-4s。 + +``` +Round 1 (3.8s, 28%): LLM 决策调 skill +Round 2 (3.0s, 22%): LLM 决策调 shell +Round 3 (3.8s, 28%): LLM 总结 +``` + +`rt-optimization-design.md` §6/§7 经过两轮 review 后,D2/D4 已被否决,D1/D3 也降级为"等浮油完成后再评估"。但**整份原文档都聚焦在末尾的 Round 3(总结轮)或单轮内的微优化(D4)上,完全没有正面讨论 Round 1 → Round 2 这个"中间轮"为什么会出现、能不能消掉**。 + +事实是:Round 2 之所以存在,**绝大多数情况是因为 Round 1 调用的 skill 没有返回完整答案**,模型才追加 shell 查询补全。如果 skill 设计成"一次拿到完整结果",3 轮 → 2 轮,省掉的就是 Round 2 那 ~3s — 这是与 D1 完全不重叠的收益面。 + +### 1.2 与 rt-optimization-design 的关系 + +| 减轮方向 | 命中的轮次 | 杠杆位置 | 本文档定位 | +| -------------------- | ------------------------------- | ---------------------------- | ---------------------------- | +| D1 `skipLlmRound` | 末尾总结轮 | 框架机制 + per-tool opt-in | 兜底,**放在 Layer 2 之后** | +| D2 fast 路由 | 单轮延迟 | 框架机制 | 已 defer,**不在本文档范围** | +| D3 Summarizing 状态 | 末尾总结轮(感知层) | UI 状态机 | 可选,与本方案正交 | +| D4 prevalidate | 单轮延迟 | 框架机制 | 已 defer,**不在本文档范围** | +| **本方案 Layer 1-3** | **中间决策轮 + 并发未触发的轮** | **skill 设计 + prompt 工程** | **新增方向** | + +### 1.3 核心论点 + +减轮的真正杠杆在 skill/tool 设计层,不在 agent 框架。三个理由: + +1. **§1.2 基线本身就暴露问题在 skill** — Round 1 → Round 2 的跳跃是 skill 返回不全才发生的,框架做对了,skill 做错了 +2. **框架级减轮最终也要 per-tool opt-in** — D1 的 `skipLlmRound` 必须每个工具显式标记,绕一圈回到 skill 工程,还多一套不变量修复 + 决策门控成本 +3. **ROI 局部可测、灰度容易** — 改一个 skill 就少一轮 × 该 skill 触发次数,不依赖 cache hit rate 数据,不依赖跨系统改动 + +> **实施前必须先走 §0 验收 Spec 前置评审(P-1 阶段,0.5d)** — §0.1 工程层 spec 和 §0.3 止损线在动手前必须锁定;§0.2 统计层阈值的方向也要前置确认(具体数值等 P1.5 基线后再锁)。跳过 §0 进入 P0 实施 = 默认走"做完才看指标"的反模式,文档不背书这种做法。 + +--- + +## 2. 设计原则 + +1. **不改 agent 框架** — 不动 `useGeminiStream` / `coreToolScheduler` / `geminiChat` 核心路径 +2. **数据驱动选优先级** — 先建 telemetry,让数据告诉你改哪个 skill,不靠拍脑袋 +3. **per-skill 可测可灰度** — 每个 skill 改造独立 A/B,失败局部回退 +4. **复利优先** — 收益 = 单次减轮收益 × 触发频率,高频 skill 优先 +5. **不绑定 D1** — 本方案的成功不依赖 D1 是否落地 + +--- + +## 3. 三层方案 + +### 3.1 Layer 1:减轮 Telemetry(找金矿) + +**目标**:让数据告诉你哪些 skill 最值得改 — 即"用了这个 skill 之后,模型有多大概率追加一次工具调用"。 + +**核心字段**(per-turn、per-skill-invocation): + +```typescript +interface SkillFollowupRecord { + skill_name: string; + prompt_id: string; // 关联同一 user prompt 内的所有 events + turn_index: number; // 该 skill 在 loop 里是第几轮 + followup_tool_names: string[]; // 同一 prompt_id 下,skill 之后还调了哪些工具 + followup_count: number; // followup_tool_names.length + followup_kinds: Kind[]; // Read/Edit/Execute/... + next_turn_is_terminal: boolean; // skill 之后下一轮就出文字(不再调工具) + user_followup_within_30s: boolean; // 用户在结果显示后 30s 内追加新 prompt(质量回归信号) +} +``` + +**关键指标**: + +- `skill_followup_rate = sum(followup_count > 0) / total_invocations` +- `terminal_after_skill_rate = sum(next_turn_is_terminal) / total_invocations` +- 按 `(skill_name, top followup tool)` 聚合 — 看哪些 skill 之后最常追加哪个工具 + +**金矿判定**: + +``` +(invocation_count_weekly × skill_followup_rate) ≥ threshold +↓ +该 skill 是减轮金矿,优先 Layer 2 改造 +``` + +阈值建议:top-3 按上式排序的 skill,先改前 2 个。 + +### 3.2 Layer 2:Skill 输出完整化 + +**目标**:让被识别为金矿的 skill 一次返回完整答案,消除 Round 1 → Round 2 的跳跃。 + +**改造模式(按 followup 类型分类)**: + +| Followup 模式 | 典型场景 | 改造方向 | +| --------------------------- | -------------------------- | ---------------------------------- | +| skill → `read_file` | skill 给路径,模型再读 | skill 内部直接读,返回内容 | +| skill → `grep/glob` | skill 给目录,模型再搜 | skill 内部搜好,返回匹配 | +| skill → `shell` (read-only) | skill 给命令,模型再执行 | skill 内部跑命令,返回输出 | +| skill → `shell` (write) | skill 给方案,模型再执行写 | **保留**(写操作要确认,不应合并) | +| skill → another skill | 链式调用 | **不合并**(保持组合性) | + +**改造检查清单(per-skill PR 模板)**: + +1. 在 skill 描述里**预声明输出契约**:明确写 "Returns: full file content / matched lines / command output",让模型知道不必追加查询 +2. 在 skill 内部**完成所有 read-only followup**:把 telemetry 显示 >50% 追加率的 read/search 操作内联进 skill +3. **不内联 write 操作**:写操作需要用户确认,必须单独成轮 +4. **不内联深度推理 followup**:如果 followup 是"基于此再分析",那是模型的事,不是 skill 的事 +5. **附 A/B telemetry**:改造后 2 周对比 `followup_rate` 是否下降到 <20% + +**典型改造示例(示意)**: + +改造前: + +``` +skill "list-workspaces" returns: ["ws_a", "ws_b"] +→ Round 2: model calls shell to get details for each workspace +``` + +改造后: + +``` +skill "list-workspaces" returns: + - ws_a (owner: foo, last_active: 2026-05-20, status: active) + - ws_b (owner: bar, last_active: 2026-05-01, status: archived) +description updated: "Returns workspaces with owner, last_active, status" +→ Round 2 disappears for ~80% of queries +``` + +### 3.3 Layer 3:Prompt 教育模型并发 + +**目标**:对于独立工具(多文件读、多目录搜),让模型在同一轮里并发发起 tool_calls,把 N 轮压成 1 轮。 + +**前提**:基础设施已就绪 — `tools/tools.ts:818` 的 `CONCURRENCY_SAFE_KINDS` + `coreToolScheduler` 的 `partitionToolCalls` 已经能并发执行同 batch 内的 read/search/fetch 工具。**差的只是模型主动发起并发 tool_calls 的意愿**,qwen-coder 默认偏串行。 + +**改动位置**:`packages/core/src/core/prompts.ts`(已审计过,加在 `# Final Reminder` 段 L396 附近不会破坏 cache 命中以外的事 — 仅一次性预热成本)。 + +**指导文本(示意,需 A/B 调优)**: + +``` +When you need to call multiple independent read-only tools (read_file, +grep, glob, web_fetch), emit them in a SINGLE tool_calls batch — do NOT +call them sequentially across rounds. They will execute concurrently. + +Examples: +- Reading 3 files for comparison: emit 3 read_file calls in one batch +- Searching for 2 patterns: emit 2 grep calls in one batch + +Do NOT batch when the second call depends on the first call's result. +``` + +**生效衡量**:新增 telemetry 字段 `batch_size`(同 turn 内 tool_calls 数量)— 改 prompt 前后对比分布。 + +#### 3.3.1 扩展 `CONCURRENCY_SAFE_KINDS`(Layer 3 子项) + +prompt 教育模型并发只是供给侧(模型愿意一次发多个 tool_calls),但 `tools/tools.ts:818` 的 `CONCURRENCY_SAFE_KINDS = { Read, Search, Fetch }` 决定**实际能并发执行的工具范围**:`partitionToolCalls`(`coreToolScheduler.ts:775`)会把"连续的安全工具"打包成 concurrent batch,其余各自串行。 + +如果模型按指导一次发了 3 个 tool_calls 但其中 1 个属于 `Kind.Execute` 且不在安全集合,整个 batch 就会被拆开串行执行 — Layer 3 prompt 改动的收益会被运行时调度抵消。 + +**扩展候选**(按风险递增): + +- `Kind.Think`(含 save_memory / todo_write)—— **不要加**,有隐式写入 +- 只读 shell(`isShellCommandReadOnly()` 返回 true 的 Execute)—— `partitionToolCalls` 已有特判(`coreToolScheduler.ts` `partitionToolCalls` 注释里提到 "Execute (shell) is safe only when isShellCommandReadOnly() returns true"),现状已覆盖,无需改 `CONCURRENCY_SAFE_KINDS` +- MCP 工具按 `Kind` 分类 —— 各 MCP server 行为差异大,需要在工具注册时显式 opt-in 才安全 + +**结论**:当前集合已经合理,**Layer 3 不依赖扩展 `CONCURRENCY_SAFE_KINDS`**。本节存在的意义是:在收完 `batch_size` telemetry 数据后,**如果发现"并发 batch P50 < 期望值",先检查是不是被 `partitionToolCalls` 切断而非模型不并发**。这是 Layer 3 A/B 失败时的一个诊断路径,不是必做项。 + +> 信用:codex review 提出"扩展 `CONCURRENCY_SAFE_KINDS` 是被忽略的杠杆"。核对后判断为:现状已有 `isShellCommandReadOnly` 特判覆盖最大头,扩展集合本身收益小、风险大;保留作为诊断路径。 + +--- + +## 4. 详细实施 + +### 4.1 Layer 1:Telemetry 扩展(1-2d) + +#### 4.1.1 补 `prompt_id` 到 `SkillLaunchEvent` + +**位置**:`packages/core/src/telemetry/types.ts:896` + +当前 `SkillLaunchEvent` 仅含 `skill_name` + `success`,**无 `prompt_id`** — 无法跟同一 turn 内的其他 `ToolCallEvent` 关联。 + +```typescript +// types.ts:896 +export class SkillLaunchEvent implements BaseTelemetryEvent { + 'event.name': 'skill_launch'; + 'event.timestamp': string; + skill_name: string; + success: boolean; + prompt_id: string; // 新增 + turn_index?: number; // 新增 + + constructor( + skill_name: string, + success: boolean, + prompt_id: string, // 新增 + turn_index?: number, // 新增 + ) { ... } +} +``` + +**调用方更新**:`packages/core/src/tools/skill.ts` 的 4 个 `logSkillLaunch` 调用点(L386, L399, L426, L482),传入 `this.params` 拿不到 `prompt_id` — `BaseToolInvocation` 仅持有 `params`,没有 `request.prompt_id` 字段。**实际实现**用鸭子类型方式注入:`SkillToolInvocation` 暴露 `setPromptId(id)` setter + 私有 `promptId` 字段,`CoreToolScheduler.buildInvocation`(`coreToolScheduler.ts:1253`)在 build 后 duck-type 调 `setPromptId(request.prompt_id)`,对齐既有 `setCallId` hook 的 pattern;invocation 在 `execute()` 内的 4 个 `logSkillLaunch` 都传 `this.promptId`。**早期版本的本节描述("BaseToolInvocation 已有 request.prompt_id")是错的**,已在 PR #4565 review 后更正。 + +#### 4.1.1b qwen-logger 链路修复(前置) + +补 `prompt_id` 之前要先解决一个 **既存的链路断点**:`packages/core/src/telemetry/qwen-logger/qwen-logger.ts:908` 定义了 `logSkillLaunchEvent(event)` 方法,但**全仓库无任何调用方** —— `loggers.ts:958` 的 `logSkillLaunch` 直接走 `logs.getLogger(SERVICE_NAME).emit()` 这条 OTLP 路径,绕过了 qwen-logger。 + +后果: + +- OTLP 路径上的 skill_launch 事件能到 OTLP collector(已工作),但 qwen-logger 那条专用上报链路目前是死的 +- 如果 telemetry backend 是从 qwen-logger 消费(而非 OTLP),skill_launch 事件**完全不上报** +- §4.1.2 离线 SQL 派生 `SkillFollowupRecord` 依赖 skill_launch 事件落库 —— **必须先验证现在 skill_launch 在 backend 是否可见** + +修复方向二选一: + +- **A**(推荐)在 `loggers.ts:958` 的 `logSkillLaunch` 里加一行 `QwenLogger.getInstance(config)?.logSkillLaunchEvent(event)`,对齐 `logToolCall` 的 `loggers.ts:230` 写法 +- **B** 确认 backend 只从 OTLP 消费,把 qwen-logger 里的 `logSkillLaunchEvent` 标 `@deprecated` 或删除 + +**为什么只补 QwenLogger 一条路径,不对齐 `logToolCall` 的 4 条全路径**: + +`logToolCall`(`loggers.ts:220-247`)实际有 4 条出口: + +1. `uiTelemetryService.addEvent(...)` — UI 展示 +2. `config.getChatRecordingService()?.recordUiTelemetryEvent(...)` — 聊天历史 +3. `QwenLogger.getInstance(config)?.logToolCallEvent(...)` — qwen-logger 后端遥测 +4. OTLP `logger.emit(...)` — OpenTelemetry + +skill_launch 是**纯后端遥测事件**,不需要在 UI 上展示(用户已经看到 SkillTool 的 returnDisplay)、也不需要进 ChatRecording 的 turn 历史(skill 内部的工具调用已经各自被 recordUiTelemetryEvent 记录)。因此只补第 3 条(QwenLogger),保留第 4 条(OTLP),跳过 1/2 是有意的,不是遗漏。 + +**字段透传细节**:`loggers.ts:961-966` 用 `{ ...event }` spread 自动透传新字段(`prompt_id` 加进 `SkillLaunchEvent` 后这条路自动生效),但 `qwen-logger.ts:908` 的 `logSkillLaunchEvent` 内部如果显式解构 `event.skill_name` / `event.success`,新字段不会自动纳入,需手动同步。 + +工作量:A 路径约 0.5d(含 backend 端确认);B 路径约 0.2d(删代码 + 文档说明)。 + +#### 4.1.2 派生 `SkillFollowupRecord`(离线聚合) + +不需要新事件类型 — `ToolCallEvent` 和 `SkillLaunchEvent` 都已带 `prompt_id`,离线 SQL 即可派生: + +```sql +-- 伪 SQL,按实际 telemetry backend 调整 +WITH skill_events AS ( + SELECT prompt_id, skill_name, timestamp FROM events + WHERE event_name = 'skill_launch' AND success = true +), +tool_events AS ( + SELECT prompt_id, function_name, timestamp FROM events + WHERE event_name = 'tool_call' +), +followups AS ( + SELECT s.skill_name, s.prompt_id, + COUNT(t.function_name) AS followup_count, + ARRAY_AGG(t.function_name) AS followup_tool_names + FROM skill_events s + LEFT JOIN tool_events t + ON s.prompt_id = t.prompt_id AND t.timestamp > s.timestamp + GROUP BY s.skill_name, s.prompt_id +) +SELECT skill_name, + COUNT(*) AS invocations, + AVG(followup_count) AS avg_followup, + SUM(CASE WHEN followup_count > 0 THEN 1 ELSE 0 END)::FLOAT / COUNT(*) AS followup_rate +FROM followups +GROUP BY skill_name +ORDER BY invocations * followup_rate DESC; +``` + +#### 4.1.3 跑 telemetry 1 周收数据 + +- 不变更 user-facing 行为 +- 不需要任何配置开关 — telemetry 已有 opt-in 框架(`telemetry.target` 设置项) +- 1 周后产出 skill ranking 报告 + +### 4.2 Layer 2:Skill 改造(per-skill 0.5-1d) + +按 Layer 1 数据从 top-down 改造。每个 skill 一个独立 PR,PR description 必须包含: + +1. **数据**:当前 invocation_count、followup_rate、top followup tools +2. **改造范围**:内联了哪些 followup(明确不内联什么) +3. **输出契约更新**:skill 描述里加了什么预声明 +4. **A/B 计划**:改造后 2 周再观察 followup_rate + +**注意事项**: + +- Skill 内联 read 操作不要重复 read_file 的所有边界情况处理(编码、二进制检测等)— 调用 `read_file` 工具本身,不要重写 +- Skill 内联 grep/glob 同理 +- Skill 内联 shell 命令需走 `executeToolCall` 标准路径(保留 telemetry) +- **不要让 skill 体积爆炸**:内联 followup 后 skill 描述 > 500 tokens 时,拆分 skill 而不是合并 + +### 4.3 Layer 3:Prompt 教育(0.5d 改动 + 实测调优) + +#### 4.3.1 加并发指导 + +**位置**:`packages/core/src/core/prompts.ts` `# Final Reminder` 段(L396) + +加上节 3.3 的指导文本。具体措辞需 A/B —— 先用最朴素版本,根据并发率提升程度再细化。 + +#### 4.3.2 加 `batch_size` telemetry + +**位置**:`packages/core/src/telemetry/types.ts` 的 `ToolCallEvent` 或新增轻量级 `ToolBatchEvent` + +```typescript +// 选项 A:在 ToolCallEvent 上加字段(侵入小) +export class ToolCallEvent { + ... + batch_size?: number; // 同一 batch 内 tool_call 数量 + batch_position?: number; // 在 batch 内的位置 (0-indexed) +} + +// 选项 B:新增 ToolBatchEvent(语义更清晰,需走完整新事件类型流程) +``` + +**推荐选项 A** — 改动小、查询时聚合方便。 + +**状态传递路径**(关键 — 这一步成本被早期版本低估): + +`coreToolScheduler.ts:2456` 的 `partitionToolCalls(callsToExecute)` 返回 `batches`,**但 batch 信息在调度路径上立刻丢失**: + +``` +executeToolCalls + └─ batches = partitionToolCalls(...) // 知道 batch.calls.length + └─ for batch of batches: + └─ this.runConcurrently(batch.calls, ...) // 知道 batch.calls.length + └─ executeSingleToolCall(call, ...) // ❌ 已不知道 batch + └─ ... + └─ finalizeToolCalls + └─ logToolCall(config, new ToolCallEvent(call)) // ❌ 无 batch context +``` + +`ToolCallEvent` 的构造器(`types.ts:189`)只接收单个 `CompletedToolCall`,无 batch 字段。 + +修复方向: + +- **方向 A**(推荐):在 `ScheduledToolCall` 上加 `batchSize?: number` + `batchPosition?: number`。两条分支分别填充: + - 并发分支(`coreToolScheduler.ts:2459-2460`,`batch.calls.length > 1`):`runConcurrently(batch.calls, ...)` 进入循环前给每个 `call` 写 `batchSize = batch.calls.length`、`batchPosition = i` + - 串行分支(`L2462-2464` 的 `for (const call of batch.calls)`):单工具 batch 显式设 `batchSize = 1`、`batchPosition = 0`(**不要默认 undefined**,否则下游 telemetry 聚合时会把并发未生效的轮次误判为缺失数据) + + `new ToolCallEvent(call)` 在构造器里从 `call` 读这两个字段 + +- **方向 B**:改 `ToolCallEvent` 构造器签名 `new ToolCallEvent(call, batchInfo?)`,所有调用方同步改(4 个 logToolCall 调用点 + 测试)。改动面比 A 大 + +工作量:方向 A 约 0.5d 含单测;方向 B 约 1d(调用方多)。 + +**同步衡量"模型并发意愿"** — Layer 3 改 prompts.ts 前后,对比 `batch_size > 1 的 tool_call 占比` 分布。这是 Layer 3 是否生效的关键指标,没这个数据 Layer 3 A/B 无法收尾。 + +#### 4.3.3 cache 影响评估 + +`prompts.ts` 改动会让 DashScope ephemeral cache 一次性失效(首次请求 cache miss,之后恢复)。这是已知一次性成本,参见 `rt-optimization-design.md` §7.8 的 prompt 稳态审计。 + +--- + +## 5. 验收与度量 + +> **本节是 §0 验收 Spec 的"方法论"配套** — §0 声明"算成功的指标 + 阈值前置/后置时机",§5 说明"怎么测、SQL 怎么写、A/B 怎么设计"。本节阈值是 §0.2 的当前占位,最终值在 P1.5 基线测量后锁定。 + +### 5.1 per-skill A/B 指标(改造后 2 周) + +| 指标 | 验收线 | 备注 | +| ----------------------------------------- | ------------------------ | -------------------------- | +| 该 skill 的 `followup_rate` | < 20%(改造前若为 70%+) | 主指标 | +| 该 skill 触发场景的端到端 RT P50 | 下降 ≥ 2s | 来自少一轮 LLM 调用 | +| 该 skill 的 `user_followup_within_30s` 率 | 不上升 | 用户没追问 = 答案完整 | +| 该 skill 的 `success` 率 | 不下降 | 内联 followup 没引入新失败 | + +### 5.2 整体 RT 指标 + +| 指标 | 基线 | Layer 2 改完 top-3 skill 后目标 | +| ---------------------------------- | ------------------------------------- | -------------------------------- | +| 端到端 RT P50(含 skill 的会话) | 13.4s(单次采样)/ 待补 ≥3 类场景基线 | 下降 2-3s | +| Tool batch P50 size(Layer 3) | 待测 | ≥ 1.3(>30% 调用涉及并发 batch) | +| Skill 总 followup_rate(加权平均) | 待测 | 下降 ≥ 30% | + +### 5.3 失败信号 — 什么时候放弃这个方向 + +**结果指标止损线**: + +- Layer 1 数据出来后,**top-5 skill 的加权 followup_rate < 30%** → 减轮空间小,不值得继续 Layer 2 +- Layer 2 改完 2 个 skill 后,**端到端 RT P50 下降 < 1s** → 改造方向错(可能 followup 是写操作不该合并),停下复盘 +- Layer 3 prompt 改动 2 周后 **batch_size P50 仍 = 1** → 模型不接受并发指导,放弃 Layer 3,只保留 Layer 1+2 + +**过程指标止损线(前置预警,避免方案"看起来在做、其实没收益")**: + +- **Skill 命中率(intended skill vs selected skill)下降 ≥ 5pp** → skill 描述改坏让模型选错 skill。典型场景:改造前用户问 X 总是命中 skill_a,改造后偶尔被路由到 skill_b 但没产生 error(模型用错 skill 但勉强凑出答案),结果指标看起来正常但 followup_rate 反而上升。**衡量方法**:在 telemetry 加 `skill_invocation_pattern` —— 按 user prompt 前 N 个关键词聚类,看每个 cluster 主要触发哪个 skill;改造前后对比顶 1 偏移 +- **Skill 内联 followup 失败率 ≥ 5%** → skill 改造引入了原本不存在的失败模式(如内联 `read_file` 处理大文件爆内存)。衡量:`SkillLaunchEvent.success` 改造前后对比 +- **Per-skill 用户取消率(Ctrl+C)上升 ≥ 2pp** → skill 输出变慢或变长导致用户失去耐心。衡量:`ToolCallEvent.status === 'cancelled'` 占比 + +--- + +## 6. 与 D1/D3 的衔接 + +### 6.1 与 D1 的关系 + +Layer 2 改完 top skill 后,**剩余的 followup-heavy skill 才是 D1 `skipLlmRound` 的真正适用场景** — 那些 skill 输出已经完整(不需要 Round 2),且确实是终态查询(Round 3 总结也是浪费)。 + +执行次序: + +1. Layer 1 telemetry 上线 → 1 周数据 +2. Layer 2 改造 top 2-3 skill → A/B 2 周 +3. Layer 3 prompt 并发 → 实测 1 周 +4. **此时**再评估 D1:剩余高频 skill 里有多少是"输出完整 + 终态查询"形态 → 是否值得 2-3d 框架改造 + +### 6.2 与 D3 的关系 + +D3(`StreamingState.Summarizing`)是感知层优化,与本方案完全正交。Layer 1-3 减少的是**真实轮数**,D3 减少的是**用户感知等待**。如果 Layer 2 已经把 RT 降到用户可接受的范围,D3 价值下降;反之 D3 可以叠加。 + +--- + +## 7. 限制与已知风险 + +1. **覆盖率受改造范围限制** — 改 10 个 skill 就只覆盖那 10 个的场景。但收益是确定可测有复利的 +2. **Skill 内联 followup 可能让单 skill 变重** — 描述膨胀、加载慢、复用度下降。Layer 2 检查清单第 5 条防御 +3. **Layer 3 模型可能不听并发指导** — qwen-coder 训练数据偏串行;A/B 数据可能显示 prompt 改动无效,作为已知失败模式 +4. **Telemetry 隐私边界** — `SkillFollowupRecord` 不应记录工具参数(已默认从 `ToolCallEvent.function_args` 拿,但要审计 skill_name 是否泄露用户意图) +5. **不适用于子 agent / cron / notification** — 这些路径不走 skill 系统,本方案不覆盖 +6. **基线数据单薄** — 沿用 `rt-optimization-design.md` §1.2 的单次采样,Layer 2 落地前需补 ≥3 类场景基线 +7. **`logSkillLaunch` 字段扩展会破坏既有 telemetry consumer** — 4 个调用点 + 下游 logger 都要同步改 +8. **`qwen-logger.ts:908` `logSkillLaunchEvent` 当前是死代码** — 仓库内无任何调用方,§4.1.1b 已列前置修复 + +### 7.1 与已有框架机制的边界(不在本方案范围) + +仓库已有几条与减轮间接相关的框架机制,**本方案不重新发明,也不替代**: + +| 已有机制 | 位置 | 与本方案的关系 | +| ---------------------------------------------------- | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `partitionToolCalls` + `runConcurrently`(并发执行) | `coreToolScheduler.ts:775, 2473` | Layer 3 直接复用;本方案不动它 | +| `CONCURRENCY_SAFE_KINDS`(决定哪些工具可并发) | `tools/tools.ts:818` | §3.3.1 已论证现状合理,不扩展 | +| `FileReadCache`(避免重复读同一文件) | `services/fileReadCache.ts` | 间接影响"模型重复读文件"轮次,已生效;本方案不依赖也不增强 | +| `chatCompressionService`(历史压缩) | `services/chatCompressionService.ts` | 与轮次正交(影响单轮成本而非轮数);与 `rt-optimization-design.md` §3.2 fast 路由的 `wouldTriggerCompression` gate 是同一组件 | + +列出这些是为了避免"本方案被理解为忽略了已有机制"。 + +--- + +## 8. 实施时间线 + +> **前提:本时间线从 P-1 开始,不能跳过**。P-1 是 §0 验收 Spec 的前置评审,0.5d 工作量但**强制性** — 不通过则不进入 P0。这一约束是为了避免"先写代码再补 spec"的反模式:spec 后置等于把"算成功"的判断推迟到结果出来后,容易出现"为了让指标好看而调整 spec"的偏差(参见 `rt-optimization-design.md` §7 D2 路线的覆辙)。 + +| Phase | 内容 | 投入 | 产出 | spec 锁定动作 | +| -------- | ---------------------------------------------------------------------- | --------------------- | ------------------------------ | --------------------------------------- | +| **P-1** | spec 前置评审 | 0.5d | §0.1 / §0.3 锁定 | **锁定 §0.1 工程层 spec + §0.3 止损线** | +| **P0** | qwen-logger 链路修复(§4.1.1b 前置) | 0.5d | skill_launch 事件可见性确认 | 验证 §0.1 第 1 条 | +| **P1** | Layer 1 telemetry:补 `prompt_id` 字段 + 离线 SQL | 1-2d | skill ranking 报告 | 验证 §0.1 第 2/3/4 条 | +| **P1.5** | 1 周数据收集 + 基线测量(≥3 类场景 × ≥10 次) | 1w | 决定改哪 2-3 个 skill | **锁定 §0.2 阈值 + 验证 §0.1 第 5 条** | +| **P2** | Layer 2 改造 top-1 skill(PR + A/B) | 0.5-1d 改造 + 2w 观察 | followup_rate ↓、RT P50 ↓ 验证 | **PR 内声明 §0.4 per-skill spec** | +| **P3** | Layer 3 prompt 并发指导 + `batch_size` telemetry(含 §4.3.2 状态传递) | 1-1.5d 改动 + 1w 实测 | batch_size 分布 | 验证 §0.2 第 3 条 | +| **P4** | Layer 2 继续改 top-2 / top-3 skill(并行 P3) | 0.5-1d × N | 累计 RT P50 ↓ | 每 PR 内声明 §0.4 | +| **P5** | 评估 D1 是否还有价值 | 决策会 | 路线图更新 | — | + +**关键决策点(对照 §0.3 止损线)**: + +- **P-1 末**:§0.1 / §0.3 任一项无法达成共识 → 不进入 P0 +- **P1.5 末**:触发 §0.3 结果指标 #1(top-5 加权 followup_rate < 30%)→ 终止方向;否则锁定 §0.2 阈值 +- **P2 末**:触发 §0.3 结果指标 #2(top-1 改造后 RT P50 ↓ < 1s)或任一过程指标 → 停下复盘 +- **P3 末**:触发 §0.3 结果指标 #3(batch_size P50 仍 = 1)→ 放弃 Layer 3 +- **P5**:根据剩余 skill 形态决定 D1 ROI + +--- + +## 9. 关键代码位置 + +| 文件 | 关键符号 | 位置 | +| -------------------------------------------------------- | ------------------------------------------------------------- | --------------------------------- | +| `packages/core/src/telemetry/types.ts` | `ToolCallEvent`(含 `prompt_id` / `duration_ms`) | L170 | +| `packages/core/src/telemetry/types.ts` | `SkillLaunchEvent`(需补 `prompt_id`) | L896 | +| `packages/core/src/telemetry/loggers.ts` | `logToolCall` | L220 | +| `packages/core/src/telemetry/loggers.ts` | `logSkillLaunch`(走 OTLP;缺 qwen-logger 转发) | L958 | +| `packages/core/src/telemetry/loggers.ts` | `logToolCall`(双路径:OTLP + qwen-logger,作为修复样板) | L220, L230 | +| `packages/core/src/telemetry/qwen-logger/qwen-logger.ts` | `logSkillLaunchEvent`(**当前死代码**,§4.1.1b 前置修复目标) | L908 | +| `packages/core/src/core/coreToolScheduler.ts` | `partitionToolCalls` | L775 | +| `packages/core/src/core/coreToolScheduler.ts` | `runConcurrently` / batch 调度 | L2456, L2473 | +| `packages/core/src/core/coreToolScheduler.ts` | `logToolCall` 调用点(batch_size 状态传递终点) | L3163 | +| `packages/core/src/services/fileReadCache.ts` | `FileReadCache`(已有,影响重复读取轮次) | L135 | +| `packages/core/src/tools/skill.ts` | `SkillTool` + 4 个 `logSkillLaunch` 调用点 | L386, L399, L426, L482 | +| `packages/core/src/skills/skill-manager.ts` | `SkillManager`(skill 注册/加载) | 全文件 | +| `packages/core/src/skills/skill-load.ts` | skill 描述加载(输出契约改动入口) | 全文件 | +| `packages/core/src/tools/tools.ts` | `Kind` + `CONCURRENCY_SAFE_KINDS` | L793, L818 | +| `packages/core/src/core/coreToolScheduler.ts` | `partitionToolCalls` + `runConcurrently`(已有并发基础设施) | 见 rt-optimization-design.md §5.7 | +| `packages/core/src/core/prompts.ts` | `# Final Reminder` 段(Layer 3 加并发指导处) | L396 | +| `.qwen/skills/` | 各 skill 定义目录(Layer 2 改造对象) | 目录 | diff --git a/docs/design/rt-optimization/rt-optimization-design.md b/docs/design/rt-optimization/rt-optimization-design.md new file mode 100644 index 00000000000..840c23e215a --- /dev/null +++ b/docs/design/rt-optimization/rt-optimization-design.md @@ -0,0 +1,1205 @@ +# Qwen Code Agent Loop RT 优化技术方案 + +## 1. 背景与问题定义 + +### 1.1 现状 + +Qwen Code 的 Agent Loop 为严格串行模型: + +``` +User Prompt → [LLM 决策] → Tool Execution → [LLM 决策] → Tool Execution → ... → [LLM 回复] → Idle + ~3-4s ~Xms-Ns ~3-4s ~Xms-Ns ~3-4s +``` + +每一轮 LLM 调用(含网络 RTT + 模型推理)约 3-4s,是端到端 RT 的主要成本。 + +### 1.2 实测数据 + +测试场景:"我有哪些工作空间"(3 轮 agent loop,2 次工具调用,单次采样) + +| 阶段 | 耗时 | 占比 | +| --------------------------- | --------- | ---- | +| LLM Round 1(决策调 skill) | 3.8s | 28% | +| Skill 执行 | 1ms | <1% | +| LLM Round 2(决策调 shell) | 3.0s | 22% | +| Shell 执行 | 2.5s | 19% | +| LLM Round 3(文字总结) | 3.8s | 28% | +| 框架开销(状态同步、渲染) | 0.3s | 3% | +| **总计** | **13.4s** | 100% | + +**结论**:LLM 调用占 78%,工具执行 19%,框架 3%。优化的核心是**减少 LLM 调用次数**和**降低单次 LLM 调用延迟**。 + +> 注:单次采样、单一场景。19% 工具执行是 shell 慢调用支配,read-heavy 场景下工具执行可降至 <5%。方案落地前需补 ≥3 类场景(写操作、跨工具推理、错误恢复)的基线。 + +### 1.3 当前架构关键约束 + +| 约束 | 代码位置 | 说明 | +| ------------------- | ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| 工具结果无后置控制 | `tools.ts` `ToolResult` 接口 (L422) | 仅有 `llmContent`/`returnDisplay`/`error`,无法表达"跳过 LLM" | +| 结果无条件回传 LLM | `useGeminiStream.ts` `handleCompletedTools` (L2038) → `submitQuery(ToolResult, …)` (L2355) | 所有 gemini-initiated 工具结果都回传 | +| Stream 完毕后才调度 | `useGeminiStream.ts` `processGeminiStreamEvents` (L1365) | stream 循环结束后才 `scheduleToolCalls`,无增量调度 | +| 模型层选择无策略层 | `client.ts` `modelOverride ?? getModel()` (L1305, L1598) | 基础设施已贯通至 `turn.run(model, …)` (L1707),但调用方仅在 skill 显式指定时使用 | + +### 1.4 已就绪的基础设施(本方案大量复用) + +| 能力 | 位置 | 现状 | +| ---------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------------------------------- | +| `fastModel` 配置 + `/model --fast ` | `config.ts:684`, `1987`, `2021` | 已就绪 | +| `SendMessageOptions.modelOverride` | `client.ts:142` → `1598` → `turn.run` | 端到端贯通至 `geminiChat.sendMessageStream(model, …)` | +| 钩子层 `modelOverrideRef`(承载 skill 选模型) | `useGeminiStream.ts:376`, `2225`, `1841` | 已贯通 | +| fast-model **非流式** side query 先例 | `services/toolUseSummary.ts:108`(via `runSideQuery`) | 已上线,证明 fast 模型配置健全;但**非流式路径** | +| fast-model **流式** 先例 | `followup/speculation.ts:224` | 已上线,但**用的是 forked chat**(`createForkedChat`),与主 chat 隔离 | + +**关键空白**:**没有任何生产代码**在主 chat 上以 fast model 跑 streaming。本方案 D2 是首个 case,需先做验证实验(详见 §3.2 前置条件)。 + +--- + +## 2. 设计原则 + +1. **通用性**:方案不绑定特定 tool/skill +2. **向后兼容**:现有工具无需修改即可继续工作 +3. **渐进式 + 显式信号**:策略默认 conservative,由工具作者通过显式字段 opt-in 优化 +4. **可回滚**:所有优化通过 feature flag 控制;用户级别可强制关闭 +5. **诚实的权衡**:明确标注质量风险、成本风险和适用边界 + +--- + +## 3. 优化方案 + +### 3.1 方向一:工具后置执行指令(ToolResult Post-Execution Directive) + +#### 问题 + +当前 `ToolResult` 不包含任何关于"接下来该怎么做"的信息。无论工具结果是否自解释,都无条件触发一轮 LLM。 + +#### 设计 + +扩展 `ToolResult` 接口(`packages/core/src/tools/tools.ts` L422): + +```typescript +export interface ToolResult { + llmContent: PartListUnion; + returnDisplay: ToolResultDisplay; + error?: { message: string; type?: ToolErrorType }; + + // 新增:后置执行指令 + postExecution?: { + /** + * 工具结果不回传 LLM,直接作为最终回复展示给用户。 + * 适用于结果完全自包含、不需要模型再解读的场景。 + * 是 ToolResult 局部属性。 + */ + skipLlmRound?: boolean; + + /** + * 工具结果"自包含、可直接展示给用户"——即 `returnDisplay` 已经是 + * 用户期望看到的最终形态,不需要模型加工。 + * 是 ToolResult 局部属性,**不**预测"下一轮是否 summary"。 + * 与方向三(展示解耦)联动:true → 进入 Summarizing 状态允许用户输入。 + */ + resultIsTerminal?: boolean; + }; +} +``` + +> **设计修正**:早期版本曾把单一 `selfExplanatory` 字段同时承担"工具产物属性"和"对话流预测信号"两份职责,但二者并不重合(例:用户 prompt 是"读 X 然后修 Y",read_file 输出自包含,但下一轮显然不是 summary)。**预测信号属于对话流全局属性**,不应通过工具字段表达——D2 改为完全用对话流启发式(见 §3.2)。 + +#### 行为变更 + +`handleCompletedTools` 中新增判断: + +``` +工具批次完成 + → 检查 batch 中所有工具的 postExecution.skipLlmRound + → 全部为 true? + → YES: markToolsAsSubmitted, 不调 submitQuery, 直接 idle + → NO: 保持现有行为 (submitQuery) +``` + +**重要约束**:`skipLlmRound` 仅在**当前 batch 的所有工具都声明 skip** 时才生效。混合 batch 仍然回传。 + +#### 历史不变量 + +跳过 LLM 后历史形如:`user → function_call → function_response → <无 assistant>`。 + +- 复核 `repairOrphanedToolUseTurnsInHistory`(session-load 时调用)是否容忍此形态 +- 复核 auto-compaction 在缺少 assistant 文本时的行为 +- PR #4176 刚关闭过 tool_use↔tool_result 不变量,落地前需补单测覆盖"skip 后下一轮 user message"的 alternation +- Qwen / OpenAI 风格 API 容忍;Anthropic 严格 alternation —— 后续若支持 Anthropic 直连需要兜底(向 history 注入空 assistant text) + +> **统一修复点**:此处和 §3.3(D3 中途打断 Summarizing)破坏的是**同一个历史不变量**。修复方案二选一(注入空 assistant / 接受 Qwen 容忍),两个方向必须使用相同选择。 + +#### 信号生态(Phase 2 工作) + +| 工具 | `skipLlmRound` | `resultIsTerminal` | 备注 | +| ------------------------------------- | -------------------- | ------------------ | --------------------------------------------------------------- | +| `read_file` | 配合 query-only 场景 | true | 文件内容即答案 | +| `cat`(via shell) | 视场景 | true | 同 read_file | +| `grep` / `glob` / `ls` | false | **false(默认)** | 结果常需模型挑选/排序/总结;skill 层在已知"纯查询"场景显式 true | +| `git status` / `git log`(via shell) | false | true | 输出已格式化 | +| Skill 工具 | 各 skill 自决 | 各 skill 自决 | 查询类 skill 倾向 true | +| MCP 工具 | 默认 false | 默认 false | 通过 allowlist 显式 opt-in | + +第三方/MCP 工具不可信任,默认不打标;通过 `config.toolPostExecAllowlist` 显式启用。 + +> `grep/glob/ls` 默认 false 是从严选择:避免 D2/D3 在需要模型总结排序的场景误判。 + +#### 适用与不适用 + +- **适用**:终态查询(read/cat/print 类型)、自包含结果(skill 已格式化输出) +- **不适用**:多步任务中间步骤、写操作确认、需解读的复杂日志 + +#### 风险与缓解 + +| 风险 | 严重度 | 缓解 | +| ------------------------------------------ | ------ | ------------------------------------------ | +| 工具错误设置 skipLlmRound 导致多步任务中断 | 中 | batch 级语义 + llmContent 仍在历史中可恢复 | +| 第三方工具滥用 | 中 | MCP 默认禁用,allowlist 显式开启 | +| 历史不变量破坏 | 中 | 落地前补单测;session-load 重放覆盖 | +| 用户预期不一致(期望总结但没有) | 低 | setting `alwaysSummarize: true` 可覆盖 | + +#### 收益 + +终态查询场景节省 3-4s(跳过最后一轮 LLM)。 + +--- + +### 3.2 方向二:summary 轮 fast-model 路由策略 + +#### 定位 + +**本方向不引入新管道,但需要扩展 GeminiChat 接口以支持运行时模型切换**。 + +§1.4 的基础设施提供了 fast 模型配置和 modelOverride 端到端贯通,但**主 chat 上跑 fastModel + streaming 没有先例**,需要: + +- 决策函数:何时把 `config.getFastModel()` 作为 override 传下去 +- 安全回退:`GeminiChat.retryStreamWithModel` 新接口(处理 chat 内部状态) +- 实验验证:主 chat 切换 fast/primary 不破坏 compaction / history-recording + +#### 应用范围 + +D2 仅作用于: + +- **useGeminiStream**(TUI 主路径)—— `sendMessageStream` 调用点 L1841 +- **ACP Session**(IDE 集成路径)—— `acp-integration/session/Session.ts:1182`,Phase 3 同步改造 + +D2 **不作用于**以下路径,避免在非交互或独立上下文里引入额外失败模式: + +- **Subagent 运行时**(`agents/runtime/agent-core.ts:614`):子 agent 已带独立模型配置 +- **Cron 触发 turn**(`SendMessageType.Cron`, client.ts:127):非交互,无 RT 紧迫性 +- **Notification turn**(`SendMessageType.Notification`, client.ts:129):同上 + +#### 核心难点 + +`submitQuery` 调用时**我们并不知道**模型看完结果后是发起新工具还是直接出文字。如果用 fast model 调而模型实际还要调工具——后果是**静默的**:fast 可能调错工具或参数错,错误不会有明显信号。 + +**任何工具级别的字段都无法可靠预测**"下一轮是否 summary",因为它取决于对话流(user prompt + 累计上下文),不是工具产物的局部属性。例: + +``` +用户:"读 utils.ts 然后把里面的 console.log 都改成 logger.info" + → Tool 1: read_file → 结果自包含 + → 但下一轮显然不是 summary +``` + +因此 D2 完全用**对话流启发式**预测,不依赖工具字段。 + +#### 决策函数:对话流启发式 + 否决 + +```typescript +import { Kind, MUTATOR_KINDS } from '../tools/tools.js'; + +function selectContinuationTier( + turn: Turn, + userPrompt: string, + batch: ToolCall[], +): 'fast' | 'primary' { + // ===== 用户级别强制开关(最高优先级) ===== + const userPref = config.getSummaryTierStrategy(); + if (userPref === 'always_primary') return 'primary'; + if (userPref === 'always_fast') return 'fast'; // 仍受运行时保险约束 + + // ===== 用户意图否决 ===== + // 1. user prompt 含动作动词 → 下一轮大概率还要调工具 + if (requestImpliesFurtherAction(userPrompt)) return 'primary'; + + // 2. 本轮已有 mutator 工具 → 大概率有验证/读后续 + if (batch.some((c) => MUTATOR_KINDS.includes(c.tool.kind))) return 'primary'; + + // 3. 本轮或历史有未解决 error → 模型需要 primary 诊断 + if (hasUnresolvedError(turn.toolResults, batch)) return 'primary'; + + // ===== 输出复杂度否决 ===== + // 4. user prompt 要求深度分析(解释/对比/为什么类) + if (needsDeepReasoning(userPrompt)) return 'primary'; + + // 5. 工具调用 ≥3 个不同工具 → 跨结果叙述靠 primary + if (needsCrossResultReasoning(turn)) return 'primary'; + + // 6. 工具输出过长 → 长内容总结靠 primary + if (estimateTotalToolOutputTokens(turn) > 4000) return 'primary'; + + // ===== 模型可行性否决 ===== + // 7. fast 模型 context window 不够 → 切到 fast 会触发 compression + // (compression 自身要 LLM 调用,反而拖慢且增加成本) + if (wouldTriggerCompression(turn.history, config.getFastModel())) + return 'primary'; + + // ===== 多语言兜底 ===== + if (!isPromptLanguageSupported(userPrompt)) return 'primary'; + + // ===== Session 状态兜底 ===== + if (turn.justCompacted || turn.justCleared) return 'primary'; + + return 'fast'; +} +``` + +八个否决项含义: + +- **`requestImpliesFurtherAction`**:动作动词(`改|删|加|替换|修复|实现|新建|create|fix|change|add|remove|implement|write|update`)→ 多步任务 +- **`MUTATOR_KINDS` 命中**:本轮已经写过 → 大概率紧跟一次读/校验。**复用 `tools.ts:806` 已有的 `MUTATOR_KINDS = [Edit, Delete, Move, Execute]`**(每个 Tool 实例的 `kind: Kind` 属性是权威分类,不要重新发明 `isWriteTool`) +- **`hasUnresolvedError(turnResults, currentBatch)`**:判定二段—— + - **当前批次任何 error → 总是未解决**(不假设并行批次能自我纠错) + - **历史按 `(toolName, args fingerprint)` 去重,最后一次仍 error 视为未解决**(仅按 toolName 在同名不同参数下会判错) + - shell 等需正确填 `ToolResult.error`(前置数据质量依赖) +- **`needsDeepReasoning`**:含"分析/解释/为什么/对比/诊断"类关键词 +- **`needsCrossResultReasoning`**:distinct 工具调用 ≥3(同工具同参数视为同一次) +- **输出 tokens > 4000**:经验阈值,**待 fast 模型基线实测后调整** +- **`wouldTriggerCompression`**:fast 模型 context window 通常小于 primary,相同 history 在 fast 上会更早触发 `tryCompress`(geminiChat.ts:1418)—— compression 自身需要一次 LLM 调用,可能**反向恶化 RT 和成本**。预算估算:`estimateHistoryTokens(history) > fastModelContextWindow × COMPACTION_THRESHOLD` 即视为会触发 +- **未支持语言**:仅检测中英文关键词,其他语言(日韩等)默认 primary +- **session 状态突变**:刚 `/compact` 或 `/clear` 后第一次 continuation → primary 重建 mental model + +否决方向**偏向 primary**(宁可多 2s 不要降质)。 + +#### 关键实现:`GeminiChat.retryStreamWithModel` + +**问题**:直接 abort + 调 `client.sendMessageStream` 会破坏 chat 状态: + +1. `geminiChat.ts:1428` 在 stream 启动时就 push `userContent` 到 history;重起会**再 push 一次**导致 history 出现重复 `function_response` +2. `sendPromise` 锁(`geminiChat.ts:1392, 1398`)—— abort 后需要确保 `streamDoneResolver` 被调用 +3. `pendingPartialState` 等 PR #4176 引入的不变量 marker 需要正确清理 +4. Telemetry span 的 model 属性需要更新 + +**新增接口**(`packages/core/src/core/geminiChat.ts`): + +```typescript +/** + * Retry an in-flight or just-aborted streaming send with a different model. + * Does NOT re-push userContent (kept from original send). + * Resets pendingPartialState; releases stale sendPromise; re-opens span. + */ +async retryStreamWithModel( + model: string, + signal: AbortSignal, +): Promise>; +``` + +调用契约: + +- 仅在原 send 已经 abort 后调用(不并发) +- prompt_id 复用(同一用户意图) +- 历史中已经 push 的 userContent 不再 push + +实现工作量约 1.5d 加单测。 + +#### 运行时保险 + +`selectContinuationTier` 返回 `'fast'` 但 stream 中出现 `ServerGeminiEventType.ToolCallRequest` 事件 → **立即 abort 当前流,调 `retryStreamWithModel(primaryModel)`**。 + +这覆盖"预测为 summary 实际仍需工具"的唯一静默放错场景。代价:一次 fast 调用浪费的 tokens(成本归因见 §5.3)。 + +#### 与 skill `modelOverride` 解耦 + +`useGeminiStream.modelOverrideRef`(L376, L2225)当前承载 **skill 显式选择的模型**,属"业务语义"。本方向的 fast 路由属"优化语义",两者**必须分离**: + +```typescript +// 新增独立 ref +const summaryTierRef = useRef<'fast' | 'primary' | undefined>(undefined); + +// 调用点合并(不复用 modelOverrideRef) +const stream = geminiClient.sendMessageStream( + finalQueryToSend, + abortSignal, + prompt_id!, + { + type: submitType, + notificationDisplayText: metadata?.notificationDisplayText, + modelOverride: + modelOverrideRef.current ?? // skill 显式选择优先 + (summaryTierRef.current === 'fast' ? config.getFastModel() : undefined), + }, +); +``` + +生命周期: + +| 时机 | `modelOverrideRef`(skill) | `summaryTierRef`(fast 路由) | +| ------------------------------------------ | --------------------------- | ---------------------------------------- | +| 新 user turn (`!Retry && !ToolResult`) | 清空 | 清空 | +| skill 工具返回 `modelOverride` 字段 | 写入 | 不变 | +| tool batch 完成 → `selectContinuationTier` | 不变 | 写入 | +| Runtime fallback(看到 ToolCallRequest) | 不变 | 升级为 `'primary'` | +| Retry(用户手动 Ctrl+Y) | 保留 | 升级为 `'primary'`(fast 失败不再 fast) | + +skill 显式选择**永远赢**——用户的显式意图优先于优化策略。 + +#### Telemetry 修正 + +`client.ts:1303` 的 interaction span 在 turn 启动时记录 `model` 属性。fallback 触发时 model 实际变了,span 数据失真。需要: + +```typescript +// fallback 触发时 +span.setAttribute('llm.model.requested', fastModel); +span.setAttribute('llm.model.actual', primaryModel); +span.setAttribute('llm.fallback.reason', 'tool_call_seen'); +``` + +并在 `addUserPromptAttributes` 中区分 `requested` / `actual` 模型,避免计费/审计混淆。 + +#### 用户级别强制开关 + +新增 setting(`packages/cli/src/config/settingsSchema.ts`): + +```typescript +summaryTierStrategy: 'auto' | 'always_primary' | 'always_fast'; +// default: 'auto' +``` + +- `'auto'`:使用 `selectContinuationTier`(推荐) +- `'always_primary'`:完全禁用 D2 优化(生产敏感场景) +- `'always_fast'`:跳过 vetoes,**仍受运行时保险约束**(高级用户) + +理由:D2 是质量换速度,部分用户/场景需要明确退出权。 + +#### 前置条件 + +- `config.getFastModel()` 已配置 +- **主 chat fastModel-streaming 验证实验**(编码前 1d): + - mock 一个 `resultIsTerminal=true` 工具,在主 chat 反复触发 summary 轮 + - 观察 `tryCompress` 是否被错误触发(fast 模型 context window 小可能提前触发) + - 观察 chatRecordingService 输出是否有 model mismatch + - 观察单次 fast 调用后下一次 primary 调用是否能正常读 history +- **Fast 候选模型基线测量**(1d): + - 跑 100 条 summary 轮 prompt(输入含 `function_response`),测 P50/P95 端到端延迟与 time-to-first-token + - 测 `tryCompress` 触发率 `P_compact`,验证净 RT 收益 = `(1 - P_compact) × ΔRT − P_compact × compression_RT > 0` + - 仅当 fast P50 ≤ primary P50 × 0.5 且 P95 ≤ primary P95 × 0.6 时启用 +- Fast model 与 primary model 同家族(避免 function_response 编码差异);跨家族需 `getFastModel()` 层校验拒绝 +- **`thinkingConfig` 兼容性**: + - Fast 模型必须与 primary 在 `thinkingConfig.includeThoughts` 支持上一致;或 + - Fast 路径强制 `includeThoughts: false`(与 `sideQuery.ts:118-122` 对齐) + - 验证:history 含 thought parts 时 fast 模型能正确处理(不报错、不把 thought 当用户输入) + +#### 风险与缓解 + +| 风险 | 严重度 | 缓解 | +| ------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------ | +| Fast 模型 tool-calling 静默放错 | 高 | 对话流启发式 + 运行时 ToolCallRequest abort 保险 | +| Fast 在含 error 的输入上幻觉成"对用户可见的错误回答" | **高** | `hasUnresolvedError` 否决;监控用户追问率(注:`emitToolUseSummaries` 的同类风险只影响 60 token 标签,本风险影响最终回答,量级更高) | +| Fast 路径触发 `tryCompress` → 多一次 LLM 调用,**反向恶化 RT 和成本** | **高** | `wouldTriggerCompression` 预判 gate(见决策函数 #7);前置基线测量 P_compact 阈值 | +| Compression 自身用谁的模型 | 中 | 触发 compression 即放弃 fast 路由(gate #7 兜底);避免回答出问题 | +| 主 chat 切模型让 chat 内部状态/recording 异常 | 中 | 前置验证实验覆盖;session resume 重放测试 | +| D2 与 `emitToolUseSummaries` 同时触发 concurrent fast 调用,超 rate-limit | 中 | 二选一:D2 启用时禁用 `emitToolUseSummaries`(标题不影响功能),或共享 rate-limit token bucket | +| `thinkingConfig` 在 fast / primary 间不一致导致 history 解析异常 | 中 | 同家族 + fast 路径强制 `includeThoughts: false`(见前置条件) | +| Fallback 路径反而更贵(fast tokens 浪费 + primary 全程) | 中 | `fast_tokens_consumed` 决策日志监控;fallback 率 >20% 自动关 flag | +| Telemetry span model 失真 | 中 | `requested` / `actual` 拆分(见 Telemetry 修正) | +| 上下文格式不兼容(跨家族) | 中 | `getFastModel()` 拒绝跨家族选择 | +| 与 skill modelOverride 语义冲突 | 中 | 独立 ref + skill 优先 | +| `/model` 运行时切换主模型后 `summaryTierRef` 决策失效 | 低 | `/model` 命令处理时同步清空 `summaryTierRef` | +| fast tokens/s 反而更慢 | 低 | 实测时同时测 TTFT,不只总 RT | + +#### 收益(待实测) + +- **RT**:summary 轮节省 2-3s(实测前不写入 PR 标题) +- **成本**:fast 模型单价通常显著低于 primary,高频 summary 场景下 token 成本可能下降 30-50%;但 fallback 路径浪费会抵消部分收益,需用 `fast_tokens_consumed` 实测确认净收益 + +--- + +### 3.3 方向三:结果展示与交互解耦(Presentation Decoupling) + +#### 问题 + +用户从工具完成到可以再次输入,必须等 LLM 总结轮完成: + +``` +工具完成 → [渲染结果] → [submitQuery] → [等 LLM 流式回复 3-4s] → Idle → 可输入 + ~~~~~~~~~~~~~~~~~~~~~~~~ + 用户已看到结果但无法操作 +``` + +#### 设计 + +新增 `StreamingState.Summarizing` 状态: + +```typescript +export enum StreamingState { + Idle = 'idle', + Responding = 'responding', + WaitingForConfirmation = 'waiting_for_confirmation', + Summarizing = 'summarizing', // 新增 +} +``` + +#### 状态机变更 + +``` +工具完成且结果已展示 + → 若 batch 全员 postExecution.resultIsTerminal === true: + → 进入 Summarizing(用户可输入) + → submitQuery 异步执行 + → LLM 总结追加到 history(或被用户新消息取消) + → 否则: + → 保持 Responding(用户不可输入) +``` + +#### 用户新消息处理 + +- `Summarizing` 状态下用户提交新消息 → abort 当前总结 → 处理新消息 +- 已生成的**部分总结文本丢弃**(不入 history),避免半句 assistant 污染上下文 +- `function_response` 仍保留在 history(模型知道工具执行了) +- followup suggestion 等 Summarizing 完成或被取消后再触发 + +#### Abort 时 partial text 清理清单 + +partial text 分布在多处,需**同时**清理,缺一会导致状态不一致: + +| 位置 | 清理动作 | +| -------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| `pendingHistoryItemRef.current`(useGeminiStream React state) | 置 `null`,不调 `addItem` | +| `GeminiChat.history` 内部累积 | abort 前若已 push 部分 assistant content,需通过新的 `discardPendingAssistant()` 接口回滚 | +| `ChatRecordingService` buffered turn | 标记为 cancelled,不写入 JSONL | +| `dualOutput.emitText`(如启用) | 发送 abort sentinel,sidecar 自行丢弃 | +| `loopDetectorRef` 累积 token | 重置当前 turn 计数 | + +执行顺序:abort signal 触发 → 收齐上述五处清理 → 才允许新 user message 进入 `submitQuery`。竞态测试覆盖:abort 触发瞬间正好收到最后一个 chunk。 + +#### 适用条件 + +batch 全员 `postExecution.resultIsTerminal === true`。 + +#### 历史不变量(与 §3.1 同源) + +中途打断 Summarizing 会产生: + +``` +[user_1, function_call, function_response, user_2] + ↑ 无 assistant turn +``` + +**这与 §3.1 跳过 LLM 轮破坏的是同一个不变量**,必须使用与 D1 相同的修复策略(注入空 assistant / 接受 Qwen 容忍)。 + +- 复用 D1 的不变量单测覆盖 +- session-load 重放(含 `repairOrphanedToolUseTurnsInHistory`)必须覆盖此形态 +- Anthropic alternation:直连时与 D1 同时补兜底 + +#### 风险与缓解 + +| 风险 | 严重度 | 缓解 | +| ----------------------------------- | ------ | -------------------------------------------------------------- | +| Abort 时半句 assistant 进 history | **中** | 显式丢弃 partial text;仅保留 function_response;单测覆盖 race | +| 历史不变量破坏(无 assistant 接续) | **中** | 与 D1 同源问题,统一修复(见 §3.1 历史不变量) | +| UI 状态复杂度增加 | 中 | Summarizing = Idle + 背景任务;输入路径复用 Idle | +| 用户感知收益依赖行为模式 | 低 | 用户若 3s 内不输入,summary 已完成 → 无感知收益;但**不退化** | + +#### 收益 + +- **理论上限**:3-4s 感知 RT(用户工具完成即输入) +- **实际中位数**:取决于用户输入间隔——读结果 2-5s 后才输入的用户不会感受到差异,但**绝不会更慢** + +--- + +### 3.4 方向四:流式提前调度(Stream-Ahead Scheduling) + +#### 问题 + +`processGeminiStreamEvents` 在 stream 完全结束后才批量调度工具。`ToolCallRequest` 事件可能在 stream 中期就已 yield。 + +#### 设计 + +在 stream 事件处理中对 `ToolCallRequest` 立即开始**前置验证**(不执行): + +```typescript +case ServerGeminiEventType.ToolCallRequest: + toolCallRequests.push(event.value); + scheduler.prevalidate(event.value, signal); // 新增 + break; +``` + +`CoreToolScheduler.prevalidate(request)`: + +1. 查找工具注册 +2. 构建 invocation +3. 执行 `shouldConfirmExecute`(缓存结果) +4. `schedule()` 时直接使用缓存结果 + +#### 纯度契约与 Allowlist + +`prevalidate` 要求 `shouldConfirmExecute` 是 side-effect-free **且**结果在 prevalidate→schedule 间隙不会被外部修改使之失效。 + +**直接复用 `tools.ts:818` 的 `CONCURRENCY_SAFE_KINDS`**: + +```typescript +export const CONCURRENCY_SAFE_KINDS: ReadonlySet = new Set([ + Kind.Read, + Kind.Search, + Kind.Fetch, +]); +``` + +这是项目已有的"无副作用 + 可并发"分类,正好匹配 prevalidate 需求。 + +| 工具 Kind | 是否在 allowlist | 理由 | +| ----------------------------- | ----------------------- | ------------------------------------------------------- | +| `Read`(read_file 等) | ✅ | 纯读 | +| `Search`(grep / glob) | ✅ | 纯读 | +| `Fetch`(web_fetch 等) | ✅ | 远程读,无写副作用 | +| `Edit` | **❌**(见下文 TOCTOU) | shouldConfirmExecute 纯只读,但 diff 在调度间隙可能失效 | +| `Delete` / `Move` / `Execute` | ❌ | MUTATOR_KINDS | +| `Think` | ❌ | 含 save_memory / todo_write 等隐式写 | +| MCP 工具 | ❌ | 不可信 | + +**TOCTOU:为什么 Edit 不进 allowlist** + +理论上 Edit 的 `shouldConfirmExecute` 是纯只读(读文件、算 diff)。但 prevalidate 与 schedule 之间存在时间窗: + +``` +T=0 stream 收到 Edit(file=a.ts, ...) → prevalidate +T=10ms shouldConfirmExecute 读 a.ts,缓存 diff_v0 +T=300ms stream 结束,scheduler.schedule() +T=305ms 期间其他工具/IDE/外部进程修改 a.ts +T=310ms scheduler 用 diff_v0 展示给用户 +T=320ms 用户基于 v0 确认 +T=330ms Edit 应用旧 params 到 v1 文件 → 内容损坏 / merge 失败 +``` + +这是 TOCTOU。修复方向: + +- **A(推荐)**:Edit 不进 allowlist,prevalidate 仅覆盖 `CONCURRENCY_SAFE_KINDS` 三类。代价:收益从"50-200ms(Edit 主导)"降到"50-100ms(仅读类)" +- **B(可选加强)**:Edit 进入 allowlist 但缓存附 `(mtime, size, content_hash)`;schedule() 时校验未变才用缓存,否则重算 + +文档暂选 A。 + +#### 与现有并行调度的交互 + +`coreToolScheduler.attemptExecutionOfScheduledCalls`(L2436+)使用 `partitionToolCalls` 把工具分成"并发安全 batch"和"串行 batch",并发 batch 通过 `runConcurrently`(L2473)执行。 + +prevalidate 必须与这个分批模型对齐: + +- 缓存按 `callId` 索引(不是 `(toolName, args)`,避免并发同名调用冲突) +- prevalidate 失败的 call → 不影响其他 call,schedule 时该 call 走原始 `shouldConfirmExecute` 路径 +- stream 取消时按 `signal` 级联 abort 所有 in-flight prevalidate + +#### 风险 + +| 风险 | 严重度 | 缓解 | +| ------------------------------------------ | ------ | ---------------------------------------------------------------------- | +| 缓存 diff 与确认时实际文件不一致(TOCTOU) | 高 | 方案 A:Edit 不进 allowlist;方案 B:缓存附 `(mtime, size, hash)` 校验 | +| prevalidate 失败影响调度 | 低 | 失败/超时退回原 `shouldConfirmExecute` 路径,缓存缺失 ≡ 未启用 | +| 并发 prevalidate 共享 fd / 资源争抢 | 低 | `QWEN_CODE_MAX_TOOL_CONCURRENCY` 已限并发上限(默认 10) | + +#### 收益 + +50-100ms/轮(仅 `CONCURRENCY_SAFE_KINDS` 范围)。若选方案 B 含 Edit,理论收益 100-200ms。 + +--- + +## 4. 综合评估与路线图 + +### 4.1 综合评估 + +| 方向 | RT 收益 | 实施复杂度 | 质量风险 | 依赖 | 优先级 | +| -------------------- | ----------------------------- | ------------------------ | -------- | ------------------------------------------- | ------ | +| D1 工具后置指令 | 3-4s/终态轮 | 低(2-3d) | 低 | 无 | **P0** | +| D2 summary fast 路由 | 2-3s/summary 轮(待实测) | **中-高(9d)** | 中-高 | D2 自带启发式 + 主 chat 验证实验 + ACP 同步 | **P1** | +| D3 展示解耦 | 3-4s 感知改善(依赖用户行为) | 中(3-5d,含不变量修复) | 中 | D1 历史不变量修复 | **P1** | +| D4 流式提前调度 | 50-200ms/轮 | 高(5-7d) | 极低 | 无 | P2 | + +#### D2 工作量细分 + +| 子任务 | 估时 | +| ------------------------------------------------------------------------------------------ | ------ | +| 主 chat fastModel-streaming 验证实验(含 P_compact 测量) | 1d | +| Fast 候选模型基线测量(含 TTFT、P95、`thinkingConfig` 兼容性) | 1d | +| `selectContinuationTier` + `summaryTierRef` 接入(useGeminiStream) | 0.5d | +| 启发式实现(含 `MUTATOR_KINDS` 复用 / `wouldTriggerCompression` 估算 / 多语言 / 状态突变) | 1d | +| `GeminiChat.retryStreamWithModel` + `discardPendingAssistant` 接口实现 | 1.5d | +| ACP Session 同步改造(acp-integration/session/Session.ts) | 1d | +| Telemetry span 修正(`requested` / `actual` 拆分) | 0.5d | +| User-level setting `summaryTierStrategy` + JSON schema + `/config` 集成 | 0.5d | +| 单测(race、abort 时机、history 不变量、fallback 路径、ACP 路径) | 2d | +| **合计** | **9d** | + +> 注:早期估时 6.5d 未含 ACP 路径、`wouldTriggerCompression` gate、清理清单、settings schema 工程化等成本。 + +### 4.2 实施路线 + +#### Phase 1:D1 工具后置指令(1 周) + +- 扩展 `ToolResult.postExecution`(tools.ts L422):`skipLlmRound` + `resultIsTerminal` +- `handleCompletedTools` 实现 `skipLlmRound` 短路(useGeminiStream.ts L2038) +- 单测覆盖历史不变量 +- **Phase 1 不消费 `resultIsTerminal`**(留给 Phase 3) + +#### Phase 2:信号生态建设(2 周,与 Phase 4 并行) + +- 内置工具陆续打标 `skipLlmRound` / `resultIsTerminal`(见 §3.1 表) +- 验证打标覆盖率 ≥60%(按 turn 数加权,非按调用次数) +- 收集 production 数据,校准 §3.2 否决 gate 阈值 +- Phase 2 末期跑 §3.2 主 chat 验证实验和基线测量 + +#### Phase 3:D2 + D3(约 3 周,含 ACP 同步) + +> **修正**:早期路线图估 1 周,未含 fastModel-streaming 验证实验、`retryStreamWithModel` 实现、不变量统一修复、ACP 路径同步。 + +- 编码前:完成主 chat 验证实验 + 基线测量(含 `P_compact` 与 thinkingConfig 兼容性) +- 新增 `summaryTierRef` + `selectContinuationTier`(含 `wouldTriggerCompression` gate) +- 新增 `GeminiChat.retryStreamWithModel` + `discardPendingAssistant` +- **同步改造 ACP Session 路径**(acp-integration/session/Session.ts)使用同一决策函数 +- 新增 `StreamingState.Summarizing` + 输入路径复用 + abort 清理清单 +- 历史不变量统一修复(D1+D3 同源) +- Feature flag `experimental.summaryRoundFastModel: false`,**Release N 默认关** +- User setting `summaryTierStrategy` +- Telemetry span 修正 +- 运行时保险(ToolCallRequest abort + retryStreamWithModel) + +#### Phase 4:D4 流式提前调度(可独立插入) + +- `CoreToolScheduler.prevalidate` + allowlist +- `processGeminiStreamEvents` 增量调度 + +--- + +## 5. 度量、验收与限制 + +### 5.1 性能指标 + +| 指标 | 基线 | Phase 1 | Phase 3 | +| -------------------------- | ----- | ------- | ------------------------- | +| 端到端 RT P50(3 轮 loop) | 13.4s | <10s | <8s(待实测) | +| 端到端 RT P95 | - | <13s | <12s(fallback 路径上限) | +| 用户感知首结果时间 P50 | 13.4s | <10s | <5s(D3 启用) | +| 用户感知首结果时间 P95 | - | <13s | <8s | +| LLM 调用次数(可跳过场景) | 3 | 2 | 2(更快) | + +> 注:基线为单次采样,落地前需补 ≥3 类场景。 + +### 5.2 质量指标 + +| 指标 | 基线 | 允许退化 | +| -------------------------------------------- | ---- | ------------------------ | +| Tool-calling 准确率(fast model summary 轮) | 100% | ≥98% | +| skipLlmRound 误用率(用户追问"再详细些") | - | <1% | +| Fast model fallback_triggered 率 | - | <10%(>20% 自动关 flag) | +| Summarizing 状态下半句 assistant 入 history | 0 | 0(硬性) | + +### 5.3 成本指标 + +| 指标 | 基线 | Phase 3 目标 | +| --------------------------------- | ---- | ------------------------------------------------------------ | +| 每千会话 token 成本(summary 轮) | 100% | <70% | +| Fallback 路径浪费 tokens 占比 | 0 | <15%(fallback 率 × 单次 fast tokens / 单次 primary tokens) | + +### 5.4 决策日志 schema + +每次 `selectContinuationTier` 与 `handleCompletedTools` 的关键判定写一条结构化日志: + +``` +{ + turn_id, prompt_id, + decision: 'skip' | 'fast' | 'primary', + tier_requested: 'fast' | 'primary', // 决策(fallback 前) + tier_actual: 'fast' | 'primary', // 实际跑(fallback 后) + signal_skipLlmRound: bool, + signal_resultIsTerminal: bool, + user_strategy: 'auto' | 'always_primary' | 'always_fast', + veto_reason: 'further_action' | 'write_tool' | 'unresolved_error' | + 'deep_reasoning' | 'cross_result' | 'output_tokens' | + 'lang_unsupported' | 'compact_or_clear' | null, + tool_count, distinct_tool_count, + has_write_tool: bool, + has_error: bool, has_cancel: bool, + output_tokens_est: int, + user_prompt_classification: 'query' | 'action' | 'analysis', + fast_ttft_ms, primary_ttft_ms, // fallback 时双份 + fast_tokens_consumed: int, // fallback 浪费的 tokens(成本归因) + total_rt_ms, + fallback_triggered: bool, + fallback_reason: 'tool_call_seen' | 'timeout' | 'error' | null, +} +``` + +观察指标: + +- fast 触发率(预期 30-50%) +- fallback_triggered 率(预期 <10%;>20% 提示在下个 release 关 default flag) +- 各 veto 占比(识别过严/过松) +- fast_tokens_consumed × fallback_rate(成本反向风险) +- 用户追问"再详细些"频次(fast 质量回归信号) + +**`fast_tokens_consumed` 测量说明**: + +abort 中断的 stream **大概率收不到 `finishReason` / `usageMetadata`**——后者只在 stream 完整结束时填充。实现需估算: + +- 优先:abort 前尝试 `stream.return()` 让生成器走 finally 路径,可能拿到 partial usage +- 兜底:累计已收 chunk 的文本长度 × 4 估算 output tokens;input tokens 用 history 估算 +- 标注:日志字段附 `tokens_source: 'usage' | 'estimated'`,事后分析需区分 + +### 5.5 验证方法与发布策略 + +#### 验证 + +- 复用 `/tmp/tool-timing.log` 计时框架 +- 新增 `T_userIdle`(用户可再次输入时刻) +- 新增 `T_firstToken`(流式首 token 时刻) +- A/B 测试对比各 Phase 前后的 RT 与 cost 分布 + +#### 发布策略(适配本地 CLI) + +Qwen Code 是本地 CLI,**没有运行时下发能力**——传统"5% / 25% / 100% 灰度"不适用。采用**阶段性 release 推进**: + +| 阶段 | Release 节点 | feature flag 默认值 | 触发条件 | +| --------------------- | ---------------------- | ------------------- | ----------------------------------------------------------- | +| Phase 3a:dogfood | Release N | `false` | 内部用户用 `summaryTierStrategy=always_fast` 自启用 | +| Phase 3b:opt-in 默认 | Release N+1(≥2 周后) | `false`(不变) | dogfood 阶段决策日志达标:fallback <10%、净 RT/cost 收益 >0 | +| Phase 3c:默认开启 | Release N+2(≥4 周后) | `true` | Phase 3b 用户层面无质量回归报告 | +| 回滚 | Release N+3(如需) | `true → false` | 大规模 fallback >20% 或质量指标退化 | + +**回滚机制**: + +- 无运行时下发,**回滚 = 发新 release 关 default flag** +- 用户级 `summaryTierStrategy=always_primary` 始终提供"我要立刻退出"通道,不依赖新 release +- 决策日志的 `fallback_rate` / `cost_regression` 在每个 Release 周期评估,决定下一步 + +### 5.6 已知限制 + +1. **基线数据单薄**:单次采样不能覆盖全部任务模式,落地前需补场景 +2. **fast 模型前提**:不存在显著更快且 tool-calling 达标的同家族模型 → D2 不启用 +3. **`skipLlmRound` 是质量换速度**:跳过 LLM = 放弃模型理解和纠错,仅适用确定性高场景 +4. **D2 是质量+成本换速度**:fast 模型质量低于 primary;fallback 路径反而更贵——必须以决策日志实测净收益 +5. **`tryCompress` 触发可能反向恶化**:fast 模型 context 小,compression 自身耗 LLM 调用——`wouldTriggerCompression` gate 是必备防御 +6. **展示解耦改变交互模型**:新模式需要用户适应;用户行为决定实际感知收益 +7. **网络延迟不可控**:本方案减少调用次数,非优化单次调用 +8. **Anthropic 直连未覆盖**:当前 alternation 容忍度依赖 Qwen / OpenAI 风格 API +9. **主 chat 上 fastModel-streaming 是首次落地**:无生产先例,需独立验证实验 +10. **本地 CLI 无运行时下发**:发布策略只能阶段性 release 推进,不支持快速灰度调节 +11. **D2 仅作用于交互路径**:Subagent / Cron / Notification 不享收益,刻意如此 +12. **混合模型 history 长期影响未知**:D2 启用后 session 内 turn 在 fast/primary 间切换,长会话 resume 与上下文连贯性需观察 +13. **D4 收益缩水**:Edit 退出 allowlist 后,prevalidate 仅覆盖纯读类工具(50-100ms 收益);含 Edit 的 200ms 收益需方案 B 的 mtime/hash 校验机制 + +### 5.7 关键代码位置 + +| 文件 | 关键符号 | 位置 | +| ----------------------------------------------------- | -------------------------------------------------------- | ------------------------ | +| `packages/core/src/tools/tools.ts` | `ToolResult` interface | L422 | +| `packages/core/src/tools/tools.ts` | `Kind` enum + `MUTATOR_KINDS` + `CONCURRENCY_SAFE_KINDS` | L793, L806, L818 | +| `packages/core/src/tools/tools.ts` | `DeclarativeTool.kind: Kind`(每个 Tool 实例都带) | L165 | +| `packages/core/src/core/client.ts` | `SendMessageOptions.modelOverride` | L142 | +| `packages/core/src/core/client.ts` | `sendMessageStream` | L1216 | +| `packages/core/src/core/client.ts` | `modelOverride ?? getModel()` | L1305, L1598 | +| `packages/core/src/core/client.ts` | `turn.run(model, …)` | L1707 | +| `packages/core/src/core/geminiChat.ts` | `sendMessageStream(model, …)` | L1387 | +| `packages/core/src/core/geminiChat.ts` | `history.push(userContent)` | L1428 | +| `packages/core/src/core/geminiChat.ts` | `sendPromise` 锁 | L1392 | +| `packages/cli/src/ui/hooks/useGeminiStream.ts` | `modelOverrideRef`(skill 选模型) | L376, L2225 | +| `packages/cli/src/ui/hooks/useGeminiStream.ts` | `processGeminiStreamEvents` | L1365 | +| `packages/cli/src/ui/hooks/useGeminiStream.ts` | `sendMessageStream` 调用点 | L1841 | +| `packages/cli/src/ui/hooks/useGeminiStream.ts` | `handleCompletedTools` | L2038 | +| `packages/cli/src/ui/hooks/useGeminiStream.ts` | `submitQuery(ToolResult, …)` | L2355 | +| `packages/core/src/services/toolUseSummary.ts` | fast-model side query(非流式先例) | L108 | +| `packages/core/src/followup/speculation.ts` | fast-model streaming(forked chat 先例) | L224 | +| `packages/core/src/config/config.ts` | `fastModel` + `getFastModel` + `setFastModel` | L684, L1987, L2021 | +| `packages/core/src/core/coreToolScheduler.ts` | `attemptExecutionOfScheduledCalls` | L2436 | +| `packages/core/src/core/coreToolScheduler.ts` | `runConcurrently` + `partitionToolCalls` | L2473 | +| `packages/cli/src/acp-integration/session/Session.ts` | `sendMessageStream` 调用点(ACP / IDE 路径) | L705, L965, L1182, L1423 | +| `packages/core/src/agents/runtime/agent-core.ts` | Subagent `sendMessageStream`(不受 D2 影响) | L614 | + +--- + +## 6. Review 验证记录(2026-05-26) + +### 6.1 验证方法 + +针对设计文档中**只声明、未量化**的几条前置数据质量假设与收益估算,启动 4 个并行 Explore subagent 做只读代码调研。每个 subagent 只回答一个事实问题,不做判断,不给优化建议。调研基于当前 `main` 分支(HEAD: `026f2f768`)。 + +| 验证问题 | 关联章节 | +| ---------------------------------------------------------------------- | ---------------------------------- | +| Q3 当前所有工具的 `ToolResult.error` 字段填充率 | §3.2 `hasUnresolvedError` 前置依赖 | +| Q4 stream abort 后 `usageMetadata` 实际可得性 | §5.4 `fast_tokens_consumed` 测量 | +| Q5 "用户追问 / clarification" 埋点存在性 | §5.2 fast 质量回归监控信号 | +| Q6 `CONCURRENCY_SAFE_KINDS` 工具 `shouldConfirmExecute` 实际 IO 工作量 | §3.4 D4 收益估算 | + +### 6.2 发现 1:`hasUnresolvedError` 启发式存在 32% 工具盲区(影响 D2) + +**事实**:在 22 个有错误路径的工具中,**15 个(68%)规范填 `ToolResult.error` 字段**(shell、read-file、write-file、edit、grep、glob、ls、web-fetch、mcp-tool、cron-\* 等核心 I/O 工具齐备),**7 个(32%)仅把错误塞进 `llmContent` 字符串**:`askUserQuestion`、`monitor`、`skill`、`lsp`、`exitPlanMode`、`todoWrite` 等。 + +**不存在**统一的 `createErrorResult` helper,每个工具独立实现错误构造。 + +**对设计的影响**: + +- §3.2 的 `hasUnresolvedError` 否决项若仅检查 `ToolResult.error` 字段,**这 7 个工具的失败永远不会触发"切回 primary"**——下一轮仍会被路由到 fast model +- 其中 **`skill` 工具的失败被 fast model 错误总结**是高优风险场景(本仓库大量 skill 驱动的工作流会被影响) +- §3.2 列出的"shell 等需正确填 ToolResult.error(前置数据质量依赖)" **范围太窄**,shell 实际已规范,真正漏报的是 skill / lsp / todoWrite 等 + +**建议修正**:把 "**将 7 个仅靠 `llmContent` 传错的工具改造为规范填 `error` 字段**" 列为 D2 的硬前置依赖(§3.2 前置条件),估时 ~2d;不接受 "用 `llmContent.match(/^Error:/i)` 兜底" 的脏路径(误判风险高)。 + +### 6.3 发现 2:`fast_tokens_consumed` 指标实现成本被低估(影响 D2 / §5.3) + +**事实**: + +- `turn.ts` 的 abort 路径(L289-291)直接 `return`,**没有 finally 块,也没有 `stream.return()` 调用**——文档 §5.4 暗示的 "abort 前 `stream.return()` 让生成器走 finally" 在当前代码中不存在该入口 +- `geminiChat.ts:processStreamResponse` 的 `for await` 循环只在完整遍历时记录 turn(L1286),abort 中断意味着最后的 usage-only chunk(通常携带完整 metadata)**被直接丢弃** +- 主聊天路径**无任何 chunk-level token 累计兜底**;仅 subagent 层(`agent.ts:731-744`)有累计,无法复用 +- 结论:abort 时 `usageMetadata` **零获取**,只能靠 `chars/4` 估算(±20% 误差) + +**对设计的影响**: + +- §5.4 末尾的"优先 / 兜底 / 标注"三层方案中,**"优先" 路径在当前代码不可达**——需先改 `sendMessageStream` 生成器结构加 finally,工作量约 1d,设计文档没体现这笔成本 +- §5.3 把 "每千会话 token 成本 <70%" 列为 Phase 3 目标,但若指标本身 ±20% 误差,**"70%" 与 "82%" 落在测量噪声内** + +**建议修正**: + +- §5.3 改写为**趋势指标**,不作为 release gate;改用 "决策日志的 `fallback_triggered` 率 + `fast_tokens_consumed` 同向趋势" 双指标联合判断 +- §5.4 增补:`fast_tokens_consumed` 实现需先改造 turn.ts abort 路径加 finally + `stream.return()`,作为 §3.2 工作量补充(+1d) + +### 6.4 发现 3:`user_prompt_classification` 与"用户追问"埋点需新建(影响 D2 / §5.2) + +**事实**: + +- `packages/core/src/followup/` 已存在 `speculation.ts` / `suggestionGenerator.ts` / `followupState.ts`,但其 telemetry(`PromptSuggestionEvent`)记录的是 **"系统建议被采纳/忽略"**,不是"用户主动追问" +- `ChatRecordingService` 存储用户消息但**不打分类标签** +- 全仓库 grep 无 `user_prompt_classification`、无中英文追问模式匹配、无 `clarif*` / `intentDetect` 类机制 + +**对设计的影响**: + +- §5.4 决策日志 schema 里 `user_prompt_classification: 'query' | 'action' | 'analysis'` 字段**没有数据源**——既不能从现有 PromptSuggestionEvent 推导,也不能从 ChatRecord 读出 +- §5.2 "用户追问'再详细些'频次" 监控信号同上,**最接近的现有锚点 `followupState.onOutcome` 不可复用** + +**建议修正**: + +- §3.2 前置条件中追加"用户输入分类器最小实现"(中英文模式匹配,~3d),否则 §5.4 决策日志的 `user_prompt_classification` 与 `requestImpliesFurtherAction` 都缺数据 +- 或者**接受**在 Phase 3a dogfood 阶段没有这两个信号,仅靠 `fallback_triggered` 率监控质量回归——成本低但风险高 + +### 6.5 发现 4:D4 设计内在矛盾——allowlist 与收益归因不对齐(影响 D4 / §3.4) + +**事实**: + +- `Kind.Read`(read_file)、`Kind.Search`(glob / grep)、`Kind.Fetch`(web_fetch)三类工具的 `shouldConfirmExecute` / `getConfirmationDetails`,**绝大多数继承 `BaseToolInvocation` 默认实现,做零 IO**(read_file / glob / grep 完全没 override,web_fetch 只做 5-10 行字符串解析 URL hostname) +- 真正有 IO 的是 `Edit` / `WriteFile`(`calculateEdit` + `readTextFile` + `Diff.createPatch`,典型 ~20ms),但 §3.4 方案 A 把它们排除出 allowlist 以规避 TOCTOU +- **结果**:留在 allowlist 里的三类工具,prevalidate 与不 prevalidate 工作量基本相同——allowlist 实际拦截的是"唯一有 IO 可省的 Edit",留下"本来就零成本的工具" + +**对设计的影响**: + +- §3.4 的"前置 IO 验证"叙事**不成立**:50-100ms 收益的真正来源是 **"stream 完全结束 → 才批量 schedule" 这段调度等待被消除**,与工具端 IO 几乎无关 +- 收益归因错误会带来两个问题: + 1. **allowlist 可以更宽**——凡是 idempotent prevalidate 的工具都行,不必绑定 `CONCURRENCY_SAFE_KINDS` + 2. **5-7d 投入难以自洽**——如果真实收益只有调度模型改变的 ~50ms,Edit 又不在 allowlist 里,这笔投入的 ROI 比设计文档暗示的低 + +**建议修正**:§3.4 重写收益归因—— + +- 拆分为两部分:(a) 调度模型改变省下的 stream 等待 ~50ms,(b) 工具端 IO 前置可省的工作量 ~0ms(allowlist 内)/ ~20ms(若 Edit 入 allowlist) +- 在 §4.1 综合评估表里把 D4 RT 收益从 "50-200ms" 改为 "30-80ms(方案 A,主要来自调度模型)/ 100-200ms(方案 B,含 Edit)" +- 在 §4.2 路线图中把 D4 进一步降级——纯调度模型改造可独立做,不必强行绑定 prevalidate 概念 + +### 6.6 对路线图的合并影响 + +| 章节 | 原估时 | 验证后估时 | 增量来源 | +| ----------------------------- | ------ | ------------ | ------------------------------------------------------------------------------------------------ | +| D2 §3.2 工作量(§4.1 细分表) | 9d | **14-16d** | +2d(发现 1 前置工具改造)+1d(发现 2 turn.ts finally 改造)+3d(发现 3 输入分类器,如取硬路径) | +| D4 §3.4 综合评估 | 5-7d | 5-7d(不变) | 工作量不变,但 **RT 收益归因从"工具端 IO"改为"调度模型"**,投入 ROI 下调 | +| Phase 3 总时长(§4.2) | ~3 周 | **~4-5 周** | D2 工作量上调 + 前置工具改造 PR 单独走 review 周期 | + +**对原路线图的修正建议**: + +1. **保持 D1(P0)和 D3 紧随其后**——本次验证未触及它们的核心假设,ROI 判断不变 +2. **D2 启动条件加严**——把发现 1/2/3 的前置工作(共 ~6d)作为 "D2 启动 gate",未完成不进入 §3.2 前置实验 +3. **D4 重新评估优先级**——既然真实收益是调度模型改变而非工具端 IO,要么 (a) 接受 30-80ms 把 D4 降到 P3 后置,要么 (b) 考虑方案 B(Edit + mtime/hash)拿回 100-200ms 但额外 5-7d +4. **不修改 §1.2 单次采样基线**——但 §5.1 P95 一栏在 D1 落地、补完 ≥3 类场景基线之前不写具体数字 + +### 6.7 验证未覆盖的追问点 + +以下追问点属于主观判断或作者意图问题,本次验证未通过 subagent 处理,留作后续 design review 讨论: + +- D2 实施次序应否后置于 D3(主观次序) +- D1/D3 是否应合并到 Phase 1 一起做(实施策略) +- §3.2 `needsCrossResultReasoning` 阈值 ≥3 是否反向拟合 §1.2 基线场景(作者意图) +- §5.7 关键代码位置表的行号锚点是否应改为符号锚点(文档稳定性) + +--- + +## 7. 浮油评估与下一步(2026-05-26 二次 review) + +### 7.1 触发本次重排的事实 + +§6 验证之后,又发现两个**改变 ROI 判断的事实**: + +1. **DashScope `cache_control` 已实装**(`packages/core/src/core/openaiContentGenerator/provider/dashscope.ts:172-181`) + - streaming 请求标记 `system + 最后一条 message + 最后一个 tool definition` + - 命中数据 `cached_tokens` 已采集到 `usageMetadata.cachedContentTokenCount`(`converter.ts:1124-1149`) + - 这是 prefix cache 机制:Round N+1 自动命中 Round N 写入的前缀 + - **summary 轮恰好是命中前缀最长的一轮** + +2. **system prompt 已经稳态**(`prompts.ts` 审计结果) + - 没有 cwd / timestamp / git status / 文件列表 / LSP 状态等"每 turn 都变"的硬伤 + - `process.cwd()` 仅用作 `isGitRepository()` 开关,不写入 prompt 内容 + - 唯一动态点:`save_memory` 工具触发 / `/model` 切换 / MCP 动态加载(均事件性,低频) + +### 7.2 这两条事实改变了 D2 的 ROI 判断 + +§3.2 文档假设 "fast model 比 primary 快 ~2s",对照基线是 **primary uncached vs fast uncached**。 + +但现实运行中 primary 是 **cached**(summary 轮恰好命中最强),所以正确对照是: + +> primary cached vs fast uncached + +| 路由 | 估算延迟 | 备注 | +| ----------------------------- | --------- | ------------------------ | +| primary 命中 80% 前缀 cache | ~1.8-2.2s | summary 轮的当前实际表现 | +| fast 无 cache(跨模型不共享) | ~1.5-2s | D2 切换后的实际表现 | + +**净差距:几百毫秒,甚至可能 fast 反而慢**。叠加 14-16d 工程成本 + 质量风险 + fallback 浪费,**D2 净收益接近 0 或负**。 + +§3.2 前置条件**必须新增**:基线测量必须对比 primary **cached** vs fast **uncached**,且 `T_primary_cached < T_fast_uncached × 1.5` 时 D2 不应启用。 + +### 7.3 候选清单(按浮油性重排) + +**真·浮油(立刻动手,< 1d 投入,极低风险,确定收益)**: + +| 项 | 投入 | 收益 | 操作位置 | +| ----------------------------- | ----- | --------------------------------- | --------------------------------------------------------------------------- | +| 简洁回复指令 | 30min | ~2s/summary 轮(输出 token 减半) | `prompts.ts` Final Reminder 段加一句 | +| 暴露 cache hit rate telemetry | 0.5d | 0s 直接,是后续决策 **enabler** | `cachedContentTokenCount` 已采集,缺暴露;并应识别 `save_memory` 后单独打标 | + +**近浮油(等数据决定,0.5-1d 投入)**: + +| 项 | 投入 | 收益 | 决策前置 | +| ------------------------------- | --------------------- | --------------------------------------- | --------------------------------------------------------------------- | +| summary 轮 `tool_choice='none'` | 0.5-1d | 0.3-1s(sampling 跳过 tool_call token) | 需"是 summary 轮"判定逻辑,错判风险低 | +| summary 轮关 thinking | 1d | 0.5-2s | 仅对启用 thinking 的模型有意义(qwen3.5-plus、glm-4.7、kimi-k2.5 等) | +| UI 渲染层 chunk batching | 0.5d 调研 + 0.5d 实施 | 待验证 | 假设:长 summary 的 `useGeminiStream` token 渲染累计开销不小 | + +**待调研(可能是大鱼)**: + +| 项 | 调研投入 | 潜在收益 | 关键未知 | +| ------------------------------------ | ------------------------ | ------------------- | ------------------------------------------------------------------------------------------ | +| ~~DashScope `scope: 'global'` 支持~~ | ~~0.5d 文档 + 0.5d A/B~~ | ~~跨 session 命中~~ | **已调研,结论 (c) 不可行**(见 §7.4 发现 B 调研结果)。此行保留作为决策记录,不要重启调研 | + +**中等改造(不算浮油,单独评估)**: + +| 项 | 投入 | 风险 | 收益 | +| --------------------------------- | ---------------- | ---- | ----------- | +| D1 `skipLlmRound`(终态查询场景) | 2-3d | 中 | 3-4s/终态轮 | +| summary 轮工具结果裁剪(D5 子集) | 2d | 中 | 1-2s | +| D3 `Summarizing` 状态 | 3-5d | 中 | 感知改善 3s | +| system prompt 减肥 | 2-3d 含 A/B 测试 | 中 | 0.5-1s | + +**已废弃方向(不要再做)**: + +| 项 | 废弃原因 | +| ------------------------------------------ | ------------------------------------------------------ | +| D2 fast model 路由 | 被 DashScope cache 抵消,净收益接近 0 或负 | +| D4 prevalidate | 收益归因错(真实仅 ~50ms 来自调度模型),5-7d 投入不值 | +| system prompt 稳定化 | 已稳态,无事可做 | +| 流式提前 terminal(提前 abort 收尾客套话) | 高误判风险,用户感知答案被切断 | + +### 7.4 三个值得展开的新发现 + +#### 发现 A:`tool_choice='none'` 的真实机制 + +OpenAI / DashScope API 里 `tool_choice='none'` 不仅是"禁止调工具"——模型 sampling 阶段会**完全跳过 `` 特殊 token 的概率分配**,decoder 直接走自然语言生成路径。收益不在"省一两次 retry",而在 sampling 本身更快。 + +#### 发现 B:`scope: 'global'` 在仓库已有 Anthropic 先例 + +`packages/core/src/core/anthropicContentGenerator/converter.test.ts:85, 1543` 已有 `cache_control: { type: 'ephemeral', scope: 'global' }` 用法。但 `provider/dashscope.ts:288` 标 cache_control 时**没传 scope**: + +```typescript +cache_control: { type: 'ephemeral' }, // 没有 scope +``` + +若 DashScope 服务端识别 `scope: 'global'`: + +- system + tools 升级为 global cache(TTL 远大于 ephemeral 的 5min) +- **跨 session 命中**,启动延迟也降 +- 单这一条收益可能超过原 D2 全部假设收益 + +##### 调研结果(2026-05-26,结论:(c) 不可行,关闭此线) + +通过查阿里云百炼官方文档 `help.aliyun.com/zh/model-studio/context-cache` 得到的事实清单: + +| 问题 | 结论 | 证据 | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `scope` 字段支持 | **不支持**。仅识别 `type: 'ephemeral'`,任何 `scope`/`persistent`/`global` 会被 silently dropped | 官方文档原文:"仅支持将 `type` 设置为 `ephemeral`" | +| ephemeral 实际 TTL | **5 分钟滑动窗口**(命中后重置) | 百炼文档明确说明 | +| 长 TTL / 全局机制 | **无任何公有云 API 端机制**。无 `persistent` type 值、无独立预上传 API、无 `prompt_cache_key`;唯一"全局持久"产品是 PAI 全局上下文缓存(自部署 + vLLM + 灵骏 + 共享 Redis),与 DashScope API 无关 | PAI 文档 | +| 跨 session 共享 | 同账号 + 同模型 + 内容匹配 → 已经命中(这就是 `ephemeral` 已经在做的);不同账号绝对不共享 | 百炼文档 | +| 定价 | cache write 125%、显式 cache read 10%、**隐式 cache read 20%**(无 `cache_control` 标记也能拿到隐式 20% 折扣) | 百炼定价文档 | +| 最小可缓存 prompt | **1024 tokens** | 百炼文档 | +| 模型支持(显式 cache) | qwen3.7-max / qwen3.6-plus / qwen3.5-plus / qwen3-coder-plus / qwen3-vl-plus / deepseek-v3.2 / kimi-k2.5 / glm-5.1 均显式列出。**qwen3.6-plus 与 qwen3.7-max 同样享受 90% 显式 cache 折扣** | 百炼模型列表(2026-05-26 重核) | + +**几条副发现的连带意义**: + +1. **TTL 滑动窗口** 对 agent loop 是好消息——loop 内连续调用间隔通常 < 30s,**cache 永远新鲜,不会 5min 失效** +2. **隐式 cache 20% 折扣** 是免费红利——即使没标 `cache_control` 也能拿;但精细控制需要显式 +3. ~~`qwen3.6-plus` 未在显式列表~~ —— **更正(2026-05-26)**:经重核,qwen3.6-plus **确实在显式 cache 列表里**,享受 90% 折扣。前一轮报告此处错误,已于本节首张表更正 +4. **`dashscope.ts:288` 当前做法已经是 DashScope 公有云 API 的能力上限**——没有继续榨的空间 + +**对 §7.2 D2 判断的连带加强**: + +TTL 滑动窗口意味着 agent loop 内 summary 轮**几乎 100% 命中** primary 的 cache(前几轮刚刚命中过、5min 内)。D2 切 fast model 不仅会打碎累计的 cache 写入链,**还会让 summary 轮从"近 100% 命中"退化为"完全 miss"**——净收益判断比 §7.2 原假设更明确为负。 + +#### 发现 C:UI 渲染层是被忽视的盲区 + +§1.2 基线把"框架开销"标为 0.3s(3%),但这是粗估。Ink 7 + React 19.2 在每个 chunk 触发 setState → re-render,长 summary 累计可能 200-500ms。需要查 `useGeminiStream` 怎么处理 token 流,有没有 `requestAnimationFrame` / `useDeferredValue` 合并 chunk。 + +### 7.5 待数据 checkpoint —— 数据到了该看哪个决策 + +本节是**这份文档的活动入口**:后续有任何度量数据,对照下表决定该回看哪个决策。 + +#### Checkpoint 1:cache hit rate 数据出来后 + +**触发条件**:浮油"暴露 cache hit rate telemetry"上线 ≥3 天,决策日志含 `cached_tokens` / `prompt_tokens` 分布。 + +**该看的数据**: + +- 整体命中率(cached / prompt)的 P50、P90 分布 +- 按轮次划分:Round 1 / Round 2 / Round 3 (summary) 各自命中率 +- `save_memory` 触发后下一轮命中率(应该接近 0) +- `/model` 切换后下一轮命中率(应该接近 0) + +**决策路径**: + +| 整体命中率 | 含义 | 行动 | +| ---------- | -------------------- | --------------------------------------------------------------------------- | +| > 70% | 现状已经接近理论上限 | 只做 #1 简洁指令 + 发现 B 调研;其余浮油按需 | +| 40-70% | 还有空间但来源不明 | 分析按轮次命中率,找出哪一段在 miss | +| < 40% | 有动态点在打 cache | 重新审计 system prompt / userMemory 触发频率;可能 `save_memory` 比预期频繁 | + +#### Checkpoint 2:DashScope `scope: 'global'` 文档调研结果 ✅ 已完成(2026-05-26) + +**结果**:**完全不识别**。详见 §7.4 发现 B 的"调研结果"段。 + +**已执行行动**:接受现状,跳过此项。`dashscope.ts:288` 维持现有 `ephemeral` 标记,无需改造。 + +**后续不要重新启动此调研**——除非 DashScope 官方公告新增持久化机制。 + +#### Checkpoint 3:UI 渲染层调研结果 + +**触发条件**:发现 C 调研完成(看 `useGeminiStream` token 流处理 + Ink/React DevTools 实测)。 + +**决策路径**: + +| 结果 | 行动 | +| ---------------------------------- | ------------------------------------------------ | +| 长 summary stream 渲染累计 > 200ms | 改用 batching(`useDeferredValue` 或自定义节流) | +| 渲染开销 < 100ms | 关闭此线索 | + +#### Checkpoint 4:完成"真·浮油"后的二次基线测量 + +**触发条件**:#1 简洁指令 + Checkpoint 1/2/3 决策完成 ≥1 周。 + +**该看的数据**: + +- 端到端 RT P50 与 §1.2 单次采样基线(13.4s)对比 +- summary 轮单独的 P50 / P95 +- 用户追问率(如果浮油 A 顺带做了用户输入分类) + +**决策路径**: + +| 累计节省 | 行动 | +| ---------------------------- | ----------------------------------------------------------------------------- | +| > 4s(达到 9.6s 端到端 P50) | 评估 D1 `skipLlmRound`(再省 3-4s/终态轮) | +| 2-4s | 接受现状,评估 D3 感知改善是否值得做 | +| < 2s | 重新审视:是否浮油本身被高估,还是有未识别的瓶颈(网络 RTT、provider 端延迟) | + +### 7.6 与 §3 各方向的最终判定 + +基于 §6 验证 + 本节 ROI 重排: + +| 方向 | §3 原优先级 | 本节判定 | 理由 | +| -------------------- | ----------- | ------------------------------------ | -------------------------------------------------- | +| D1 工具后置指令 | P0 | **P0 保留**,但等浮油完成后再评估 | ROI 仍然好,但不再"立刻就做"——先把更便宜的浮油拿掉 | +| D2 summary fast 路由 | P1 | **Defer / Won't Fix** | 被 DashScope cache 抵消,14-16d 投入换接近 0 收益 | +| D3 展示解耦 | P1 | **保留为可选**,看 Checkpoint 4 数据 | 感知改善确定,但绝对 RT 不变,依赖用户行为 | +| D4 流式提前调度 | P2 | **Defer** | 收益归因错,真实 ~50ms 不值 5-7d | + +### 7.7 推荐执行顺序 + +**Day 1**(可单人单日完成): + +- ✅ `prompts.ts` 加简洁回复指令(30min) +- ✅ `cachedContentTokenCount` 暴露到 telemetry + `save_memory` / `/model` 切换打标(0.5d) +- ✅ 启动发现 B 调研:DashScope `scope: 'global'` 文档查询 + 现有 Anthropic 用法对照(0.5d) + +**Day 2-3**: + +- 收第一批 cache hit rate 数据 +- 启动发现 C 调研:`useGeminiStream` 的 React 渲染路径 +- 根据 Checkpoint 2 决定要不要做 `scope: 'global'` 改造 + +**Week 1 末**: + +- Checkpoint 1 数据决策(看分布) +- 决定要不要做 `tool_choice='none'` / 关 thinking(根据 hit rate 数据) + +**Week 2-3**: + +- Checkpoint 4 二次基线测量 +- 决定是否启动 D1(最大的非浮油项,3-4s/终态轮) + +**始终不做**:D2 / D4 / system prompt 稳定化。 + +### 7.8 `prompts.ts` 动态内容审计(2026-05-27) + +§7.1 给出 "system prompt 已稳态" 的结论时只做了粗略 grep。本节是对 `packages/core/src/core/prompts.ts`(1169 行)的系统性审计,列清单作为后续 cache 命中率分析与浮油决策的依据。 + +**审计方法**:枚举所有 `${...}` 插值表达式、IIFE、`process.*` / `new Date` / `Date.now` / `Math.random` / `fs.*` 调用,对每一处判断"在同一 session 内是否会变化"。 + +#### 完全没有(常被怀疑的硬伤) + +| 候选 | 代码事实 | +| ---------------------------------- | ----------------------------------------------------------------------------------- | +| `Date.now()` / `new Date()` | 全文 **零次出现**(`rg` 全无匹配) | +| `Math.random()` | **零次出现** | +| `process.cwd()` 值写入 prompt | 仅 L366 `if (isGitRepository(process.cwd())) { ... }`,**值不写入字符串**,只作开关 | +| git status / git branch 子进程调用 | **零次**,git 段是静态指导文本 | +| 当前文件列表 / 项目结构注入 | **零次** | +| LSP 状态 / 错误数 | **零次** | +| 用户输入历史 | **零次**(history 走 messages,不在 system) | + +#### 启动时一次,session 内不变 + +| 位置 | 内容 | 何时可能变 | +| -------- | ------------------------------------------------------------------------------------------------ | ------------------------- | +| L190 | `process.env['QWEN_SYSTEM_MD']` 决定 basePrompt 来源(默认 vs 用户 system.md) | 进程内不变 | +| L342-343 | `process.env['SANDBOX']` 决定 sandbox 段选哪一版(Seatbelt / Sandbox / Outside) | 进程内不变 | +| L366 | `isGitRepository(process.cwd())` 决定 git 段是否插入 | cwd 同 session 内通常不变 | +| L871 | `process.env['QWEN_CODE_TOOL_CALL_STYLE']` 决定 tool call 风格(qwen-coder / qwen-vl / general) | 进程内不变 | + +#### 事件触发(低频) + +| 参数 | 触发条件 | 频率估计 | +| ------------------------------------------------- | ------------------------------------------------- | ------------------ | +| `userMemory`(`getCoreSystemPrompt` 第 1 参) | `save_memory` 工具 / `/memory refresh` / 扩展加载 | 0-3 次/session | +| `model` 名(影响 `getToolCallExamples` 选哪一支) | `/model` 切换 | 罕见 | +| `appendInstruction` | 配置项,session 内基本不变 | 几乎从不 | +| `deferredTools`(`buildDeferredToolsSection`) | MCP 工具动态加载 | session 启动期居多 | + +#### 一个隐蔽的小坑 + +L207-209:若设置了 `QWEN_SYSTEM_MD` env,**每次** `getCoreSystemPrompt` 都会 `fs.readFileSync(systemMdPath)`: + +```typescript +const basePrompt = systemMdEnabled + ? fs.readFileSync(systemMdPath, 'utf8') + : `...`; +``` + +- 文件不变时内容稳定 → cache 命中不受影响 +- 但每轮 LLM 调用都有一次同步 IO(默认 `.qwen/system.md`,网络挂载文件会更慢) +- 不影响本节"cache 友好性"结论,仅作为已知性能小坑记录 + +#### 连带结论 + +1. **system prompt 在稳态 session 内每次产出 byte-for-byte 一致** → DashScope ephemeral cache key(基于内容 hash)整段稳定 → **system 段 cache 命中率几乎 100%** +2. 唯一打 cache 的事件是 `save_memory`——核心功能,不能为 cache 让路 +3. **浮油 #1(简洁回复指令)的代价分析**:把指令加到 Final Reminder 段(L389-390)→ system prompt 内容改变一次 → **首次请求 cache miss(一次性预热成本),之后所有请求继续命中** +4. **§7 的 "system prompt 稳定化" 已废弃判断得到正式证据支持**——不仅没必要做,连"理论上做了能进一步降低 cache miss 率"都不成立,因为本来就 ≈ 0 +5. 本审计可作为后续相关讨论的引用基线,避免重复 grep;若 prompts.ts 有大改动,本节需要同步更新 diff --git a/docs/design/structured-output/structured-output.md b/docs/design/structured-output/structured-output.md new file mode 100644 index 00000000000..8938d9506c2 --- /dev/null +++ b/docs/design/structured-output/structured-output.md @@ -0,0 +1,426 @@ +# Structured Output (`--json-schema`) — Design + +This document captures the implementation decisions behind the +`--json-schema` headless feature. User-facing usage lives in +[`docs/users/features/structured-output.md`](../../users/features/structured-output.md). + +## Goal + +In headless runs (`qwen -p`, piped stdin, or positional prompt), let +the caller constrain the model's final answer to a user-supplied JSON +Schema and surface the validated payload as machine-readable output +that scripts and downstream tooling can consume directly. The model's +incidental prose during planning is allowed, but the run must +terminate with a payload that conforms to the schema, not with +free-form text. + +## Approach: synthetic tool whose parameter schema IS the user schema + +When `--json-schema` is set, `Config.createToolRegistry` registers a +synthetic `structured_output` tool +([`syntheticOutput.ts`](../../../packages/core/src/tools/syntheticOutput.ts)). +Its `parametersJsonSchema` is exactly the schema the user passed; its +`execute()` returns a stop-message `llmContent`. The tool-call +infrastructure already validates args against `parametersJsonSchema` +client-side (via Ajv in `BaseDeclarativeTool.build()`), so "the model +returned an answer conforming to the schema" reduces to "the model +successfully called `structured_output`." + +Three properties fall out of this for free: + +1. **No bespoke validator path.** Ajv-backed `validateToolParams` + already runs inside `BaseDeclarativeTool.build()` and rejects + non-conforming args before `execute()` ever fires. +2. **Standard retry behavior.** A validation failure surfaces to the + model as a tool-call error the same way any other tool's args error + does. The model sees the Ajv message and can correct in the next + turn. +3. **Provider-agnostic.** Gemini, OpenAI, and Anthropic all serialize + tool param schemas the same way (via the `DeclarativeTool` + abstraction); the synthetic tool plugs into all three. + +The tool is registered with `alwaysLoad: true` so the ToolSearch +on-demand-loading infrastructure (introduced in #3589 — keeps the +exposed tool surface small by deferring rarely-used tools behind a +search call, only mounting their full schemas when the model asks) +never hides it from the model. Without that flag, the model wouldn't +know the terminal contract exists. + +## Parse-time validation pipeline + +`resolveJsonSchemaArg(raw)` in +[`packages/cli/src/config/config.ts`](../../../packages/cli/src/config/config.ts) +runs four checks before the schema reaches `Config.createToolRegistry`: + +1. **Source resolution.** Accept either an inline JSON literal or + `@path/to/file`. The `@path` form `stat`s the resolved path first, + refuses non-regular files (FIFOs, character devices, directories), + caps size at 4 MiB, and on JSON parse failure emits a generic error + (no file-content prefix in stderr). +2. **JSON shape.** Parsed result must be a non-array object — + primitives, booleans, and arrays are rejected with a clear + message. +3. **Root accepts objects** — + [`schemaRootAcceptsObject`](../../../packages/cli/src/config/config.ts). + Function-calling APIs always pass objects as tool args; a root + schema like `{type: "array"}` would register an unusable tool. + The walk handles `type`, `const`, `enum`, `anyOf`, `oneOf`, + `allOf`, `not`, `if` / `then` / `else`, and root `$ref`. +4. **Strict Ajv compile** — + [`SchemaValidator.compileStrict`](../../../packages/core/src/utils/schemaValidator.ts). + A dedicated Ajv instance with `strictSchema: true` surfaces + typos like `propertees` that the lenient runtime validator would + silently swallow. + +### `schemaRootAcceptsObject` boundaries + +The walk is intentionally best-effort. It catches the unambiguous +"this can never accept an object" cases, and defers anything that +needs whole-schema satisfiability analysis to Ajv at runtime. + +**Decided at parse time:** + +| Pattern | Outcome | +| ------------------------------------------------------ | ----------------------------------------------------------------- | +| `type` present, doesn't include `"object"` | reject | +| `type: ["object", "null"]` etc. | accept | +| `const`: non-object value | reject | +| `enum`: no object members (incl. empty) | reject | +| `anyOf`/`oneOf`: empty array | reject | +| `anyOf`/`oneOf`: no branch admits object | reject | +| `allOf`: any branch is `false` or rejects object | reject | +| Root `$ref` (with or without sibling `type`) | reject | +| `not`: bare `{type: "object"}` (no narrowing keywords) | reject | +| `not`: `{type: "object", required: […], …}` etc. | accept (narrowing keywords leave some objects satisfiable; defer) | +| `if: true` + `then` rejects object | reject | +| `if: false` + `else` rejects object | reject | + +**Deferred to Ajv at runtime:** + +- `$ref` inside `anyOf` / `oneOf` / `allOf` branches (opaque — local + `$ref` resolution would need cycle detection, JSON Pointer escapes, + and `$defs` vs `definitions` handling; the cost outweighs the + benefit for a parse-time best-effort check). +- `if` whose value is an object schema (decidable only against a + candidate value). +- Negated `anyOf` / `oneOf` / `const` patterns more complex than + `not.type`. +- Arbitrary `pattern` ReDoS exposure (user-supplied; the threat model + is narrow because the flag is a CLI argument, not a network input). + +The `maxSessionTurns` exit path appends a `--json-schema`-specific +hint pointing users at the common stuck-run symptom (model never +called `structured_output`) and its two likely causes (tool denied +via permissions / schema unsatisfiable) so the runtime fallthrough +has user-visible diagnostics. + +## Runtime: turn dispatch + +[`packages/cli/src/nonInteractiveCli.ts`](../../../packages/cli/src/nonInteractiveCli.ts) +handles the runtime dispatch. The structured-output specifics: + +### Pre-scan + sibling suppression + +When the model emits `structured_output` alongside other tools in the +same assistant turn, the synthetic call is the terminal contract. The +pre-scan in `processToolCallBatch` filters `requestsToExecute` to +**only** `structured_output` calls, so side-effecting siblings +(`write_file`, `run_shell_command`, `edit`, …) never run. + +Example batches (when `--json-schema` is active): + +| Model emits | Behavior | +| -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[write_file(…), structured_output(…)]` | `write_file` is skipped. `structured_output` validates, run ends. | +| `[structured_output(bad-args), structured_output(good)]` | First fails Ajv validation; second succeeds. Run ends with the second call's args. | +| `[structured_output(bad-args), write_file(…)]` | `structured_output(bad)` fails. `write_file` is also skipped (it was suppressed up front). The model sees both: Ajv's error message for the structured call, and a synthesised `"Skipped: …"` tool_result for the side-effect call. Next turn, the model may re-issue both or correct the structured call alone. | +| `[other_tool_a, other_tool_b]` (no `structured_output`) | Pre-scan is inert. Both tools run normally; the run does NOT terminate. | + +The synthesised "Skipped:" body has two variants: + +- **Success path** (a structured call captured the contract this turn): + `"Skipped: this turn's structured_output contract took precedence as +the terminal output."` — short, because the session terminates + immediately and no consumer (model or SDK) acts on it. +- **Retry path** (no structured call captured, the model gets another + turn): adds `"Re-issue this call in a separate turn if needed."` — + this is the only model-actionable case. + +### Main-turn / drain-turn parity + +`processToolCallBatch(batchRequests, setModelOverride)` is defined +inside `runNonInteractive` and called from both: + +- The main-turn loop (top of the function). +- `drainOneItem` (cron-prompt / background-task notification reply + loop). + +The drain turn matters because `structured_output` is registered for +the whole session, so a cron job or a notification reply MIGHT also +fire the tool. The helper handles both call sites identically at +invocation time; the only call-site-specific binding is which +`modelOverride` variable to write to — passed in as a setter. + +The **post-helper termination flow** differs between the two sites: +the main-turn path directly calls `return emitStructuredSuccess()`, +while the drain-turn path requires a two-hop termination +(`processToolCallBatch` captures the result into the closure-scoped +`structuredSubmission`; `drainLocalQueue` checks it to stop the drain +loop, then the holdback loop checks it to break out and call +`emitStructuredSuccess`). Both converge on the same terminal block, +but the extra indirection in the drain path is load-bearing — +without it the drain loop would continue processing queued items +after the structured result was captured. + +### Structured success terminal block + +`emitStructuredSuccess()` (also defined inside `runNonInteractive`) is +the shared "we got a valid call, shut down" path: + +1. `registry.abortAll()` aborts in-flight background agents — the + structured-output contract is single-shot and shouldn't race + `task_notification`s into the terminal emit. +2. Bounded holdback (`STRUCTURED_SHUTDOWN_HOLDBACK_MS = 500` ms) so + the natural cancel handlers of just-aborted agents have a chance + to emit their terminal `task_notification` and land it in + `localQueue`. The loop guard is + `Date.now() < deadline && registry.hasUnfinalizedTasks()`, so the + wait exits immediately when nothing is in flight (typical path) + and never blocks longer than the cap. The 500 ms ceiling is + best-effort — orphaned `task_started` events remain possible under + load if a particular agent's abort handler exceeds the budget. + The loop does **not** poll the abort signal: a SIGINT received + during holdback or during the emit path that follows will not + short-circuit the result that was already captured. Without the + holdback, stream-json consumers would routinely see `task_started` + events without matching `task_notification`. +3. `flushQueuedNotificationsToSdk(localQueue)` drains everything still + queued. +4. `finalizeOneShotMonitors()` (idempotent — safe to call twice; the + drain-turn path already invoked it). +5. `adapter.emitResult({ structuredResult: …, isError: false, … })`. + +### Failure paths + +| Cause | Exit code | Surface | +| ----------------------------------------------------------------- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Model emits plain text only | 1 | Error with turn count + truncated `Output preview`. | +| Model never calls `structured_output` for `maxSessionTurns` turns | 53 | `Reached max session turns` + `--json-schema` hint pointing at the common stuck-run symptom and its two likely causes. | +| Validation fails repeatedly | (eventually 53 via max-turns) | Each failure surfaces to the model on the next turn with the Ajv message. | +| Abort / SIGINT | 130 | Cancellation path. A structured result is normally not emitted, but `emitStructuredSuccess()`'s holdback loop does not poll the abort signal — a SIGINT that arrives after capture but before/during the stdout emit may still flush the result. Exit code is the reliable signal. | + +## Output envelope + +The adapter pipeline in +[`BaseJsonOutputAdapter.buildResultMessage`](../../../packages/cli/src/nonInteractive/io/BaseJsonOutputAdapter.ts) +treats the presence of `structuredResult` (tracked via `'structuredResult' in options`, +not `!== undefined`, so the contract is preserved even when the model +called `structured_output` with no args under an empty schema): + +- `result` is forced to `JSON.stringify(payload)` — overriding any + free-text summary the adapter accumulated. +- A top-level `structured_result` field carries the raw object for + consumers that don't want to re-parse the stringified form. +- `undefined` payloads normalize to `null` (rendered as the literal + JSON `null` in both fields) so the field can't silently disappear. + In practice this fallback is rarely reached: upstream, `turn.ts` + applies `(fnCall.args || {})` before storing the submission, so a + zero-arg call against an empty schema lands as `{}` and renders as + `{}` on stdout, not `null`. The `?? null` step is defence-in-depth + for the strictly-undefined case. + +TEXT mode writes just the `result` field + newline to stdout (any +incidental assistant prose accumulated during the run is discarded — +not mirrored to stderr). JSON mode emits the full event log as a +JSON array; `structured_result` lives on the final `type: "result"` +element of that array, not at the document root. Stream-json mode +emits each message on its own line as JSONL; the terminating `result` +line carries `structured_result`. + +## Privacy: cross-surface redaction + +The args submitted via `structured_output` ARE the structured payload. +On the success path they're already on stdout; on validation-failure +retries they may never reach stdout at all. Either way, persisting +them on durable on-device surfaces (or exporting them off-device +through telemetry) is duplication that leaks the payload into +longer-lived storage than the user asked for. The redaction rule is +therefore "never persist any args from this synthetic tool, regardless +of outcome," not just "dedup what's already on stdout." + +Two surfaces have to redact, and both share the same placeholder +constant +[`STRUCTURED_OUTPUT_REDACTED_ARGS`](../../../packages/core/src/tools/syntheticOutput.ts): + +- `ToolCallEvent.function_args` (telemetry) — covers OTLP exports, + QwenLogger, ui-telemetry, and the chat-recording UI event mirror. +- `redactStructuredOutputArgsForRecording` (used by + `recordAssistantTurn` in `geminiChat.ts`) — covers the on-disk + chat-recording JSONL at + `~/.qwen/projects//chats/.jsonl`. + Validation-failure retries land here too — each retry's args also + get the same placeholder. + +The shared constant prevents drift between the two surfaces. Tool-call +metrics (duration, success, decision) are preserved. + +Hooks (`PreToolUse`, `PostToolUse`, `PostToolUseFailure`) are +intentionally **not** redacted — they receive the raw `tool_input` +because the hook contract is "see what the tool sees." This is +documented in the user-doc Privacy section as a "Hooks see raw args" +callout so operators can filter on `tool_name` or add hook-side +redaction before running `--json-schema` against sensitive data. + +The redaction is intentionally scoped to **on-device** persistence +surfaces (telemetry exports + chat-recording JSONL). The schema +itself still travels to the model provider on every request as the +`structured_output` function declaration's `parameters` block — no +provider-side redaction is possible, since the model needs the +schema to satisfy the tool-call contract. The user-doc Privacy +section warns users to keep `enum` / `const` / `default` / +`examples` / `description` payloads free of secrets for the same +reason. + +## Permission gating + +`structured_output` is deliberately excluded from +`PermissionManager.CORE_TOOLS` (the set of tools subject to the +`--core-tools` allowlist check) — alongside the other synthetic +tools (`agent`, `exit_plan_mode`, `ask_user_question`, `task_stop`, +`send_message`). Dynamically discovered tools (`skill`, MCP) are a +separate exclusion category that also bypasses the allowlist for +unrelated reasons. The synthetic tool only exists when `--json-schema` +is set; adding it to the allowlist machinery would mean +`--core-tools read_file --json-schema X` silently drops the terminal +contract. + +Explicit `permissions.deny` rules and `--exclude-tools` settings still +apply via `PermissionManager.evaluate` → `isToolEnabled`. Both use +the same deny mechanism and both prevent registration — the tool +declaration is stripped from the registry, so the model never sees +the tool. The typical outcome is that the model answers in plain text +(exit 1). If the model loops through other tools without producing +text, it eventually hits `maxSessionTurns` (exit 53) and the +`--json-schema` hint in `handleMaxTurnsExceededError` tells the user +where to look. + +**`--bare` interaction.** Bare mode short-circuits the settings → CLI +config bridge: `packages/cli/src/config/config.ts` builds +`mergedDeny` as `[...(bareMode ? [] : settings.permissions.deny), ...]`, +so settings-level denies (and `tools.exclude`) are dropped under +`--bare`. Argv-level `--exclude-tools` is unconditionally appended +into `mergedDeny`, so it still applies. The synthetic tool is +registered independently of all this (driven by `jsonSchema`, not by +the deny list), so a settings-only deny of `structured_output` +silently no-ops under `--bare` while the tool remains callable. + +## Subagent contexts + +`Config.createToolRegistry` accepts a `forSubAgent: true` option that +suppresses the synthetic registration. Subagent overrides reuse the +parent Config via prototype delegation (`createApprovalModeOverride` / +`buildSubagentContextOverride` → `Object.create(base)`), and +`this.jsonSchema` propagates through the prototype chain. Without the +flag, the synthetic tool would register in the subagent's registry +too, and a subagent calling it would receive the "session ends now" +llmContent — but only `runNonInteractive`'s main / drain loops detect +that as terminal, so the subagent would keep running and burn tokens +on a tool whose contract its loop can't honor. + +> **Maintainer note.** This suppression hangs on the single call path +> through `createToolRegistry(forSubAgent: true)`. Any future subagent +> spawn mechanism that bypasses this path will leak the synthetic +> tool into the subagent's registry and reintroduce the +> burn-tokens-forever failure mode. The fail-safe complement would be +> a runtime guard inside `syntheticOutput.execute()` that returns a +> `fatalError` (or no-op) when invoked from a subagent context. Land +> one if a second leak path appears. + +## MCP shadow-tool guard + +`tool-registry.ts:registerTool` checks the lazy `factories` map for +name collisions, not just the eager `tools` map. If an MCP server +discovers a tool literally named `structured_output`, the +auto-qualification path that exists for eager-tool collisions fires +for factory collisions too: the MCP tool gets renamed to +`mcp____structured_output` and the synthetic factory keeps +the bare name. Without this guard, an MCP server could silently hijack +the structured-output contract. + +## Compatibility surface + +| Combination | Status | Rationale | +| -------------------------------------------------------- | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `--json-schema` + `-p` (or stdin, or positional) | Supported | Primary headless path. | +| `--json-schema` + `--output-format text` (default) | Supported | `JSON.stringify(payload)` + newline. | +| `--json-schema` + `--output-format json` / `stream-json` | Supported | `structured_result` field carries the raw object. | +| `--json-schema` + `--bare` | Supported | `--bare` restricts the registry to `read_file`, `edit`, `run_shell_command`; the synthetic tool is registered alongside that minimal set. | +| `--json-schema` + `-i` | Rejected at parse time | TUI has no terminal contract for the synthetic tool. | +| `--json-schema` + `--input-format stream-json` | Rejected at parse time | Single-shot contract vs. long-lived protocol. | +| `--json-schema` + `--acp` / `--experimental-acp` | Rejected at parse time | ACP loop is independent. | +| `--json-schema` + `--prompt-interactive` | Rejected at parse time | Same as `-i`. | +| `--json-schema` + no prompt + no piped stdin | Rejected at parse time | Headless requires a prompt. | + +## Alternatives considered + +**Schema-aware response prompting (no synthetic tool).** Asking the +model to "respond with JSON matching this schema" via the system +prompt and parsing the final assistant message instead. Rejected +because the model has no syntactic guarantee — the output might be +fenced, prefixed with chatter, or hallucinate fields. Tool-call +validation is enforced by the function-calling layer before +`execute()`, which gives us a hard syntactic + semantic guard. + +**OpenAI's `response_format: {type: "json_schema", …}`.** Provider- +specific; would require parallel implementations for Gemini and +Anthropic. The synthetic-tool approach is provider-agnostic. + +**Reorder structured_output to the front of the batch instead of +filtering.** Lets side-effecting siblings run if the structured call +fails validation. Rejected because the contract for `--json-schema` is +"produce structured output" — if the model is in this mode, sibling +side-effects are probably a mistake. Suppressing them entirely is +safer; the model sees a "Skipped:" tool_result and can re-issue them +in a separate turn. + +**Local `$ref` resolution inside `schemaRootAcceptsObject`.** Would +catch schemas like `{anyOf: [{$ref: "#/$defs/String"}], $defs: {…}}` +at parse time. Rejected for now because the cost (cycle detection, +JSON Pointer syntax, `$defs` vs `definitions`, partial pointers, +remote refs) outweighs the benefit; the `maxSessionTurns` hint already +points users at "schema is unsatisfiable" as a likely cause. + +## Open work + +- Schema-aware response validation could grow a `pattern`-based + ReDoS guard if real users hit catastrophic-backtracking patterns + in `--json-schema` arguments. +- SDK protocol additions (Python / TypeScript / Java SDKs exposing a + typed `structured_result` field) — track separately; + [PR #4001](https://github.com/QwenLM/qwen-code/pull/4001) (closed + unmerged on 2026-05-11) covered that scope before the cli/core work + landed and was superseded. + +## File index + +- `packages/cli/src/config/config.ts` — `resolveJsonSchemaArg`, + `schemaRootAcceptsObject`, yargs `.check` mutex rules. +- `packages/cli/src/gemini.tsx` — TUI guard, exit-code plumbing. +- `packages/cli/src/nonInteractiveCli.ts` — + `processToolCallBatch`, `emitStructuredSuccess`, + `suppressedOutputBody`, plain-text failure path. +- `packages/cli/src/nonInteractive/io/BaseJsonOutputAdapter.ts` — + `structuredResult` → `result` + `structured_result` envelope. +- `packages/core/src/config/config.ts` — registration with + `registerStructuredOutputIfRequested`, `forSubAgent` skip. +- `packages/core/src/tools/syntheticOutput.ts` — synthetic tool + + `STRUCTURED_OUTPUT_REDACTED_ARGS` placeholder. +- `packages/core/src/tools/tool-registry.ts` — factory-collision + rename for MCP shadow tools. +- `packages/core/src/telemetry/types.ts` — `function_args` redaction. +- `packages/core/src/core/geminiChat.ts` — + `redactStructuredOutputArgsForRecording`. +- `packages/core/src/utils/schemaValidator.ts` — `compileStrict` + with strict Ajv instance. +- `packages/cli/src/utils/errors.ts` — + `handleMaxTurnsExceededError`'s `--json-schema` hint. diff --git a/docs/design/telemetry-llm-request-timing-design.md b/docs/design/telemetry-llm-request-timing-design.md new file mode 100644 index 00000000000..4a41b082d16 --- /dev/null +++ b/docs/design/telemetry-llm-request-timing-design.md @@ -0,0 +1,538 @@ +# LLM Request Timing Decomposition Design (P3 Phase 4) + +> Issue #3731 — Phase 4 of hierarchical session tracing. Adds time-to-first-token, request-setup duration, sampling duration, and per-attempt retry telemetry to the `qwen-code.llm_request` span so operators can answer "why was this LLM call slow?" without guessing. +> +> Builds on Phase 1 (#4126), Phase 1.5 (#4302), Phase 2 (#4321). Independent of Phase 3 (#4410, in review) — recommended to land Phase 3 first so Phase 4's per-attempt fields aggregate cleanly under subagent subtrees. + +## Problem + +`qwen-code.llm_request` spans today carry only `model`, `prompt_id`, `input_tokens`, `output_tokens`, `success`, `error`, `duration_ms`. Operators reading a single trace cannot tell: + +1. **How much of `duration_ms` was the model thinking vs the network setup.** A 12-second `duration_ms` could be 11s of retries followed by 1s of fast generation, or 100ms of setup followed by 12s of slow streaming — the trace doesn't say. +2. **When the user saw the first token.** TTFT (time-to-first-token) is the standard latency SLO for chat UIs. We can't compute it; we don't capture it. +3. **What happened during retries.** `retryWithBackoff` (`utils/retry.ts:285`) only calls `debugLogger.warn` — no OTel event, no span attribute. The 4 LLM call sites that go through it (`client.ts:1540`, `baseLlmClient.ts:193,282`, `geminiChat.ts:1039`) have zero retry visibility in traces or metrics. `ContentRetryEvent` exists for content-recovery retries inside `geminiChat.ts:806,830` but not for the more common rate-limit / 5xx retries. +4. **That `api.request.breakdown` is dead code.** The metric is defined at `metrics.ts:242-251` with 4 `ApiRequestPhase` values, exported from `index.ts:117`, tested in `metrics.test.ts:646-675` — but `recordApiRequestBreakdown()` has zero callers in production code. The metric infrastructure is paid for; the data flow was never connected. + +These gaps make `qwen-code.llm_request` the least informative span in the trace tree. Tool spans (#4126/#4321) and subagent spans (#4410) both surface lifecycle phases; LLM spans collapse the entire request into one opaque duration. + +## Existing surface (no change) + +| Component | Location | Why we don't touch it | +| ------------------------------------------------------------ | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| LLM request span lifecycle | `session-tracing.ts` `startLLMRequestSpan` / `endLLMRequestSpan` | Phase 1 (#4126) established the helpers. We extend the metadata interface, don't restructure | +| Active span propagation into provider generators | `loggingContentGenerator.ts:213,287` | Phase 1 (#4126) replaced `withSpan('api.*')` with native helpers; the active context already reaches the stream wrapper | +| `ContentRetryEvent` schema + consumers | `types.ts:626`, `qwen-logger.ts:947`, `loggers.ts:717` | Existing event keeps its shape and downstreams; we add a sibling event class for the `retryWithBackoff` path | +| `LogToSpanProcessor` log-bridge spans | `log-to-span-processor.ts` | ContentRetryEvent's existing bridge continues to nest under the active LLM span. Phase 4 does not change this | +| `ApiRequestPhase` enum | `metrics.ts:330-334` | Public surface (4 values). We populate 3 of the 4 from production code; leave the enum unchanged for backward compatibility | +| Per-provider chunk normalization → `GenerateContentResponse` | `loggingContentGenerator.ts:286-393` | Each provider already normalizes to Google's `GenerateContentResponse` shape before LoggingContentGenerator sees the stream. TTFT detection runs centrally over this normalized shape; no per-provider code | +| `retryWithBackoff` general-purpose retry | `utils/retry.ts:140` | Used by both LLM callers and non-LLM (`channels/weixin/src/api.ts`). We extend with an opt-in `onRetry` callback rather than hard-coupling to LLM telemetry | +| Non-streaming `generateContent` | `loggingContentGenerator.ts:212` | TTFT is not meaningful for non-streaming; the new fields stay `undefined`. Span lifecycle and existing attrs unchanged | + +## Out-of-scope (deferred) + +- **SDK-level retries** (openai SDK `maxRetries=3`, google-genai SDK internal retries). These happen entirely inside the third-party SDK; observing them requires disabling SDK retries and reimplementing in `retryWithBackoff`. Separate decision, not Phase 4. +- **Per-token streaming metrics** (inter-token latency, per-chunk size). Useful for inference-engine perf debugging, not for the user-perceived latency questions Phase 4 targets. +- **Separate TTFT for reasoning/thinking blocks.** "First token" includes thinking content (see D1). A future enhancement could split `ttft_to_reasoning_ms` vs `ttft_to_answer_ms`, but only after we know there's demand. +- **Sampling phase as a dedicated child span.** Computable from `duration_ms - ttft_ms - request_setup_ms`; child span adds nothing for OTel-only backends (claude-code uses one for Perfetto only). Stored as a span attribute instead — see D6. +- **Persistent retry mode (`QWEN_CODE_UNATTENDED_RETRY`) event-level rate limiting.** A single LLM request can produce 50+ `ContentRetryEvent` / `ApiRetryEvent` records under persistent retry. Capping emission is a follow-up — Phase 4 emits all events; if production volumes prove unbearable, add a per-span emission cap with a "+N more attempts (truncated)" summary event in a follow-up PR. +- **`TOKEN_PROCESSING` breakdown phase.** Enum value exists but qwen-code has no real post-stream local processing worth measuring (<10ms typical). Skipped in production callers; enum value retained for future use or for callers we don't control. +- **Migrating `ContentRetryEvent` onto LLM span as span events.** Same reasoning as Phase 3's `subagent_execution` LogRecord: existing consumers (qwen-logger RUM, future metrics) are tightly coupled to the LogRecord. Bridge-span coverage is good enough. + +## References (decision evidence) + +| Source | Key takeaway | +| --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| claude-code (Anthropic) `claude.ts:1762, 1789, 1982, 2882` | TTFT captured as `Date.now() - start` on `message_start` SSE event; `start` reset per retry attempt. `requestSetupMs = start - startIncludingRetries`. `attemptStartTimes` array preserved per attempt. Confirms feasibility of the approach; their TTFT semantic is "first stream event" (we diverge to "first content" — see D1) | +| claude-code `perfettoTracing.ts:549-671` | Renders Request Setup → Attempt N (retry) → First Token → Sampling as nested B/E pairs. Demonstrates the visual decomposition; qwen-code does the same decomposition with OTel attributes since we have no Perfetto | +| claude-code `sessionTracing.ts:447` | Only `ttft_ms` makes it onto the OTel span (not `requestSetupMs`, not `samplingMs`, not per-attempt timing). We deliberately put more on the span — claude-code has Perfetto for visualization; we don't | +| opencode (sst/opencode) `session/llm.ts`, `route/client.ts` | No TTFT measurement. Single `LLM.run` Effect span covers everything. Validates that the gap exists across competing tools; not a reference for what to do | +| [OTel GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) (status: Development / Experimental) | `gen_ai.usage.input_tokens` (Stable), `gen_ai.usage.output_tokens` (Stable), `gen_ai.usage.cached_tokens` (Experimental), `gen_ai.request.model` (Stable), `gen_ai.server.time_to_first_token` (Experimental, seconds as double). Dual-emit pattern follows #4410 precedent | +| [OTel Trace Spec — Span Events](https://opentelemetry.io/docs/specs/otel/trace/api/#add-events) | "Events SHOULD NOT be used to record information that's better captured as Span Attributes." Confirms per-attempt info belongs on the LLM span attributes + log-bridge spans, not as Span Events on the parent | +| Phase 3 design doc (`telemetry-subagent-spans-design.md`) | Established the dual-emit pattern (`qwen-code.subagent.id` + `gen_ai.agent.id`) and the "private name is authoritative" rule. Phase 4 follows the same convention for TTFT and token fields | + +## Design — seven decisions, each justified + +### D1 — TTFT semantic: "first chunk containing user-visible content" + +TTFT measures wall-clock from the **successful attempt's** request dispatch to the **first stream chunk that contains user-visible output**. A chunk is "user-visible" if any normalized `Part` in `candidates[0].content.parts` is one of: + +- `text` with non-empty string +- `functionCall` (tool use) +- `inlineData` (image, binary) +- `executableCode` +- `thought` / reasoning content (whatever the provider surfaces — Gemini's `thought`, Anthropic's `` block, OpenAI o1 reasoning chunk) + +Chunks containing only `role` metadata or only `usageMetadata` (final usage-summary chunk) do not trigger TTFT. + +**Why not "first stream event of any kind" (claude-code's choice)**: claude-code measures TTFT at `message_start`, an Anthropic-specific metadata event that fires 50–300ms before any actual content. Their internal `headlessProfiler.ts` already separates `time_to_first_response_ms` for the "user saw something" semantic, acknowledging the distinction. qwen-code spans multiple providers (Anthropic, OpenAI, Gemini, Qwen) — picking the metadata-event semantic means TTFT for Anthropic is fundamentally different from TTFT for OpenAI (which has no analogous metadata-only first event). The user-visible-content semantic is uniform across all 4 providers and matches "time-to-first-token" literally. + +**Why include `thought` / reasoning**: from the operator's perspective, reasoning chunks are still "the model produced output." Excluding them would understate TTFT for reasoning-heavy models (o1, Qwen thinking variants). Future split into `ttft_to_reasoning_ms` vs `ttft_to_answer_ms` is possible; not Phase 4. + +**Why include tool-call-only chunks**: agent tool-decision LLM calls (one `tool_use`, no text) are common in qwen-code's workflow. Excluding them means TTFT is undefined for these requests. The `functionCall` Part is meaningful output. + +**Cross-product comparison note**: design doc explicitly states `qwen-code.ttft_ms ≈ claude-code.time_to_first_response_ms ≠ claude-code.ttft_ms`. Operators comparing across products should align on the user-visible-content semantic. + +### D2 — TTFT measurement site: method-local variables in `LoggingContentGenerator.generateContentStream` + +The first-chunk detection runs inside the existing stream wrapper at `loggingContentGenerator.ts:393` (`async function* processStreamGenerator`). Per-call variables (`start`, `ttftMs`) live in the method's closure; **never as instance fields**. + +**Why never instance fields**: `LoggingContentGenerator` is instantiated **once per `ContentGenerator`** (`contentGenerator.ts:377`) and shared across all concurrent `generateContentStream` calls — subagent fan-out, warmup queries, side-queries from `geminiChat`. An instance field would be overwritten across concurrent calls, producing nonsense TTFT for one of every two interleaved requests. + +**Why not AsyncLocalStorage**: ALS would work but adds a context-management layer for a piece of state that doesn't need to escape the method. Method-local is simpler, zero overhead, zero risk of leakage. + +```ts +// loggingContentGenerator.ts — inside generateContentStream +const attemptStart = Date.now(); // per-call local +const requestEntryTime = Date.now(); // also per-call local — see D3 +let ttftMs: number | undefined; +const attemptStartTimes: number[] = [attemptStart]; +let retryTotalDelayMs = 0; +let finalAttempt = 1; +// stream wrapper inspects each chunk; first one matching hasUserVisibleContent: +// ttftMs = Date.now() - attemptStart; +``` + +`hasUserVisibleContent(chunk)` is a small standalone helper colocated with the wrapper, exported for tests: + +```ts +function hasUserVisibleContent(chunk: GenerateContentResponse): boolean { + const parts = chunk.candidates?.[0]?.content?.parts; + if (!parts?.length) return false; + return parts.some( + (p) => + (typeof p.text === 'string' && p.text.length > 0) || + p.functionCall !== undefined || + p.inlineData !== undefined || + p.executableCode !== undefined || + // @ts-expect-error — `thought` is not on all SDK versions but providers emit it + p.thought !== undefined, + ); +} +``` + +### D3 — `request_setup_ms` computation: entry-time vs successful-attempt-start + +`request_setup_ms` measures wall-clock from `generateContentStream`/`generateContent` entry to the **start of the successful attempt** — including all failed retries, backoff sleeps, and any pre-retry preparation work. + +```ts +request_setup_ms = attemptStart_of_successful_attempt - requestEntryTime; +``` + +When `attempt === 1` and no retries happened, `request_setup_ms` is small (just SDK setup). When retries occurred, it captures the entire retry-budget overhead. + +**Putting it on the OTel span (diverges from claude-code, which puts it only on Perfetto)**: rationale at three levels: + +1. **No Perfetto** — qwen-code has no out-of-band visualization layer. OTel attributes are the only channel. +2. **Single-trace debug** — operator sees `duration_ms=12000, request_setup_ms=11500, ttft_ms=200, sampling_ms=300` → instantly diagnoses "retries ate 11.5s, model itself was fast." Computing `request_setup_ms` from other fields requires also exposing `sampling_ms`, which we do anyway (D6). +3. **Negligible cost** — 1 INT64 attribute. Same order of magnitude as the existing `input_tokens`, `output_tokens` attributes. Backend ingest cost is not material. + +### D4 — Retry telemetry: `onRetry` callback option on `retryWithBackoff` + new `ApiRetryEvent` + +`retryWithBackoff` currently calls `logRetryAttempt` (`retry.ts:343`) which only writes to `debugLogger.warn`. We extend the `RetryOptions` interface with an opt-in callback: + +```ts +// utils/retry.ts +interface RetryOptions { + // ... existing fields ... + /** + * Optional. Called once per failed attempt, before the backoff sleep. + * Receives the attempt number (1-based), the error, and the delay before + * the next attempt. Use this to emit telemetry events for LLM call sites; + * leave undefined for non-LLM callers (e.g., channels/weixin) so they + * stay silent in LLM-specific telemetry channels. + */ + onRetry?: (info: RetryAttemptInfo) => void; +} + +interface RetryAttemptInfo { + attempt: number; // 1-based, matches debugLogger output + error: unknown; + errorStatus?: number; + delayMs: number; // backoff delay before next attempt +} +``` + +The 4 LLM call sites (`client.ts:1540`, `baseLlmClient.ts:193,282`, `geminiChat.ts:1039`) register a callback that emits a new `ApiRetryEvent`: + +```ts +// types.ts — new event class, sibling to ContentRetryEvent +export class ApiRetryEvent implements BaseTelemetryEvent { + 'event.name': typeof EVENT_API_RETRY; + 'event.timestamp': string; + model: string; + prompt_id?: string; + attempt_number: number; // 1-based + error_type: string; + error_message: string; // truncated to 256 chars + status_code?: number; + retry_delay_ms: number; + // ... duration_ms set to retry_delay_ms so LogToSpanProcessor renders + // a bridge span of meaningful width + duration_ms: number; +} +``` + +**Why a new event class, not extending `ContentRetryEvent`**: + +- `ContentRetryEvent` has 2 downstream consumers (qwen-logger, log-record export). Changing its payload risks breaking them. +- The naming "content retry" semantically refers to content-recovery retries (invalid stream, schema repair) — extending it to cover rate-limit retries would muddy the schema. +- New event is additive; no consumer surprise. + +**Why not embed callback IN `retry.ts`**: `retry.ts` is called by `channels/weixin/src/api.ts` too (microsoft messaging API retries). Hard-coupling LLM telemetry inside retry.ts would emit `ApiRetryEvent` for non-LLM retries. The `onRetry` callback is opt-in per caller — LLM callers opt in, weixin caller doesn't. + +**ContentRetryEvent coexistence**: ContentRetryEvent stays as-is for content-recovery retries inside `geminiChat.ts:806,830`. ApiRetryEvent covers the rate-limit / 5xx retries from `retryWithBackoff`. The two events fire from different layers and never duplicate. Existing log-bridge behavior for both events is preserved via `LogToSpanProcessor` — both events nest under the active LLM span automatically (Phase 1 wiring ensures the LLM span is active during retries). + +**Persistent retry mode (`QWEN_CODE_UNATTENDED_RETRY`)**: a single 429-loop request may emit 50+ events. Out of scope to rate-limit emission in Phase 4 — if production volumes prove unbearable, add a per-span cap with summary event in a follow-up PR. The aggregated `attempt` and `retry_total_delay_ms` on the parent LLM span (D5) remain accurate regardless of event cap. + +### D5 — Parent LLM span aggregation: scalar attributes only (no map-typed attrs) + +OTel span attributes are scalars (`string | number | boolean | array of these`). Map-typed attributes (like `retry_count_by_status: {429:2, 503:1}`) require JSON serialization and are awkward to query. Skip them. + +| Attribute | Type | Semantic | +| -------------------------- | ------ | ----------------------------------------------------------------------------------- | +| `attempt` | int | 1-based final attempt count (`attemptStartTimes.length`) | +| `retry_total_delay_ms` | int | Sum of all `delayMs` reported by `onRetry`; 0 if no retries | +| `ttft_ms` | int | TTFT per D1; undefined for non-streaming or aborted-before-first-chunk requests | +| `request_setup_ms` | int | Per D3 | +| `sampling_ms` | int | Per D6 | +| `output_tokens_per_second` | double | Derived; `output_tokens / (sampling_ms / 1000)`; undefined when `sampling_ms === 0` | + +Per-attempt status-code distribution (e.g., "2 of the 3 attempts were 429s") is queryable from log-bridge spans of `ApiRetryEvent` records. No need to duplicate it as a flattened attribute on the parent. + +**Why `sampling_ms` and `output_tokens_per_second` on the span**: derivable but cumbersome to compute in backend queries when summing across many spans. Same cost-benefit as `request_setup_ms` (D3). + +### D6 — Activate `recordApiRequestBreakdown()` for 3 of 4 phases + +In `endLLMRequestSpan` (or the wrapper that calls it), after computing TTFT/setup/sampling, emit: + +```ts +recordApiRequestBreakdown(config, model, [ + { phase: ApiRequestPhase.REQUEST_PREPARATION, durationMs: requestSetupMs }, + { phase: ApiRequestPhase.NETWORK_LATENCY, durationMs: ttftMs }, // ttftMs = network + first-token-generation + { phase: ApiRequestPhase.RESPONSE_PROCESSING, durationMs: samplingMs }, +]); +``` + +**Why skip `TOKEN_PROCESSING`**: qwen-code does stream chunk processing inline (consolidation happens in the wrapper at `loggingContentGenerator.ts:644`); the post-stream wrap-up phase is <10ms and not architecturally distinct. Filling it with a meaningless value pollutes the histogram. Leaving the enum value unused is safe — `apiRequestBreakdownHistogram.record(value, {model, phase})` is just a histogram with `phase` as a label; missing labels are simply absent in queries. + +**Why not redefine `NETWORK_LATENCY`**: the spec name is slightly misleading (it's network + first-token-generation, not pure network latency), but: + +- The enum is part of `metrics.ts:330-334` which is exported from `index.ts:117` and tested. +- Backend dashboards may already reference these phase names. +- Renaming or adding a new phase would be a breaking change for trivially marginal accuracy improvement. + +Document the semantic in the design doc; leave the enum unchanged. + +**Why on the span path, not parallel**: keeps `recordApiRequestBreakdown` colocated with span attribute writes — single gated emission point (see D7 idempotency), single ordering invariant. + +### D7 — `endLLMRequestSpan` idempotency: metric recording gated on existing double-end guard + +Phase 1.5 (#4302) established that `endLLMRequestSpan` may be called twice (abort path + error path collision). The existing guard at `session-tracing.ts:~470` (`if (!activeSpans.has(...)) return;`) prevents double `span.end()`. Phase 4 metric recording (D6) **must sit inside the same guarded block**, before `span.end()`: + +```ts +// session-tracing.ts — endLLMRequestSpan +const llmCtx = activeSpans.get(spanRef); +if (!llmCtx) return; // already ended — double-end guard +activeSpans.delete(spanRef); // claim the end + +// ... compute duration, set attributes ... +if (metadata) { + recordApiRequestBreakdown(config, llmCtx.attributes.model, [...]); // NEW — gated + recordTokenUsageMetrics(...); // existing +} + +span.end(); +``` + +This guarantees metric is recorded **exactly once** per LLM request, matching the span lifecycle. + +**Why not record in `loggingContentGenerator`**: it doesn't see the abort path. Recording at the span lifecycle layer ensures every LLM request that opens a span produces exactly one breakdown sample, regardless of success/failure/abort. + +### D8 — GenAI semantic conventions dual-emit (private name authoritative) + +Each Phase 4 attribute that corresponds to an OTel GenAI semconv attribute is written twice on the span: + +| qwen-code private (authoritative) | GenAI semconv (compat layer) | Unit conversion | Spec status | +| ------------------------------------------ | ----------------------------------------------- | --------------- | ------------ | +| `ttft_ms` (ms, int) | `gen_ai.server.time_to_first_token` (s, double) | `ttftMs / 1000` | Experimental | +| `input_tokens` (int) | `gen_ai.usage.input_tokens` (int) | identical | Stable | +| `output_tokens` (int) | `gen_ai.usage.output_tokens` (int) | identical | Stable | +| `cached_input_tokens` (int) (when present) | `gen_ai.usage.cached_tokens` (int) | identical | Experimental | +| `qwen-code.model` (string) | `gen_ai.request.model` (string) | identical | Stable | + +**Existing token attribute names** on the LLM span (set in `endLLMRequestSpan` before Phase 4): qwen-code uses bare `input_tokens` and `output_tokens` already. Phase 4 adds the `gen_ai.usage.*` siblings to match #4410's pattern. The bare names stay; **don't rename**. + +Fields with no GenAI semconv equivalent — `request_setup_ms`, `sampling_ms`, `retry_total_delay_ms`, `attempt`, `output_tokens_per_second` — are emitted only under the qwen-code namespace. + +**Why "private authoritative, semconv as compat"**: + +- Internal dashboards, SLOs, debugLogger output, qwen-logger RUM, ARMS queries — all reference `ttft_ms` etc. Treating those as canonical avoids a flag-day migration. +- The Experimental GenAI semconv may rename `gen_ai.server.time_to_first_token` before reaching Stable. If/when it does, we update the semconv emission; the qwen-code names don't move. +- Future spec-aware backends (Datadog AI views, Honeycomb AI, ARMS GenAI dashboards) auto-pick up the `gen_ai.*` attributes without our involvement. + +**Why dual-emit unit conversion** (ms ↔ seconds): GenAI semconv chose seconds-as-double for latency; qwen-code chose ms-as-int (matches `duration_ms` already on the span). Both representations have value; the conversion is cheap. + +## Helper API (additive to `session-tracing.ts`) + +```ts +// session-tracing.ts — LLMRequestMetadata interface extended (additive) +export interface LLMRequestMetadata { + // ... existing fields: inputTokens, outputTokens, cachedInputTokens, success, error, ... + + /** Time from successful attempt start to first user-visible content chunk (ms). Undefined for non-streaming or aborted-before-first-chunk requests. */ + ttftMs?: number; + + /** Time from generateContent entry to start of successful attempt (ms). Includes all failed retries + backoff. */ + requestSetupMs?: number; + + /** Final attempt number (1-based). 1 = no retries. */ + attempt?: number; + + /** Sum of all backoff delays before the successful attempt (ms). */ + retryTotalDelayMs?: number; +} + +// No new exported helpers — Phase 4 reuses startLLMRequestSpan / endLLMRequestSpan with extended metadata. +``` + +```ts +// types.ts — new event class +export class ApiRetryEvent implements BaseTelemetryEvent { + 'event.name': typeof EVENT_API_RETRY = EVENT_API_RETRY; + 'event.timestamp': string; + model: string; + prompt_id?: string; + attempt_number: number; + error_type: string; + error_message: string; + status_code?: number; + retry_delay_ms: number; + duration_ms: number; // = retry_delay_ms, drives LogToSpanProcessor bridge span width + + constructor(opts: { model: string; promptId?: string; attemptNumber: number; error: unknown; statusCode?: number; retryDelayMs: number }) { ... } +} + +// constants.ts +export const EVENT_API_RETRY = 'qwen-code.api_retry'; + +// loggers.ts +export function logApiRetry(config: Config, event: ApiRetryEvent): void { ... } +``` + +```ts +// utils/retry.ts — RetryOptions extension +interface RetryOptions { + // ... existing ... + onRetry?: (info: RetryAttemptInfo) => void; +} + +interface RetryAttemptInfo { + attempt: number; + error: unknown; + errorStatus?: number; + delayMs: number; +} + +// Inside retryWithBackoff, where logRetryAttempt is called today: +options.onRetry?.({ attempt, error, errorStatus, delayMs: actualDelay }); +logRetryAttempt(attempt, error, errorStatus); // existing debugLogger call unchanged +``` + +## Lifecycle wiring + +### Streaming path (the common case) + +```ts +// loggingContentGenerator.ts:283 — generateContentStream +async generateContentStream(req, userPromptId): Promise> { + const requestEntryTime = Date.now(); + let attemptStart = requestEntryTime; + const attemptStartTimes: number[] = [attemptStart]; + let retryTotalDelayMs = 0; + let finalAttempt = 1; + + // Use existing startLLMRequestSpan (Phase 1) + // Pass onRetry callback to whatever retry layer is in use: + const onRetry: RetryAttemptInfo & { invoke: ... } = (info) => { + finalAttempt = info.attempt + 1; // we're about to start attempt N+1 + retryTotalDelayMs += info.delayMs; + attemptStart = Date.now() + info.delayMs; // approximate; actual reset is at top of next attempt + attemptStartTimes.push(attemptStart); + // emit ApiRetryEvent + logApiRetry(this.config, new ApiRetryEvent({ + model: req.model, + promptId: userPromptId, + attemptNumber: info.attempt, + error: info.error, + statusCode: info.errorStatus, + retryDelayMs: info.delayMs, + })); + }; + + // stream wrapper detects first user-visible chunk: + return this.processStreamGenerator(stream, ..., { + onFirstUserVisibleChunk: (now) => { + ttftMs = now - attemptStart; + }, + }); +} +``` + +At span end (already in Phase 1's `endLLMRequestSpan` flow), include the new fields in `LLMRequestMetadata`: + +```ts +endLLMRequestSpan(llmSpan, { + success: true, + inputTokens, + outputTokens, + cachedInputTokens, + ttftMs, + requestSetupMs: attemptStart - requestEntryTime, + attempt: finalAttempt, + retryTotalDelayMs, +}); +``` + +### Non-streaming path + +`generateContent` (`loggingContentGenerator.ts:212`) does not produce streaming chunks. TTFT is `undefined`; `request_setup_ms` is still meaningful (captures retry overhead). The breakdown metric records 2 phases (REQUEST_PREPARATION + RESPONSE_PROCESSING where `RESPONSE_PROCESSING = duration_ms - request_setup_ms`), not 3. + +### Retry layer integration (4 sites) + +Each of the 4 LLM `retryWithBackoff` call sites adds `onRetry`: + +```ts +// client.ts:1540 (similar at baseLlmClient.ts:193, 282, geminiChat.ts:1039) +const result = await retryWithBackoff(apiCall, { + ...existingOptions, + onRetry: (info) => { + logApiRetry( + this.config, + new ApiRetryEvent({ + model, + promptId: userPromptId, + attemptNumber: info.attempt, + error: info.error, + statusCode: info.errorStatus, + retryDelayMs: info.delayMs, + }), + ); + // also feed back into LoggingContentGenerator's local retry accumulator + // (when in scope — for callers that don't go through LoggingContentGenerator, + // the LLM span still gets `attempt` and `retry_total_delay_ms` via the + // metadata path because endLLMRequestSpan is called at the LLM layer) + }, +}); +``` + +The non-LLM caller (`channels/weixin/src/api.ts`) **does not register `onRetry`** — no `ApiRetryEvent` is emitted for its retries, matching today's behavior. + +## Concurrent safety — the headline guarantee + +`LoggingContentGenerator` instance is shared (one per `ContentGenerator`, `contentGenerator.ts:377`). Three concurrent `generateContentStream` calls (e.g., 3 subagents fan out via `coreToolScheduler.runConcurrently`) execute three independent closures of `generateContentStream`: + +``` +call_A: attemptStart_A, ttftMs_A, ... (closure) +call_B: attemptStart_B, ttftMs_B, ... (closure) +call_C: attemptStart_C, ttftMs_C, ... (closure) +``` + +Per-call locals never overlap. Stream chunks are detected against the local `attemptStart` of each call. Span attributes are set at each call's own `endLLMRequestSpan`. + +`AsyncLocalStorageContextManager` (registered by NodeSDK at `sdk.ts:273`) already ensures the active OTel context — and thus the parent span passed to `startLLMRequestSpan` — is correct per fiber. + +## Files to change + +| File | Change | LOC est | +| -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `packages/core/src/telemetry/constants.ts` | Add `EVENT_API_RETRY` constant | +2 | +| `packages/core/src/telemetry/types.ts` | Add `ApiRetryEvent` class + union member | +40 | +| `packages/core/src/telemetry/loggers.ts` | Add `logApiRetry()` function | +20 | +| `packages/core/src/telemetry/qwen-logger/qwen-logger.ts` | Add `logApiRetryEvent()` for RUM downstream consistency | +20 | +| `packages/core/src/telemetry/session-tracing.ts` | Extend `LLMRequestMetadata` (ttftMs, requestSetupMs, attempt, retryTotalDelayMs); extend `endLLMRequestSpan` to set new attrs + breakdown metric + dual-emit gen_ai.\* | +60 | +| `packages/core/src/telemetry/metrics.ts` | Wire `recordApiRequestBreakdown` callsite inside `endLLMRequestSpan` (no change to the existing recorder) | 0 | +| `packages/core/src/utils/retry.ts` | Add `onRetry?: (info: RetryAttemptInfo) => void` to RetryOptions; export `RetryAttemptInfo`; invoke callback in the existing logRetryAttempt site | +25 | +| `packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts` | TTFT capture: method-local accumulators + `hasUserVisibleContent` helper + first-chunk detection in stream wrapper; pass new metadata to `endLLMRequestSpan` | +80 | +| `packages/core/src/core/client.ts` | Wire `onRetry` callback at `retryWithBackoff` call site (`client.ts:1540`) | +15 | +| `packages/core/src/core/baseLlmClient.ts` | Wire `onRetry` callback at 2 `retryWithBackoff` call sites | +25 | +| `packages/core/src/core/geminiChat.ts` | Wire `onRetry` callback at `retryWithBackoff` call site (`geminiChat.ts:1039`) | +15 | +| `packages/core/src/telemetry/session-tracing.test.ts` | `endLLMRequestSpan` sets ttft_ms / request_setup_ms / attempt / retry_total_delay_ms / sampling_ms / output_tokens_per_second + gen_ai dual-emit + breakdown metric (each phase) + idempotent end | +120 | +| `packages/core/src/core/loggingContentGenerator/loggingContentGenerator.test.ts` | `hasUserVisibleContent` (text / functionCall / inlineData / executableCode / thought / role-only / usage-only); concurrent calls don't cross-contaminate; TTFT undefined when aborted before first chunk; TTFT undefined on non-streaming | +100 | +| `packages/core/src/utils/retry.test.ts` | `onRetry` invoked per failed attempt with correct `attempt`, `delayMs`, `error`, `errorStatus`; absence of `onRetry` is silent (no telemetry emitted) | +50 | +| `packages/core/src/telemetry/loggers.test.ts` | `logApiRetry` emits LogRecord with expected payload; bridges through LogToSpanProcessor to nested span under active LLM span | +40 | + +Total: 14 files, ~610 LOC. Larger than Phase 2 (#4321) but comparable to Phase 3 (#4410) and justified by the breadth of integration (4 retry sites + telemetry plumbing + streaming wrapper). + +If review pushes back on size: split into **Phase 4a + 4b + 4c**: + +- **4a** (~200 LOC): TTFT capture + extended `LLMRequestMetadata` + dual-emit. Self-contained value (TTFT visibility from day one). +- **4b** (~250 LOC): `onRetry` callback + `ApiRetryEvent` + 4 caller wiring. **Independently a bug fix** for the `retryWithBackoff` telemetry gap. +- **4c** (~160 LOC): `recordApiRequestBreakdown` activation + parent span aggregation attrs (`attempt`, `retry_total_delay_ms`, `sampling_ms`, `output_tokens_per_second`). Depends on 4a + 4b. + +## Testing strategy + +| Test | What it proves | +| -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | +| `hasUserVisibleContent` returns true for text/functionCall/inlineData/executableCode/thought | D1 semantics across part types | +| `hasUserVisibleContent` returns false for role-only and usage-only chunks | D1 negative cases | +| streaming: TTFT measured from attempt start to first user-visible chunk | End-to-end TTFT detection | +| streaming: TTFT undefined if stream aborts before any user-visible chunk | Edge case | +| streaming: TTFT computed from final attempt's start (not first attempt) | D3 — TTFT reset on retry | +| non-streaming: TTFT remains undefined | S3 decision | +| concurrent `generateContentStream` calls don't cross-contaminate TTFT | D2 — method-local guarantee | +| `endLLMRequestSpan` sets all Phase 4 attrs (ttft_ms, request_setup_ms, sampling_ms, attempt, retry_total_delay_ms, output_tokens_per_second) | Attribute presence | +| `endLLMRequestSpan` dual-emits gen_ai.server.time_to_first_token + gen_ai.usage.\* + gen_ai.request.model | D8 dual-emit | +| `endLLMRequestSpan` records breakdown metric with 3 phases for streaming, 2 for non-streaming | D6 | +| `endLLMRequestSpan` called twice: metric recorded exactly once, attrs not re-set | D7 idempotency | +| `retryWithBackoff` with `onRetry`: callback invoked per failed attempt with correct args | D4 callback contract | +| `retryWithBackoff` without `onRetry`: no telemetry emitted (silent for non-LLM callers) | P2 — channels/weixin scope protection | +| `client.ts` / `baseLlmClient.ts` / `geminiChat.ts` retry callsites emit `ApiRetryEvent` on retry | Integration of D4 at 4 sites | +| `ApiRetryEvent` LogRecord bridges via LogToSpanProcessor to a child span under active LLM span | Trace tree correctness | +| LLM span `attempt` field correctly reflects final attempt number under retries | D5 aggregation | +| LLM span `retry_total_delay_ms` correctly sums onRetry delays | D5 aggregation | +| `output_tokens_per_second` undefined when `sampling_ms === 0` (no streaming) | Avoid divide-by-zero | + +## Edge cases + +| Case | Handling | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Stream aborts before any chunk arrives | `ttftMs = undefined`, `sampling_ms = undefined`, `output_tokens_per_second = undefined`. `attempt`, `request_setup_ms` still set. `success = false` | +| Stream aborts after first chunk | `ttftMs` set; `sampling_ms` = `duration_ms - ttftMs - request_setup_ms`; reflects partial response time. `success = false` | +| Retry succeeds on attempt 1 (no retries) | `attempt = 1`, `retry_total_delay_ms = 0`, no `ApiRetryEvent` emitted, breakdown metric records `request_setup_ms` close to 0 | +| Persistent retry mode 50+ attempts | 50+ `ApiRetryEvent` records emitted (out-of-scope cap deferred); LLM span `attempt = 51`, `retry_total_delay_ms = sum of all delays`. Operator sees aggregated view on span; full per-attempt detail in log-bridge spans | +| Non-LLM `retryWithBackoff` caller (channels/weixin) | No `onRetry` registered; only existing `debugLogger.warn` fires. No `ApiRetryEvent`; no breakdown metric (caller isn't an LLM site) | +| `endLLMRequestSpan` called twice (abort + error race) | Phase 1.5 guard at `activeSpans.delete()` returns early on second call; `recordApiRequestBreakdown` is inside the guard, recorded exactly once | +| Anthropic `message_start` chunk arrives before content | `hasUserVisibleContent` returns false for it (no parts with text/functionCall/etc.); TTFT not triggered until subsequent `content_block_delta` chunk | +| OpenAI first chunk with empty `delta.content` but `role` only | `hasUserVisibleContent` returns false; TTFT not triggered until first chunk with non-empty delta | +| Tool-call-only response (no text) | First chunk with `functionCall` Part triggers TTFT; `output_tokens_per_second` computed against tool-call token count | +| Concurrent subagents (3 calls in flight) | Each call's closure has its own `attemptStart`, `ttftMs`, `attemptStartTimes`. Per-call span receives its own metadata at `endLLMRequestSpan`. No interleaving (D2) | +| SDK-level retries inside openai-sdk (`maxRetries=3`) | Invisible to qwen-code telemetry — happens entirely inside SDK before retryWithBackoff sees the request. `attempt` reflects retryWithBackoff attempts only. Out of scope (see Out-of-scope) | +| `gen_ai.server.time_to_first_token` spec renames before reaching Stable | Single-file update: `session-tracing.ts:endLLMRequestSpan`. The qwen-code-native `ttft_ms` stays authoritative — no downstream impact | +| Subagent's LLM request | Parent is the subagent span (Phase 3). Phase 4 fields nest correctly. Aggregations grouped by `qwen-code.subagent.id` give per-subagent LLM perf — design-doc-future, easy follow-up | +| Reasoning model with long thought blocks | First `thought` Part triggers TTFT; `sampling_ms` includes both thinking + answer phases. Split into separate metrics deferred | + +## Rollback + +The change is additive at the OTel and metric level — every new attribute is optional, every new event is a new class. Existing dashboards that don't filter on the new fields keep working unchanged. + +Behavior-affecting changes: + +- New `ApiRetryEvent` LogRecord starts flowing → log volume increases proportional to retry rate (typically <1% of requests retry). Mitigate by sampling LogRecord at the SDK layer if needed. +- New breakdown metric `qwen-code.api.request.breakdown` starts producing time series → mild Prometheus cardinality bump (`{model, phase}` — bounded). +- `output_tokens_per_second` derived attribute may appear unusual on dashboards filtering "all attributes" — document. + +Rollback path: revert the single PR (or each of 4a/4b/4c independently). All new fields use defensive defaults (undefined / 0) and don't change span structure. + +## Sequencing + +- **After Phase 3 (#4410, in review)**: not a hard dependency. Phase 4 attributes attach to `qwen-code.llm_request` spans regardless of whether they're under a `qwen-code.subagent` (Phase 3) or `qwen-code.interaction` (Phase 1) parent. Recommend Phase 3 land first so per-attempt aggregation under subagent subtrees works naturally. +- **Independent of #4384** (`traceparent` + `X-Qwen-Code-Session-Id` outbound propagation). They touch the HTTP layer; Phase 4 touches the stream/retry/metric layer. +- **Independent of `clearDetailedSpanState` chat-compression follow-up** (#4097 follow-up). Different surface. + +## Open questions + +1. **`onRetry` callback firing semantics**: invoked **before** backoff sleep (current proposal) or **after** (when the next attempt is about to start)? Before is simpler — callback has all the info immediately; after would require capturing the just-completed delay separately. Pre-sleep is the recommendation; document in callback contract. +2. **Per-attempt timing on the LLM span**: should we add `attempt_durations_ms: number[]` array? OTel supports array-of-primitive attributes. Useful for "which attempt of N was slow" diagnostics. Defer until production data shows demand — log-bridge spans already carry the equivalent. +3. **Persistent retry mode emission cap**: at what `attempt > N` threshold should we start sampling? `N = 5` then 1-in-10? `N = 10` then summary-only? Defer until we have production volume data. +4. **`TOKEN_PROCESSING` phase**: keep enum value dormant or wire it to something (e.g., consolidation time)? Defer — wait for a real use case. +5. **Subagent-level LLM rollups**: trivial follow-up once Phase 4 lands — sum `ttft_ms`/`output_tokens`/`input_tokens` per subagent subtree. Not Phase 4 scope but the data flow enables it. diff --git a/docs/design/telemetry-outbound-propagation-design.md b/docs/design/telemetry-outbound-propagation-design.md new file mode 100644 index 00000000000..91fea0d835a --- /dev/null +++ b/docs/design/telemetry-outbound-propagation-design.md @@ -0,0 +1,878 @@ +# Telemetry: Outbound Trace Context & Session ID Header Propagation + +> 配套 issue: [#4384](https://github.com/QwenLM/qwen-code/issues/4384) +> 父 issue: [#3731](https://github.com/QwenLM/qwen-code/issues/3731) (P3 deeper observability) +> 前置 PR: #4367 (resource attributes — merged 2026-05-21, commit `64401e1`) +> 基于 2026-05-21 对 qwen-code main 分支 + 直接验证的 claude-code 源码 + +## 修订历史 + +| 修订 | 日期 | 触发 | 摘要 | +| ---- | ---------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| R1 | 2026-05-21 | 初稿 | 全广播:所有出站 LLM 请求都带 `X-Qwen-Code-Session-Id` + `traceparent` | +| R2 | 2026-05-22 | wenshao R2/R3 review | 边界安全:URL normalize、port matching、quote 对齐、staticCorrelationHeaders try/catch、host:port fallback strip | +| R3 | 2026-05-23 | LaZzyMan REQUEST_CHANGES | **重大语义改动**:`X-Qwen-Code-Session-Id` 默认作用域收窄到 first-party(Alibaba/DashScope)host 白名单。详见 §11 | +| R4 | 2026-05-25 | LaZzyMan round-8 follow-up (scope conflation) | **PR scope 大幅收窄**:本 PR 仅保留 client HTTP span + OTLP loop guard;`traceparent` 默认 off(NoopTextMapPropagator);新增 `outboundCorrelation.*` 顶级 namespace 放安全相关 toggle;R3 落地的整套 `X-Qwen-Code-Session-Id` 机器**移除本 PR**,搬到独立 follow-up PR。详见 §12 | + +**特别提示**:阅读 §3.1(目标)/ §3.2(非目标)/ §4.3(Part B 设计)/ §4.4(配置 schema 影响)/ §5(文件改动清单)/ §9(与 claude-code 对比)/ §10(未来工作)/ §11(R3 host-allowlist scoping)时,请同时参考 §12 —— **R4 修订让 R1-R3 关于"本 PR 同时落地 traceparent + session id header"的论断不再成立**:本 PR 现仅为 telemetry observability + 独立的 outbound trace-context toggle,所有 outbound correlation header 工作(包括 R3 的 host allowlist)整体搬到独立 follow-up PR。R3 工作代码本身没浪费,挪到 follow-up PR 即可复用。 + +## 1. 背景 + +#4367 解决了**emitted telemetry 上的 attribute 与 cardinality**(操作员能给 span/log/metric 打 `user.id`/`tenant.id` 这类标签)。但有一类东西它没碰:**outbound LLM 请求的 HTTP header**。今天 qwen-code 发往 DashScope / OpenAI / Gemini / Anthropic 的请求**完全不带任何 cross-process correlation header**——既没有 W3C `traceparent`,也没有 session id。 + +后果: + +1. trace context 在 qwen-code 进程边界断开。若模型服务(如 ARMS Tracing 接入的 DashScope)本身有 OTel instrumentation,它产生的 span 与 qwen-code 的 trace 彼此独立,端到端 trace tree 不存在。 +2. 没有 session id 在 wire 上。后端要把 qwen-code 的 metric/log 与服务端日志关联,需要离线匹配 trace id 或时间戳,远不如直接读 header 简单。 +3. 本地 trace 缺一层 client-side HTTP span。今天只能看 `api.generateContent` 的总耗时,看不到网络 TTFB / 响应体大小 / 重试次数。 + +## 2. 现状 + +### 2.1 仅启用了 `HttpInstrumentation` + +`packages/core/src/telemetry/sdk.ts:330`: + +```ts +instrumentations: [new HttpInstrumentation()], +``` + +`HttpInstrumentation` 只 hook Node 内建的 `http`/`https` 模块,**不**覆盖 `globalThis.fetch` / undici 路径。 + +### 2.2 两套 LLM SDK 都走 fetch / undici + +| SDK | HTTP 实现 | `HttpInstrumentation` 是否覆盖 | +| ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | +| `openai@5.11.0` | `globalThis.fetch`(Node 18+ 即 undici)。证据:`node_modules/openai/internal/shims.mjs` 报错 `'fetch' is not defined as a global` | ❌ | +| `@google/genai@1.30.0` | `globalThis.fetch` + `new Headers()`。证据:`dist/node/index.mjs` 内的 `new Headers()` 调用 | ❌ | +| `@anthropic-ai/sdk`(anthropicContentGenerator) | 同样基于 fetch | ❌ | + +### 2.3 代码库零 manual propagation + +``` +grep -rn "propagation\.\|setGlobalPropagator\|W3CTraceContext\|traceparent" packages/core/src --include="*.ts" | grep -v "\.test\." +``` + +→ 空。没有任何 `propagation.inject()` 调用,没有手动 traceparent 注入。 + +### 2.4 各 provider 的 `defaultHeaders` 现状 + +OpenAI 家族(用 `openai` SDK): + +所有 OpenAI 子 provider 都 `extends DefaultOpenAICompatibleProvider`。**buildHeaders override 行为分两类**(已 grep audit 验证): + +| Provider | 文件 | `buildHeaders()` 行为 | 影响 | +| ---------- | ---------------------- | --------------------------------------------------------------------------------------- | ---------------------------------------------- | +| 基类 | `default.ts:63-74` | 提供 `{ 'User-Agent' }` + customHeaders | 改这里 | +| DashScope | `dashscope.ts:110-124` | **`override` 但不 call `super`**——返回 `User-Agent` + `X-DashScope-*` 全新对象 | **必须单独改这里**,否则 correlation header 丢 | +| OpenRouter | `openrouter.ts:20-30` | `override` 但**先 `const baseHeaders = super.buildHeaders()`** | 改基类自动继承 ✅ | +| DeepSeek | `deepseek.ts` | 不 override `buildHeaders`(只 override `buildRequest` / `getDefaultGenerationConfig`) | 改基类自动继承 ✅ | +| Minimax | `minimax.ts` | 同 deepseek | 自动继承 ✅ | +| Mistral | `mistral.ts` | 同 deepseek | 自动继承 ✅ | +| ModelScope | `modelscope.ts` | 同 deepseek | 自动继承 ✅ | + +→ **OpenAI 家族需要触动 2 个文件**:`default.ts` 和 `dashscope.ts`。其余 5 个自动继承。 + +Google Gemini: + +| Provider | 文件 | 头注入路径 | +| -------- | ------------------------------ | -------------------------------------------------------------- | +| Gemini | `geminiContentGenerator.ts:59` | `new GoogleGenAI({ httpOptions: { headers } })` — SDK 原生支持 | + +Anthropic: + +| Provider | 文件 | 头注入路径 | +| --------- | ------------------------------------------------------------------------------------------------------ | ---------------- | +| Anthropic | `anthropicContentGenerator.ts:177` (`buildHeaders`) + `:212` (`defaultHeaders` arg to `new Anthropic`) | `defaultHeaders` | + +**总计 4 个 SDK 构造点**需要注入 session id header。所有 SDK 都已支持 `defaultHeaders` / `httpOptions.headers`,无需 fetch wrapper。 + +### 2.5 已有的 proxy 与 fetch 配置 + +`provider/default.ts:87-89`: + +```ts +const runtimeOptions = buildRuntimeFetchOptions( + 'openai', + this.cliConfig.getProxy(), +); +``` + +`buildRuntimeFetchOptions` 在用户配 proxy 时返回 `{ fetch: customFetch }` 或类似,触发 `setGlobalDispatcher(new ProxyAgent(...))`(见 `config.ts:1126-1128`)。**undici 全局 dispatcher 模式与 `UndiciInstrumentation` 兼容**——它通过 monkey-patch `globalThis.fetch` 与 undici 的 channel diagnostics 协作,不依赖具体 dispatcher。 + +## 3. 目标 / 非目标 + +### 3.1 目标 + +- 所有 outbound LLM 请求自动带 W3C `traceparent` header(OTel SDK 默认的 `W3CTraceContextPropagator`) +- ~~所有~~ 出站 LLM 请求带 `X-Qwen-Code-Session-Id` header(claude-code 同款产品命名空间) — **R3 修订**:默认仅向 first-party (Alibaba/DashScope) host 注入,第三方 provider 默认不发;详见 §11 +- 自动避免对 OTLP exporter endpoint 自身的 trace(feedback loop) +- 给 LLM 请求加一层精确的 client span(网络耗时 vs 模型耗时分离) +- 覆盖 4 个 provider 构造点:OpenAI 基类、DashScope override、Gemini、Anthropic +- streaming 请求 / proxy 模式 / 重试场景全部不退化 +- 与 #4367 的设计哲学一致:通过 `defaultHeaders` 这种 SDK-native 选项 — **R1 修订**:因 staleness 问题转用 fetch wrapper;**R3 修订**:fetch wrapper 内再叠加 host gate + +### 3.2 非目标 + +- **`baggage` header**:标准 SDK 已支持,但 qwen-code 没调 `propagation.setBaggage()`,默认不会发送。本设计不主动开启。 +- **subprocess `TRACEPARENT` env var 继承**:claude-code 给 Bash/PowerShell 子进程注入 `TRACEPARENT`。qwen-code 的 `BashTool` 没做。是独立 follow-up sub-issue。 +- **inbound `TRACEPARENT` / `TRACESTATE` 读取**:claude-code 的 `-p` 模式和 Agent SDK 从 env 读 traceparent 接续父进程 trace。qwen-code 没做。独立 follow-up。 +- **`X-Qwen-Code-Request-Id`**:claude-code 有 `x-client-request-id`,对超时容错 correlation 有用。本期不做,可作为下一个 sub-issue。 +- **自定义 propagator(B3 / Jaeger / X-Ray)**:默认 W3C 已覆盖 99% 场景。可作为 future config option。 +- ~~**per-endpoint 选择性注入**:claude-code 对第三方 endpoint (Bedrock / Vertex) 不发 traceparent;qwen-code 没有第三方区分需要,统一发即可。~~ — **R3 修订**:此论断已被推翻。LaZzyMan review 指出 qwen-code 是开源 CLI 连接多个第三方 provider(OpenAI / Anthropic / OpenRouter / 等),claude-code 的 first-party→first-party 类比不适用;session id header 必须按 host 区分。详见 §11。`traceparent` 仍按 R1 设计全注入(OTel 标准 header,且 trace id 是 `sha256(sessionId)` 哈希值),可作为独立 follow-up 加 per-destination toggle(`telemetry.propagateTraceContext`)。 + +## 4. 设计 + +### 4.1 总体分层 + +``` +┌─ qwen-code process ────────────────────────────────────────────┐ +│ │ +│ ┌─ session-tracing.ts ─┐ │ +│ │ active span ctx │ │ +│ └──────┬───────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─ propagation.inject() (called by undici instrumentation) ─┐│ +│ │ writes `traceparent: 00---01` to headers ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ │ │ +│ ┌──────▼──────────────────────────────────────────────────┐ │ +│ │ fetch() — undici, instrumented │ │ +│ │ creates HTTP client span │ │ +│ │ injects traceparent into request headers │ │ +│ │ (skipped via ignoreRequestHook if endpoint is OTLP) │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ │ +│ │ ┌─ defaultHeaders (per SDK constructor) ───────┐ │ +│ │ │ { 'X-Qwen-Code-Session-Id': sessionId, ... } │ │ +│ └───┴────────────────────────────────────────────────┘ │ +│ │ │ +└─────────────┼──────────────────────────────────────────────────┘ + │ + ▼ outbound HTTP + POST /v1/chat/completions + traceparent: 00-... + X-Qwen-Code-Session-Id: ... + ... (existing User-Agent, X-DashScope-*, etc.) +``` + +两条注入路径独立、互不依赖: + +| Layer | 何时注入 | 由谁注入 | +| ------------------------ | ------------------------------------- | ------------------------------------------------------------- | +| `traceparent` | 每次 fetch 调用时 | `UndiciInstrumentation` 自动(来自 OTel SDK 默认 propagator) | +| `X-Qwen-Code-Session-Id` | SDK 构造时一次性写入 `defaultHeaders` | 应用代码 | + +### 4.2 Part A — `traceparent` via undici instrumentation + +**改动点**:`packages/core/src/telemetry/sdk.ts` + +```ts +import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici'; + +// ... +const otlpUrls = [ + config.getTelemetryOtlpEndpoint(), + config.getTelemetryOtlpTracesEndpoint(), + config.getTelemetryOtlpLogsEndpoint(), + config.getTelemetryOtlpMetricsEndpoint(), +] + .filter((u): u is string => !!u) + .map((u) => u.replace(/\/$/, '')); + +instrumentations: [ + new HttpInstrumentation(), + new UndiciInstrumentation({ + ignoreRequestHook: (request) => { + // request.origin = "https://collector:4318", request.path = "/v1/traces" + const url = `${request.origin}${request.path}`; + return otlpUrls.some((e) => url.startsWith(e)); + }, + }), +], +``` + +#### 为什么 `ignoreRequestHook` 必须 + +OTel SDK 自己用 fetch 把数据 POST 到 OTLP collector。如果不跳,UndiciInstrumentation 会给"上报数据"的请求也建一个 span → 这个新 span 会被再次上报 → 无限循环 / 巨量噪声。每个 OTel 项目都踩过这个坑,OTel 文档明确推荐这种 hook。 + +#### 默认 propagator + +OTel SDK `NodeSDK` 不传 `textMapPropagator` 时默认是 `CompositePropagator([W3CTraceContextPropagator, W3CBaggagePropagator])`。无需显式设置。 + +#### `traceparent` 格式 + +``` +traceparent: 00-<32hex traceId>-<16hex spanId>-<01 sampled | 00 not sampled> + ─┬─ ─┬─ + version (固定 00) flags +``` + +固定 55 bytes,无 padding。 + +#### `tracestate` 与 `baggage` + +- `tracestate`: 上游传过来才续传;自己 inject 不会主动加(OTel SDK 行为)。 +- `baggage`: 仅当 `propagation.setBaggage(ctx, ...)` 被调用过才有。qwen-code 不调,所以不会发送。 + +### 4.3 Part B — `X-Qwen-Code-Session-Id` via fetch wrapper(OpenAI / Anthropic)+ static headers(Gemini) + +> **R3 修订**:以下设计描述的是 fetch wrapper 的 staleness 解决和 4 个 provider 集成点 — 这些都保留。但 wrapper 内部增加了一道 host allowlist gate,`staticCorrelationHeaders` 也加了 `destinationUrl` 参数。带 host gate 的最新实现代码与 default allowlist 见 §11。 + +#### Critical:staleness 问题与方案选择 + +天真做法(`defaultHeaders` 直接 bake-in `getSessionId()`)有**真 bug**: + +1. `pipeline.ts:60` 在 contentGenerator 构造时一次性 `this.client = this.config.provider.buildClient()`,SDK client 的 `defaultHeaders` 在那一刻 capture 当时的 session id +2. `config.ts:1850` 的 session reset(用户 `/clear` 时触发)更新 `this.sessionId` 并 `refreshSessionContext()`,但**不重建 contentGenerator** +3. 后续 LLM 调用仍走旧 client → wire header 仍是旧 session id → 后端 correlation 错位 + +→ 必须读取 session id **per-request**,不能 bake at构造时。 + +#### 方案 + +``` + ┌─ fetch 支持 ─┐ 方案 +OpenAI SDK │ ✅ │ fetch wrapper (per-request 读 sessionId) ✅ +Anthropic SDK │ ✅ │ fetch wrapper ✅ +@google/genai SDK │ ❌ │ static httpOptions.headers + 接受 staleness + └──────────────┘ +``` + +`@google/genai`'s `HttpOptions` interface 不支持 `fetch`(已 grep `node_modules/@google/genai/dist/genai.d.ts` 验证:只有 `baseUrl`/`apiVersion`/`headers`/`timeout`/`extraParams`)。所以 Gemini 走 static headers,与 OpenAI/Anthropic 不一致——这是 **known limitation**,见 §8.6。 + +#### 集中辅助函数(per-request fetch wrapper) + +新文件 `packages/core/src/telemetry/llm-correlation-fetch.ts`: + +```ts +import type { Config } from '../config/config.js'; + +/** + * Wrap a fetch implementation so every outbound request gets correlation + * headers (`X-Qwen-Code-Session-Id`) populated from the **current** session + * id, not the value captured when the SDK client was constructed. + * + * Matches claude-code's pattern (src/services/api/client.ts:370-390 — + * `buildFetch()`). Per-request injection is necessary because `/clear` + * resets the session id mid-process; SDK clients (and their static + * `defaultHeaders`) are NOT recreated on reset. + * + * Caller responsible for choosing the base fetch — usually + * `runtimeOptions?.fetch ?? globalThis.fetch` so proxy-aware fetch is + * preserved when ProxyAgent is in use. + * + * If telemetry is disabled, returns baseFetch unchanged (no correlation + * header is added, matching the privacy stance of §3.1). + */ +export function wrapFetchWithCorrelation( + baseFetch: typeof fetch, + config: Config, +): typeof fetch { + return async function correlationFetch(input, init) { + if (!config.getTelemetryEnabled()) { + return baseFetch(input, init); + } + const sid = config.getSessionId(); + if (!sid) { + // Defensive: empty header value is rejected by some HTTP middleware. + // Skip injection rather than send `X-Qwen-Code-Session-Id: `. + return baseFetch(input, init); + } + const headers = new Headers(init?.headers); + headers.set('X-Qwen-Code-Session-Id', sid); + return baseFetch(input, { ...init, headers }); + }; +} +``` + +Companion helper for the SDKs that can only take static headers (Gemini): + +```ts +/** + * Static correlation headers. Captures the session id at call time — + * **subject to staleness** if the host SDK keeps these headers in a + * captured-at-construction slot (e.g. `@google/genai`'s `httpOptions.headers`). + * Prefer `wrapFetchWithCorrelation` whenever the SDK exposes a `fetch` hook. + */ +export function staticCorrelationHeaders( + config: Config, +): Record { + if (!config.getTelemetryEnabled()) return {}; + return { 'X-Qwen-Code-Session-Id': config.getSessionId() }; +} +``` + +#### 集成点 1: `provider/default.ts` (OpenAI 基类) + +`buildClient()` 改动——compose 现有 `runtimeOptions.fetch`(proxy)与我们的 wrapper: + +```ts +buildClient(): OpenAI { + // ... existing ... + const runtimeOptions = buildRuntimeFetchOptions('openai', this.cliConfig.getProxy()); + const baseFetch = + (runtimeOptions as { fetch?: typeof fetch } | undefined)?.fetch + ?? globalThis.fetch; + return new OpenAI({ + apiKey, + baseURL: baseUrl, + timeout, + maxRetries, + defaultHeaders, + ...(runtimeOptions || {}), + // After spread, override `fetch` so our correlation wrapper wraps the + // proxy-aware fetch (or globalThis.fetch when no proxy). + fetch: wrapFetchWithCorrelation(baseFetch, this.cliConfig), + }); +} +``` + +`buildHeaders()` itself unchanged. + +#### 集成点 2: `provider/dashscope.ts` (override) + +`buildClient()` 同样的 compose 模式(它本来就 override buildClient)。`buildHeaders()` 不动。 + +#### 集成点 3: `geminiContentGenerator/index.ts` (factory, NOT 构造器) + +**修正先前设计的过度声明**:`geminiContentGenerator.ts` 构造器**不需要**改签名。`index.ts:48` 的 factory 函数已经接收 `gcConfig: Config`(line 33 已经在用 `gcConfig?.getUsageStatisticsEnabled()`),只需要在 factory 里把 correlation 静态 headers merge 进 `httpOptions.headers`: + +```ts +// geminiContentGenerator/index.ts +let headers: Record = { ...baseHeaders }; +if (gcConfig?.getUsageStatisticsEnabled()) { + // ... existing x-gemini-api-privileged-user-id ... +} +headers = { ...headers, ...staticCorrelationHeaders(gcConfig) }; // ← 新增 +const httpOptions = config.baseUrl + ? { headers, baseUrl: config.baseUrl } + : { headers }; +// new GeminiContentGenerator(...) unchanged +``` + +零 signature 改动。 + +#### 集成点 4: `anthropicContentGenerator.ts` + +Anthropic SDK 同样接受 custom `fetch`(已经在用 `buildRuntimeFetchOptions`)。把 `buildClient` 路径里那个 fetch wrap 一下,方式同 OpenAI default.ts。`buildHeaders` 不变。 + +#### 优先级链 + +不变:用户的 `customHeaders` 在 `defaultHeaders` merge 中仍然赢(见 §8.2 spoofing 讨论)。fetch wrapper 注入的 `X-Qwen-Code-Session-Id` 在 SDK 的 headers list 之**后**追加到最终 `Headers` 对象上——以 Node `Headers.set()` 的语义,等于覆盖任何之前同名的(包括 user 的 customHeaders 里写的同名 header)。 + +**对 OpenAI/Anthropic(fetch wrapper 路径)**:correlation > customHeaders > SDK defaults。 +**对 Gemini(static headers 路径)**:customHeaders > correlation > SDK defaults(沿用既有 spread 顺序)。 + +差异是 fetch wrapper 路径下 spoofing 不再可能(fetch wrapper 在 SDK headers 之后跑)。这是 **bug 修复的副产品**,并非有意收紧——但更安全。要在 §8.2 明示。 + +### 4.4 配置 schema 影响 + +~~**几乎为零**。本设计不引入新 setting~~ — **R3 修订**:引入了一项新 setting `telemetry.sessionIdHeaderHosts: string[]`,用于覆盖默认的 first-party host 白名单。schema 项已加入 `packages/cli/src/config/settingsSchema.ts`,描述与 override 语法(`["*"]` 恢复广播 / `[]` 全关 / 自定义数组)见 §11。原文以下描述仅适用于 R3 之前: + +- `traceparent` 注入由 telemetry enabled 触发(已有 toggle) +- `X-Qwen-Code-Session-Id` 注入也由 telemetry enabled 触发 +- `ignoreRequestHook` 的 OTLP url 已经从现有 config 读 + +未来可以加的 setting(**out of scope**): + +- `telemetry.outboundCorrelationHeader`: 自定义 header name(默认 `X-Qwen-Code-Session-Id`) +- `telemetry.outboundPropagationDisabled`: 全局关闭(如果 LLM 服务对未知 header 严格) +- ~~per-destination header scope toggle~~ — **R3 已落地**,见 §11 + +## 5. 文件改动清单 + +| 文件 | 改动类型 | 说明 | +| ------------------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `packages/core/package.json` | 加依赖 | `@opentelemetry/instrumentation-undici` | +| `packages/core/src/telemetry/sdk.ts` | 修改 | +`UndiciInstrumentation` + `ignoreRequestHook` | +| `packages/core/src/telemetry/llm-correlation-fetch.ts` | 新文件 | `wrapFetchWithCorrelation()` (OpenAI/Anthropic) + `staticCorrelationHeaders()` (Gemini fallback) | +| `packages/core/src/core/openaiContentGenerator/provider/default.ts` | 修改 | `buildClient()` 在 `new OpenAI({...})` 里加 `fetch: wrapFetchWithCorrelation(baseFetch, cliConfig)` | +| `packages/core/src/core/openaiContentGenerator/provider/dashscope.ts` | 修改 | 同上(override `buildClient`) | +| `packages/core/src/core/geminiContentGenerator/index.ts` | 修改 | factory 函数里 merge `staticCorrelationHeaders(gcConfig)` 进 `httpOptions.headers`(**caller 已有 Config,零 signature 改动** — 修正之前的 over-specification) | +| `packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts` | 修改 | `buildClient` 路径下用 `wrapFetchWithCorrelation` 包 SDK 的 `fetch` option | + +**显式 audited 但无需改动**(避免 reviewer 怀疑漏路径): + +- `packages/core/src/qwen/qwenContentGenerator.ts` — `extends OpenAIContentGenerator`,用 `DashScopeOpenAICompatibleProvider`,**自动继承 dashscope.ts 的 buildClient 改动**。所有 Qwen OAuth 流程同样受益。 +- `packages/core/src/core/loggingContentGenerator/loggingContentGenerator.ts` — wrapper 模式,不构造 SDK client(它包装其他 contentGenerator 做 telemetry logging),无需改动。 +- `packages/core/src/core/contentGenerator.ts` — factory 入口,不持有 client。 + | `packages/core/src/telemetry/sdk.test.ts` | 修改 | 加 undici instrumentation 注册 + ignoreRequestHook 测试 | + | `packages/core/src/telemetry/llm-correlation-fetch.test.ts` | 新文件 | telemetry-on/off 行为单测 + per-request 读 sessionId 验证(critical:session reset 后 wrapped fetch 读到新 id) | + | 各 provider 的 `*.test.ts` | 修改 | 断言 SDK 构造时 `fetch` option 是 wrapped 版本(OpenAI/Anthropic);断言 Gemini 构造时 `httpOptions.headers` 含 `X-Qwen-Code-Session-Id` | + | `docs/developers/development/telemetry.md` | 修改 | 新增 "Trace context & session correlation propagation" 段 | + | `docs/design/telemetry-outbound-propagation-design.md` | 本文件 | 设计文档 | + +## 6. 分 PR 拆分 + +按 review 友好度分两个 PR(也可以合一,规模允许): + +### PR 1 — `traceparent` 自动注入(structural) + +- 加 `@opentelemetry/instrumentation-undici` 依赖 +- `sdk.ts` 加 `UndiciInstrumentation` + `ignoreRequestHook` +- 测试:SDK 注册、OTLP endpoint 不被 trace +- 文档片段 + +**风险**:低。Additive。已有 client span 是 net 增益,不会改变现有 span 结构。 + +### PR 2 — `X-Qwen-Code-Session-Id` header(结合 helper 函数) + +- 新文件 `llm-correlation-headers.ts` +- 4 个 provider 集成 +- 测试:每个 provider 断言 header 存在;telemetry-off 时不发 +- 文档片段 + +**风险**:低-中。要小心 `geminiContentGenerator` 构造器签名扩展可能波及调用方。 + +### PR 3(可选) — Docs + E2E verify + +- 完善 `telemetry.md` 段落 +- 加 E2E verify script(复用 `/tmp/verify-telemetry-pr-4367.mjs` 模式):实际跑 fetch + 抓 header + +也可以合并到 PR 2 里。 + +### 顺序偏好 + +PR 1 和 PR 2 技术上**互相独立**——不共享代码。但**推荐 PR 1 先合**: + +- `traceparent` 是 OTel **标准** header,任何 OTel-aware collector / 后端立刻识别 → 用户立即获益 +- `X-Qwen-Code-Session-Id` 是**产品自定义** header,需要后端配置识别才有价值 → 价值滞后 +- 万一 PR 2 review 周期长,PR 1 已经把 cross-process trace 跑通了 +- PR 1 是 additive structural(低风险),适合先建立信心 + +## 7. 测试计划 + +### 7.1 `sdk.ts` 单测 + +- ✅ `UndiciInstrumentation` 在 `NodeSDK` 的 `instrumentations` 中存在 +- ✅ `ignoreRequestHook` 对 `https://collector:4318/v1/traces` 返回 true +- ✅ `ignoreRequestHook` 对 `https://dashscope.aliyuncs.com/...` 返回 false +- ✅ trailing slash 与无 trailing slash 都正确匹配 + +### 7.2 `llm-correlation-fetch.ts` 单测 + +**`wrapFetchWithCorrelation`**: + +| 场景 | 期望 | +| ------------------------------------------------------- | ---------------------------------------------------------------------- | +| `getTelemetryEnabled() === false` | wrapped fetch = baseFetch(不加任何 header) | +| `getTelemetryEnabled() === true`, sessionId = "abc-123" | wrapped fetch 发出的 init.headers 含 `X-Qwen-Code-Session-Id: abc-123` | +| `init.headers` 已有 `X-Qwen-Code-Session-Id: spoof` | wrapper 后覆盖为真 sessionId(fetch wrapper 路径不允许 spoof,§8.1) | +| **session reset 后 wrapped fetch 被再次调用** | **读取新 sessionId**(regression guard for staleness fix) | +| baseFetch reject | wrapper 透传 reject 不吞 | + +**`staticCorrelationHeaders`**(Gemini path): + +| 场景 | 期望返回 | +| ------------------------------------------------------- | ---------------------------------------------------------------- | +| `getTelemetryEnabled() === false` | `{}` | +| `getTelemetryEnabled() === true`, sessionId = "abc-123" | `{ 'X-Qwen-Code-Session-Id': 'abc-123' }` | +| sessionId 中含 unicode(`會話-1`) | 原样返回——HTTP header value 由 SDK 负责编码 | +| sessionId 为空字符串 | `{ 'X-Qwen-Code-Session-Id': '' }`——业务 invariant,不在此层校验 | + +### 7.3 Per-provider 集成测试 + +每个 provider 的 `buildHeaders()` / 构造测试加: + +```ts +it('includes X-Qwen-Code-Session-Id when telemetry enabled', () => { + const config = makeFakeConfig({ + sessionId: 'sess-xyz', + telemetry: { enabled: true }, + }); + const provider = new DefaultProvider(genConfig, config); + expect(provider.buildHeaders()['X-Qwen-Code-Session-Id']).toBe('sess-xyz'); +}); + +it('omits X-Qwen-Code-Session-Id when telemetry disabled', () => { + const config = makeFakeConfig({ telemetry: { enabled: false } }); + const provider = new DefaultProvider(genConfig, config); + expect(provider.buildHeaders()).not.toHaveProperty('X-Qwen-Code-Session-Id'); +}); +``` + +### 7.4 E2E verification(tmux + local HTTP server) + +⚠️ **不要** mock `globalThis.fetch` 来抓 header:`UndiciInstrumentation` 通过 undici 的 diagnostics channel hook,monkey-patching globalThis.fetch 可能完全 bypass instrumentation(取决于 patch 顺序),让 `traceparent` 注入测不到。**正确做法是起 local HTTP server**,让 SDK 真发请求,server 端记录收到的 headers。 + +写一个仿 `/tmp/verify-telemetry-pr-4367.mjs` 的脚本: + +1. `http.createServer((req, res) => { capturedHeaders.push(req.headers); res.end('{}') })` 起本地 server +2. 启 telemetry + outfile + 把 OpenAI SDK 的 `baseURL` 指向 `http://127.0.0.1:`(或者用 mock provider 让 SDK 真发 fetch) +3. 触发一次 `client.chat.completions.create(...)`(要带最小可解析的 mock 响应,否则 SDK 解析报错——本地 server 返回合法但空的 OpenAI 响应即可) +4. 断言 `capturedHeaders[0]` 含 `traceparent: 00-...` 和 `X-Qwen-Code-Session-Id: ` +5. 另起一个 OTLP collector mock 在 different port,验证给它发的 OTLP 上报**不**触发 `traceparent` 注入(验证 `ignoreRequestHook`) +6. **额外:staleness 验证** — emit request 1 → call `config.resetSession(...)` → emit request 2 → 断言 request 2 的 `X-Qwen-Code-Session-Id` 是新 session id(**这是 #1 fix 的关键回归测试**) + +### 7.5 回归保护 + +- streaming chat completion 的 fetch(带 `stream: true`)仍正常关闭——`UndiciInstrumentation` 历史上对 streaming response 的 span lifecycle 有过 bug,**实施时需要实际跑一次 streaming completion 端到端验证 client span 正常 end + 无 leaked span + 流不被截断**;不假设具体版本号已修 +- proxy mode (`ProxyAgent`) 与 instrumentation 同时启用——`ignoreRequestHook` 仍按 endpoint 字符串匹配,proxy 不影响 +- 重试(`maxRetries`)下每次重试都得到独立 client span,但都共享同一个 `traceparent` parent(理想是 retry 作为同一个父 span 下多个 child span — 这部分由 SDK 行为决定,本设计不强制) + +## 8. 边界 / 边角 + +### 8.1 customHeaders override 与 spoofing 的不一致行为 + +不同 provider 路径的 spoofing 表面**不同**(设计后果,非原意收紧): + +| Provider 路径 | spoofing 可能? | 原因 | +| --------------------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------- | +| OpenAI / Anthropic (fetch wrapper 路径) | ❌ 不能 spoof | fetch wrapper 在 SDK headers list 之后 `headers.set('X-Qwen-Code-Session-Id', ...)`,覆盖 user customHeaders 的同名 | +| Gemini (static headers 路径) | ✅ 可 spoof | merge 顺序 `{ ...baseHeaders, ...correlationHeaders, ...customHeaders }`——customHeaders 最后赢 | + +claude-code 同样使用 fetch wrapper 路径,行为与 OpenAI/Anthropic 一致(spoofing 不能)。这是修 staleness bug 的副产品,不是原本要做的事。 + +**不打算"对齐"两条路径**——Gemini 路径的行为是 SDK 限制(没有 `fetch` hook)导致的,反向把 OpenAI 也降级到 static 不合理。 + +Session id spoofing 不是真威胁(用户控制本地,可以直接改 source code)。文档里要明示这个差异,避免 reviewer 看到 fetch wrapper 路径无法 spoof 时质疑 customHeaders 优先级。 + +### 8.2 OTLP collector URL 匹配的两类 edge case + +#### (a) Auth token in URL + +如果用户 OTLP endpoint 形如 `https://collector/path?token=secret`,`ignoreRequestHook` 的 `url.startsWith(e)` 比对应包含 query string。但 undici 给的 `request.path` 只到 path(不含 query),所以比较时 `e` 也只用到 path 部分。为安全起见,剥掉 query: + +```ts +const otlpUrls = [...] + .map((u) => u.replace(/\?.*$/, '').replace(/\/$/, '')); +``` + +#### (b) startsWith 跨 hostname 边界的理论 false positive + +若 `e = "http://collector"`(无 port),来路 url = `http://collector-fake/v1/traces` 会被 startsWith 错误匹配。 + +**实际触发概率极低**: + +- OTLP endpoint 几乎总带 port(4317 gRPC / 4318 HTTP),`http://collector:4318` 形态后 `-fake` 这种延伸不可能(port 后跟的是 `/`) +- 用户配 endpoint 不带 port 是配置错误,本来 SDK 就要默认 fallback + +**如果想 harden**:解析 URL origin + path 分别比较,不用裸 startsWith: + +```ts +const parsed = otlpUrls.map((u) => new URL(u)); +return parsed.some( + (e) => + `${request.origin}` === e.origin && request.path.startsWith(e.pathname), +); +``` + +本期不做——开销没必要,false positive 实际触发不到。 + +### 8.3 Vertex AI 模式的 Gemini + +`@google/genai` 支持 `vertexai: true` 模式(用 GCP 凭据走 Vertex 端点而非 generative ai endpoint)。两种模式都走 fetch,所以 instrumentation 都覆盖。`httpOptions.headers` 在两种模式下都有效。 + +### 8.4 Anthropic SDK 已有 `defaultHeaders` 逻辑 + +`anthropicContentGenerator.ts:177` 已经在调 `buildHeaders()` 然后传给 `new Anthropic({ defaultHeaders })`。但 staleness 同样适用——本设计改用 `fetch` wrapper 路径(与 OpenAI 一致)。 + +### 8.5 SDK 与 fetch 之间的 trailer header + +`openai` SDK 在 streaming 时可能用 `Transfer-Encoding: chunked` 和 trailer headers。这些都不影响 request-time 的 `traceparent` / `X-Qwen-Code-Session-Id` 注入——它们都是请求头,发出时一次性写入。 + +### 8.6 ⚠️ Known limitation: Gemini 的 session id 在 `/clear` 后 stale + +由于 `@google/genai` SDK 不支持 `fetch` hook(`HttpOptions` 接口只有 `baseUrl`/`apiVersion`/`headers`/`timeout`/`extraParams`),Gemini provider 走 static `httpOptions.headers` 路径——session id 在 SDK 构造时 capture,**`/clear` 触发 session reset 后不刷新**。 + +**实际影响范围**: + +- 用户启动 qwen-code → `/clear` → 用 Gemini 模型 → wire 上的 `X-Qwen-Code-Session-Id` 是旧 session id +- 后端 correlation 错位(trace id 和 log 已正确切换到新 session,但 wire header 滞后) + +**为什么不修**(本期): + +- OpenAI / Anthropic 路径**没有这个 bug**(fetch wrapper 路径 per-request 读 session id) +- Gemini fix path 有几个选项,全部超出本期 scope(见下) + +**Future fix path 选项**(按推荐顺序): + +| 选项 | 描述 | 代价 | +| --------------------------------------------- | ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------- | +| **A. Lazy invalidate** ★ 推荐 | session reset 时只 mark contentGenerator dirty,下次 LLM 调用时 lazy recreate | 小:~10 行加在 `resetSession` + LLM 调用入口;同步 API,无侵入 | +| B. Eager recreate | session reset 时立即 `await createContentGenerator(...)`,需 async 化 `resetSession` | 中:API 改动级联多处 | +| C. Proxy headers object | 给 `httpOptions.headers` 包 Proxy 拦截 getter | 风险高:`@google/genai` 内部是否 per-request 重读 headers 不可知,行为可能 silently break | +| D. 推动 `@google/genai` 上游加 `fetch` option | 提 PR 给 google-deepmind/generative-ai-js | 长期;不可控 | + +**文档要在用户面前说明**:使用 Gemini provider 时如果 `/clear` 后立刻有 LLM 调用,wire 上的 session id 在那一刻是旧的。可以靠 trace correlation 间接修正(spans/logs 上 session.id 已经是新的)。 + +应单开 follow-up sub-issue 跟踪选项 A。 + +## 9. 与 claude-code 对比 + +| 维度 | claude-code | qwen-code 本设计 | 决策依据 | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Session id header 命名 | `X-Claude-Code-Session-Id`(产品前缀) | `X-Qwen-Code-Session-Id`(产品前缀) | ✅ 同样命名空间策略 | +| Session id 注入机制 | SDK `defaultHeaders`(`client.ts:108`)+ 自定义 `buildFetch()` wrapper(`client.ts:370-390`,per-request `randomUUID()` 注入 `x-client-request-id`) | OpenAI/Anthropic 走 fetch wrapper(per-request 读 session id,避免 `/clear` staleness);Gemini 走 static `httpOptions.headers`(SDK 限制) | 与 claude-code 的 fetch wrapper 模式对齐。claude-code 也用 fetch wrapper 才能 per-request 加 `x-client-request-id` | +| Session id 持久性 | claude-code 没有 `/clear`-式 session reset;session = process | 有 `/clear` reset → fetch wrapper 路径自动跟随;static headers 路径会 stale(§8.6) | qwen-code 独有的复杂度 | +| Session id 编码 | HTTP header(不是 baggage) | HTTP header | ✅ 同——backend 友好 | +| `traceparent` 注入 | 闭源;公开 docs 描述存在;开源 repo 无 `propagation.inject` / `UndiciInstrumentation` 引用 | `@opentelemetry/instrumentation-undici` 自动 | claude-code 怎么实现的不可见。我们选 OTel 官方推荐路径,更轻 | +| `traceparent` 发送范围 | 仅第一方 Anthropic API;不发 Bedrock/Vertex/Foundry | 发给所有出站 fetch (W3C 标准;trace id 是 `sha256(sessionId)` 哈希)。**R3 修订**:session id header 仅向 first-party (Alibaba/DashScope) 白名单注入,第三方默认不发。详见 §11 | R3 后 qwen-code 的 session header 与 claude-code 同样的 first-party-only 语义;`traceparent` 仍待 per-destination toggle follow-up | +| `x-client-request-id` (随机) | 有,自动 | 暂不做(独立 follow-up sub-issue 价值更高) | 范围控制 | +| 子进程 `TRACEPARENT` env | 文档承认存在(实现闭源) | 不做(独立 follow-up) | 范围控制 | +| 入站 `TRACEPARENT` 读取 | 文档承认存在(`-p` / Agent SDK 模式) | 不做(独立 follow-up) | 范围控制 | + +**verified vs documented 注解**: + +| claim | 验证状态 | +| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `X-Claude-Code-Session-Id` via `defaultHeaders` | ✅ Open source `src/services/api/client.ts:108` 已读 | +| `x-client-request-id` via fetch wrapper | ✅ Open source `src/services/api/client.ts:370-390` 已读 | +| `traceparent` 注入 | ⚠️ 仅 docs.claude.com/docs/en/monitoring-usage.md 提到;开源 repo `grep -rn "propagation\.inject\|UndiciInstrumentation\|traceparent" src` 返回空 | + +## 10. 未来工作 + +挂在 #3731 P3 下,本设计**不**包含但与之相关: + +- **`X-Qwen-Code-Request-Id`** 随机 UUID per request(claude-code 等价:`x-client-request-id`)。对超时/timeout error correlation 有用——超时时服务端可能还没 assign request id,客户端先发的 id 是唯一关联手段。R3 修订后这个建议变得更有意义:per-request UUID 没有"跨请求行为画像"风险,可以作为"对所有 LLM provider 发送的支持/调试 header"。 +- **`traceparent` 的 per-destination scope toggle** — R3 修订仅处理了 session id header 的作用域;`traceparent` 仍向所有出站 fetch 注入。可以加 `telemetry.propagateTraceContext: 'trusted-hosts' | 'all' | 'none'`,使用与 §11 同一份 allowlist 决定行为。 +- **Gemini 的 session id staleness lazy-invalidate fix**(§8.6 选项 A):`/clear` 时 mark contentGenerator dirty,下次 LLM 调用 lazy recreate。让 Gemini 路径也享受 fetch wrapper 的实时性。 +- **子进程 `TRACEPARENT` env**:给 `BashTool` 执行子进程时注入 env,让外部工具能续传 trace。需要单独看 tool execution lifecycle。 +- **入站 `TRACEPARENT`**:`--prompt` 模式启动时读 env,让 CI / 外部 orchestrator 能把 qwen-code 接到更大的 trace。 +- **可配置 `correlationHeader` name**:让企业 ops 自定义 header(默认 `X-Qwen-Code-Session-Id`)。 +- **`baggage` propagation 策略**:是否主动 set baggage 让 `user.id` / `tenant.id` 等也走 baggage 传到下游。本期不做,等需求明确。 + +## 11. R3 修订 — Host-Allowlist Scoping for `X-Qwen-Code-Session-Id` + +> 触发:[LaZzyMan 在 PR #4390 的 REQUEST_CHANGES review](https://github.com/QwenLM/qwen-code/pull/4390) +> 落地 commit:`1c8528a56` (核心实现) + `cb162e716` (Vertex baseUrl fail-closed + `["*"]` trim 容错) + +### 11.1 触发与论证 + +R1 设计把 `X-Qwen-Code-Session-Id` 向**所有**出站 LLM 请求注入,仅由 `telemetry.enabled` 控制。LaZzyMan review 指出了三个递进的问题: + +1. **标签错位**:`feat(telemetry):` + `telemetry/` 路径 + `getTelemetryEnabled()` gate 让用户合理理解为"自家可观测性数据流向自家 collector"。但 `X-Qwen-Code-Session-Id` 不会到达 OTLP 后端,它走在 LLM API 请求里发给 DashScope / OpenAI / Anthropic / Gemini / OpenRouter / MiniMax / ModelScope / Mistral。两种不同的数据出口决策绑在一个开关上。 + +2. **claude-code 类比不成立**:R1 在 §9 把命名空间策略和 fetch wrapper 模式都"对齐"了 claude-code。但 claude-code 是 Anthropic 一方 → Anthropic 一方(single vendor, single direction),qwen-code 是开源 CLI → 多个第三方 provider。"一个稳定 cross-request UUID 广播到所有第三方"是 R1 没正面回答的问题。 + +3. **traceparent 是同一指纹的另一通道**:trace id = `sha256(sessionId).slice(0, 32)`,对接收方来说仍是稳定 per-session 标识符(哈希后不可逆,但同一 session 仍稳定)。 + +LaZzyMan 标定 severity:session id `high` / traceparent `medium`。 + +### 11.2 解法概要 + +**收窄默认作用域到 first-party hosts**。新增一项 setting: + +```jsonc +"telemetry": { + "sessionIdHeaderHosts": ["*"] // 恢复 R1 广播行为 + "sessionIdHeaderHosts": [] // 全关 header + "sessionIdHeaderHosts": ["api.mycompany.com", + "*.gateway.mycompany.internal"] +} +``` + +默认值(来自 `packages/core/src/telemetry/trusted-llm-hosts.ts:DEFAULT_SESSION_ID_HEADER_HOSTS`): + +``` +dashscope.aliyuncs.com +dashscope-intl.aliyuncs.com +*.dashscope.aliyuncs.com +*.dashscope-intl.aliyuncs.com +*.alibaba-inc.com +*.aliyun-inc.com +``` + +这个集合的语义是"LLM provider、ARMS Tracing 后端、qwen-code distribution 同一法律实体"——也就是 claude-code 那个 single-vendor / single-direction 关系在 qwen-code 的对应集合。第三方 provider(OpenAI / Anthropic / OpenRouter / 等)默认**不**接收 header。 + +### 11.3 Pattern 语法(intentionally tiny) + +`matchesTrustedHost(hostname, patterns)` 只支持两种模式,与 `DashScopeOpenAICompatibleProvider.isDashScopeProvider` 对齐: + +- bare hostname → 精确匹配(case-insensitive) +- `*.suffix` → 匹配 `suffix` 自身 **AND** 任何子域;dot-anchored 拒绝 `evil-alibaba-inc.com` / `alibaba-inc.com.attacker.tld` 等 typo-suffix 攻击向量 + +不引入 regex、不引入端口/scheme 感知 globbing —— 让 settings 里的字符串就是它字面看起来的语义。 + +### 11.4 实现差异 vs R1 + +#### `wrapFetchWithCorrelation` (OpenAI / Anthropic) + +R1 的 wrapper 只有 telemetry-enabled + sessionId 两个 gate。R3 在两者之间插入第三个 gate: + +```ts +const trustedHosts = + config.getTelemetrySessionIdHeaderHosts?.() ?? + DEFAULT_SESSION_ID_HEADER_HOSTS; +const broadcastAll = trustedHosts.some((p) => p.trim() === '*'); + +return async function correlationFetch(input, init) { + if (!config.getTelemetryEnabled()) return baseFetch(input, init); + if (!broadcastAll) { + const host = extractRequestHost(input); + if (!host || !matchesTrustedHost(host, trustedHosts)) { + return baseFetch(input, init); // host gate + } + } + const sid = config.getSessionId(); + if (!sid) return baseFetch(input, init); + // ... header injection +}; +``` + +`trustedHosts` 在 wrap 时一次性 snapshot(与 session id 的"每请求实时读"不同)。中途修改 `telemetry.sessionIdHeaderHosts` 需要重建 contentGenerator 才生效。`[" * "]` 之类带空格的写法通过 `.trim()` 兜底成 broadcast,避免 settings.json 手敲笔误沉默退化。 + +#### `staticCorrelationHeaders` (Gemini) + +签名加一个 `destinationUrl?: string` 参数: + +```ts +export function staticCorrelationHeaders( + config: Config, + destinationUrl?: string, +): Record { + if (!config.getTelemetryEnabled()) return {}; + if (!destinationUrl) return {}; // fail-closed: 不知道目的地就不发 + if (!matchesTrustedHost(new URL(destinationUrl).hostname, trustedHosts)) { + return {}; + } + return { [SESSION_ID_HEADER]: config.getSessionId() }; +} +``` + +#### Gemini factory 集成 + +Gemini SDK 有两个不可见 default endpoint(`generativelanguage.googleapis.com` 与 `{region}-aiplatform.googleapis.com`,由 `vertexai` 决定),factory 层无法准确还原其中之一。R3 选择"`config.baseUrl` 没设就传 `undefined`",让 helper fail-closed → 不发 header。运营商想要相关性必须显式设 `baseUrl`(也是 SDK 自己用来解 destination 的同一输入)。这一改动避免了猜错 Vertex destination 后被允许列表错误命中。 + +### 11.5 新文件 / 新代码 + +| 文件 | 说明 | +| -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `packages/core/src/telemetry/trusted-llm-hosts.ts` (NEW) | `DEFAULT_SESSION_ID_HEADER_HOSTS` + `matchesTrustedHost` + `extractRequestHost` | +| `packages/core/src/telemetry/trusted-llm-hosts.test.ts` (NEW) | 单测,含 TLD-suffix 攻击向量、IPv6 fail-closed、port/userinfo/query 提取 | +| `packages/core/src/telemetry/llm-correlation-fetch.ts` | 加 host gate;`staticCorrelationHeaders` 加 `destinationUrl` 参数 | +| `packages/core/src/telemetry/llm-correlation-fetch.test.ts` | 加 host-gate 8 个 case;`mockConfig` 用 `'hosts' in opts` 区分 "default allowlist" vs "broadcast" | +| `packages/core/src/telemetry/config.ts` (`resolveTelemetrySettings`) | 透传 `sessionIdHeaderHosts` | +| `packages/core/src/config/config.ts` | `TelemetrySettings.sessionIdHeaderHosts` + `getTelemetrySessionIdHeaderHosts()` getter | +| `packages/core/src/core/geminiContentGenerator/index.ts` | 传 `config.baseUrl` 给 helper;fail-closed when undefined | +| `packages/core/src/core/geminiContentGenerator/index.test.ts` | 重写 telemetry-on Gemini 测试以匹配新 fail-closed 语义 | +| `packages/cli/src/config/settingsSchema.ts` | `sessionIdHeaderHosts` JSON schema 入口 | +| `packages/vscode-ide-companion/schemas/settings.schema.json` | 由 `npm run generate:settings-schema` 重新生成 | +| `docs/developers/development/telemetry.md` | "Session correlation header" 段落改写 + 默认 scope + override 语法 | + +### 11.6 对各 LazzyMan 论点的回应 + +| LazzyMan 论点 | R3 回应 | +| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ① telemetry 标签错位 | **化解**:在 DashScope 用例下,session id header 字面就是发给 ARMS Tracing 后端(同一法律实体),`telemetry.enabled` 语义对齐 | +| ② cross-vendor stable identifier 广播 | **化解**:默认 allowlist 只含阿里系 first-party host;广播退化为 opt-in (`["*"]`) | +| ③ traceparent 是同一指纹的另一通道 | **暂保留**:traceparent 仍按 R1 全注入。理由:W3C 标准、trace id 是 sha256 哈希、in-vendor trace 续接是 W3C 的核心设计场景。per-destination traceparent toggle 列入 §10 future work | + +### 11.7 已知遗留 + 跟进 + +- **traceparent scope** — 见上文第 ③ 点,列入 §10 +- **Per-request random UUID** (`X-Qwen-Code-Request-Id`) — LazzyMan 提的替代方案,列入 §10 +- **Gemini staleness lazy-invalidate** (§8.6 选项 A) — 与 R3 解耦,独立 sub-issue +- **`matchesTrustedHost` IPv6 支持** — 当前 IPv6 destination 永不在 allowlist 上(`URL.hostname` 返回 `[::1]` 带方括号,pattern 语法无对应形式)。当前满足"命名 first-party endpoint"用例。若将来有 raw IP allowlist 需求再扩展。 + +## 12. R4 修订 — Scope Conflation Split + +> 触发:[LaZzyMan round-8 follow-up review on PR #4390](https://github.com/QwenLM/qwen-code/pull/4390) +> 落地:本 PR 收窄;R3 落地的 session-id 整套挪到独立 follow-up PR + +### 12.1 触发与论证 + +R3 化解了 LaZzyMan 第一轮 review 的「广播稳定指纹给第三方 provider」担忧(severity: high)。但在 round-8 follow-up 中他升级到更深的架构原则反对: + +> "Telemetry is not a container for adjacent features. The `traceparent` cross-process propagation and the `X-Qwen-Code-Session-Id` header injection are **not telemetry**. They are outbound-identity / outbound-correlation work that uses some OTel APIs internally as an implementation detail." + +他的核心元论点: + +- **"telemetry" namespace 暗示 recipient = 用户自己的 OTLP collector** +- 但 `traceparent` 和 `X-Qwen-Code-Session-Id` 的 recipient = **第三方 LLM provider** +- 两类不同 recipient 应该有两类不同的同意决策树 +- 即使默认行为安全(R3 已实现),把 wire-level 行为放在 `telemetry.*` 下**设了坏先例**:未来 telemetry PR 可以继续偷渡 wire 行为给第三方 +- "If we accept that principle, the split is mechanical. If we don't, this PR is the wrong place to debate it because the technical fixes are already in." + +### 12.2 解法概要("方案 C" hybrid split) + +经过几轮内部讨论(含 yiliang 提出的 customHeader 模板替代方案,最终判定 customHeader 不能携带 runtime-dynamic 值),决定走 **方案 C**: + +**本 PR 留下**: + +- `UndiciInstrumentation` 注册(产 client HTTP span → 用户自家 OTLP collector) +- OTLP feedback-loop guard(前者的必要副作用) +- **`NoopTextMapPropagator` 默认安装** → `propagation.inject()` 是 no-op → outbound `fetch` 上**不再有 `traceparent`** +- **新增 `outboundCorrelation.propagateTraceContext: bool` (默认 false)** 作为独立 namespace 顶级设置;设 true 时安装默认 W3C composite propagator +- 整套 `R3 session-id` 代码(`llm-correlation-fetch.ts` / `trusted-llm-hosts.ts` / `telemetry.sessionIdHeaderHosts` setting / 4 个 provider 集成点 / 所有相关测试)**全部移除** + +**搬到 follow-up PR**: + +- `X-Qwen-Code-Session-Id` header 整套机器(R3 实现复用) +- 进入新 `outboundCorrelation.*` namespace(具体 setting key TBD,但**不会**叫 `telemetry.*`) +- Follow-up PR 自带:threat model section、独立 review、security-relevant 标注的 docs +- `X-Qwen-Code-Request-Id` per-request UUID(LazzyMan 在 R3 round 提出的替代设计)也归入此 follow-up 的考虑范围 + +### 12.3 与 R3 R1 论点的映射 + +| R1/R3 论点 | R4 后状态 | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| §3.1 "所有出站 LLM 请求带 traceparent" | ❌ **R4 默认 off**;需 `outboundCorrelation.propagateTraceContext: true` 才开 | +| §3.1 "所有出站 LLM 请求带 `X-Qwen-Code-Session-Id`" | ❌ **R4 整套移出本 PR**,搬到 follow-up PR | +| §4.3 fetch wrapper 注入 session id | ❌ 整段代码不在本 PR;复用到 follow-up PR | +| §11 host allowlist (R3 设计) | ❌ 同上;整体迁移 follow-up PR | +| §4.4 不引入新 setting | ❌ **本 PR 新增 `outboundCorrelation.propagateTraceContext`** 一个 boolean;session id 相关 setting 在 follow-up PR | +| §10 future work "`X-Qwen-Code-Request-Id`" | ✅ 仍是 future work;与 session-id follow-up 一起设计 | + +### 12.4 新 namespace 设计意图 + +`outboundCorrelation.*` 顶级 namespace 在本 PR 只有一个 boolean (`propagateTraceContext`),看起来过度结构化。但这是**精心选择的**: + +- **建立命名空间作为承诺**:让后续 session-id / request-id / etc. 自然进入这个 namespace +- **标注为 security-relevant**:`settingsSchema.ts` description 显式写 "SECURITY-RELEVANT",文档化为"安全设置"而非"observability 设置" +- **defaults 全部 off**:符合 LazzyMan 提出的"open-source 客户端不应未经显式同意向第三方发稳定 id"原则 +- **与 telemetry.\* 解耦**:用户读 settings.json 看到 `outboundCorrelation.*` 立刻能识别这是出站 wire 行为,不是 observability + +#### 隐性依赖:`telemetry.enabled` + +虽然 namespace 与 `telemetry.*` 解耦,**运行时生效仍依赖 `telemetry.enabled: true`** —— OTel SDK 只在 telemetry 启用时初始化,没有 SDK 就没有 propagator 安装、没有 `propagation.inject()` 调用,flag 等于沉默 no-op。容易踩的 footgun:运营商加 `propagateTraceContext: true` 却忘开 telemetry,trap server 上看不到任何 `traceparent`,无 error / 无 warning。 + +两个面向用户的面板都显式标注此依赖: + +- `telemetry.md` 的 `propagateTraceContext` 段附完整双 flag JSON 示例 +- `settingsSchema.ts` 的 description string **首句**即 "Requires `telemetry.enabled: true`"(前置以避免 VS Code 设置 UI 长描述折叠后看不到) + +未来若添加 session-id header 或其他 `outboundCorrelation.*` setting,**同一依赖关系适用** —— 都得在 telemetry 启用前提下才有意义(因为它们都通过 OTel instrumentation/SDK 注入)。Follow-up PR 应继承此 footgun 提示模式。 + +### 12.5 实施 + +| 文件 | 改动 | +| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `packages/core/src/telemetry/llm-correlation-fetch.ts` | **删除** | +| `packages/core/src/telemetry/llm-correlation-fetch.test.ts` | **删除** | +| `packages/core/src/telemetry/trusted-llm-hosts.ts` | **删除** | +| `packages/core/src/telemetry/trusted-llm-hosts.test.ts` | **删除** | +| `packages/core/src/telemetry/sdk.ts` | + `NoopTextMapPropagator`;按 `getOutboundCorrelationPropagateTraceContext()` 决定 SDK textMapPropagator | +| `packages/core/src/core/openaiContentGenerator/provider/default.ts` | 移除 `wrapFetchWithCorrelation` 引用 | +| `packages/core/src/core/openaiContentGenerator/provider/dashscope.ts` | 同上 | +| `packages/core/src/core/anthropicContentGenerator/anthropicContentGenerator.ts` | 同上 | +| `packages/core/src/core/geminiContentGenerator/index.ts` | 移除 `staticCorrelationHeaders` 引用 | +| 上述 4 个 provider 的 `*.test.ts` | 删 session-id 相关测试 case | +| `packages/core/src/config/config.ts` | 删 `TelemetrySettings.sessionIdHeaderHosts`、`getTelemetrySessionIdHeaderHosts`;**新增 `OutboundCorrelationSettings` 接口 + `outboundCorrelationSettings` 字段 + `getOutboundCorrelationPropagateTraceContext()` getter** | +| `packages/core/src/telemetry/config.ts` | 删 `resolveTelemetrySettings` 中 sessionIdHeaderHosts 透传 | +| `packages/cli/src/config/settingsSchema.ts` | 删 `sessionIdHeaderHosts` schema;**新增 `outboundCorrelation` 顶级 schema 项** | +| `packages/cli/src/config/config.ts` | 透传 `outboundCorrelation: settings.outboundCorrelation` 进 `ConfigParameters` | +| `packages/vscode-ide-companion/schemas/settings.schema.json` | `npm run generate:settings-schema` 重新生成(description 后续更新时同步刷新) | +| `docs/developers/development/telemetry.md` | 重写 "Trace context propagation" → "Client-side HTTP span on outbound fetch";删 "Session correlation header" 整节;新增 "Outbound correlation (SECURITY-RELEVANT)" 顶级 section;附 `telemetry.enabled` 依赖说明 + JSON 配置示例 | +| `docs/design/telemetry-outbound-propagation-design.md` | 本节 + R4 表头 + 修订指针 | +| `packages/core/src/config/config.test.ts` | **新增 `OutboundCorrelation Configuration` describe block**,`it.each` 4 个 case 锁定 `getOutboundCorrelationPropagateTraceContext` 的 default-false 安全不变性(omitted / `{}` / explicit true / explicit false) | + +### 12.6 对 LazzyMan 元论点的回应 + +| 论点 | R4 后状态 | +| ----------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| "Telemetry namespace 暗示自家 collector 接收方" | ✅ wire 行为已搬出 `telemetry.*`;新 `outboundCorrelation.*` namespace 显式标识"出站第三方"语义 | +| "默认行为不应未经显式同意向第三方发标识符" | ✅ `propagateTraceContext` 默认 false;session-id 整套 follow-up PR 也将默认 off | +| "telemetry PR 不应偷渡 wire-level 行为" | ✅ 本 PR 不再添加任何"telemetry 控制 wire 行为"的代码路径;wire 行为统一由 `outboundCorrelation.*` 管 | +| "split is mechanical, work isn't wasted" | ✅ R3 落地代码物理删除自本 branch,留在 git history 里给 follow-up PR 复用(或 cherry-pick) | + +### 12.7 follow-up PR 大纲(信息性,不在本 PR 范围) + +未来 follow-up PR 应包含: + +- `outboundCorrelation.sessionIdHeader: { enabled, trustedHosts }` 或类似 setting +- 复用 R3 已实现的 `wrapFetchWithCorrelation` / `matchesTrustedHost` / `DEFAULT_SESSION_ID_HEADER_HOSTS` 代码骨架 +- threat model 一节,明确:recipient 集合、稳定 id 的去匿名化窗口、可选 per-request UUID 配套 +- **默认 off**(无 default allowlist —— 比 R3 更严,符合 LazzyMan 的开源 CLI 原则) +- security-relevant 标注 + docs/users/configuration/settings.md 收录 diff --git a/docs/design/telemetry-resource-attributes-design.md b/docs/design/telemetry-resource-attributes-design.md new file mode 100644 index 00000000000..01ec84efa5b --- /dev/null +++ b/docs/design/telemetry-resource-attributes-design.md @@ -0,0 +1,762 @@ +# Telemetry: Custom Resource Attributes + Metric Cardinality Controls + +> 配套 issue: [#4365](https://github.com/QwenLM/qwen-code/issues/4365) +> 父 issue: [#3731](https://github.com/QwenLM/qwen-code/issues/3731) +> 基于 2026-05-21 对 qwen-code main 分支的代码复核 + +## 1. 背景 + +qwen-code 已经接入 OpenTelemetry SDK,但 Resource 构造方式让它在两个常见生产场景下不可用: + +1. **无法附加自定义维度**:运维侧想给所有 telemetry 数据打 `team` / `env` / `cost_center` / `user_id` 标签,今天没有任何机制可以做到。即使设置标准的 `OTEL_RESOURCE_ATTRIBUTES` 环境变量也**完全不生效**。 +2. **指标基数(cardinality)失控**:`session.id` 被注入到了 Resource 层,会自动附着到每条 metric 数据点。每个 CLI session 产生一个新值,指标后端(Prometheus / 阿里云 ARMS Metric / VictoriaMetrics)会被无界 time-series 撑爆。 + +这两个问题耦合在一起:解决前者会让用户**更容易**给数据加高基数的字段,所以必须配套提供后者。 + +## 2. 现状 + +### 2.1 Resource 构造 + +`packages/core/src/telemetry/sdk.ts:156-161`: + +```ts +const resource = resourceFromAttributes({ + [SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME, + [SemanticResourceAttributes.SERVICE_VERSION]: + config.getCliVersion() || 'unknown', + 'session.id': config.getSessionId(), +}); +``` + +`sdk.ts:274-278`: + +```ts +sdk = new NodeSDK({ + resource, + // Disable async host/process/env resource detectors: they leave attributes + // pending and trigger an OTel diag.error on any resource attribute read + // before the detectors settle (e.g. during HttpInstrumentation span creation). + autoDetectResources: false, + ... +}); +``` + +`autoDetectResources: false` 关闭了标准 OTel 的 `envDetector`——也就是平时会读取 `OTEL_RESOURCE_ATTRIBUTES` 和 `OTEL_SERVICE_NAME` 的那一层。这是有原因的(detector 异步,会在 settle 前触发 `diag.error`),但副作用是这两个标准环境变量在 qwen-code 里**完全无效**。 + +### 2.2 `session.id` 实际是三重注入 + +| 位置 | 行号 | 影响 | +| --------------------------- | ------------------------ | ------------------------------------- | +| Resource | `sdk.ts:160` | 所有 signal(spans / logs / metrics) | +| Per-span | `session-tracing.ts:169` | spans | +| Per-log | `loggers.ts:128` | logs | +| **`getCommonAttributes()`** | `metrics.ts:57` | **每条 metric record 显式叠加** | + +也就是说**单独把 `session.id` 从 Resource 拿掉是不够的**——`metrics.ts:57` 的 `baseMetricDefinition.getCommonAttributes()` 会被 30+ 个 metric 调用点 `...spread` 进去,再次塞回 `session.id`。 + +```ts +// metrics.ts:55-59 +const baseMetricDefinition = { + getCommonAttributes: (config: Config): Attributes => ({ + 'session.id': config.getSessionId(), + }), +}; +``` + +好消息:所有 metric 调用点(30+ 个)都走这一个函数,是天然的 chokepoint。 + +### 2.3 config resolver 模式 + +`packages/core/src/telemetry/config.ts:resolveTelemetrySettings()` 用统一的优先级链: + +``` +argv (highest) > QWEN_* env > OTEL_* env > settings.json (lowest) +``` + +新加项照搬这个 pattern。 + +### 2.4 settings schema 现状 + +`packages/cli/src/config/settingsSchema.ts:998-1018` 定义 `telemetry` 的 JSON schema: + +```ts +telemetry: { + type: 'object', + // ... + jsonSchemaOverride: { + type: 'object', + properties: { + includeSensitiveSpanAttributes: { ... }, + }, + additionalProperties: true, // ← 今天对其他 telemetry.* key 不校验 + }, +} +``` + +`additionalProperties: true` 意味着今天 schema 对 `otlpEndpoint` / `otlpProtocol` / `resourceAttributes` 等其他字段全部放行不校验。新加 `resourceAttributes` / `metrics` 字段时,应同步在这里补 schema,方便 IDE 自动补全和 settings UI 渲染。 + +### 2.5 不在本设计范围的代码路径 + +`packages/core/src/telemetry/qwen-logger/qwen-logger.ts` 是 qwen-code 的**第一方使用上报通道**(基于阿里 RUM 内部协议 `RumResourceEvent`),与 OTel SDK 完全独立。它有自己的 endpoint、proxy 和数据模型,**不受本设计影响**。详见第 3 节。 + +### 2.6 已支持 / 未支持的 `OTEL_*` 环境变量 + +| 环境变量 | 现状 | +| --------------------------------------------------- | --------------------------------- | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | ✅ 支持(`config.ts:79`) | +| `OTEL_EXPORTER_OTLP_{TRACES,LOGS,METRICS}_ENDPOINT` | ✅ 支持 | +| `OTEL_EXPORTER_OTLP_HEADERS` | ✅ 底层 exporter 直接读取 | +| `OTEL_TRACES_SAMPLER` | ✅ 支持(`tracer.ts:247`) | +| **`OTEL_RESOURCE_ATTRIBUTES`** | ❌ 完全不支持 | +| **`OTEL_SERVICE_NAME`** | ❌ 完全不支持 | +| **`OTEL_METRICS_INCLUDE_*`** | ❌ 完全不支持(claude-code 风格) | + +## 3. 目标 / 非目标 + +### 3.1 目标 + +- 让运维通过标准 `OTEL_RESOURCE_ATTRIBUTES` 和自家 `settings.json` 给所有 OTLP 导出的 span / log / metric 附加自定义 resource attributes +- 让 `OTEL_SERVICE_NAME` 按 OTel 规范工作(包括与 `OTEL_RESOURCE_ATTRIBUTES` 里的 `service.name` 的优先级) +- 默认情况下,metric 上**不**携带 `session.id`(保护后端基数) +- 提供显式开关让需要 metric-level session correlation 的用户重新打开 +- 保留 spans 和 logs 上的 `session.id`(trace correlation 必须) +- 保留 `autoDetectResources: false`,不退化 `diag.error` 那个已修的 bug +- 配套更新 `settingsSchema.ts` 让新字段对 settings UI 和 IDE 可见 + +### 3.2 非目标 + +- **`qwen-logger` 第一方上报**:完全独立的 RUM 通道,不在本设计范围。其上报字段(device id、user agent 等)由 RUM 协议决定,不应被用户 resource attribute 干扰。若未来要给 `qwen-logger` 增加自定义维度,是另一条独立的设计。 +- **Per-span 动态 attribute hook**:让用户写代码 / hook 给每个 span 计算 attribute。claude-code 也没解决这块,复杂度高、收益低。 +- **`service.version` cardinality 控制**:版本变化频率有限(月级),time series 增长可控。需要时走 v2,引入 OTel View API。 +- **Agent SDK 形态的 per-query resource attrs**:qwen-code 目前没有 SDK 调用场景。 +- **OTLP 请求头(auth headers)配置**:是另一条 issue 线(#3731 P1),与本设计独立。 +- **CLI flag 形式的 resource attribute**:env var + settings.json 已覆盖临时与基线两种场景,CLI flag 会让命令行变得啰嗦,无明显增益。 + +## 4. 设计 + +### 4.1 总体分层 + +``` +┌─ Resource(sdk.ts:156)────────────────────────────────────────┐ +│ service.name ← OTEL_SERVICE_NAME │ +│ > OTEL_RESOURCE_ATTRIBUTES.service.name│ +│ > 'qwen-code' │ +│ service.version ← config.getCliVersion() [reserved] │ +│ ...user attrs ← OTEL_RESOURCE_ATTRIBUTES │ +│ + settings.resourceAttributes │ +│ ✗ session.id 移走 │ +└────────────────────────────────────────────────────────────────┘ + │ + ├──→ Spans + session.id(session-tracing.ts:169,保留) + ├──→ Logs + session.id(loggers.ts:128,保留) + └──→ Metrics + getCommonAttributes() — 默认 {} + toggle ON: { session.id } +``` + +### 4.2 优先级 / merge 顺序 + +#### 一般 attribute + +低 → 高: + +1. `OTEL_RESOURCE_ATTRIBUTES`(标准 OTel env var) +2. `settings.telemetry.resourceAttributes` +3. 内建保留键(覆盖以上任何同名) + +**理由**:环境变量是 ops-time 临时覆盖(CI / 单机 debug),settings.json 是 fleet-baked 基线,内建是产品契约——基线优先级应高于临时变量,内建优先级应高于一切。 + +#### `service.name` 特殊处理 + +`service.name` 必须遵守 [OTel 规范](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/): + +> **`OTEL_SERVICE_NAME` takes precedence over `service.name` defined with the `OTEL_RESOURCE_ATTRIBUTES` variable.** + +因此对 `service.name` 单独应用这条优先级链(高 → 低): + +1. `OTEL_SERVICE_NAME`(最高,标准 OTel 规范规定) +2. `settings.resourceAttributes.service.name`(settings 优先于 env,沿用本设计一般规则) +3. `OTEL_RESOURCE_ATTRIBUTES.service.name` +4. 内建默认 `'qwen-code'` + +`service.name` 允许通过 settings 覆盖——它是 service 身份,企业 fleet 用统一 settings.json 配置 service.name 是常见且合理的做法,禁止反而会阻断 GitOps 分发场景。`OTEL_SERVICE_NAME` 作为标准 OTel 规范规定的"最高优先级"通道,仍然可以在 CI / 单机调试时临时覆盖 settings。 + +具体规则: + +| 来源 | 写入 `service.name` 是否生效 | +| ------------------------------------------------------- | -------------------------------------- | +| `OTEL_SERVICE_NAME=foo` | ✅ 最高优先级(覆盖任何其他来源) | +| `settings.resourceAttributes={ "service.name": "foo" }` | ✅ 仅在没有 `OTEL_SERVICE_NAME` 时生效 | +| `OTEL_RESOURCE_ATTRIBUTES=service.name=foo` | ✅ 仅在以上两者都没有时生效 | + +### 4.3 保留键策略 + +| 键 | 用户能否覆盖 | 理由 | +| ----------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `service.name` | ✅ env var + settings 都可(见 §4.2 优先级链) | service 身份,应允许 ops 控制 | +| `service.version` | ❌ 任何来源都丢弃 + warn | 遥测可信度——不允许用户谎报版本 | +| `session.id` | ❌ 任何来源都丢弃 + warn(在 metric 上额外有 toggle 控制 runtime 注入) | runtime-only;用户写到 Resource 会绕过 metric cardinality toggle(Resource attr 自动附到所有 signal) | +| `qwen.*` 前缀 | ⚠️ 不强制保留,但 docs 建议留给产品自用 | 避免未来内建 attr 与用户 attr 冲突 | + +**保留键以常量集中维护**: + +```ts +// telemetry/resource-attributes.ts (new file) +/** Keys that cannot be overridden from any source (env or settings). */ +export const RESERVED_RESOURCE_ATTRIBUTE_KEYS = new Set([ + 'service.version', + 'session.id', +]); +``` + +`service.name` **不**在 RESERVED 列表里——它走自己的优先级链(§4.2),不属于"全局禁止覆盖"语义。RESERVED 是"任何来源写了都警告并丢弃",统一适用于 env 和 settings 两个入口。 + +### 4.4 `OTEL_RESOURCE_ATTRIBUTES` 解析 + +同步实现,绕开 OTel 自带的异步 envDetector: + +```ts +function parseOtelResourceAttributes( + raw: string | undefined, +): Record { + if (!raw) return {}; + const out: Record = {}; + for (const pair of raw.split(',')) { + const trimmed = pair.trim(); + if (!trimmed) continue; + const idx = trimmed.indexOf('='); + if (idx <= 0) { + diag.warn( + `Skipping malformed OTEL_RESOURCE_ATTRIBUTES entry: ${trimmed}`, + ); + continue; + } + const key = trimmed.slice(0, idx).trim(); + const valueRaw = trimmed.slice(idx + 1).trim(); + if (!key) continue; + let value: string; + try { + value = decodeURIComponent(valueRaw); + } catch { + diag.warn( + `Invalid percent-encoding in OTEL_RESOURCE_ATTRIBUTES for key "${key}", using raw value`, + ); + value = valueRaw; + } + out[key] = value; // duplicate keys: last wins (matches OTel reference impls) + } + return out; +} +``` + +格式严格按 OTel 规范:`key1=val1,key2=val2`,值 percent-encoded。 + +### 4.5 Metric attribute filter + +唯一改动点 `metrics.ts:55-59`: + +```ts +const baseMetricDefinition = { + getCommonAttributes: (config: Config): Attributes => { + const out: Attributes = {}; + if (config.getTelemetryMetricsIncludeSessionId()) { + out['session.id'] = config.getSessionId(); + } + return out; + }, +}; +``` + +调用点(30+ 个)零改动——`...spread` 一个空对象等价于不展开任何字段。 + +### 4.6 边界情况与校验 + +| 输入 | 行为 | +| ---------------------------------------------------------------- | ----------------------------------------------------------------------- | +| `OTEL_RESOURCE_ATTRIBUTES=""` (空字符串) | 返回 `{}`,正常启动 | +| `OTEL_RESOURCE_ATTRIBUTES="a"` (无 `=`) | 跳过该项 + `diag.warn`,继续解析其余 | +| `OTEL_RESOURCE_ATTRIBUTES="=val"` (空 key) | 跳过该项,继续解析其余 | +| `OTEL_RESOURCE_ATTRIBUTES="a=,b=2"` (空 value) | `a=''`, `b='2'`(OTel 规范允许空 value) | +| `OTEL_RESOURCE_ATTRIBUTES="a=val%ZZbad"` (无效 percent-encoding) | 保留原始 `val%ZZbad` + `diag.warn` | +| `OTEL_RESOURCE_ATTRIBUTES="a=1,a=2"` (duplicate key) | 后写胜出 `a=2`(与 OTel SDK 参考实现一致) | +| `OTEL_RESOURCE_ATTRIBUTES="a=1, b=2 "` (含空格) | 自动 trim | +| `OTEL_RESOURCE_ATTRIBUTES=service.version=x` | 静默丢弃 `service.version` + `diag.warn`,保留其他键 | +| `settings.resourceAttributes={ "service.name": "x" }` | 接受(settings 可设 service.name,见 §4.2) | +| `settings.resourceAttributes={ "service.version": "x" }` | 静默丢弃 + `diag.warn` | +| `settings.resourceAttributes={ "team": 123 }` (非 string) | TypeScript 类型阻挡;runtime 传入则 settings JSON schema validator 拒绝 | +| Resource 总大小 > OTel 限制 (4KB?) | 由底层 OTel SDK 处理,不在本层校验 | + +**为什么不在本层做 attribute key 命名校验**(如 OTel 推荐的 `[a-z][a-z0-9_.]*` 模式):OTel SDK 自己会在 export 时校验,本层重复校验既慢又容易和 SDK 行为偏移。我们只做格式解析,不做语义校验。 + +**RESERVED 键的强制保护对两个入口都生效**: + +```ts +// 应用于 env-parsed attrs +for (const k of RESERVED_RESOURCE_ATTRIBUTE_KEYS) { + if (k in envAttrs) { + diag.warn(`OTEL_RESOURCE_ATTRIBUTES cannot override "${k}"; ignoring`); + delete envAttrs[k]; + } +} + +// 应用于 settings attrs +for (const k of RESERVED_RESOURCE_ATTRIBUTE_KEYS) { + if (k in settingsAttrs) { + diag.warn( + `settings.telemetry.resourceAttributes cannot override "${k}"; ignoring`, + ); + delete settingsAttrs[k]; + } +} +``` + +### 4.7 生命周期与多进程 + +- **SDK init 时机**:Resource 在 `initializeTelemetry()` 时一次性构造,**进程内不可变**。这与 OTel SDK 设计一致。 +- **Subagent fork**:qwen-code 的 subagent 是同进程内的 (`subagent-runtime.ts`),共享 Resource。若未来引入跨进程 subagent,子进程会**重新 init SDK**,重新读 env var 和 settings——只要 env 透传过去,行为一致。 +- **Hot reload**:settings 修改后**不会重新构造 Resource**。需要操作员重启 CLI 才能生效。文档应明确说明。 +- **`refreshSessionContext()`** (`sdk.ts:306`):仅刷新 session ALS context,**不重建 Resource**——因为 Resource 上已经没有 `session.id` 了(本设计的核心改动之一)。 + +## 5. Config schema 改动 + +### 5.1 `TelemetrySettings` 接口(`packages/core/src/config/config.ts:293`) + +```ts +export interface TelemetrySettings { + // ... existing fields + /** Static resource attributes attached to every span/log/metric. */ + resourceAttributes?: Record; + /** Per-signal cardinality controls. */ + metrics?: { + /** Include session.id on metric data points (default: false). */ + includeSessionId?: boolean; + }; +} +``` + +### 5.2 `Config` getter(同文件) + +```ts +class Config { + getTelemetryResourceAttributes(): Record { + return this.telemetrySettings.resourceAttributes ?? {}; + } + getTelemetryMetricsIncludeSessionId(): boolean { + return this.telemetrySettings.metrics?.includeSessionId ?? false; + } +} +``` + +### 5.3 `resolveTelemetrySettings()` 新增 + +```ts +const envResourceAttrs = parseOtelResourceAttributes( + env['OTEL_RESOURCE_ATTRIBUTES'], +); +const settingsResourceAttrs = { ...(settings.resourceAttributes ?? {}) }; + +// Strip RESERVED keys from both sources (warn if user tried to set them). +for (const k of RESERVED_RESOURCE_ATTRIBUTE_KEYS) { + if (k in envResourceAttrs) { + diag.warn(`OTEL_RESOURCE_ATTRIBUTES cannot override "${k}"; ignoring`); + delete envResourceAttrs[k]; + } + if (k in settingsResourceAttrs) { + diag.warn( + `settings.telemetry.resourceAttributes cannot override "${k}"; ignoring`, + ); + delete settingsResourceAttrs[k]; + } +} + +// Merge: env < settings (settings wins on conflict). +const merged: Record = { + ...envResourceAttrs, + ...settingsResourceAttrs, +}; + +// service.name precedence: OTEL_SERVICE_NAME (env-only escape) wins over +// everything else. settings already overwrote env in the spread above. +if (env['OTEL_SERVICE_NAME']) { + merged['service.name'] = env['OTEL_SERVICE_NAME']; +} + +const resourceAttributes = merged; + +const metricsIncludeSessionId = + parseBooleanEnvFlag(env['QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID']) ?? + settings.metrics?.includeSessionId ?? + false; + +return { + // ... existing fields + resourceAttributes, + metrics: { includeSessionId: metricsIncludeSessionId }, +}; +``` + +### 5.4 `sdk.ts` Resource 构造改动 + +```ts +const userAttrs = config.getTelemetryResourceAttributes(); +// service.version is always built-in; service.name flows through userAttrs +// (it was already resolved with OTEL_SERVICE_NAME precedence in resolver). +const builtinServiceName = userAttrs['service.name'] ?? SERVICE_NAME; +const { 'service.name': _, 'service.version': __, ...nonReserved } = userAttrs; + +const resource = resourceFromAttributes({ + ...nonReserved, + [SemanticResourceAttributes.SERVICE_NAME]: builtinServiceName, + [SemanticResourceAttributes.SERVICE_VERSION]: + config.getCliVersion() || 'unknown', + // session.id deliberately NOT placed on Resource — see design doc §4.1 +}); +``` + +### 5.5 `settingsSchema.ts` 改动 + +`packages/cli/src/config/settingsSchema.ts:998-1018` 的 `telemetry.jsonSchemaOverride.properties` 加: + +```ts +{ + // ... existing includeSensitiveSpanAttributes + resourceAttributes: { + type: 'object', + additionalProperties: { type: 'string' }, + description: + 'Static resource attributes attached to all telemetry data. ' + + 'Keys must be strings; values must be strings. ' + + 'Reserved keys (service.name, service.version) are silently dropped.', + default: {}, + }, + metrics: { + type: 'object', + additionalProperties: false, + properties: { + includeSessionId: { + type: 'boolean', + default: false, + description: + 'Include session.id on every metric data point. ' + + 'WARNING: each CLI session creates a new value, causing unbounded ' + + 'metric time-series fan-out. Only enable for short-term debugging.', + }, + }, + }, +} +``` + +也要把 `additionalProperties: true` 重新评估——目前是 permissive,可以保留也可以转 strict。建议保留 permissive,避免对其他未在 schema 中声明的 `telemetry.*` 字段产生破坏性变更,但 docs 里明确"未声明字段会被忽略"。 + +## 6. 文件改动清单 + +| 文件 | 改动 | +| -------------------------------------------------------------- | -------------------------------------------------------------------------- | +| `packages/core/src/telemetry/sdk.ts` | 改 Resource 构造(合并 user attrs,删 `session.id`) | +| `packages/core/src/telemetry/resource-attributes.ts` (新文件) | `parseOtelResourceAttributes()` + `RESERVED_RESOURCE_ATTRIBUTE_KEYS` 常量 | +| `packages/core/src/telemetry/config.ts` | resolver 加 `resourceAttributes` + `metrics.includeSessionId` 解析与 merge | +| `packages/core/src/telemetry/metrics.ts` | `getCommonAttributes()` 加 toggle gate | +| `packages/core/src/config/config.ts` | `TelemetrySettings` schema + 两个 getter | +| `packages/cli/src/config/settingsSchema.ts` | `jsonSchemaOverride` 加 `resourceAttributes` + `metrics` | +| `docs/developers/development/telemetry.md` | 加 "Resource attributes" + "Cardinality controls" 两节 + 迁移说明 + 示例 | +| `packages/core/src/telemetry/resource-attributes.test.ts` (新) | 解析器单元测试(覆盖 §4.6 全部用例) | +| `packages/core/src/telemetry/sdk.test.ts` | merge 优先级 / 保留键 / `OTEL_SERVICE_NAME` | +| `packages/core/src/telemetry/metrics.test.ts` | toggle off/on 时 `session.id` 出现与否 | +| `packages/core/src/telemetry/config.test.ts` | env / settings 合并 | +| `CHANGELOG.md` 或 release notes | PR 2 的 breaking change 说明 | + +## 7. 分 PR 拆分 + +按 review 友好性与 blast radius 分三个 PR: + +### PR 1 — Custom resource attributes(additive,零破坏) + +- 新文件 `resource-attributes.ts`:`parseOtelResourceAttributes()` + `RESERVED_RESOURCE_ATTRIBUTE_KEYS` +- `TelemetrySettings.resourceAttributes` 字段 + resolver merge 逻辑 +- `OTEL_SERVICE_NAME` / `OTEL_RESOURCE_ATTRIBUTES` 接入,按 §4.2 优先级 +- 合并进 Resource(`sdk.ts`) +- `settingsSchema.ts` 加 `resourceAttributes` JSON schema +- **不动** `session.id` 在 Resource 上的位置 +- Docs 加 "Resource attributes" 一节 + +**风险**:低。完全 additive,不改任何现有行为。除非用户主动设置环境变量或 settings,否则导出的数据无变化。 + +### PR 2 — Cardinality controls(semantic break) + +- 从 Resource 删 `session.id` (`sdk.ts:160` 那一行) +- 加 `metrics.includeSessionId` toggle(settings + env)+ `getCommonAttributes()` gate +- `settingsSchema.ts` 加 `metrics` JSON schema +- CHANGELOG / 迁移说明 +- 快照测试锁定 metric attribute 集合(防回归) +- Docs 加 "Cardinality controls" 一节 + 迁移指南 + +**风险**:中等。任何依赖 metric 上 `session.id` 的 Prometheus query / Grafana dashboard / 告警规则会失效。需要显式 release note 与 1-2 个版本的迁移窗口。 + +**Opt-in 过渡方案**(候选,本期建议**不采用**): + +> PR 2 可先以"opt-out"形式落地——默认仍把 `session.id` 注入 metric,但加 warn log "this default will flip in v0.X"。一个 release 后再翻转默认。 + +不建议采用的原因:(1)当前 qwen-code 用户群不大,破坏面有限;(2)这是 cardinality bug,越早默认安全越好;(3)双段式发布会增加文档负担。如果父 issue owner 想要保守一些,可以采纳。 + +### PR 3 — Docs polish + samples(cleanup) + +- `docs/developers/development/telemetry.md` 补示例(见 §10) +- 阿里云 ARMS / Prometheus / Grafana 接入示例 +- 把所有典型 use case 的 settings.json 片段加进去 + +## 8. 测试计划 + +### 8.1 `parseOtelResourceAttributes()` 单元测试 + +参数化覆盖 §4.6 表格全部行(建议用 vitest `it.each`): + +```ts +it.each([ + ['', {}], + ['a=1', { a: '1' }], + ['a=1,b=2', { a: '1', b: '2' }], + ['a=hello%20world', { a: 'hello world' }], + ['a=val%ZZbad', { a: 'val%ZZbad' }], // invalid percent + ['malformed', {}], + ['=val', {}], + ['a=', { a: '' }], + ['a=1,a=2', { a: '2' }], + [' a = 1 , b = 2 ', { a: '1', b: '2' }], +])('parses %j → %j', (input, expected) => { + expect(parseOtelResourceAttributes(input)).toEqual(expected); +}); +``` + +### 8.2 Resolver merge 测试 + +| 场景 | 期望 `service.name` | 期望 user attr | +| ----------------------------------------------------------------------- | ----------------------------------------------------- | ------------------------------------ | +| 全空 | `'qwen-code'` | 不存在 | +| 仅 env `OTEL_SERVICE_NAME=A` | `'A'` | — | +| 仅 env `OTEL_RESOURCE_ATTRIBUTES=service.name=B` | `'B'` | — | +| `OTEL_SERVICE_NAME=A` + `OTEL_RESOURCE_ATTRIBUTES=service.name=B` | `'A'`(OTEL_SERVICE_NAME 优先) | — | +| `OTEL_SERVICE_NAME=A` + `settings={service.name:C}` | `'A'`(OTEL_SERVICE_NAME 优先) | — | +| `OTEL_RESOURCE_ATTRIBUTES=service.name=B` + `settings={service.name:C}` | `'C'`(settings 优先于 env,无 OTEL_SERVICE_NAME 时) | — | +| `OTEL_RESOURCE_ATTRIBUTES=team=x` + `settings={team:y}` | `'qwen-code'` | `team='y'`(settings 优先) | +| `OTEL_RESOURCE_ATTRIBUTES=service.version=fake` | `'qwen-code'` + warn | service.version 仍为真实 cli version | +| `settings={service.version:fake}` | `'qwen-code'` + warn | service.version 仍为真实 cli version | + +### 8.3 Resource 内容快照测试 + +用 `InMemorySpanExporter` 拿一个 span,断言: + +```ts +expect(span.resource.attributes['service.name']).toBe('qwen-code'); +expect(span.resource.attributes['service.version']).toBe(EXPECTED_VERSION); +expect(span.resource.attributes['session.id']).toBeUndefined(); // 关键 +expect(span.resource.attributes['team']).toBe('platform'); // 用户加的 +``` + +### 8.4 Metric attribute toggle 测试 + +```ts +it('does not emit session.id on metrics by default', async () => { + // emit one tool call counter + recordToolCallMetrics(...); + const data = await metricReader.collect(); + const dp = data.resourceMetrics.scopeMetrics[0].metrics[0].dataPoints[0]; + expect(dp.attributes['session.id']).toBeUndefined(); +}); + +it('emits session.id when toggle is true', async () => { + config.telemetrySettings.metrics = { includeSessionId: true }; + recordToolCallMetrics(...); + const data = await metricReader.collect(); + const dp = data.resourceMetrics.scopeMetrics[0].metrics[0].dataPoints[0]; + expect(dp.attributes['session.id']).toBe(KNOWN_SESSION_ID); +}); +``` + +### 8.5 Spans / Logs 行为保持测试 + +- spans 仍有 `session.id`(不受 metric toggle 影响) +- logs 仍有 `session.id`(不受 metric toggle 影响) + +### 8.6 回归保护 + +- `autoDetectResources: false` 保持不变(assertion on config) +- 启动期间不出现新增 `diag.error`(捕获 OTel diag 日志做 assertion) +- 现有所有 telemetry 测试通过(CI) + +### 8.7 Diag warn 测试 + +校验下列输入都触发 `diag.warn` 一次: + +- `settings.resourceAttributes = { 'service.version': 'x' }`(reserved) +- `OTEL_RESOURCE_ATTRIBUTES=service.version=x`(reserved,env 也要 warn) +- `OTEL_RESOURCE_ATTRIBUTES=malformed`(无 `=`) +- `OTEL_RESOURCE_ATTRIBUTES=a=val%ZZ`(无效 percent-encoding) + +校验下列输入**不**触发 warn(合法路径): + +- `settings.resourceAttributes = { 'service.name': 'x' }`(settings 允许设 service.name) +- `OTEL_SERVICE_NAME=foo` + `settings.resourceAttributes = { 'service.name': 'bar' }`(OTEL_SERVICE_NAME 优先即可,不需要 warn) + +## 9. 迁移 / 破坏性变更 + +### 9.1 破坏性变更(PR 2) + +**指标上的 `session.id` 默认消失**。这会影响: + +- Prometheus query 中 `by (session_id)` / `group_left(session_id)` 的聚合 +- Grafana dashboard 中按 session 切片的图 +- 任何按 session.id 做告警分组的规则 + +注:spans 和 logs 上的 `session.id` **不受影响**。 + +### 9.2 迁移路径 + +文档里给两个选项: + +**选项 A**:恢复旧行为(短期 debug 推荐) + +```bash +export QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID=true +``` + +或 `settings.json`: + +```json +{ + "telemetry": { + "metrics": { "includeSessionId": true } + } +} +``` + +⚠️ **警告**:长期开启会让 metric time-series 数量 = 历史 session 数量,撑爆后端。仅短期 debug 用。 + +**选项 B**:改用 spans / logs 做 session 切片(推荐) + +- spans / logs 上仍有 `session.id`,可在 trace backend(如 Jaeger / Aliyun ARMS Tracing)/ log backend(如 Loki / SLS)按 session 切片 +- 这两类数据本来就是 per-event 存储,cardinality 不会爆炸 +- 适合做 session-level drill-down 分析 + +### 9.3 Release note 模板 + +``` +**Breaking change (metric attribute):** + +The `session.id` attribute is no longer attached to metric data +points by default. This protects metric backends from unbounded +time-series fan-out. + +- Spans and logs are unaffected — `session.id` is still present. +- To restore the previous behavior (short-term debugging only), set + `QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID=true` or in settings.json: + `telemetry.metrics.includeSessionId: true`. +- For long-term session correlation, query against trace / log + backends instead of metric backends. + +See docs/developers/development/telemetry.md "Migration" for details. +``` + +## 10. 示例配置(用于文档) + +### 10.1 按 team / env 切片所有 telemetry + +```bash +export OTEL_RESOURCE_ATTRIBUTES="team=platform,env=prod,cost_center=eng-123" +``` + +效果:所有 span / log / metric 都带 `team=platform` `env=prod` `cost_center=eng-123`。 + +### 10.2 用 `OTEL_SERVICE_NAME` 在共享 collector 中路由 + +```bash +export OTEL_SERVICE_NAME=qwen-code-ci +``` + +效果:`service.name=qwen-code-ci`,多租户 OTel collector 可按 service.name 路由到不同后端。 + +### 10.3 Fleet baseline + 单机 override + +公司 fleet 的 `~/.qwen/settings.json`(GitOps 分发): + +```json +{ + "telemetry": { + "resourceAttributes": { + "deployment.environment": "production", + "service.namespace": "engineering-tooling" + } + } +} +``` + +单机 ops 临时覆盖(不修改 settings): + +```bash +export OTEL_RESOURCE_ATTRIBUTES="debug_run=true" +# settings 里的 deployment.environment / service.namespace 仍然生效 +# 同时这次运行额外带 debug_run=true +``` + +### 10.4 短期 debug 打开 metric session.id + +```bash +# 一次性 debug run +QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID=true qwen "投资分析" +``` + +完事即关闭,不要持久化到 settings。 + +### 10.5 阿里云 ARMS Metric 接入(推荐配置) + +```json +{ + "telemetry": { + "enabled": true, + "otlpEndpoint": "http:///api/v1/...", + "otlpProtocol": "http", + "resourceAttributes": { + "team": "platform", + "deployment.environment": "production" + }, + "metrics": { + "includeSessionId": false + } + } +} +``` + +## 11. 与 claude-code 实现的对比 + +| 维度 | claude-code | qwen-code 本设计 | 决策依据 | +| -------------------------- | ------------------------------------------------ | ------------------------------------------------ | -------------------------------------------------- | +| 标准 OTel env var | `OTEL_RESOURCE_ATTRIBUTES` / `OTEL_SERVICE_NAME` | ✅ 一致 | 标准契约 | +| `OTEL_SERVICE_NAME` 优先级 | 遵守 OTel 规范 | ✅ 遵守 | spec 明确规定 | +| Cardinality 开关命名 | `OTEL_METRICS_INCLUDE_*` | `QWEN_TELEMETRY_METRICS_INCLUDE_*` | 不污染标准 OTel 命名空间 | +| 开关作用域 | 仅 metric | ✅ 仅 metric | spans / logs 是 per-event,无 cardinality 爆炸问题 | +| 默认值 | 高基数 attribute 默认 false | ✅ 默认 false | 安全优先 | +| Per-attribute granularity | 每 attribute 一个 toggle | ✅ 一致 | 灵活,符合实际诊断需求 | +| settings.json 等价物 | ❌ 无 | ✅ 有 `telemetry.resourceAttributes` + `metrics` | 企业 fleet 部署 base config | +| Per-span 动态 hook | ❌ 无 | ❌ 无 | 复杂度高,claude-code 也没解,本期不做 | +| 多租户 `account_uuid` | 有 | ❌ 无 | qwen-code metric 里没有此 attr | +| Agent SDK `options.env` | 有 | ❌ 无 | qwen-code 没有等价模式 | +| 保留键策略 | 不允许覆盖 built-in id | ✅ 一致 | 遥测可信度 | +| 第一方上报通道 | claude-code 也有独立第一方通道(与 OTel 隔离) | ✅ qwen-logger 同样隔离 | 第一方与第三方通道职责分离 | + +**最值得借的两点**: + +1. **命名约定**:`*_INCLUDE_*` 一眼能看出语义,比反义命名(`*_EXCLUDE_*` / `*_DROP_*`)清晰 +2. **范围克制**:只 gate metric,不 gate span/log——claude-code 显然踩过这个边界,我们直接受益 + +**qwen-code 做得更好的点**: + +- settings.json 支持:claude-code 完全靠 env var,对企业 fleet 场景不友好 +- 明确的保留键策略(`service.version` 不可覆盖):减少遥测被污染的可能 +- 第一方上报隔离:qwen-logger 走独立通道,与用户 OTLP 设置完全解耦 + +## 12. 未来工作(v2 + 候选) + +- **`service.version` cardinality 控制**:用 OTel View API 在 metric 层 drop attribute +- **更多 cardinality toggle**:未来若 metric 上引入 `user.account_uuid` / `model` 等,按需补 toggle +- **Per-span 动态 attribute hook**:可借鉴 qwen-code 自家 hooks 系统,加 `OnSpanStart(span, context) => attrs` 回调。需要独立设计。 +- **Resource attribute schema 校验**:限制 key 命名空间(如禁止覆盖 `service.*` 前缀以外的内建 attr),目前靠保留键列表硬编码够用。 +- **Hot reload Resource**:当 settings.json 在进程内被修改(设想 qwen-serve daemon 场景),目前不会重建 Resource。若 daemon 场景成熟,可以增加一条 reload 路径。 +- **跨进程 subagent context 传播**:subagent 跨进程时,把 parent 的 trace context(包括 resource)通过 OTel context propagation 标准 header 传过去。需要独立设计。 diff --git a/docs/design/virtual-viewport/README.md b/docs/design/virtual-viewport/README.md new file mode 100644 index 00000000000..9ba6bcb71d8 --- /dev/null +++ b/docs/design/virtual-viewport/README.md @@ -0,0 +1,368 @@ +# Virtual viewport for long conversations on ink 7 + +Status: **implemented**, PR #4146 ships: +core viewport, ASCII scrollbar with auto-hide animation, SGR mouse-wheel, `ui.useTerminalBuffer` gate, keyboard scroll keys. +Scrollbar drag / in-app search / alt-buffer mode / dual-write to host scrollback are scoped out to V.3+ (see §7). +Author: 秦奇 +Tracking branch: `feat/virtual-viewport-on-ink7` (base: `main`) + +## 1. Problem + +Several user-reported flicker / lag issues all bottom-out in the same architectural fact: ink's `` is **append-only** and qwen-code's `MainContent.tsx` feeds the _entire_ `mergedHistory` through it on every render. For a 1000-turn conversation, that is 1000 `HistoryItemDisplay` React renders + ink layout passes per state change. + +The current symptoms this enables: + +| Issue | Symptom | Current contributor | +| --------------- | -------------------------------------------------- | ------------------------------------------------------------- | +| #2950 | Long session shows continuous up/down scroll storm | full Static remount on every refresh | +| #3118 | Switching back to window keeps flickering | `clearTerminal` + `historyRemountKey++` triggers full remount | +| #3007 | Generic interface flickering | same as #3118 | +| #3838 (UI side) | Scrollbar grows unboundedly | each cumulative-delta render adds rows; no viewport eviction | +| #3899 → #3905 | Ctrl+O froze terminal for seconds | the partially-fixed case, sealed with `setImmediate` chunking | + +PR #3905 explicitly notes: + +> Discussion of alternatives (sealed prefix + live tail, **true viewport virtualization**, ANSI-output caching) was considered but each changes UX or requires an architectural rewrite. + +That architectural rewrite is what this design proposes. + +## 2. Reference implementations + +Surveyed two open-source ink-based CLIs that already solved (or worked around) the same problem: + +### 2.1 claude-code (`/Users/gawain/Documents/codebase/opensource/claude-code`) + +Maintains its **own forked ink** at `src/ink/`: + +- `ink.tsx` — 1722 LoC custom main loop +- `log-update.ts` — 773 LoC custom diff renderer with scroll-region (`DECSTBM`) optimization, full-frame fallback when scrollback would be touched +- `screen.ts` / `frame.ts` — explicit Screen / Frame objects, `cellAt` / `diffEach` cell-level diffing +- `render-to-screen.ts` — exposes `renderToScreen(node)` to render ANY node tree to a `Screen` object out of band. This is the underlying capability for "render once, cache, replay" — i.e. virtualization +- `screens/REPL.tsx`: + - `visibleStreamingText = streamingText.substring(0, streamingText.lastIndexOf('\n') + 1) || null` — only complete lines exposed to renderer + - `ScrollBox` with `scrollRef`, `cursorNavRef` + - `Markdown.tsx` `StreamingMarkdown` splits content at last top-level block boundary, memoizes stable prefix, only re-parses unstable suffix +- `Markdown.tsx` token cache (LRU-500) — survives unmount→remount, so virtual-scroll re-mounts hit cache without re-lexing + +**Why we don't replicate this approach**: forking ink wholesale is unsustainable maintenance (1722 LoC `ink.tsx` alone, plus a custom reconciler). Every upstream ink fix has to be hand-merged. That cost is justified for claude-code's scale; not for qwen-code. + +### 2.2 gemini-cli (`/Users/gawain/Documents/codebase/opensource/gemini-cli`) + +Uses `@jrichman/ink@6.6.9` (a smaller fork that adds `ResizeObserver` and `StaticRender` exports), and ships **a complete virtualized list as plain components**: + +| File | LoC | Role | +| --------------------------------------- | --- | ---------------------------------------------------------------------- | +| `components/shared/VirtualizedList.tsx` | 764 | Core viewport + measurement + scroll-anchor + per-item resize tracking | +| `components/shared/ScrollableList.tsx` | 278 | Wraps `VirtualizedList`, adds keypress nav + smooth scroll + scrollbar | +| `contexts/ScrollProvider.tsx` | 469 | Mouse drag, scroll lock, focus context | +| `hooks/useBatchedScroll.ts` | 35 | Coalesces same-tick scroll updates | +| `hooks/useAnimatedScrollbar.ts` | 130 | Scrollbar fade-in/out animation | + +`MainContent.tsx` switches between two render paths via a `isAlternateBufferOrTerminalBuffer` flag: + +```tsx +if (isAlternateBufferOrTerminalBuffer) { + return ; +} + +return , ...staticHistoryItems, ...lastResponseHistoryItems]}>...; +``` + +`HistoryItemDisplay` is wrapped in `React.memo` so unchanged items don't re-render. + +**This is the production-grade reference.** + +## 3. ink 7 capability check + +qwen-code is on the in-flight `chore/upgrade-ink-7` branch. Inspected `node_modules/ink/build/index.d.ts` exports: + +- ✅ `useBoxMetrics(ref): {width, height, left, top, hasMeasured}` — auto-updates on layout change. **Functional equivalent of `ResizeObserver`.** +- ✅ `measureElement(node)` — single-shot imperative measure +- ✅ `useWindowSize` — terminal resize +- ✅ `useAnimation` — for scrollbar fade +- ✅ `Static`, `Box`, `Text`, etc. +- ❌ `ResizeObserver` (component/class) — needs adaptation +- ❌ `StaticRender` — needs custom implementation + +**Conclusion**: ink 7 has every primitive needed. No fork swap required. + +## 4. Strategic decision + +**Port gemini-cli's `ScrollableList` + `VirtualizedList` + supporting hooks/contexts to qwen-code, adapting `ResizeObserver` → `useBoxMetrics` and rolling a custom `StaticRender`.** + +Rejected alternatives: + +| Alternative | Why rejected | +| --------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| Fork ink like claude-code | Unsustainable maintenance burden | +| Switch to `@jrichman/ink` | Reverses the in-flight ink 7 upgrade; loses ink 7's React 19.2 + reconciler 0.33 + new diff renderer improvements | +| Build virtualization from scratch | Reinvents ~1700 LoC of proven design; gemini-cli's reference exists and works | + +## 5. Architecture + +### File map after PR #4146 + +``` +packages/cli/src/ui/ +├── components/shared/ +│ ├── VirtualizedList.tsx [NEW] core viewport + ASCII scrollbar +│ ├── ScrollableList.tsx [NEW] keyboard + mouse-wheel wrapper +│ └── StaticRender.tsx [NEW] React.memo wrapper (replaces gemini-cli's ink fork export) +├── hooks/ +│ ├── useBatchedScroll.ts [NEW] coalesce same-tick scroll updates +│ ├── useMouseEvents.ts [NEW] enable SGR mouse mode + parse stdin events +│ └── useAnimatedScrollbar.ts [NEW] thumb flash on scroll + idle auto-hide +├── utils/ +│ └── mouse.ts [NEW] SGR + X11 mouse-event parser (port from gemini-cli) +├── components/MainContent.tsx [MOD] add virtualized branch + stability refs +└── AppContainer.tsx [MOD] feed scroll-related UI state into context + gate refreshStatic +``` + +Deferred to follow-up PRs: + +- **Scrollbar drag + click-to-position** — needs screen-absolute element coords, blocked on a stock-ink-7 limitation (see V.4 / V.7). +- **In-app `/` search** — claude-code's `TranscriptSearchBar` pattern (V.5). +- **Alternate-buffer mode** — `contexts/ScrollProvider.tsx`-style focus / lock, with full alt-screen takeover (V.6). + +### Setting (V.2) + +```ts +// settings schema +ui: { + /** + * Enables virtualized history rendering for long conversations. + * When true, only items in the visible viewport are rendered through React; + * scrolled-out items remain in the terminal scrollback buffer. + * + * Default: false. Opt-in until proven stable on long conversations. + */ + useTerminalBuffer?: boolean; // alias kept compat with gemini-cli +} +``` + +`MainContent.tsx` reads the setting and switches paths: + +```tsx +const useTerminalBuffer = uiState.settings?.ui?.useTerminalBuffer ?? false; + +if (useTerminalBuffer) { + return ; // virtualized +} + +return ; // existing path, untouched +``` + +The legacy `` path stays as-is — no regression risk for users who don't opt in. + +## 6. Key adaptations from gemini-cli source + +### 6.1 `ResizeObserver` → `useBoxMetrics` + +gemini-cli's container observer (imperative pattern): + +```ts +const containerObserverRef = useRef(null); + +const containerRefCallback = useCallback((node: DOMElement | null) => { + containerObserverRef.current?.disconnect(); + containerRef.current = node; + if (node) { + const observer = new ResizeObserver((entries) => { + const entry = entries[0]; + if (entry) { + const newHeight = Math.round(entry.contentRect.height); + const newWidth = Math.round(entry.contentRect.width); + setContainerHeight((prev) => (prev !== newHeight ? newHeight : prev)); + setContainerWidth((prev) => (prev !== newWidth ? newWidth : prev)); + } + }); + observer.observe(node); + containerObserverRef.current = observer; + } +}, []); +``` + +Our adaptation (declarative ink 7 hook): + +```ts +const containerRef = useRef(null); +const { width: containerWidth, height: containerHeight } = + useBoxMetrics(containerRef); +``` + +`useBoxMetrics` already handles attach/detach + layout-change subscription; the imperative bookkeeping disappears. + +### 6.2 Per-item resize tracker (`itemsObserver`) + +Harder. gemini-cli observes N item nodes via a single `ResizeObserver` and routes the entry → key via a `WeakMap`: + +```ts +const nodeToKeyRef = useRef(new WeakMap()); +const itemsObserver = useMemo( + () => + new ResizeObserver((entries) => { + setHeights((prev) => { + let next = null; + for (const entry of entries) { + const key = nodeToKeyRef.current.get(entry.target); + if (key && prev[key] !== Math.round(entry.contentRect.height)) { + if (!next) next = { ...prev }; + next[key] = Math.round(entry.contentRect.height); + } + } + return next ?? prev; + }); + }), + [], +); +``` + +`useBoxMetrics` is **single-ref-per-hook**, so we cannot 1:1 replace this. Two options: + +**Option A — push measurement down to `VirtualizedListItem`** + +Each `VirtualizedListItem` already runs as its own component (memoized). Add `useBoxMetrics` inside it; report height up via a callback prop: + +```tsx +const VirtualizedListItem = memo(({ itemKey, onHeightChange, ...props }) => { + const ref = useRef(null); + const { height, hasMeasured } = useBoxMetrics(ref); + useEffect(() => { + if (hasMeasured) onHeightChange(itemKey, height); + }, [itemKey, height, hasMeasured, onHeightChange]); + return {...}; +}); +``` + +**Option B — use `measureElement` + `useLayoutEffect`** in the parent + +Parent stores refs for visible items, runs a layout-effect after each render to measure them. Less reactive but simpler: + +```ts +useLayoutEffect(() => { + const newHeights: Record = { ...heights }; + let changed = false; + for (const [key, ref] of itemRefs.current) { + if (ref) { + const { height } = measureElement(ref); + if (newHeights[key] !== height) { + newHeights[key] = height; + changed = true; + } + } + } + if (changed) setHeights(newHeights); +}); +``` + +**Recommendation: Option A.** Cleaner separation, leverages ink 7's built-in change detection. Avoids the "measure storm" risk where every render measures everything. + +### 6.3 `StaticRender` — custom implementation + +gemini-cli imports `StaticRender` from `@jrichman/ink`. Looking at usage in `VirtualizedList.tsx`: + +```tsx +{shouldBeStatic ? ( + + {content} + +) : ( + content +)} +``` + +Semantics: render `content` once at the given width; subsequent renders with the same key + width return the cached render. + +For ink 7, the equivalent is plain `React.memo` with a stable component that the parent guarantees not to re-render. Custom implementation: + +```tsx +import { memo } from 'react'; +import { Box } from 'ink'; + +interface StaticRenderProps { + children: React.ReactElement; + width?: number | string; +} + +const StaticRender = memo( + ({ children, width }: StaticRenderProps) => ( + + {children} + + ), + (prev, next) => prev.children === next.children && prev.width === next.width, +); +``` + +Combined with the parent's stable `key` prop (`${itemKey}-static-${width}`), changing children or width causes a fresh mount; otherwise React skips re-rendering. + +This is the core capability: items that ARE static (e.g. completed Gemini messages) get measured + rendered once and never re-walk through React. + +### 6.4 Memoize `HistoryItemDisplay` + +gemini-cli does: + +```ts +const MemoizedHistoryItemDisplay = memo(HistoryItemDisplay); +``` + +Same pattern in qwen-code. Required for virtualization to actually skip re-renders. + +## 7. PR sequence + +| PR | Title (draft) | Scope | Lines | Dependencies | Risk | +| --------- | --------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | ------------ | ---------------------------------------------- | +| **#4146** | feat(cli): virtual viewport for long conversations on ink 7 | core primitives + ASCII scrollbar with **auto-hide animation** + SGR **mouse-wheel** + `ui.useTerminalBuffer` gate + `MainContent`/`AppContainer` wiring + tests | ~2800 LoC | `main` | ✅ **shipped** — typecheck clean, vitest green | +| **V.3** | test(integration): capture-suite regressions for streaming / resize / shell | port 3 capture scripts from PR #3663 | ~2000 (test-only) | #4146 | pending | +| **V.4** | feat(cli): scrollbar drag + click-to-position | SGR mouse hit-test on scrollbar column. Needs screen-absolute coords — either upstream `getBoundingBox` to ink 7 or own yoga walker. Auto-hide animation already shipped in #4146. | ~400 | #4146 | deferred — coord blocker | +| **V.5** | feat(cli): in-app `/` search | viewport-bound highlight + n/N navigation (claude-code's `TranscriptSearchBar` pattern) | ~300 | #4146 | deferred | +| **V.6** | feat(cli): alternate-buffer mode (full alt-screen takeover) | additional setting `ui.useAlternateBuffer` | ~500 | #4146 | deferred — separate UX decision required | +| **V.7** | research: preserve host terminal scrollback (dual-write) | `@jrichman/ink`'s `overflowToBackbuffer` is fork-only. Options: upstream PR to ink 7, own dual-write, or accept loss. Investigation. | — | #4146 | structurally blocked on stock ink 7 | + +V.3 (integration tests) is the remaining critical-path item before flipping the default. V.4–V.6 close the remaining gemini-cli-parity gaps; V.7 is open research because the underlying ink prop we'd need (`overflowToBackbuffer`) only exists in gemini-cli's `@jrichman/ink` fork. + +## 8. Verification plan + +Per-PR (mandatory before any "ready for review"): + +- `npm run typecheck --workspace=@qwen-code/qwen-code` — clean +- `npm run lint --workspace=@qwen-code/qwen-code` — clean +- `cd packages/cli && npx vitest run` — all green +- Multi-round directionless audit per project workflow + +End-to-end (after V.3): + +- Long-conversation benchmark: 1000-turn session, measure + - First-paint time (initial mount + paint) + - Ctrl+O toggle latency + - Resize latency + - Per-frame render time during streaming +- Compare `useTerminalBuffer: false` (legacy) vs `true` (virtualized) + +## 9. Open questions / decisions needed + +1. **Setting name**: `ui.useTerminalBuffer` (gemini-cli compat) vs `ui.virtualizedHistory` (more descriptive)? +2. **Default value**: ship as `false` (opt-in) or stage rollout via env var first? +3. **Static-item heuristic**: gemini-cli marks only `header` as static. Should we also mark completed Gemini messages, tool results that are no longer in `pendingHistoryItems`, etc.? +4. **Mouse support**: gemini-cli's `ScrollProvider` includes mouse drag for scrollbar. Worth porting now or skip until V.4? +5. **Compatibility with #3905**: ~~PR #3905 (Ctrl+O freeze fix) is open and modifies the same `MainContent.tsx`. Coordinate merge order — likely V.2 rebases on top of #3905.~~ **Resolved**: #3905's progressive-replay landed in `main` and is preserved in the legacy `` branch of `MainContent.tsx`; the VP branch supersedes it for opt-in users because the freeze trigger (full Static remount) no longer applies. +6. **Compatibility with `chore/re-upgrade-ink-7-0-3`**: PR #4146 stacks on it. After #4119 (the ink 7.0.3 re-upgrade PR) merges to `main`, PR #4146's base will re-target to `main`. + +## 10. Risks + +| Risk | Likelihood | Mitigation | +| ------------------------------------------------------------------------- | ---------- | ------------------------------------------------------------------------------------------------------- | +| `useBoxMetrics` per-item creates measurement storms on long lists | medium | Option A in §6.2 already memoizes per-item; only items in render window pay the cost. Benchmark in V.3. | +| `StaticRender` custom impl misses an edge case the @jrichman fork handled | medium | Audit gemini-cli's StaticRender source if available; otherwise rely on functional tests + benchmark. | +| `` legacy path drift as the new path evolves | low | Feature-flag gate keeps both paths active; CI runs both via setting matrix. | +| ink 7 still has unfilled bugs upstream | low | We're already on ink 7 via `chore/upgrade-ink-7`; this PR doesn't introduce additional ink risk. | +| Long-running sessions accumulate memory in measurement caches | medium | Add LRU eviction on `heights` Record once size exceeds N×viewport (e.g. 5×). V.3 benchmarks this. | + +## 11. Approval checklist + +- [x] Architectural direction approved — port from gemini-cli (§4) +- [x] Setting name + default decided — `ui.useTerminalBuffer`, default `false` (opt-in) +- [x] Static-item heuristic — `isStaticItem={(item) => item.id > 0}` (completed history items) +- [x] Mouse-support scope — deferred to V.4; keyboard-only scroll in #4146 +- [x] Merge ordering with #3905 (§9.5) — #3905 already in `main`; #4146 preserves the legacy progressive-replay path and supersedes it only for VP users +- [x] PR #4146 implementation complete diff --git a/docs/design/workflow-tracing-gaps.md b/docs/design/workflow-tracing-gaps.md new file mode 100644 index 00000000000..b0247e9d807 --- /dev/null +++ b/docs/design/workflow-tracing-gaps.md @@ -0,0 +1,376 @@ +# Workflow 级 Span 粒度不足分析 (P1) + +> 基于 2026-05-13 对 qwen-code origin/main 的复核 + +## 现状 + +qwen-code 已具备 tracing 基础设施: + +| 组件 | 位置 | 说明 | +| ------------- | ------------------------------------------------ | -------------------------------------------------------- | +| Span 类型定义 | `packages/core/src/telemetry/session-tracing.ts` | `interaction`、`llm_request`、`tool`、`tool.execution` | +| Tracer 工具 | `packages/core/src/telemetry/tracer.ts` | session root context、`withSpan`、`startSpanWithContext` | +| 交互入口 | `packages/core/src/core/client.ts` | 顶层交互显式启动 `interaction` span | +| 生命周期管理 | — | AsyncLocalStorage + WeakRef + TTL cleanup | + +当前 runtime 中稳定接入的主要是两类 generic spans: + +- `api.generateContent` / `api.generateContentStream` +- `tool.` + +**结论:已进入"有 tracing 主干"阶段,但尚未把 agent workflow 的阶段边界完整编码进 trace 树。** + +### 对比:claude-code 已实现的 span 类型 + +参考 `claude-code/src/utils/telemetry/sessionTracing.ts` (line 49): + +- `interaction` +- `llm_request` +- `tool` +- `tool.blocked_on_user` +- `tool.execution` +- `hook` + +## 缺失项 + +| 缺失 span / 机制 | 影响 | +| ------------------------------------------ | ----------------------------------------------- | +| `permission_wait` / `blocked_on_user` span | 无法区分审批等待 vs 工具执行耗时 | +| `hook` span | hook 耗时被折叠进 tool span,定位边界不清 | +| `subagent` root span | subagent 内部 llm/tool 调用无法形成 trace 子树 | +| `tool.execution` 真实接线 | helper 已定义但主链路未调用 | +| 稳定的 parent-child wiring | spans 多为 session root 下的 sibling 而非层级树 | + +## 逐项分析 + +### 1. 用户审批等待不在 trace 中 + +工具调用等待审批时,状态迁移路径为 `awaiting_approval` → `scheduled` → 执行。 + +- "等待用户确认"只是状态迁移,不是 trace 节点 +- trace 上看不到审批等待耗时 +- 工具慢时无法区分是"卡在等用户"还是"工具本身执行慢" + +### 2. Hook 有事件记录但没有独立 span + +Pre/Post hook 执行后产出 `HookCallEvent`,走 `logHookCall()`,但不建立独立 OTel span。 + +- hook 变慢时表现为外层 tool span 变慢 +- hook 失败时表现为 "tool 失败" +- trace 无法回答"时间花在 hook 还是 tool.execution 上" + +### 3. Subagent 是 log/metric 而非 trace subtree + +subagent 启动/完成时记录 `SubagentExecutionEvent` 并进入 log/metric,但没有形成显式 span 子树。 + +- 能统计"哪个 subagent 跑过" +- 不能顺着 trace 看"这个 subagent 触发了哪些 llm/tool 调用" +- 并发 subagent 场景下因果链不清 + +### 4. tool.execution helper 已定义但未接入主链路 + +`session-tracing.ts` 中已有 `startToolExecutionSpan()` / `endToolExecutionSpan()`,但非测试代码中未见调用点。 + +当前实际 trace 树: + +``` +session-root + interaction + api.generateContent + tool.Bash + subagent_execution (log/metric) + hook_call (event/QwenLogger) +``` + +理想 trace 树: + +``` +interaction + llm_request + tool + tool.blocked_on_user + hook(pre) + tool.execution + hook(post) + subagent + interaction + llm_request + tool +``` + +### 5. Parent-child wiring 不够稳定 + +interaction span 已存在,但很多运行中的 spans 挂在 session root 下作为 sibling,而不是 interaction 的子节点。 + +- 调用树偏平 +- 节点间因果关系不直观 +- 从一个用户轮次追到内部 llm/tool/hook/subagent 的体验不连续 + +## 影响 + +- traces 有基础价值,但不足以支撑 workflow 级排障 +- 无法直接回答"这轮慢在等用户、hook,还是 tool 真执行" +- 无法把 subagent 运行过程还原为可阅读的 trace 子树 +- hook 问题被折叠进 tool span,定位边界不清 +- 在 Jaeger / Tempo / ARMS 上的树比 claude-code 更平、更难读 + +--- + +## claude-code 方案复用分析 + +> 基于 2026-05-13 对 claude-code 源码的深度对比 + +### claude-code 的 tracing 架构 + +claude-code 在 `src/utils/telemetry/sessionTracing.ts` 中实现了一个**统一的、基于双 ALS 的 span 管理系统**: + +``` + interactionContext (ALS) toolContext (ALS) + │ │ + ▼ ▼ + ┌─────────────────────┐ ┌─────────────────────┐ + │ interaction span │ │ tool span │ + │ (session root) │ │ (child of intxn) │ + └─────────────────────┘ └─────────────────────┘ + ▲ parent of ▲ parent of + │ │ + ┌───────┴───────┐ ┌──────────┼──────────┐ + │ │ │ │ │ + llm_request tool blocked execution hook + _on_user +``` + +**核心机制:** + +| 机制 | 实现 | +| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 双 ALS | `interactionContext` 存当前 interaction span;`toolContext` 存当前 tool span | +| parent 解析 | 每种 span 类型硬编码从哪个 ALS 取 parent:`llm_request`/`tool` 取 `interactionContext`;`blocked_on_user`/`execution`/`hook` 取 `toolContext`;`hook` 有 fallback 到 `interactionContext` | +| 生命周期 | enterWith 注入 → span 运行 → enterWith(undefined) 清除 | +| 查找 span | 非 ALS 存储的 span(如 blocked_on_user)通过 `activeSpans` Map 按 `span.type` 反查 | +| 内存管理 | ALS 持有的 span 用 WeakRef;非 ALS 持有的 span 用 strongRef 防 GC;TTL 30min 自动清理 | + +**claude-code tool span 完整生命周期** (`toolExecution.ts`): + +``` +startToolSpan(name, attrs) // → toolContext.enterWith(spanCtx) + startToolBlockedOnUserSpan() // → parent = toolContext.getStore() + [permission resolution / user prompt] + endToolBlockedOnUserSpan(decision, source) + startToolExecutionSpan() // → parent = toolContext.getStore() + [tool.call()] + endToolExecutionSpan({ success }) +endToolSpan(result) // → toolContext.enterWith(undefined) +``` + +**claude-code hook span** (`hooks.ts`): + +``` +startHookSpan(event, name, count, defs) // → parent = toolContext ?? interactionContext + [parallel hook execution] +endHookSpan(span, { success, blocking, ... }) +``` + +### qwen-code 现有架构 vs claude-code + +#### 根本差异:两套断裂的 span 创建路径 + +这是 qwen-code 当前最关键的架构问题: + +| 层 | 文件 | 用法 | parent 解析 | +| ------------------ | -------------------- | ------------------------------------------------------------------------------------------- | --------------------------------------------------------- | +| session-tracing 层 | `session-tracing.ts` | `startInteractionSpan` / `startLLMRequestSpan` / `startToolSpan` / `startToolExecutionSpan` | 显式从 `interactionContext` ALS 取 parent | +| tracer 层 | `tracer.ts` | `withSpan` / `startSpanWithContext` | 从 `context.active()` 取 parent,fallback 到 session root | + +**runtime 实际调用情况:** + +- `startInteractionSpan` → **已接入** (`client.ts` line 956),写入 `interactionContext` ALS +- `startLLMRequestSpan` / `endLLMRequestSpan` → **未接入**,runtime 用的是 `withSpan('api.generateContent', ...)` (在 `loggingContentGenerator.ts`) +- `startToolSpan` / `endToolSpan` → **未接入**,runtime 用的是 `withSpan('tool.${name}', ...)` (在 `coreToolScheduler.ts`) +- `startToolExecutionSpan` / `endToolExecutionSpan` → **未接入** + +**后果:** + +`withSpan` 的 `getParentContext()` 先检查 `context.active()`(OTel 原生 context),找不到活跃 span 时回退到 session root context。它**完全不读取 `interactionContext` ALS**。 + +因此 interaction span 和 LLM/tool spans 变成了 session root 下的**平级 sibling**,而不是 parent-child 树: + +``` +session-root + ├── interaction (来自 session-tracing, 写入了 interactionContext ALS) + ├── api.generateContent (来自 withSpan, 不读 interactionContext → 挂到 session root) + ├── tool.Bash (来自 withSpan, 同上) + └── tool.Read (来自 withSpan, 同上) +``` + +**而 claude-code 中,只有一套 span 创建路径(sessionTracing.ts),所有 span 都走同一套 ALS → OTel context 转换逻辑,所以树是完整的。** + +#### 逐项复用评估 + +##### 1. 双 ALS + 显式 parent 解析 — 可复用,是核心修复 + +| 维度 | claude-code | qwen-code | +| ------------ | ----------------------------------------------------- | -------------------------------------------- | +| ALS 数量 | 2 (`interactionContext` + `toolContext`) | 1 (`interactionContext`,无 `toolContext`) | +| parent 解析 | 每种 span 类型显式指定从哪个 ALS 取 parent | `withSpan` 统一走 `context.active()` | +| context 注入 | `trace.setSpan(otelContext.active(), parentCtx.span)` | `withSpan` 内部由 `startActiveSpan` 隐式注入 | + +**复用方案:** + +qwen-code 的 `session-tracing.ts` 已经实现了与 claude-code **几乎相同的 parent 解析模式**: + +```typescript +// qwen-code session-tracing.ts (已有但未用) +export function startLLMRequestSpan(model, promptId): Span { + const parentCtx = interactionContext.getStore(); + const ctx = parentCtx + ? trace.setSpan(otelContext.active(), parentCtx.span) + : otelContext.active(); + // ... +} +``` + +这段代码与 claude-code 的 `startLLMRequestSpan` 逻辑**完全一致**。 + +**核心修复路径:废弃 runtime 中的 `withSpan('api.*')` / `withSpan('tool.*')` 调用,改为调用 session-tracing 的 typed helpers。** 不需要重写 session-tracing 层——它的 API 已经就绪。 + +需要新增的只有: + +- 增加 `toolContext` ALS(仿 claude-code) +- 增加 `blocked_on_user` 和 `hook` span 类型及 helper 函数 + +##### 2. tool.blocked_on_user — 需要适配审批流差异 + +| 维度 | claude-code | qwen-code | +| ------------- | ------------------------------------------ | -------------------------------------------------------------------------- | +| 审批位置 | 在 `toolExecution.ts` 内,tool span 内部 | 在 `coreToolScheduler._schedule()` 内,tool span 之前 | +| 审批模式 | 同步等待 `resolveHookPermissionDecision()` | 状态机驱动:`validating` → `awaiting_approval` → `scheduled` → `executing` | +| span 覆盖范围 | tool span 包含 blocked + execution | tool span(`withSpan`) 只包含 execution(从 `executeSingleToolCall` 开始) | + +**关键差异:** qwen-code 的 `executeSingleToolCall` 入口检查 `toolCall.status !== 'scheduled'` 才继续——也就是说调用到这里时审批已经完成。Tool span 的 `withSpan` 包不住审批等待。 + +**适配方案(两种):** + +**方案 A — 前移 tool span 起点(推荐):** + +将 `startToolSpan` 调用从 `executeSingleToolCall` 移到 `_schedule` 中审批检查之前,使 tool span 覆盖完整生命周期。在进入 `awaiting_approval` 状态时 `startToolBlockedOnUserSpan`,在审批完成(`scheduled`)时 `endToolBlockedOnUserSpan`。 + +``` +_schedule(): + startToolSpan(name) // ← 新增 + startToolBlockedOnUserSpan() // ← 新增,进入 awaiting_approval 时 + [状态机等待] + endToolBlockedOnUserSpan(decision) // ← 新增,进入 scheduled 时 +executeSingleToolCall(): + startToolExecutionSpan() // ← 接入已有 helper + [hook + execute] + endToolExecutionSpan() + endToolSpan() // ← 需要在 finally 中 +``` + +**方案 B — 保持 tool span 位置不变,单独追踪审批:** + +在 `_schedule` 中独立创建 `approval_wait` span(不作为 tool 的 child),挂到 interaction 下。好处是改动更小,坏处是与 claude-code 模型不一致、trace 树可读性差。 + +**建议采用方案 A**,因为: + +- 与 claude-code 的 trace 树结构一致 +- trace 上一个 tool 节点就能看到"等了多久 + 执行了多久" +- 状态机驱动的特性只影响 span start/end 的触发时机,不影响 parent-child 建模 + +##### 3. hook span — 可直接复用 + +| 维度 | claude-code | qwen-code | +| ------------- | ----------------------------------- | -------------------------------------------------------------------- | +| hook 执行入口 | `executeHooks()` in `hooks.ts` | `firePreToolUseHook`/`firePostToolUseHook` via `hookEventHandler.ts` | +| 现有记录方式 | OTel span + Perfetto span | `HookCallEvent` → `QwenLogger` (无 OTel) | +| parent | `toolContext ?? interactionContext` | — | + +**复用方案:** + +1. 在 `session-tracing.ts` 新增 `startHookSpan` / `endHookSpan`(parent = `toolContext ?? interactionContext`,与 claude-code 一致) +2. 在 `coreToolScheduler.ts` 的 `executeSingleToolCall` 中,pre/post hook 调用前后分别 start/end hook span +3. 保留现有 `logHookCall` 事件记录(两套并行,不互斥) + +改动量低,不影响现有 hook 逻辑。 + +##### 4. tool.execution — 已有 helper,只需接线 + +qwen-code 的 `startToolExecutionSpan(parentToolSpan)` / `endToolExecutionSpan(span, metadata)` 已经完整实现,只需在 `executeSingleToolCall` 中调用: + +```typescript +// coreToolScheduler.ts executeSingleToolCall 内部 +const toolSpan = startToolSpan(toolName, attrs); +// ... hook pre ... +const execSpan = startToolExecutionSpan(toolSpan); +try { + // ... invocation.execute() ... + endToolExecutionSpan(execSpan, { success: true }); +} catch (e) { + endToolExecutionSpan(execSpan, { success: false, error: e.message }); +} +// ... hook post ... +endToolSpan(toolSpan); +``` + +注意:qwen-code 的 `startToolExecutionSpan` 接收显式 `parentToolSpan` 参数,而 claude-code 的是从 `toolContext` ALS 隐式获取。这不影响功能,只是风格差异。如果引入 `toolContext` ALS,可以统一改为隐式获取。 + +##### 5. subagent trace tree — 双方都不完整,不建议直接复用 + +| 维度 | claude-code | qwen-code | +| --------------- | ----------------------------------------------------------------------- | ---------------------------------------------------- | +| OTel trace 传播 | **无** — subagent 的 interaction 是新 root | **无** — subagent 无显式 trace 传播 | +| 身份关联 | Perfetto metadata(agent process/thread)+ `teammateContextStorage` ALS | `subagentNameContext` ALS + `SubagentExecutionEvent` | +| 并发隔离 | OTel ALS 有泄漏风险(`enterWith` 是进程级,并发 subagent 会互覆盖) | 同样的风险 | + +claude-code 在 subagent OTel tracing 上**自己也没解决好**: + +- `interactionContext.enterWith()` 是进程级的,并发 subagent 会覆盖彼此的 ALS 值 +- 真正的 agent 层级树只存在于 Perfetto(一个 Anthropic 内部 feature-flagged 的系统),不在 OTel 中 + +**建议:** + +- 短期:沿用 qwen-code 现有的 `subagentNameContext` + 事件日志方案 +- 中期:在 subagent 启动时创建一个 `subagent` span(parent = 当前 toolContext),并用 `context.with()` 而非 `enterWith()` 来隔离并发 subagent 的 OTel context +- 这是需要独立设计的工作项,不建议直接照搬 claude-code + +##### 6. LLM request span — 路径明确 + +qwen-code 当前在 `loggingContentGenerator.ts` 中用 `withSpan('api.generateContent', ...)` 和 `startSpanWithContext('api.generateContentStream', ...)`。 + +改为调用 `startLLMRequestSpan` / `endLLMRequestSpan`(session-tracing 层已有实现)即可。streaming 场景需要注意: + +- `startLLMRequestSpan` 返回 `Span` 对象 +- 需要手动传入 `endLLMRequestSpan(span, metadata)` 终结 +- 这与 `startSpanWithContext` 的手动管理模式兼容 + +### 复用总结 + +| 改造项 | 可复用程度 | 改动量 | 优先级 | +| ------------------------------------------------------------------------- | ------------------------------------- | --------------------------------------------- | ------ | +| 统一 span 创建路径(废弃 runtime `withSpan`,用 session-tracing helpers) | **核心修复** — 解决 parent-child 断裂 | 中(~5 个调用点) | P0 | +| 新增 `toolContext` ALS | 直接照搬 claude-code 模式 | 低(session-tracing.ts 内部) | P0 | +| tool.blocked_on_user span | 方案 A 需适配状态机 | 中(\_schedule + executeSingleToolCall 协调) | P1 | +| tool.execution 接线 | helper 已有,只需调用 | 低(executeSingleToolCall 内 3 行) | P1 | +| hook span | 新增 helper + 调用点 | 低 | P1 | +| LLM request span 切换 | 替换 withSpan 为 typed helper | 低(2 个调用点) | P1 | +| subagent trace tree | **不建议直接复用** — 需独立设计 | 高 | P2 | + +### 推荐实施顺序 + +``` +Phase 1 — 修复 trace 树结构 (P0) +├── 1a. session-tracing.ts 新增 toolContext ALS + blocked_on_user / hook span helpers +├── 1b. loggingContentGenerator.ts: withSpan → startLLMRequestSpan/endLLMRequestSpan +└── 1c. coreToolScheduler.ts: withSpan → startToolSpan/endToolSpan + +Phase 2 — 补齐 workflow span (P1) +├── 2a. coreToolScheduler._schedule: blocked_on_user span 接入 +├── 2b. coreToolScheduler.executeSingleToolCall: tool.execution span 接入 +└── 2c. hook pre/post 调用处: hook span 接入 + +Phase 3 — Subagent trace tree (P2) +├── 3a. 设计 context.with() 隔离方案(替代 enterWith) +├── 3b. subagent 启动时创建 subagent root span +└── 3c. 并发 subagent 场景验证 +``` diff --git a/docs/design/worktree.md b/docs/design/worktree.md index 4de9968b7aa..b6187da5477 100644 --- a/docs/design/worktree.md +++ b/docs/design/worktree.md @@ -8,23 +8,24 @@ qwen-code 目前仅有面向 Arena 多模型对比场景的内部 worktree 实 ## 现状对比 -| 功能 | qwen-code | claude-code | -| --------------------------------- | --------------- | ----------- | -| `EnterWorktree` 工具 | ❌ | ✅ | -| `ExitWorktree` 工具 | ❌ | ✅ | -| AgentTool `isolation: 'worktree'` | ❌ | ✅ | -| worktree 会话状态持久化与恢复 | ❌ | ✅ | -| 过期 worktree 自动清理 | ❌ | ✅ | -| Post-creation setup(hooks 配置) | ❌ | ✅ | -| StatusLine worktree 状态展示 | ❌ | ✅ | -| WorktreeExitDialog(退出提示) | ❌ | ✅ | -| 符号链接目录(node_modules 等) | ❌ | ✅ | -| sparse checkout | ❌ | ✅ | -| `--worktree` CLI 启动标志 | ❌ | ✅ | -| tmux 集成 | ❌ | ✅ | -| Arena 多模型 worktree 隔离 | ✅(qwen 独有) | ❌ | -| 脏状态覆盖(stash + copy) | ✅ | ✅ | -| Baseline commit 追踪 | ✅(qwen 独有) | ❌ | +| 功能 | qwen-code | claude-code | 阶段 | +| --------------------------------- | --------------- | ----------- | ------- | +| `EnterWorktree` 工具 | ✅(Phase A) | ✅ | — | +| `ExitWorktree` 工具 | ✅(Phase A) | ✅ | — | +| AgentTool `isolation: 'worktree'` | ✅(Phase B) | ✅ | — | +| 过期 worktree 自动清理 | ✅(Phase B) | ✅ | — | +| worktree 会话状态持久化与恢复 | ❌ | ✅ | Phase C | +| Post-creation setup(hooks 配置) | ❌ | ✅ | Phase C | +| StatusLine worktree 状态展示 | ❌ | ✅ | Phase C | +| WorktreeExitDialog(退出提示) | ❌ | ✅ | Phase C | +| `--worktree` CLI 启动标志 | ✅(Phase D) | ✅ | — | +| 符号链接目录(node_modules 等) | ✅(Phase D) | ✅ | — | +| PR 引用(`--worktree=#123`) | ✅(Phase D) | ✅ | — | +| sparse checkout | ❌ | ✅ | Future | +| tmux 集成 | ❌ | ✅ | Future | +| Arena 多模型 worktree 隔离 | ✅(qwen 独有) | ❌ | — | +| 脏状态覆盖(stash + copy) | ✅ | ✅ | — | +| Baseline commit 追踪 | ✅(qwen 独有) | ❌ | — | ## 设计原则 @@ -54,12 +55,13 @@ AgentTool 的 `isolation: 'worktree'` 只走通用路径,Arena 内部不经过 Arena 的 worktree 路径由 `agents.arena.worktreeBaseDir` 控制,默认 `~/.qwen/arena`(`ArenaManager.ts:125`),与通用路径完全独立,不做任何改动。 -### 扩展配置(暂缓至 Phase C/D) +### 扩展配置 -| 配置项 | 类型 | 用途 | 阶段 | -| ----------------------------- | ---------- | -------------------------------------------------------------- | ------- | -| `worktree.symlinkDirectories` | `string[]` | 符号链接指定目录(如 `node_modules`)到 worktree,避免磁盘浪费 | Phase C | -| `worktree.sparsePaths` | `string[]` | git sparse-checkout cone 模式,大型 monorepo 只写入指定路径 | Phase D | +| 配置项 | 类型 | 用途 | 阶段 | +| --------------------------------- | ---------- | ---------------------------------------------------------------- | ------- | +| `ui.hideBuiltinWorktreeIndicator` | `boolean` | 隐藏 Footer 中内置 `⎇ worktree-… (…)` 行,留给 custom statusline | Phase C | +| `worktree.symlinkDirectories` | `string[]` | 符号链接指定目录(如 `node_modules`)到 worktree,避免磁盘浪费 | Phase D | +| `worktree.sparsePaths` | `string[]` | git sparse-checkout cone 模式,大型 monorepo 只写入指定路径 | Future | Phase A / B 不新增任何配置项。 @@ -187,27 +189,209 @@ _无需改动:_ --- -### Phase C:体验优化(Post-creation setup + UI) +### Phase C:会话完整性(SessionService 持久化 + UI 安全网) -**目标:** worktree 创建后自动初始化环境,状态在界面上可见。 +**目标:** worktree 状态在会话中断后可恢复,用户在界面上始终知道自己在哪个 worktree 里,退出会话时有安全提示。 **要实现的功能:** -- Post-creation setup:配置 `core.hooksPath` 指向主仓库(qwen-code 无 `settings.local.json` 概念,不需要复制) -- StatusLine 展示当前 worktree 名称 / 分支 -- WorktreeExitDialog:会话退出时(检测到 worktree 仍活跃)提示用户选择 keep 或 remove -- 新增 `worktree.symlinkDirectories` 配置项,实现目录符号链接 +_SessionService worktree 状态持久化 + `--resume` 恢复:_ + +- `SessionService` 扩展 `WorktreeSession` 字段,记录 `{ slug, worktreePath, worktreeBranch, originalCwd, originalBranch }` +- `EnterWorktreeTool` 调用 `sessionService.setWorktreeSession()` 写入状态 +- `ExitWorktreeTool` 调用 `sessionService.clearWorktreeSession()` 清除状态 +- `--resume` 启动路径读取该字段,恢复 `targetDir` 并向模型注入上下文提示 + +_Post-creation setup:_ + +- 创建 worktree 后自动执行 `git config core.hooksPath /.git/hooks`,确保 worktree 内的提交与主仓库 hooks 行为一致 + +_StatusLine worktree 展示:_ + +- `UIStateContext` 新增 `activeWorktree` 字段(从 session 状态读取),在会话进入 / 退出 worktree 时更新 +- `StatusLineCommandInput` payload 新增 `worktree?: { slug: string; branch: string }` 字段,供用户 statusline 脚本使用 +- `Footer` 在 `activeWorktree` 非空时内置展示一行 `⎇ ()`,无需用户配置 statusline 脚本即可获得基本可见性 + +_WorktreeExitDialog:_ + +- 新增 `WorktreeExitDialog.tsx` 组件,参考现有 Dialog 写法 +- 修改退出键(Ctrl+C / Ctrl+D)处理逻辑:检测到 `activeWorktree` 非空时,拦截第二次确认,展示 Dialog 提示用户选择 keep 或 remove +- keep / remove 操作复用 `ExitWorktreeTool` 的现有路径 + +**影响文件:** + +| 文件 | 变更类型 | +| ------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| `packages/core/src/services/sessionService.ts` | 新增 `WorktreeSession` 字段及读写方法 | +| `packages/core/src/tools/enter-worktree.ts` | 调用 `sessionService.setWorktreeSession()` | +| `packages/core/src/tools/exit-worktree.ts` | 调用 `sessionService.clearWorktreeSession()` | +| `packages/core/src/services/gitWorktreeService.ts` | `createUserWorktree()` / `createAgentWorktree()` 后追加 `core.hooksPath` 配置 | +| `packages/cli/src/ui/contexts/UIStateContext.tsx` | 新增 `activeWorktree` 字段及 set/clear action | +| `packages/cli/src/ui/hooks/useStatusLine.ts` | `StatusLineCommandInput` 新增 `worktree` 字段 | +| `packages/cli/src/ui/components/Footer.tsx` | 内置 worktree 行展示 | +| `packages/cli/src/ui/components/WorktreeExitDialog.tsx` | 新建 | +| `packages/cli/src/ui/components/DialogManager.tsx` | 注册 `WorktreeExitDialog` | +| `packages/cli/src/ui/components/ExitWarning.tsx` 或退出键处理 | 检测 `activeWorktree` 并拦截退出 | --- -### Phase D:高级功能 +### Phase D:启动时配置(`--worktree` CLI 标志 + 目录符号链接 + PR 引用) -**目标:** 对齐 claude-code 的完整特性集。 +**目标:** 支持在启动时直接进入 worktree、通过目录符号链接减少大型项目的磁盘开销,以及通过 PR 引用快速基于一个 pull request 创建 worktree。 -**要实现的功能:** +**范围:** 三个功能在一个阶段一起落地,因为它们都挂在同一个启动入口上,且 symlink / PR fetch 两者都需要在 worktree 创建之后立即执行 — 单独拆分会重复改 bootstrap 序列。 + +#### D-1:`--worktree [name]` CLI 启动标志 + +**参数形态:** yargs 选项接受三种形式: + +| 形式 | 行为 | +| ------------------------- | ---------------------------------------------------- | +| `qwen --worktree` | bare flag,自动生成 slug(`{形容词}-{名词}-{6hex}`) | +| `qwen --worktree my-name` | 显式 slug,沿用 `EnterWorktreeTool` 的 slug 校验规则 | +| `qwen --worktree=my-name` | 等价于上一种 | + +不提供短别名 `-w`(qwen-code 短别名只保留给最高频参数,避免命名冲突)。 + +**启动序列:** worktree 在以下位置创建: + +1. `parseArguments()` 解析 argv(已有) +2. resume picker(已有,line 588-629 of `gemini.tsx`) +3. `loadCliConfig()` 初始化 Config + auth(已有,line 643-653) +4. **新增:** 若 `argv.worktree !== undefined`,调用 `createUserWorktree()` + - 写入 sidecar(`writeWorktreeSession()`) + - 设置 `process.chdir(worktreePath)` 同时 `Config.setTargetDir(worktreePath)` + - 同一 worktree 的 re-attach 路径:跳过 `git worktree add` 并就地 chdir(Phase 6 修复)。跨 projectHash 的 `--resume` × `--worktree` 组合在 session lookup 阶段会失败,详见下文"与 `--resume` 的优先级"。 +5. 主循环(TUI / headless `-p` / ACP 三种入口都要走第 4 步) + +**与 Phase A 简化的差异:** Phase A 的 `EnterWorktreeTool` **不**修改 `Config.targetDir`,依赖模型从工具结果里读到绝对路径并继续使用。Phase D 的 CLI flag 在启动期就生效,没有运行中的模型上下文需要兼容,所以**直接切换 `targetDir` 和 `process.cwd()`** —— 这是更强的隔离保证。两条路径行为不同,需要在用户文档里说明。 + +**退出行为:** 复用现有 `WorktreeExitDialog`(Phase C 已实现)。Ctrl+C/D 两次触发 → 用户在 keep / remove / cancel 之间选择。不需要新代码路径。 + +**与 `--resume` 的优先级:** + +由于 session 存储以 `projectHash(process.cwd())` 为 key,而 `--worktree` 在 resume picker / `loadCliConfig` 之前就 chdir 到 worktree,所以"在 worktree X 启动的 session,从 worktree Y 内 resume"是**架构上不可达**的(两者的 projectHash 不同,session 文件落在不同目录)。下表反映 D-1 实现 + Phase 6 re-attach 修复后的实际行为: + +| `--resume` 状态 | `--worktree` 状态 | 结果 | +| ---------------------------- | -------------------------- | ------------------------------------------------------------------------------------------ | +| 无 | 无 | 普通会话,无 worktree | +| 无 | 有(新 slug) | 新建 worktree | +| 无 | 有(已存在的 slug) | **re-attach** 到已有 worktree(Phase 6 修复) | +| 有 | 无 | 恢复旧 worktree(Phase C 行为,sidecar 命中则注入 reminder) | +| 有(sid 出自同一 worktree) | 有(同一 slug,re-attach) | re-attach + session 命中:正常 resume | +| 有(sid 出自 main checkout) | 有(任意 slug) | **session lookup 失败**:`No saved session found with ID …`,exit 1。documented limitation | +| 有(sid 出自 worktree X) | 有(slug Y, X != Y) | 同上,session 跨 projectHash 不可寻 | + +跨 projectHash override 的语义(`--worktree` 在不同 worktree / 主 checkout 的 session 之间转移)需要 storage 锚定到 repo root 而非 cwd-derived projectHash,属于未来 Config 重构范畴。`persistStartupWorktreeSidecar` 内的 `overrodeResumedWorktree` 分支代码保留是为该重构落地后能自动生效,目前在生产路径不会触发。 + +#### D-2:`worktree.symlinkDirectories` 配置项 + +**schema:** + +```jsonc +{ + "worktree": { + "symlinkDirectories": ["node_modules", "dist", ".turbo"], + }, +} +``` + +- 类型:`string[]`,默认 `undefined`(不开启,opt-in) +- 顶层 namespace `worktree` 是新增的(在 `settingsSchema.ts` 中按字母序插在 `tools` 与 `ui` 之间) +- 路径**相对于主仓库根**,绝对路径或包含 `..` 的路径被路径遍历守卫拒绝 + +**作用范围:** 所有由通用层创建的 worktree,包括: + +- `EnterWorktreeTool`(Phase A) +- `AgentTool` `isolation: 'worktree'`(Phase B) +- `--worktree` CLI flag(Phase D-1) + +Arena 的 worktree 不走通用层,**不**受此配置影响。 + +**实现位置:** `GitWorktreeService.performPostCreationSetup()` —— 紧跟现有的 `configureHooksPath()`(Phase C 已建立的模式)。新增 `symlinkConfiguredDirectories()` 方法,遍历配置项调用 `fs.symlink(absSource, absDest, 'dir')`。 + +**错误处理(fail-open):** + +| 场景 | 行为 | +| ----------------------------- | ------------------------------ | +| 源目录不存在(ENOENT) | 静默跳过,debug log | +| 目标路径已存在(EEXIST) | 静默跳过,debug log(不覆盖) | +| 路径遍历(`../`、绝对路径等) | 拒绝该项,debug log warn | +| 其他 I/O 错误 | debug log warn,继续处理后续项 | + +worktree 创建本身**不会**因为 symlink 失败而中止 —— 与 `configureHooksPath()` 相同的"best-effort post-creation setup"原则。 + +#### D-3:PR 引用解析(`--worktree=#` / 全 URL) + +**支持形式:** + +| 形式 | 解析后的 PR 号 | +| --------------------------------------------------------------- | -------------- | +| `--worktree=#123` | 123 | +| `--worktree '#123'` | 123 | +| `--worktree https://github.com/foo/bar/pull/123` | 123 | +| `--worktree https://gh.enterprise.com/foo/bar/pull/123?baz=qux` | 123 | + +**slug 与分支命名:** + +- slug:`pr-`(特殊保留前缀,与用户 slug 区分) +- 分支:`worktree-pr-`(沿用 qwen-code 现有 `worktree-` 命名规则;不采用 claude-code 的 `pr-` 直接命名,避免与本地 `pr-` 分支冲突) + +**fetch 策略:** + +``` +git fetch origin pull//head +→ 用 FETCH_HEAD 作为新 worktree 的 base +``` + +不依赖 `gh` CLI —— 纯 git fetch,支持任何 GitHub 实例(公网或企业版),只要 `origin` 远程指向 GitHub。 + +**错误路径:** + +| 场景 | 错误消息 | +| ------------------------ | ---------------------------------------------------------------------------- | +| `origin` 远程缺失 | `--worktree=# requires an "origin" remote that points at GitHub.` | +| `git fetch` 失败 | `Failed to fetch PR #: PR may not exist or origin remote is unreachable.` | +| 网络超时(30s) | 同上,加 `(timeout)` | +| `origin` 远程不是 GitHub | 不做主动检查,由 `git fetch` 自然失败(PR 协议是 GitHub 特有的) | + +**与 D-2 的关系:** PR worktree **同样**应用 `symlinkDirectories`(用户期望在 PR 上立刻能跑测试,依赖目录需要复用)。 + +#### 影响文件 + +| 文件 | 变更类型 | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `packages/cli/src/config/config.ts` | yargs 新增 `--worktree` 选项;`CliArgs` 接口加 `worktree?: string \| boolean` | +| `packages/cli/src/gemini.tsx` | `loadCliConfig()` 之后、主循环之前调用新的 `setupStartupWorktree()` helper | +| `packages/cli/src/startup/worktreeStartup.ts` | 新建:`setupStartupWorktree()` 处理 slug 解析、PR fetch、sidecar 写入、cwd 切换 | +| `packages/cli/src/nonInteractiveCli.ts` | 复用同一 helper(已有 `restoreWorktreeContext` 注入逻辑,无须改) | +| `packages/cli/src/acp-integration/acpAgent.ts` | 复用同一 helper | +| `packages/core/src/services/gitWorktreeService.ts` | 新增 `parsePRReference()`、`fetchPullRequestRef()`、`symlinkConfiguredDirectories()`;`createUserWorktree()` 接受可选 `baseBranchRef` 参数 | +| `packages/cli/src/config/settingsSchema.ts` | 新增 `worktree.symlinkDirectories: string[]` 顶层项 | +| `packages/vscode-ide-companion/schemas/settings.schema.json` | 重新生成 | +| `docs/users/features/worktree.md` | 新增 Quick Start CLI flag 章节、Settings 表新增一行 | + +#### 安全与回滚 + +- **fail-open vs fail-close:** symlink / hooks 失败 **不** 中止 worktree 创建(同 Phase C 既定模式);PR fetch 失败 **中止** 启动(无 base ref 就无法创建 worktree);slug 校验失败 **中止** 启动(与 `EnterWorktreeTool` 一致)。 +- **path traversal:** `symlinkDirectories` 项必须解析后仍在 `repoRoot` 内,否则拒绝该项并 log。 +- **PR fetch 超时:** 30 秒硬超时,避免无响应的网络拖死启动。 +- **cwd 切换的副作用:** 切 `process.cwd()` 之后,相对路径(如 `--prompt-file ./foo.txt`)的解析会受影响。**对策:** 在切 cwd 之前先解析所有相对路径参数(具体在 `setupStartupWorktree()` 入口处做一次 normalize)。 + +#### 开放问题 + +1. **`--worktree-keep-on-exit`?** claude-code 没有,qwen-code 是否需要一个 CLI flag 让 Exit Dialog 默认选 keep?建议**先不加**,等用户反馈。 +2. **`worktree.symlinkDirectories` 是否需要 per-project override?** 当前 settings 已经支持 user/workspace/project 三级合并,无需特殊处理。 +3. **PR fetch 是否要拉取 `merge` ref(`pull//merge`,即与 base 合并后的 ref)而非 `head`?** claude-code 选 `head`,理由是用户通常想看 PR 的实际改动。沿用此选择。 + +--- + +### Future:高级功能(按需实现) + +以下功能面向更特定的使用场景,当前阶段不纳入排期,待用户需求明确后再评估实现。 -- `--worktree [name]` CLI 启动标志:启动时直接创建 worktree,整个会话在隔离环境中运行 -- sparse checkout 支持:新增 `worktree.sparsePaths` 配置项 -- `.worktreeinclude` 文件:支持将 gitignore 的文件复制到 worktree -- tmux 集成:`--worktree --tmux` 在 tmux 会话中启动 -- PR 引用解析:`--worktree=#123` 自动 fetch 并基于 PR 创建 worktree +| 功能 | 说明 | +| ----------------------- | ----------------------------------------------------------------------------------------- | +| sparse checkout | `worktree.sparsePaths` 配置项,大型 monorepo 只 checkout 指定路径,缩短创建时间和磁盘占用 | +| `.worktreeinclude` 文件 | 将 gitignore 的文件(`.env`、`secrets.json` 等)自动复制进 worktree | +| tmux 集成 | `--worktree --tmux` 在新 tmux 窗口启动 worktree 会话 | diff --git a/docs/developers/daemon-client-adapters/channel-web.md b/docs/developers/daemon-client-adapters/channel-web.md new file mode 100644 index 00000000000..44caca3afd5 --- /dev/null +++ b/docs/developers/daemon-client-adapters/channel-web.md @@ -0,0 +1,120 @@ +# Channel And Web Backend Daemon Adapter Draft + +## Goal + +Let channel adapters and web chat backends consume `qwen serve` through +`DaemonSessionClient` while keeping existing channel ACP subprocess behavior as +the default. + +This draft covers server-side clients only: + +- Channel bot backend -> `qwen serve` +- Web browser -> web backend / BFF -> `qwen serve` + +It explicitly does not allow browser JavaScript to call the daemon directly. +The daemon currently rejects browser `Origin` requests by design. + +## Proposed Entry Points + +Channel backend: + +```bash +QWEN_CHANNEL_DAEMON_URL=http://127.0.0.1:4170 qwen channel start telegram +``` + +Web backend: + +```bash +QWEN_WEB_DAEMON_URL=http://127.0.0.1:4170 qwen web-chat-backend +``` + +Shared optional variables: + +```bash +QWEN_DAEMON_TOKEN=... +QWEN_DAEMON_WORKSPACE=/repo +``` + +## Minimal Channel Flow + +This PR adds `DaemonChannelBridge`, a locally verifiable server-side bridge for +channel and web-backend adapters. It keeps the existing ACP bridge as the +default and owns daemon session state inside the backend process. + +1. Resolve channel sender/thread to a channel session key. +2. Use `DaemonClient` + `DaemonSessionClient.createOrAttach()`. +3. Submit inbound user text with `session.prompt()`. +4. Subscribe to `session.events()` and collect assistant text chunks. +5. Send final text back through the platform adapter. +6. Cast permission votes through `session.respondToPermission()`. +7. Cancel active work through `session.cancel()`. + +## Minimal Web Backend Flow + +1. Browser opens a websocket or HTTP stream to the web backend. +2. Backend owns `DaemonSessionClient`. +3. Backend translates browser messages to daemon prompts. +4. Backend translates daemon SSE events to browser-safe app events. +5. Backend stores the daemon `sessionId` and last seen event id server-side. + +Browser clients must not receive daemon bearer tokens. + +## Session Isolation Constraint + +Current daemon Stage 1 behavior is effectively `sessionScope: single` at the +daemon setting level. Until per-request `sessionScope` lands, multi-user channel +or web deployments must choose one of these safe shapes: + +- one daemon per channel thread / web room +- one daemon per user workspace +- single-user demo only + +Do not silently multiplex unrelated channel threads into one daemon session. + +## Event Mapping Contract + +| Daemon event | Channel/web backend handling | +| ---------------------------------------- | -------------------------------------- | +| `session_update` / `agent_message_chunk` | Append assistant text | +| `session_update` / `agent_thought_chunk` | Optional hidden/debug stream | +| `session_update` / `tool_call` | Emit tool status card/message | +| `permission_request` | Platform-specific approval interaction | +| `permission_resolved` | Close/update approval interaction | +| `model_switched` | Update backend session metadata | +| `session_died` | Notify user and stop stream | + +Unknown daemon events must be ignored or forwarded as debug metadata, not fatal. + +The bridge is not wired into `qwen channel start` yet. Existing Telegram, +Weixin, Dingtalk, plugin channel, and browser behavior remains unchanged. + +## Explicit Non-Goals + +- No browser direct-to-daemon fetch or EventSource. +- No CORS relaxation in this adapter PR. +- No default migration of Telegram, Weixin, Dingtalk, or plugin channels. +- No file CRUD, memory CRUD, MCP restart, or provider mutation. +- No sessionScope emulation in the client when daemon-side support is absent. + +## Merge Safety + +- Default off. +- Existing ACP channel bridge remains the default. +- Web backend is an explicit BFF layer, not a daemon security change. +- No channel adapter should import daemon tokens into frontend/browser code. + +## Validation Plan + +- Unit-test channel session-key to daemon-session binding. +- Unit-test daemon event to channel/web message mapping. +- Unit-test prompt, cancel, model switch, and permission response forwarding. +- Smoke-test one single-user channel backend against local `qwen serve`. +- Smoke-test browser -> BFF -> daemon without exposing daemon token. + +## Blockers Before Default Migration + +- Per-request `sessionScope`. +- Session metadata + close/delete lifecycle. +- Daemon-stamped client identity. +- Session-scoped permission route. +- Read-only diagnostics for MCP, skills, providers, and environment. diff --git a/docs/developers/daemon-client-adapters/ide.md b/docs/developers/daemon-client-adapters/ide.md new file mode 100644 index 00000000000..4084fce619e --- /dev/null +++ b/docs/developers/daemon-client-adapters/ide.md @@ -0,0 +1,122 @@ +# IDE Daemon Adapter Draft + +## Goal + +Let the VS Code companion extension dogfood Mode B by connecting from the +extension host to `qwen serve` through `DaemonSessionClient`. + +The webview must not call the daemon directly. The extension host owns daemon +URL, token, session id, and SSE replay state, then forwards sanitized app events +to the webview. + +## Proposed Entry Point + +VS Code settings: + +```json +{ + "qwen-code.experimentalDaemon.enabled": true, + "qwen-code.experimentalDaemon.url": "http://127.0.0.1:4170", + "qwen-code.experimentalDaemon.token": "" +} +``` + +Environment fallback for local dogfood: + +```bash +QWEN_IDE_DAEMON_URL=http://127.0.0.1:4170 code . +``` + +## Minimal Flow + +1. Extension host creates `DaemonClient`. +2. Fetch `/capabilities` and verify workspace compatibility. +3. Create or attach with `DaemonSessionClient.createOrAttach()`. +4. Subscribe to `session.events()` in the extension host. +5. Translate daemon events into existing webview messages. +6. Send user prompts through `session.prompt()`. +7. Route cancel/model switch through `session.cancel()` and + `session.setModel()`. +8. Route permission decisions through `session.respondToPermission()`. + +## Relationship To Existing ACP Connection + +The first implementation introduces a sibling connection path, not replace +`AcpConnection`: + +```text +QwenAgentManager + current default -> AcpConnection -> qwen --acp child + experimental -> DaemonIdeConnection -> qwen serve HTTP/SSE +``` + +Both paths should feed the same higher-level webview callbacks where practical. +If an event cannot be faithfully mapped yet, the daemon path should surface a +clear unsupported-state warning rather than silently pretending parity. + +This PR adds `DaemonIdeConnection` as the locally verifiable extension-host +adapter spike. It is not wired into the default `QwenAgentManager` path yet, so +existing VS Code behavior remains ACP subprocess based. + +## Event Mapping Contract + +| Daemon event | IDE handling | +| ---------------------------------------- | -------------------------------------------- | +| `session_update` / `agent_message_chunk` | Existing assistant stream callback | +| `session_update` / `agent_thought_chunk` | Existing thinking stream callback | +| `session_update` / `tool_call` | Existing tool-call update callback | +| `permission_request` | Existing approval UI callback | +| `permission_resolved` | Close/update approval UI | +| `model_switched` | Existing model-state callback where possible | +| `session_died` | Disconnect UI + reconnect affordance | + +Unknown events must be ignored or logged as debug metadata. + +## Runtime Locality UX + +The extension must make daemon locality visible: + +- workspace/files are daemon-host paths +- MCP servers run on the daemon host +- skills load from the daemon filesystem +- provider credentials are resolved in the daemon process environment + +Do not imply that local VS Code extensions, local browser profile, local +localhost services, or local SSH/kube credentials are automatically available to +the daemon. + +## Explicit Non-Goals + +- No default migration away from `AcpConnection`. +- No webview direct-to-daemon transport. +- No daemon-side file CRUD through the IDE until file service boundaries land. +- No reverse RPC for editor/browser/clipboard yet. +- No full remote-control integration. + +## Merge Safety + +- Default off behind setting/env. +- Additive sibling connection path. +- Existing VS Code ACP subprocess path unchanged. +- Daemon token never crosses into webview JavaScript. + +## Validation Plan + +- Unit-test daemon session factory connection and SSE event consumption. +- Unit-test daemon event to existing extension-host callback mapping. +- Unit-test prompt, cancel, model switch, and permission response forwarding. +- Unit-test settings/env resolution when the feature flag is wired. +- Smoke-test local extension host against `qwen serve`: + - prompt streams into chat + - cancel works + - permission UI can resolve a request + - SSE reconnect uses tracked `Last-Event-ID` + +## Blockers Before Default Migration + +- Typed daemon event schema. +- Daemon-stamped client identity. +- Session-scoped permission route. +- Read-only runtime diagnostics. +- FileSystemService boundary and safe file read routes. +- Output sink refactor for CLI/TUI parity. diff --git a/docs/developers/daemon-client-adapters/tui.md b/docs/developers/daemon-client-adapters/tui.md new file mode 100644 index 00000000000..c9d223b9262 --- /dev/null +++ b/docs/developers/daemon-client-adapters/tui.md @@ -0,0 +1,96 @@ +# TUI Daemon Adapter Draft + +## Goal + +Add a flag-gated TUI transport that talks to `qwen serve` through +`DaemonSessionClient` instead of creating an in-process `Config` + agent +runtime. + +This is a dogfood path for Mode B client migration. It must not replace the +default TUI path until output sinks, typed daemon events, session-scoped +permission, and lifecycle diagnostics are stable. + +## Proposed Entry Point + +```bash +QWEN_DAEMON_URL=http://127.0.0.1:4170 qwen --experimental-daemon-tui +``` + +Optional: + +```bash +QWEN_DAEMON_TOKEN=... QWEN_DAEMON_WORKSPACE=/repo qwen --experimental-daemon-tui +``` + +The CLI should refuse this mode unless both are true: + +- `QWEN_DAEMON_URL` or `--daemon-url` is set. +- `GET /capabilities` advertises `session_create`, `session_prompt`, and + `session_events`. + +## Minimal Flow + +1. Create `DaemonClient` with daemon URL and token. +2. Fetch `/capabilities`. +3. Create or attach with `DaemonSessionClient.createOrAttach()`. +4. Subscribe to `session.events()`. +5. Submit user prompts through `session.prompt()`. +6. Route cancel through `session.cancel()`. +7. Route model switch through `session.setModel()`. +8. Route permission votes through `session.respondToPermission()`. + +## Rendering Contract + +The first implementation adds `DaemonTuiAdapter`, a locally verifiable reducer +and transport spike. It maps only these daemon events: + +| Daemon event | TUI handling | +| ---------------------------------------- | -------------------------------------------- | +| `session_update` / `agent_message_chunk` | Append assistant text | +| `session_update` / `agent_thought_chunk` | Append thinking text | +| `session_update` / `tool_call` | Show tool call lifecycle | +| `permission_request` | Show existing confirmation UI where possible | +| `permission_resolved` | Close or update confirmation UI | +| `model_switched` | Update footer/model display | +| `session_died` | Show disconnected state and stop streaming | + +Unknown events must be ignored, not fatal. Typed event reducers will land in a +later protocol PR. + +The adapter is not wired into the default Ink app yet. Existing interactive TUI, +JSONL, stream-json, and dual-output behavior remains unchanged. + +## Explicit Non-Goals + +- Do not remove the current TUI in-process runtime. +- Do not change JSONL, stream-json, or dual-output behavior in this PR. +- Do not expose file CRUD, MCP management, memory CRUD, or provider/auth + mutation through TUI yet. +- Do not make browser/web direct-to-daemon assumptions; this is terminal only. + +## Merge Safety + +- Default off. +- Additive code path. +- No existing CLI flags change behavior. +- If the daemon is unavailable, the experimental path fails before starting the + TUI and tells the user to run `qwen serve`. + +## Validation Plan + +- Unit-test event-to-TUI-state mapping with synthetic daemon events. +- Unit-test prompt, cancel, model switch, and permission vote forwarding. +- Unit-test flag/env parsing when the feature flag is wired. +- Smoke-test against a local `qwen serve`: + - prompt text streams into the TUI + - cancel resolves the active prompt + - permission request can be accepted or rejected + - reconnect sends the tracked `Last-Event-ID` + +## Blockers Before Default Migration + +- Typed daemon event schema. +- Session-scoped permission route. +- Output sink refactor for JSONL / stream-json / dual-output parity. +- Session lifecycle close/delete semantics. +- Runtime diagnostics for MCP, skills, providers, and workspace env. diff --git a/docs/developers/development/telemetry.md b/docs/developers/development/telemetry.md index 1ebc8881f58..266ce05cc83 100644 --- a/docs/developers/development/telemetry.md +++ b/docs/developers/development/telemetry.md @@ -65,22 +65,51 @@ These settings can be overridden by environment variables or CLI flags. | `otlpMetricsEndpoint` | `QWEN_TELEMETRY_OTLP_METRICS_ENDPOINT` | - | Per-signal endpoint override for metrics (HTTP only) | URL string | - | | `outfile` | `QWEN_TELEMETRY_OUTFILE` | `--telemetry-outfile ` | Save telemetry to file (overrides OTLP export) | file path | - | | `logPrompts` | `QWEN_TELEMETRY_LOG_PROMPTS` | `--telemetry-log-prompts` / `--no-telemetry-log-prompts` | Include prompts in telemetry logs | `true`/`false` | `true` | -| `includeSensitiveSpanAttributes` | `QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES` | - | Include sensitive attributes in log-to-span bridge spans | `true`/`false` | `false` | +| `includeSensitiveSpanAttributes` | `QWEN_TELEMETRY_INCLUDE_SENSITIVE_SPAN_ATTRIBUTES` | - | Include user prompts, system prompts, tool I/O, and model output as native span attributes (in addition to log-to-span bridge spans) | `true`/`false` | `false` | +| `resourceAttributes` | `OTEL_RESOURCE_ATTRIBUTES` (+ `OTEL_SERVICE_NAME`) | - | Static resource attributes attached to every exported span / log / metric. See [Resource attributes](#resource-attributes) below. | `key=value,…` | `{}` | +| `metrics.includeSessionId` | `QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID` | - | Include `session.id` on metric data points. **Disabled by default** to protect metric backends from time-series fan-out. | `true`/`false` | `false` | **Note on boolean environment variables:** For the boolean settings (`enabled`, `logPrompts`, `includeSensitiveSpanAttributes`), setting the corresponding environment variable to `true` or `1` will enable the feature. Any other value will disable it. -**Sensitive log-to-span attributes:** When Qwen Code exports HTTP traces but has -no logs endpoint, log records are bridged into trace spans. By default, the -bridge drops `prompt`, `function_args`, and `response_text` from span attributes. -Set `includeSensitiveSpanAttributes` to `true` only when you explicitly want -those fields in bridged spans. This setting only controls the log-to-span -bridge. It does not disable sensitive data in OTel logs or other telemetry -sinks; non-internal API response telemetry can populate `response_text`, so OTel -logs, UI telemetry, and chat recording may receive response text independently -of this bridge setting. QwenLogger does not include `response_text`. +**Sensitive span attributes:** When `includeSensitiveSpanAttributes` is enabled, +two things happen: + +1. **Native span attributes (`qwen-code.interaction`, `api.generateContent*`, + `tool.`)** carry verbatim conversation content: + - User prompts (`new_context`) + - System prompts (`system_prompt` — full text once per session, deduped by + SHA-256 hash; subsequent spans only carry `system_prompt_hash` + + `system_prompt_preview` + `system_prompt_length`) + - Tool schemas (emitted as `tool_schema` events, also hash-deduped) + - Tool inputs (`tool_input`) and tool results (`tool_result`) + - Model output (`response.model_output`) + + Each value is truncated at 60 KB; `*_truncated` and `*_original_length` + flags surface when truncation occurs. + +2. **Log-to-span bridge spans** (used when HTTP traces are exported without a + logs endpoint) keep their existing `prompt`, `function_args`, and + `response_text` fields, instead of being dropped. + +⚠️ **Security warning:** enabling this flag streams full conversation history, +file contents read by `read_file`, shell commands and their output (including +secrets in env vars or arguments), and model responses to the configured OTLP +backend. Treat the backend as a privileged data sink. The flag defaults to +`false`. + +**Cost / payload size:** A heavy turn (60 KB system prompt + 10 tool calls, +each up to 60 KB input + 60 KB result, plus 60 KB model output) can produce up +to ~1.5 MB of attribute payload before OTLP compression. When pointing tools +that read large files (`read_file`, etc.) at long-running sessions, monitor +exporter throughput. + +This setting does not disable sensitive data in OTel logs or other telemetry +sinks; non-internal API response telemetry can populate `response_text`, so +OTel logs, UI telemetry, and chat recording may receive response text +independently of this setting. QwenLogger does not include `response_text`. **HTTP OTLP signal routing:** When using HTTP protocol (`otlpProtocol: "http"`), Qwen Code automatically appends signal-specific paths (`/v1/traces`, `/v1/logs`, @@ -98,6 +127,234 @@ The `QWEN_TELEMETRY_OTLP_*` variants take precedence over the `OTEL_*` variants. For detailed information about all configuration options, see the [Configuration Guide](./cli/configuration.md). +### Resource attributes + +Resource attributes are static key-value pairs attached to every span, log, +and metric exported via OTLP. Use them to slice telemetry by team, environment, +deployment region, or any other dimension your backend cares about. + +Two sources, merged in priority order (lowest → highest): + +1. The standard `OTEL_RESOURCE_ATTRIBUTES` env var +2. `telemetry.resourceAttributes` in `.qwen/settings.json` (overrides env on + key conflict) + +`OTEL_SERVICE_NAME` is a separate escape hatch — when set, it overrides +`service.name` from any other source (per the OpenTelemetry spec). + +#### Examples + +**Slice all telemetry by team / environment:** + +```bash +export OTEL_RESOURCE_ATTRIBUTES="team=platform,env=prod,cost_center=eng-123" +``` + +**Route to a per-tenant collector via `service.name`:** + +```bash +export OTEL_SERVICE_NAME=qwen-code-ci +``` + +**Fleet baseline (`~/.qwen/settings.json`) + per-host override:** + +```json +{ + "telemetry": { + "resourceAttributes": { + "deployment.environment": "production", + "service.namespace": "engineering-tooling" + } + } +} +``` + +```bash +# Add a one-off tag without touching settings: +export OTEL_RESOURCE_ATTRIBUTES="debug_run=true" +``` + +#### Reserved keys + +Some keys are runtime-controlled and cannot be overridden: + +- `service.version` — always set to the running CLI version. Setting it from + any source is silently dropped with a warning. +- `session.id` — runtime-injected per session. User-provided values from + either env or settings are dropped with a warning. The reason is that + Resource attributes auto-attach to every metric data point; allowing user + override would bypass [Cardinality controls](#cardinality-controls) below. + Spans and logs always carry `session.id`. + +`service.name` is **not** reserved; it follows the precedence chain above. + +#### Format + +`OTEL_RESOURCE_ATTRIBUTES` follows the OpenTelemetry spec: +`key1=value1,key2=value2` with values percent-encoded. Spaces in values must +be encoded as `%20`, **commas as `%2C`** (unencoded commas split the value at +the wrong boundary and the second half is dropped as malformed). Malformed +pairs are skipped with a warning rather than failing telemetry startup. + +#### Troubleshooting: when a user-provided attribute appears not to take effect + +Reserved keys (`service.version`, `session.id`), malformed pairs, non-string +settings values, and invalid percent-encoding are all silently dropped with a +warning logged via the OpenTelemetry diagnostics channel. That channel routes +to the debug log file (`~/.qwen/log/otel-*.log`), **not** the console, so the +behavior can look like silent failure. + +If a custom resource attribute isn't appearing on exported telemetry: + +1. Check `~/.qwen/log/otel-*.log` for lines matching `cannot override` (reserved + key dropped), `Skipping malformed` (bad env var pair), or `must be a string` + (non-string settings value). +2. Verify the env var is set in the qwen-code process's environment (not just + your shell) and that values are percent-encoded. +3. Confirm `telemetry.enabled` is `true` — telemetry init only runs if enabled. + +### Cardinality controls + +Metrics are aggregated by attribute set at the backend — every distinct +combination of attribute values produces a new time series. Attaching a +high-cardinality field like `session.id` to a metric causes time-series fan-out +proportional to the number of sessions, which quickly exhausts metric backend +storage. + +To prevent this, Qwen Code keeps high-cardinality attributes off metric data +points by default. Spans and logs are per-event and unaffected, so they +continue to carry `session.id` for trace and log correlation. + +#### `telemetry.metrics.includeSessionId` (default: `false`) + +Setting this to `true` (via settings or +`QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID=true`) re-attaches `session.id` to +every metric data point. + +⚠️ **Warning:** each CLI session creates a new value. Leaving this on for a +fleet will blow up metric storage. Recommended only for short-term debugging. +For long-term session correlation, query trace or log backends instead. + +#### Migration from earlier versions + +Prior to this release, `session.id` was attached to metrics by default. If +your Prometheus queries / Grafana dashboards / alert rules reference +`session_id` on a metric, you have two options: + +**Option A** — restore the previous behavior for short-term debugging: + +```bash +export QWEN_TELEMETRY_METRICS_INCLUDE_SESSION_ID=true +``` + +or: + +```json +{ + "telemetry": { + "metrics": { "includeSessionId": true } + } +} +``` + +**Option B (recommended)** — move session-level analysis off metrics. Spans +and logs still carry `session.id`, and trace / log backends (Jaeger, Tempo, +Loki, Aliyun SLS / ARMS Tracing) handle per-session slicing natively without +cardinality pressure. + +### Client-side HTTP span on outbound fetch + +When telemetry is enabled, Qwen Code registers `UndiciInstrumentation` +which creates a client-side HTTP span for every outbound `fetch()` +request originated by the process — including the LLM SDKs (`openai`, +`@google/genai`, `@anthropic-ai/sdk`), the MCP StreamableHTTP client, the +`WebFetch` tool, and any IDE-extension out-of-process calls. The span +lets you see network latency (TTFB / response body transfer) separately +from upstream model processing time, which the existing +`api.generateContent` span alone can't distinguish. + +These spans go to your **own** OTLP collector (or file outfile) just like +the rest of the telemetry — they do not affect what is written onto the +outbound HTTP request itself. Whether the W3C `traceparent` header is +also written into the outgoing request stream is controlled by a +**separate, security-relevant setting** documented in +[outbound correlation](#outbound-correlation-security-relevant) below. + +**Feedback-loop avoidance.** OTel SDK uses `fetch` internally to upload OTLP +data. Without protection, instrumenting `fetch` would trace those uploads, +which would themselves be uploaded, causing an infinite loop. Qwen Code's +undici instrumentation is configured with an `ignoreRequestHook` that skips +URLs matching the configured `telemetry.otlpEndpoint` / +`telemetry.otlpTracesEndpoint` / `telemetry.otlpLogsEndpoint` / +`telemetry.otlpMetricsEndpoint` prefixes. In file-outfile mode there are no +outbound HTTP uploads, so the hook is a no-op. + +## Outbound correlation (SECURITY-RELEVANT) + +These settings live in a **separate top-level namespace** from `telemetry.*` +on purpose: telemetry controls data flow into the operator's own +observability backend, while `outboundCorrelation.*` controls what +client-side correlation data qwen-code writes **into outbound LLM API +request streams** that reach third-party LLM provider endpoints +(DashScope, OpenAI, Anthropic, etc.). Different recipients, different +consent decision. **All values default to off.** See PR #4390 review +discussion for the framing rationale. + +### `outboundCorrelation.propagateTraceContext` + +```jsonc +"outboundCorrelation": { + "propagateTraceContext": false // default +} +``` + +When `false` (default), Qwen Code installs a no-op `TextMapPropagator` on +the OTel SDK. UndiciInstrumentation still creates client HTTP spans for +your OTLP collector, but `propagation.inject()` is a no-op so **no +`traceparent` is written onto outbound requests**. Trace IDs stay +internal to the operator's collector. + +When `true`, the SDK's default W3C composite propagator +(`tracecontext` + `baggage`) is installed and the standard `traceparent` +header is written on every outbound `fetch`: + +``` +traceparent: 00-<32-hex traceId>-<16-hex parentSpanId>-<01-sampled | 00-not-sampled> +``` + +Opt in only when the LLM provider also reports into your OTel collector +for cross-process trace stitching — e.g. ARMS Tracing serving DashScope. +For most operators the value is `false`; cross-vendor trace continuation +is niche. + +**Depends on `telemetry.enabled: true`.** The OTel SDK only initializes +when telemetry is enabled, so `propagateTraceContext` only takes effect +in that state. Setting it to `true` while telemetry is disabled is a +silent no-op — no SDK, no propagator, no `traceparent` on the wire. +Verify both flags when wiring an ARMS+DashScope correlation setup: + +```jsonc +{ + "telemetry": { + "enabled": true, + "otlpTracesEndpoint": "http://tracing-analysis-...", + }, + "outboundCorrelation": { + "propagateTraceContext": true, + }, +} +``` + +### Other outbound correlation headers + +`X-Qwen-Code-Session-Id` and `X-Qwen-Code-Request-Id` are **not part of +this PR**. They will be designed and proposed in their own follow-up +PR(s) under the same `outboundCorrelation.*` namespace, each with its +own threat model and operator-consent flow. PR #4390 review (LaZzyMan) +established the principle: "telemetry's scope of work doesn't include +sending identifiers to LLM providers"; correlation-header work moves to +its own design discussion rather than landing under telemetry. + ## Aliyun Telemetry ### Manual OTLP Export diff --git a/docs/developers/examples/daemon-client-quickstart.md b/docs/developers/examples/daemon-client-quickstart.md index a72069aca47..733a9fad78c 100644 --- a/docs/developers/examples/daemon-client-quickstart.md +++ b/docs/developers/examples/daemon-client-quickstart.md @@ -9,9 +9,11 @@ In one terminal: ```bash cd your-project/ qwen serve --port 4170 -# → qwen serve listening on http://127.0.0.1:4170 (mode=http-bridge) +# → qwen serve listening on http://127.0.0.1:4170 (mode=http-bridge, workspace=/path/to/your-project) ``` +Per [#3803](https://github.com/QwenLM/qwen-code/issues/3803) §02 each daemon binds to one workspace at boot (the current `cwd`, or override with `--workspace /path/to/dir`). The daemon's bound path is advertised on `/capabilities.workspaceCwd` so clients can pre-flight check + omit `cwd` from `POST /session`. + In another: ```bash @@ -28,13 +30,23 @@ const client = new DaemonClient({ // token: process.env.QWEN_SERVER_TOKEN, // required for non-loopback binds }); -// 1. Confirm we can reach the daemon and gate UI on its features. +// 1. Confirm we can reach the daemon, gate UI on its features, and +// read back the daemon's bound workspace (#3803 §02). const caps = await client.capabilities(); console.log('Daemon features:', caps.features); - -// 2. Spawn-or-attach a session for the current workspace. +console.log('Daemon workspace:', caps.workspaceCwd); // canonical bound path + +// 2. Spawn-or-attach a session. Two equally-valid shapes: +// (a) pass `workspaceCwd: caps.workspaceCwd` to be explicit, or +// (b) omit `workspaceCwd` entirely — the SDK then sends no `cwd` +// field and the daemon route falls back to its bound +// workspace. The (b) shape is concise but assumes you trust +// `caps.workspaceCwd` to be whatever you intended. +// A non-empty `workspaceCwd` that doesn't canonicalize to the +// daemon's bound path yields `400 workspace_mismatch` (see +// "Workspace mismatch" below). const session = await client.createOrAttachSession({ - workspaceCwd: process.cwd(), + workspaceCwd: caps.workspaceCwd, }); console.log(`session=${session.sessionId} attached=${session.attached}`); @@ -97,6 +109,30 @@ function handleEvent(event: DaemonEvent): void { } ``` +## Workspace file helpers + +File routes are workspace-scoped, not session-scoped, so they live on +`DaemonClient` directly: + +```ts +const file = await client.readWorkspaceFile('src/main.ts'); + +const updated = await client.editWorkspaceFile({ + path: 'src/main.ts', + oldText: 'timeout: 30000', + newText: 'timeout: 60000', + expectedHash: file.hash!, +}); + +console.log(updated.hash); +``` + +`expectedHash` is SHA-256 over the raw on-disk bytes. `mode: "replace"` and +`editWorkspaceFile()` require it so stale clients do not overwrite a file they +did not just read. Write/edit require bearer-token configuration even on +loopback; start the daemon with `--token` or `QWEN_SERVER_TOKEN` before using +them. + ## Reconnect with `Last-Event-ID` If your client process restarts mid-session, replay events you missed: @@ -139,9 +175,12 @@ case 'permission_request': { ## Shared-session collaboration -Two clients pointed at the same daemon and `cwd` end up on the same session: +Two clients pointed at the **same daemon** end up on the same session. Per #3803 §02 each daemon is bound to ONE workspace at boot, so the daemon launched as `qwen serve --workspace /work/repo` (or `cd /work/repo && qwen serve`) is what both clients connect to: ```ts +// Daemon was launched as `qwen serve --workspace /work/repo` so +// `caps.workspaceCwd === '/work/repo'` for both clients. + // Client A (e.g. an IDE plugin) const a = await clientA.createOrAttachSession({ workspaceCwd: '/work/repo' }); console.log(a.attached); // false — A spawned the agent @@ -154,6 +193,35 @@ console.log(a.sessionId === b.sessionId); // true Both clients see the same `session_update` / `permission_request` stream. Either can send a prompt; they FIFO-queue per the agent's "one active prompt per session" guarantee. +## Workspace mismatch + +If `workspaceCwd` doesn't match the daemon's bound workspace, `createOrAttachSession` rejects with `DaemonHttpError` carrying status `400` and a structured body: + +```ts +import { DaemonHttpError } from '@qwen-code/sdk'; + +try { + await client.createOrAttachSession({ workspaceCwd: '/some/other/project' }); +} catch (err) { + if (err instanceof DaemonHttpError && err.status === 400) { + const body = err.body as { + code?: string; + boundWorkspace?: string; + requestedWorkspace?: string; + }; + if (body.code === 'workspace_mismatch') { + console.error( + `This daemon is bound to ${body.boundWorkspace}, ` + + `not ${body.requestedWorkspace}. Start a separate daemon ` + + `for that workspace, or route to the right one.`, + ); + } + } +} +``` + +Multi-workspace deployments run one daemon per workspace on separate ports — there's no intra-daemon routing under §02. An orchestrator (or the user's launcher) picks the right daemon based on the project the client wants to talk to. + ## Authentication When the daemon was started with a token (any non-loopback bind requires one): diff --git a/docs/developers/qwen-serve-protocol.md b/docs/developers/qwen-serve-protocol.md index 4b73183e169..f6d6315827e 100644 --- a/docs/developers/qwen-serve-protocol.md +++ b/docs/developers/qwen-serve-protocol.md @@ -14,6 +14,10 @@ Without a configured token (loopback dev default) the header is optional. Token **`/health` exemption** (Bctum): on loopback binds (`127.0.0.1` / `localhost` / `::1` / `[::1]`) `/health` is registered BEFORE the bearer middleware, so liveness probes inside the pod don't need to carry the token even when the daemon was started with `--token`. Non-loopback binds (`--hostname 0.0.0.0` etc.) gate `/health` behind the bearer like every other route — see the [`GET /health`](#get-health) section for the rationale. +**`--require-auth` (#4175 PR 15).** Pass this flag at boot to extend the "must have a token" rule to loopback as well. Boot fails without a token; the `/health` exemption is dropped (so `/health` also requires `Authorization: Bearer …`). + +When the flag is on, the global `bearerAuth` middleware gates **every** route — including `/capabilities`. An **unauthenticated** client therefore cannot pre-flight `caps.features` to discover that auth is required: the discovery surface for that case is the **401 response body** itself (uniform across all routes per the [Authentication](#authentication) section). The `require_auth` capability tag is a **post-authentication confirmation** — once a client successfully authenticates and reads `/capabilities`, the tag's presence confirms the daemon was started with `--require-auth` (useful for audit / compliance UIs and for SDK clients to surface "this deployment is hardened" in a settings panel). Mutation routes that opt into per-route strict mode (Wave 4 follow-ups) refuse with `401 { code: "token_required", error: "…" }` when reached on a no-token loopback default — but with `--require-auth` enabled the global bearer middleware short-circuits the request before the per-route gate, so the legacy `Unauthorized` body is what unauthenticated callers actually see. + ## Common error shape 5xx responses carry the original error's `code` and `data` when present (JSON-RPC style — the ACP SDK forwards `{code, message, data}` from the agent): @@ -42,6 +46,19 @@ with status `400`. with status `404`. +`WorkspaceMismatchError` for a `POST /session` whose `cwd` doesn't canonicalize to the daemon's bound workspace (#3803 §02 — 1 daemon = 1 workspace) returns `400` with: + +```json +{ + "error": "Workspace mismatch: daemon is bound to \"…\" but request asked for \"…\". …", + "code": "workspace_mismatch", + "boundWorkspace": "/path/the/daemon/binds", + "requestedWorkspace": "/path/in/the/request" +} +``` + +Use this to detect mismatch pre-flight: read `workspaceCwd` off `/capabilities` and omit `cwd` from `POST /session` (it falls back to the bound workspace), or route the request to a daemon bound to `requestedWorkspace`. + `POST /session` past the daemon's `--max-sessions` cap returns `503` with a `Retry-After: 5` header and: ```json @@ -54,23 +71,85 @@ with status `404`. Attaches to existing sessions are NOT counted toward the cap, so an idle daemon's reconnects keep working even when at-capacity. +`RestoreInProgressError` — only emitted by `POST /session/:id/load` and `POST /session/:id/resume` — returns `409` with a `Retry-After: 5` header (matching `session_limit_exceeded`) and: + +```json +{ + "error": "Session \"\" is already being restored via session/; retry session/ after it completes", + "code": "restore_in_progress", + "sessionId": "", + "activeAction": "load", + "requestedAction": "resume" +} +``` + +Fired when a `session/load` is issued for an id that already has a `session/resume` in flight (or vice versa). Wait at least `Retry-After` seconds and retry — the underlying restore completes within `initTimeoutMs` (default 10s). Same-action races (`load` vs `load`, `resume` vs `resume`) coalesce instead of erroring. + ## Capabilities -Every Stage 1 daemon advertises 9 feature tags. Clients **must** gate UI off `features`, not off `mode` (per design §10). +The daemon advertises its supported feature tags from the serve capability +registry. Clients **must** gate UI off `features`, not off `mode` (per design +§10). ``` -['health', 'capabilities', 'session_create', 'session_list', - 'session_prompt', 'session_cancel', 'session_events', - 'session_set_model', 'permission_vote'] +['health', 'capabilities', 'session_create', 'session_scope_override', + 'session_load', 'unstable_session_resume', + 'session_list', 'session_prompt', 'session_cancel', 'session_events', + 'slow_client_warning', 'typed_event_schema', + 'session_set_model', 'client_identity', 'client_heartbeat', + 'session_permission_vote', 'permission_vote', 'workspace_mcp', 'workspace_skills', + 'workspace_providers', 'workspace_env', 'workspace_preflight', + 'session_context', 'session_supported_commands', + 'session_close', 'session_metadata', 'mcp_guardrails', + 'mcp_guardrail_events', + 'workspace_file_read', 'workspace_file_bytes', 'workspace_file_write', + 'session_approval_mode_control', 'workspace_tool_toggle', + 'workspace_init', 'workspace_mcp_restart'] ``` -## Routes +`session_scope_override` is the negotiation handle for the per-request `sessionScope` field on `POST /session` (see below). Older daemons silently ignore the field, so SDK clients should pre-flight `caps.features` for this tag before sending it. + +`session_load` and `unstable_session_resume` advertise the explicit-restore routes (`POST /session/:id/load` and `POST /session/:id/resume`). Older daemons return `404` for these paths, so SDK clients should pre-flight `caps.features` before calling. The `unstable_` prefix on `unstable_session_resume` mirrors the underlying ACP method (`connection.unstable_resumeSession`) — the daemon's wire shape is committed for v1, but the ACP method name itself may change before ACP marks resume stable. + +`slow_client_warning` covers two co-released SSE backpressure knobs introduced in #4175 Wave 2.5 PR 10: (a) the daemon emits a `slow_client_warning` synthetic event-stream frame when a subscriber's queue crosses 75% full, once per overflow episode (rearmed after the queue drains below 37.5%); (b) `GET /session/:id/events` accepts a `?maxQueued=N` query param (range `[16, 2048]`) to pre-size the per-subscriber backlog for cold reconnects against a large replay ring. The daemon-wide ring size is controlled by `--event-ring-size` (default **8000**, per #3803 §02). Old daemons silently lack both — pre-flight this tag before opting in. + +`typed_event_schema` advertises daemon event payloads that match the SDK's `KnownDaemonEvent` schema. Older daemons may still stream compatible frames, but SDK clients should pre-flight this tag before assuming typed event coverage. + +`client_heartbeat` advertises `POST /session/:id/heartbeat`. Older daemons return `404`; pre-flight this tag before issuing periodic heartbeats. + +`session_close` and `session_metadata` advertise `DELETE /session/:id` and `PATCH /session/:id/metadata`. Older daemons return `404`; pre-flight these tags before exposing close or rename affordances. + +`session_approval_mode_control`, `workspace_tool_toggle`, `workspace_init`, and `workspace_mcp_restart` (issue [#4175](https://github.com/QwenLM/qwen-code/issues/4175) PR 17) advertise the four mutation control routes documented under "Mutation: approval, tools, init, MCP restart" below. All four are strict-gated by the PR 15 mutation gate (a daemon configured without a bearer token rejects them with 401 `token_required`). Older daemons return `404`; pre-flight each tag before exposing the corresponding affordance. -> **Stage 1 limitation — no `DELETE /session/:id`.** Sessions live until -> the agent child crashes (`session_died`), the daemon process exits, or -> a server-side `killSession` (used internally by orphan-cleanup) fires. -> HTTP clients have no explicit "close one session" route in Stage 1. -> An explicit `DELETE /session/:id` is on the Stage 2 polish list. +`mcp_guardrails` (issue [#4175](https://github.com/QwenLM/qwen-code/issues/4175) PR 14) covers the MCP budget surface: the `clientCount` / `clientBudget` / `budgetMode` / `budgets[]` fields on `GET /workspace/mcp`, the `disabledReason` field on per-server cells, and the `--mcp-client-budget` / `--mcp-budget-mode` CLI flags. Older daemons omit the new fields entirely; SDK clients pre-flight this tag before relying on `budgets[]` semantics. The registry descriptor also carries `modes: ['warn', 'enforce']` for future feature-modes exposure — for now, clients infer mode from the snapshot's `budgetMode` field. Server refusal under `enforce` mode is deterministic by `Object.entries(mcpServers)` declaration order; a future scope-precedence layer (if qwen-code adopts one) would shift this to "lowest-precedence first" to mirror claude-code's `plugin < user < project < local` convention. + +> ⚠️ **PR 14 v1 scope: per-session, not per-workspace.** Each ACP session inside the daemon constructs its own `Config` + `McpClientManager` (via `acpAgent.newSessionConfig`). The budget caps live MCP clients **per session**; each session independently reads `QWEN_SERVE_MCP_CLIENT_BUDGET` from the forwarded env. With `--mcp-client-budget=10` and 5 concurrent ACP sessions, the actual live MCP client count can reach 5 × 10 = 50 across the daemon. The `GET /workspace/mcp` snapshot reads the **bootstrap session's** `McpClientManager` accounting only — the `budgets[0].scope: 'session'` value is the honest signal that this is per-session, not aggregated. **Wave 5 PR 23 (shared MCP pool)** will introduce a workspace-scoped manager and add a `scope: 'workspace'` cell alongside the per-session cell for true cross-session aggregation. v1 is the in-process counter + soft enforcement foundation that PR 23 builds on. + +`workspace_file_read` covers the text/list/stat/glob workspace file routes +(`GET /file`, `GET /list`, `GET /glob`, `GET /stat`). `workspace_file_bytes` +covers `GET /file/bytes`, which was added later so clients can pre-flight raw +byte-window support against PR19-era daemons. `workspace_file_write` covers +the hash-aware text mutation routes (`POST /file/write`, `POST /file/edit`). +The write tag means the route contract exists; it does not mean the current +deployment is open for anonymous mutation. Write/edit are strict mutation +routes and require a configured bearer token even on loopback. + +**Conditional tags.** A small number of feature tags are advertised only when the matching deployment toggle is on. Tag presence = behavior is on; absence = either an older daemon predating the tag, OR a current daemon where the operator did not opt in. Currently: + +| Tag | Advertised when … | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `require_auth` | the daemon was started with `--require-auth` (or `requireAuth: true` via the embedded API). Bearer token is mandatory on every route, including `/health` on loopback binds. | + +`mcp_guardrails` is **not** in this conditional table — it's an always-on tag, advertised whenever the binary supports the new `/workspace/mcp` budget fields, regardless of whether the operator configured a budget. Operators who haven't set `--mcp-client-budget` still get the new fields (with `budgetMode: 'off'`, `budgets: []`). + +`mcp_guardrail_events` (issue [#4175](https://github.com/QwenLM/qwen-code/issues/4175) PR 14b) advertises the typed SSE push events that surface MCP budget state crossings without a poll loop. Two frame types arrive on `GET /session/:id/events`: + +- `mcp_budget_warning` — fires once on the upward 75% crossing of `reservedSlots.size / clientBudget`. Re-arms only after the ratio drops below 37.5% (`MCP_BUDGET_REARM_FRACTION`). Mirrors PR 10's `slow_client_warning` hysteresis, but at the manager level rather than the per-subscriber backlog level. Payload: `{ liveCount, reservedCount, budget, thresholdRatio: 0.75, mode: 'warn' | 'enforce' }`. Fires under both `warn` and `enforce` modes; never under `off`. +- `mcp_child_refused_batch` — fires at end of each `discoverAllMcpTools*` pass when one or more servers were refused, AND as a length-1 batch on the `readResource` lazy-spawn refusal path. Payload: `{ refusedServers: [{ name, transport, reason: 'budget_exhausted' }, ...], budget, liveCount, reservedCount, mode: 'enforce' }`. `mode` is the literal `'enforce'` because `warn` mode never refuses. + +Both events live in the per-session SSE replay ring (they carry an `id`) so a client reconnecting with `Last-Event-ID` resumes through them; the snapshot at `GET /workspace/mcp` is still the source-of-truth for state-after-extended-disconnect. Always-on once advertised — there is no conditional toggle. SDK reducer state (`DaemonSessionViewState`) exposes `mcpBudgetWarningCount`, `lastMcpBudgetWarning`, `mcpChildRefusedBatchCount`, `lastMcpChildRefusedBatch` for adapters that want simple lag-style UI. + +## Routes ### `GET /health` @@ -91,16 +170,674 @@ Pass `?deep=1` (also accepts `?deep=true` or bare `?deep`) for a probe that expo ```json { "v": 1, + "protocolVersions": { + "current": "v1", + "supported": ["v1"] + }, "mode": "http-bridge", "features": ["health", "capabilities", "..."], - "modelServices": [] + "modelServices": [], + "workspaceCwd": "/canonical/path/to/workspace" } ``` Stable contract: when `v` increments the frame layout has changed in a backwards-incompatible way. +> **`protocolVersions`** describes the serve protocol versions the daemon can speak. `current` is the daemon's preferred protocol version and `supported` is the compatible set. Clients that require a specific protocol should check `supported`; feature-specific UI should still gate on `features`. Additive to v=1: older v=1 daemons omit this field, so SDK clients that target older builds should treat it as optional. + > **`modelServices` is always `[]` in Stage 1.** The agent uses its single default model service and doesn't enumerate it over the wire. Stage 2 will populate this from registered model adapters so SDK clients can build service-pickers; until then, do NOT rely on this field being non-empty. +> **`workspaceCwd`** is the canonical absolute path this daemon binds to (#3803 §02 — 1 daemon = 1 workspace). Use it to (a) detect mismatch before posting `/session` and (b) omit `cwd` on `POST /session` (the route falls back to this path). Multi-workspace deployments expose multiple daemons on different ports, each with its own `workspaceCwd`. Additive to v=1: pre-§02 v=1 daemons omit the field — clients that target older builds should null-check before consuming it. + +### Read-only runtime status routes + +These routes report daemon-side runtime snapshots. They are additive v1 routes, +do not mutate state, and do not change the serve protocol version. Workspace +status routes intentionally do **not** start the ACP child process just because +a client polls a GET route: if the daemon is idle, they return +`initialized: false` with an empty snapshot. Session status routes require a +live session and use the standard `404 SessionNotFoundError` shape for unknown +ids. + +Capability tags: + +- `workspace_mcp` → `GET /workspace/mcp` +- `workspace_skills` → `GET /workspace/skills` +- `workspace_providers` → `GET /workspace/providers` +- `workspace_env` → `GET /workspace/env` +- `workspace_preflight` → `GET /workspace/preflight` +- `session_context` → `GET /session/:id/context` +- `session_supported_commands` → `GET /session/:id/supported-commands` + +Common status cell: + +```ts +type DaemonStatus = + | 'ok' + | 'warning' + | 'error' + | 'disabled' + | 'not_started' + | 'unknown'; + +type DaemonErrorKind = + | 'missing_binary' + | 'blocked_egress' + | 'auth_env_error' + | 'init_timeout' + | 'protocol_error' + | 'missing_file' + | 'parse_error'; + +interface DaemonStatusCell { + kind: string; + status: DaemonStatus; + error?: string; + errorKind?: DaemonErrorKind; + hint?: string; +} +``` + +`errorKind` is a closed enum shared by `/workspace/preflight`, +`/workspace/env`, and (eventually) MCP guardrails so SDK clients can render +remediation per category instead of parsing free-form messages. PR 13 +(#4175) introduced the seven literals listed above; PR 14 will populate +`blocked_egress` once the egress probe lands. + +Status payloads never expose MCP env values, headers, OAuth/service-account +details, provider API keys, provider `baseUrl` / `envKey`, skill body, skill +filesystem paths, hook definitions, or values of secret environment +variables. `/workspace/env` reports the **presence** of whitelisted env +vars only; proxy URLs are stripped of credentials and reduced to +`host:port` before they hit the wire. + +### `GET /workspace/mcp` + +```json +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "discoveryState": "completed", + "servers": [ + { + "kind": "mcp_server", + "status": "ok", + "name": "docs", + "mcpStatus": "connected", + "transport": "stdio", + "disabled": false, + "description": "Documentation server", + "extensionName": "docs-ext" + } + ] +} +``` + +`discoveryState` is one of `not_started`, `in_progress`, or `completed`. +`transport` is one of `stdio`, `sse`, `http`, `websocket`, `sdk`, or +`unknown`. `errors` is omitted when discovery succeeds. + +**MCP client guardrails (issue [#4175](https://github.com/QwenLM/qwen-code/issues/4175) PR 14).** Post-PR-14 daemons extend the payload with four additive fields and one workspace-level cell: + +```jsonc +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "discoveryState": "completed", + "clientCount": 3, + "clientBudget": 2, + "budgetMode": "enforce", + "budgets": [ + { + "kind": "mcp_budget", + "scope": "session", + "status": "error", + "errorKind": "budget_exhausted", + "hint": "Raise --mcp-client-budget or remove servers from mcpServers config.", + "liveCount": 2, + "budget": 2, + "mode": "enforce", + "refusedCount": 1, + }, + ], + "servers": [ + { + "kind": "mcp_server", + "status": "ok", + "name": "a", + "mcpStatus": "connected", + "transport": "stdio", + "disabled": false, + }, + { + "kind": "mcp_server", + "status": "ok", + "name": "b", + "mcpStatus": "connected", + "transport": "stdio", + "disabled": false, + }, + { + "kind": "mcp_server", + "status": "error", + "name": "c", + "mcpStatus": "disconnected", + "transport": "stdio", + "disabled": false, + "disabledReason": "budget", + "errorKind": "budget_exhausted", + "hint": "...", + }, + ], +} +``` + +`budgetMode` is one of `enforce`, `warn`, or `off`. `clientBudget` is absent when no budget was set. `budgets[]` is **always an array** on post-PR-14 daemons (possibly empty when `budgetMode === 'off'`); pre-PR-14 daemons omit the field entirely. v1 emits one cell with `scope: 'session'` (per-session enforcement — see the capabilities section above for why). Consumers MUST tolerate additional `budgets[]` entries with unrecognized `scope` values — Wave 5 PR 23 will add `scope: 'workspace'` (or `'pool'`) alongside the per-session cell without a schema bump. + +`disabledReason` on per-server cells distinguishes operator-disabled (`'config'` — `disabledMcpServers` config list) from budget-refused (`'budget'` — discovered but never connected due to `enforce` mode). Refusals are deterministic by `Object.entries(mcpServers)` declaration order. The per-server `status: 'error', errorKind: 'budget_exhausted'` shadows the raw `mcpStatus: 'disconnected'` (which is true but not the operator-facing severity). + +Budget enforcement in PR 14 v1 is **per-session, not per-workspace**. Although Mode B daemons are `1 daemon = 1 workspace × N sessions` post-#4113 at the process level, the `McpClientManager` is constructed inside each ACP session's `Config` via `acpAgent.newSessionConfig`, so N sessions each enforce their own copy of the cap. The snapshot represents the bootstrap session's view. Wave 5 PR 23 introduces a workspace-scoped shared MCP pool that graduates this to true per-workspace enforcement. + +**Detecting budget pressure.** Two surfaces, both populated post-PR-14b: + +- **Push events** (advertised via `mcp_guardrail_events`): subscribe to `GET /session/:id/events` and narrow `mcp_budget_warning` / `mcp_child_refused_batch` frames through `KnownDaemonEvent`. The state machine fires once per upward 75% crossing (re-armed below 37.5%); refusals are coalesced once per discovery pass under `enforce` mode. +- **Snapshot poll** (advertised via `mcp_guardrails`): `GET /workspace/mcp` and inspect the per-session budget cell (`budgets[0]`): + +- `budgets[0].status === 'warning'` ⇔ `liveCount >= 0.75 * clientBudget` (matches the hysteresis threshold PR 14b's push event will use). +- `budgets[0].status === 'error'` ⇔ `refusedCount > 0` (one or more servers refused this discovery pass). +- `budgets[0].status === 'ok'` ⇔ below the 75% threshold AND no refusals. + +Recommended poll cadence: aligned with whatever already polls `/workspace/mcp`; the snapshot is cheap and the budget cell carries no extra discovery cost. SDK clients that subscribe to push events still benefit from the snapshot for state-after-extended-disconnect (the SSE replay ring depth is finite — `--event-ring-size`, default 8000 — so a client offline longer than the ring's coverage falls back to snapshot resync). + +### `GET /workspace/skills` + +```json +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "skills": [ + { + "kind": "skill", + "status": "ok", + "name": "review", + "description": "Review code", + "level": "project", + "modelInvocable": true, + "argumentHint": "[path]" + } + ] +} +``` + +`level` is one of `project`, `user`, `extension`, or `bundled`. `errors` is +omitted when discovery succeeds. + +### `GET /workspace/providers` + +```json +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "current": { "authType": "qwen", "modelId": "qwen3(qwen)" }, + "providers": [ + { + "kind": "model_provider", + "status": "ok", + "authType": "qwen", + "current": true, + "models": [ + { + "modelId": "qwen3(qwen)", + "baseModelId": "qwen3", + "name": "Qwen 3", + "description": null, + "contextLimit": 4096, + "isCurrent": true, + "isRuntime": false + } + ] + } + ] +} +``` + +Models are grouped by auth type. Provider connection diagnostics live on +`/workspace/preflight`'s `providers` cell; environment preflight lives on +`/workspace/preflight` and `/workspace/env` (below). `errors` is omitted +when snapshot construction succeeds. + +### `GET /workspace/env` + +Reports the daemon process's runtime, platform, sandbox, proxy, and the +**presence** of whitelisted secret environment variables. Always answers +from `process.*` state — the daemon never spawns an ACP child to serve +this route, and the response is identical whether ACP is up or idle. The +`acpChannelLive` field is informational only. + +```json +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "acpChannelLive": false, + "cells": [ + { "kind": "runtime", "name": "node", "status": "ok", "value": "22.4.0" }, + { "kind": "platform", "name": "darwin", "status": "ok", "value": "arm64" }, + { + "kind": "sandbox", + "name": "SANDBOX", + "status": "disabled", + "present": false + }, + { + "kind": "proxy", + "name": "HTTPS_PROXY", + "status": "ok", + "present": true, + "value": "proxy.internal:1080" + }, + { + "kind": "proxy", + "name": "NO_PROXY", + "status": "disabled", + "present": false + }, + { + "kind": "env_var", + "name": "OPENAI_API_KEY", + "status": "ok", + "present": true + }, + { + "kind": "env_var", + "name": "ANTHROPIC_BASE_URL", + "status": "disabled", + "present": false + } + ] +} +``` + +Cell shape: + +```ts +type DaemonEnvKind = + | 'runtime' // name: 'node' | 'bun' | 'unknown'; value: process.versions.node + | 'platform' // name: process.platform; value: process.arch + | 'sandbox' // name: 'SANDBOX' | 'SEATBELT_PROFILE'; value optional + | 'proxy' // name: HTTP_PROXY | HTTPS_PROXY | NO_PROXY | ALL_PROXY; value: redacted host + | 'env_var'; // presence-only; value field is ALWAYS omitted + +interface DaemonEnvCell extends DaemonStatusCell { + kind: DaemonEnvKind; + name: string; + present?: boolean; + value?: string; +} +``` + +**Redaction policy.** `kind: 'env_var'` cells never include a `value` +field; clients see `present: boolean` only. `kind: 'proxy'` cells run the +raw env value through credential redaction (`redactProxyCredentials`) and +then through `URL` parsing so the wire only carries `host:port`. `NO_PROXY` +is passed through redaction verbatim because it is a host list rather than +a URL. The whitelist of enumerated secret env vars currently includes +`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `GOOGLE_API_KEY`, +`DASHSCOPE_API_KEY`, `OPENROUTER_API_KEY`, and `QWEN_SERVER_TOKEN`. Other +env vars are not enumerated, so accidentally-set secrets stay invisible. + +### `GET /workspace/preflight` + +Reports daemon readiness checks. **Daemon-level cells** (`node_version`, +`cli_entry`, `workspace_dir`, `ripgrep`, `git`, `npm`) are always +populated from `process.*` and `node:fs`. **ACP-level cells** (`auth`, +`mcp_discovery`, `skills`, `providers`, `tool_registry`, `egress`) +require a live ACP child — when the daemon is idle they emit +`status: 'not_started'` placeholders. The route never spawns ACP solely +to populate cells; the corresponding cells fall back to `not_started`. + +Idle response (no ACP child): + +```json +{ + "v": 1, + "workspaceCwd": "/canonical/path", + "initialized": true, + "acpChannelLive": false, + "cells": [ + { + "kind": "node_version", + "status": "ok", + "locality": "daemon", + "detail": { "version": "22.4.0", "required": ">=22" } + }, + { + "kind": "cli_entry", + "status": "ok", + "locality": "daemon", + "detail": { "path": "/usr/local/bin/qwen", "source": "process.argv[1]" } + }, + { + "kind": "workspace_dir", + "status": "ok", + "locality": "daemon", + "detail": { "path": "/canonical/path" } + }, + { "kind": "ripgrep", "status": "ok", "locality": "daemon" }, + { + "kind": "git", + "status": "ok", + "locality": "daemon", + "detail": { "version": "2.45.0" } + }, + { + "kind": "npm", + "status": "ok", + "locality": "daemon", + "detail": { "version": "10.7.0" } + }, + { + "kind": "auth", + "status": "not_started", + "locality": "acp", + "hint": "spawn a session to populate" + }, + { + "kind": "mcp_discovery", + "status": "not_started", + "locality": "acp", + "hint": "spawn a session to populate" + }, + { + "kind": "skills", + "status": "not_started", + "locality": "acp", + "hint": "spawn a session to populate" + }, + { + "kind": "providers", + "status": "not_started", + "locality": "acp", + "hint": "spawn a session to populate" + }, + { + "kind": "tool_registry", + "status": "not_started", + "locality": "acp", + "hint": "spawn a session to populate" + }, + { + "kind": "egress", + "status": "not_started", + "locality": "acp", + "hint": "egress probing lands in PR 14 (#4175)" + } + ] +} +``` + +Cell shape: + +```ts +type DaemonPreflightKind = + | 'node_version' + | 'cli_entry' + | 'workspace_dir' + | 'ripgrep' + | 'git' + | 'npm' + | 'auth' + | 'mcp_discovery' + | 'skills' + | 'providers' + | 'tool_registry' + | 'egress'; + +interface DaemonPreflightCell extends DaemonStatusCell { + kind: DaemonPreflightKind; + locality: 'daemon' | 'acp'; + detail?: Record; +} +``` + +`errorKind` semantics: + +- `missing_binary` — Node version below required, missing `QWEN_CLI_ENTRY`, + ripgrep / git / npm not on PATH (warnings rather than errors for the + optional binaries). +- `missing_file` — `boundWorkspace` does not exist or is not a directory; + skill parse error pointing at a missing or unreadable file. +- `parse_error` — `SKILL.md` parse failure, malformed config JSON. +- `auth_env_error` — `validateAuthMethod` returned a non-null failure + string, or a `ModelConfigError` subclass propagated from provider + resolution. +- `init_timeout` — `withTimeout` reject in the bridge (an actual timeout + while waiting on an ACP roundtrip). Recognized via the + `BridgeTimeoutError` typed class. Note: a transient `mcp_discovery` + `warning` cell with `connecting > 0` does NOT carry this kind — that's + a normal handshake-in-progress state, distinct from a real timeout. +- `protocol_error` — ACP `extMethod` rejected because the channel closed + mid-request, or because tool registry was unexpectedly absent. +- `blocked_egress` — reserved for PR 14 (#4175). PR 13 leaves the + `egress` cell as `status: 'not_started'`. + +If the bridge fails to reach the ACP child while serving a preflight +request (e.g. a mid-request channel close), the envelope's `errors` array +carries a single `ServeStatusCell` describing the failure and the cells +fall back to `not_started` ACP placeholders. Daemon-level cells are still +returned. + +### Workspace file routes + +All file paths are resolved through the daemon's bound workspace. Responses use +workspace-relative paths and never return absolute filesystem paths for normal +success cases. Successful file responses include: + +```http +Cache-Control: no-store +X-Content-Type-Options: nosniff +``` + +Filesystem errors use this JSON shape: + +```json +{ + "errorKind": "hash_mismatch", + "error": "expected sha256:..., found sha256:...", + "hint": "re-read the file and retry with the latest hash", + "status": 409 +} +``` + +`errorKind` values include `path_outside_workspace`, `symlink_escape`, +`path_not_found`, `binary_file`, `file_too_large`, `untrusted_workspace`, +`permission_denied`, `parse_error`, `hash_mismatch`, +`file_already_exists`, `text_not_found`, and `ambiguous_text_match`. + +#### `GET /file` + +Reads a text file. Query params: `path` (required), `maxBytes`, `line`, and +`limit`. The daemon rejects binary files and files above the text read cap. +The response includes `hash`, a SHA-256 digest over the raw on-disk bytes for +the whole file, even when `line`, `limit`, or `maxBytes` returned a slice. + +```json +{ + "kind": "file", + "path": "src/index.ts", + "content": "export {};\n", + "encoding": "utf-8", + "bom": false, + "lineEnding": "lf", + "sizeBytes": 11, + "returnedBytes": 11, + "truncated": false, + "hash": "sha256:...", + "matchedIgnore": null, + "originalLineCount": null +} +``` + +#### `GET /file/bytes` + +Reads raw bytes from a file without decoding. Query params: `path` (required), +`offset` (default `0`), and `maxBytes` (default `65536`, max `262144`). This +route supports bounded windows on large binary files without slurping the whole +file. The response includes `hash` only when the returned window covers the +entire file. + +```json +{ + "kind": "file_bytes", + "path": "assets/logo.png", + "offset": 0, + "sizeBytes": 3912, + "returnedBytes": 3912, + "truncated": false, + "contentBase64": "...", + "hash": "sha256:..." +} +``` + +#### `POST /file/write` + +Creates or replaces a text file. This is a strict mutation route: on loopback +without a configured token it returns `401 { "code": "token_required" }`. +With `--require-auth`, the global bearer middleware rejects unauthenticated +requests before the route runs. + +Body: + +```json +{ + "path": "src/new.ts", + "content": "export const value = 1;\n", + "mode": "create" +} +``` + +```json +{ + "path": "src/existing.ts", + "content": "export const value = 2;\n", + "mode": "replace", + "expectedHash": "sha256:..." +} +``` + +`mode` must be `create` or `replace`. `create` never overwrites an existing +file (`409 file_already_exists`). `replace` requires `expectedHash`; missing or +malformed hashes are `400 parse_error`, and stale hashes are +`409 hash_mismatch`. `expectedHash` is `sha256:` plus 64 lowercase hex +characters, computed over raw on-disk bytes. + +`bom`, `encoding`, and `lineEnding` may be supplied. Replacement preserves the +existing file's encoding profile by default; explicit fields override it. +Binary writes are out of scope. + +The daemon writes to a random temp file in the target directory, fsyncs where +supported, re-checks the current hash immediately before `rename()`, then +renames into place. This prevents partial-file observation and serializes +daemon-originated writes to the same file, but it is not a cross-process +kernel compare-and-swap: an external editor can still race in the tiny window +between final hash check and rename. + +```json +{ + "kind": "file_write", + "path": "src/existing.ts", + "mode": "replace", + "created": false, + "sizeBytes": 24, + "hash": "sha256:...", + "encoding": "utf-8", + "bom": false, + "lineEnding": "lf", + "matchedIgnore": null +} +``` + +#### `POST /file/edit` + +Applies one exact text replacement to an existing text file. This is also a +strict mutation route and requires `expectedHash`. + +```json +{ + "path": "src/config.ts", + "oldText": "timeout: 30000", + "newText": "timeout: 60000", + "expectedHash": "sha256:..." +} +``` + +`oldText` must be non-empty and occur exactly once. No match returns +`422 text_not_found`; multiple matches return `422 ambiguous_text_match`. +The route preserves encoding, BOM, and line endings, and re-checks +`expectedHash` immediately before the atomic rename. + +Explicit writes/edits to ignored paths are allowed because the authenticated +caller named the path. Success responses and audit events include +`matchedIgnore: "file" | "directory" | null`. + +```json +{ + "kind": "file_edit", + "path": "src/config.ts", + "replacements": 1, + "sizeBytes": 128, + "hash": "sha256:...", + "encoding": "utf-8", + "bom": false, + "lineEnding": "lf", + "matchedIgnore": null +} +``` + +### `GET /session/:id/context` + +```json +{ + "v": 1, + "sessionId": "", + "workspaceCwd": "/canonical/path", + "state": { + "models": {}, + "modes": {}, + "configOptions": [] + } +} +``` + +`state` mirrors the same ACP model/mode/config-option shapes used by +`POST /session`, `POST /session/:id/load`, and `POST /session/:id/resume`. + +### `GET /session/:id/supported-commands` + +```json +{ + "v": 1, + "sessionId": "", + "availableCommands": [ + { + "name": "init", + "description": "Initialize the project", + "input": null, + "_meta": { "source": "builtin" } + } + ], + "availableSkills": ["review"] +} +``` + +`availableCommands` is the same command snapshot used by the +`available_commands_update` SSE notification. `availableSkills` lists skill +names only; clients must not expect skill bodies or paths over this route. + ### `POST /session` Spawn a new agent or attach to an existing one (under `sessionScope: 'single'`, the default). @@ -110,14 +847,16 @@ Request: ```json { "cwd": "/absolute/path/to/workspace", - "modelServiceId": "qwen-prod" + "modelServiceId": "qwen-prod", + "sessionScope": "thread" } ``` | Field | Required | Notes | | ---------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `cwd` | yes | Absolute path. Relative paths return `400`. Workspace paths are canonicalized via `realpathSync.native` (with a resolve-only fallback for non-existent paths) so case-insensitive filesystems don't fork sessions per spelling. | +| `cwd` | no | Absolute path matching the daemon's bound workspace. If omitted, the route falls back to `boundWorkspace` (read it off `/capabilities.workspaceCwd`). A mismatched non-empty `cwd` returns `400 workspace_mismatch` (#3803 §02 — 1 daemon = 1 workspace). Workspace paths are canonicalized via `realpathSync.native` (with a resolve-only fallback for non-existent paths) so case-insensitive filesystems don't reject sessions per spelling. | | `modelServiceId` | no | Selects which configured _model service_ the agent will route through (the back-end provider — Alibaba ModelStudio, OpenRouter, etc). If omitted the agent uses its default. If the workspace already has a session, this calls `setSessionModel` on the existing one and broadcasts `model_switched`. Distinct from `modelId` on `POST /session/:id/model`, which selects the model **within** an already-bound service. The `modelServices` array on `/capabilities` is reserved for advertising configured services; in Stage 1 it is always `[]` (the agent's default service is used and not enumerated over HTTP). | +| `sessionScope` | no | Per-request override for session sharing. `'single'` (the daemon-wide default) makes a second same-workspace `POST /session` reuse the existing session (`attached: true`); `'thread'` forces a fresh distinct session every call. Omit to inherit the daemon-wide default. Values outside the enum return `400 { code: 'invalid_session_scope' }`. Old daemons (pre-#4175 PR 5) silently ignore the field — pre-flight `caps.features.session_scope_override` before sending. The daemon-wide default is hardcoded to `'single'` in production today; #4175 may add a `--sessionScope` CLI flag in a follow-up. | Response: @@ -146,6 +885,60 @@ Concurrent `POST /session` calls for the same workspace are **coalesced** to one > event (covers the spawn-time `model_switch_failed` even if the > subscribe lands a few ms after the create response). +### `POST /session/:id/load` + +Restore a persisted ACP session by id and replay its history through SSE. The path id is authoritative; any `sessionId` field in the body is ignored. Pre-flight `caps.features.session_load` — older daemons return `404` for this route. + +Request: + +```json +{ + "cwd": "/absolute/path/to/workspace" +} +``` + +| Field | Required | Notes | +| ----- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `cwd` | no | Same canonicalization + `workspace_mismatch` rules as `POST /session`. Omit to inherit `/capabilities.workspaceCwd`. `mcpServers` is intentionally NOT accepted here — daemon-wide MCP is settings-driven (matches `POST /session`). | + +Response: + +```json +{ + "sessionId": "persisted-1", + "workspaceCwd": "/canonical/path", + "attached": false, + "state": { + "models": { ... }, + "modes": { ... }, + "configOptions": [ ... ] + } +} +``` + +`state` mirrors ACP's `LoadSessionResponse` — `models` is a `SessionModelState`, `modes` a `SessionModeState`, `configOptions` an array of `SessionConfigOption`. Missing fields are agent-decided. Late attachers (the `attached: true` paths below) get the SAME `state` snapshot the original load caller saw — the daemon caches it on the entry; runtime mutations (e.g. `model_switched`) are delivered on the SSE stream, not on subsequent attach responses. + +`attached: true` means the session was already live (either from a prior `session/load`/`session/resume`, or because a coalesced concurrent caller raced just ahead). + +**History replay over SSE.** While `loadSession` is in flight on the agent side, the agent emits `session_update` notifications for every persisted turn. The daemon buffers them onto the session's event-bus before the route response returns, so subscribers that immediately call `GET /session/:id/events` with `Last-Event-ID: 0` see the full replay. **The replay ring is bounded** (default 4000 frames per session). Long histories with many tool-call / thought-stream turns can exceed that — the oldest frames are dropped silently. Clients that need full history should subscribe immediately after `load` returns; alternatively they can persist the SSE event ids and use `Last-Event-ID` to resume from a later turn boundary. + +**Errors:** + +- `404` — persisted session id doesn't exist (`SessionNotFoundError`). +- `400` — `workspace_mismatch` (same shape as `POST /session`). +- `503` — `session_limit_exceeded` (counts against `--max-sessions`; in-flight restores are accounted for too). +- `409` — `restore_in_progress` (a `session/resume` for the same id is already in flight). `Retry-After: 5`. Same-action races (two concurrent `session/load` for the same id) coalesce — exactly one returns `attached: false`, the rest return `attached: true` with the same `state`. + +### `POST /session/:id/resume` + +Restore a persisted ACP session by id WITHOUT replaying history through SSE. The model context is restored internally on the agent side (via `geminiClient.initialize` reading `config.getResumedSessionData`); the SSE stream stays clean for clients that already have history rendered. Pre-flight `caps.features.unstable_session_resume`. + +Same request shape as `/load`. Same response shape — `state` mirrors ACP's `ResumeSessionResponse`. Same error envelope, including `409 restore_in_progress` (which fires when a `session/load` is in flight; `session/resume` racing behind another `session/resume` coalesces). + +Use `/load` when the client has no history rendered (cold reconnect, picker → open). Use `/resume` when the client already has the turns on screen and only needs the daemon-side handle back. + +> ⚠️ **Why `unstable_` on the capability tag?** The route is wire-stable for the daemon's v1, but it's backed by ACP's `connection.unstable_resumeSession` which is still subject to ACP-side breaking changes. The daemon insulates the wire shape from those changes; the prefix is a courtesy signal so SDK consumers know the underlying agent contract is not yet locked. + ### `GET /workspace/:id/sessions` List all live sessions whose canonical workspace matches `:id` (URL-encoded absolute cwd). @@ -158,7 +951,16 @@ Response: ```json { - "sessions": [{ "sessionId": "", "workspaceCwd": "/canonical/path" }] + "sessions": [ + { + "sessionId": "", + "workspaceCwd": "/canonical/path", + "createdAt": "2026-05-17T08:30:00.000Z", + "displayName": "My Session", + "clientCount": 2, + "hasActivePrompt": false + } + ] } ``` @@ -211,6 +1013,72 @@ curl -X POST http://127.0.0.1:4170/session/$SID/cancel > **Multi-prompt contract:** cancel only affects the active prompt. Any prompts the same client previously POSTed and are still queued behind the active one will continue to execute. Multi-prompt queueing is a daemon-introduced behavior (not in ACP spec); the contract for queued prompts is "they keep running unless you cancel each, or kill the session via channel exit". +### `DELETE /session/:id` + +Explicitly close a live session. Force-closes even when other clients are attached — cancels any active prompt, resolves pending permissions as cancelled, publishes `session_closed` event, closes the EventBus, and removes the session from daemon maps. On-disk persisted sessions are NOT deleted — they can be reloaded via `POST /session/:id/load`. Pre-flight `caps.features.session_close`. + +```bash +curl -X DELETE http://127.0.0.1:4170/session/$SID +# → 204 No Content +``` + +Idempotent: returns `404` for unknown sessions (same `SessionNotFoundError` shape as other routes). + +> **`session_closed` event.** SSE subscribers receive a terminal `session_closed` event with `{ sessionId, reason: 'client_close', closedBy?: '' }` before the stream ends. SDK reducers treat this identically to `session_died` (sets `alive: false`, clears `pendingPermissions`). + +### `PATCH /session/:id/metadata` + +Update mutable session metadata. Currently supports `displayName` only. Pre-flight `caps.features.session_metadata`. + +Request: + +```json +{ "displayName": "My Investigation Session" } +``` + +| Field | Required | Notes | +| ------------- | -------- | ------------------------------------------------------------------------------ | +| `displayName` | no | String, max 256 characters. Empty string clears the name. Omit to leave as-is. | + +Response: + +```json +{ "sessionId": "", "displayName": "My Investigation Session" } +``` + +Publishes a `session_metadata_updated` event on the session's SSE stream with `{ sessionId, displayName }`. + +### `POST /session/:id/heartbeat` + +Bump the daemon's last-seen bookkeeping for this session. Long-lived adapters (TUI/IDE/web) ping this on an interval so future revocation policy (Wave 5 PR 24) can distinguish dead clients from quiet ones. + +Headers: + +| Header | Required | Notes | +| ------------------ | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `X-Qwen-Client-Id` | no | Echoes the daemon-issued id from `POST /session`. Identified clients also bump their per-client timestamp; anonymous heartbeats only bump the per-session watermark. Must satisfy the same `[A-Za-z0-9._:-]{1,128}` shape as elsewhere. | + +Request body is empty (`{}` is fine — no fields are read today). + +Response: + +```json +{ + "sessionId": "", + "clientId": "", + "lastSeenAt": 1700000000123 +} +``` + +`clientId` is echoed only when a trusted `X-Qwen-Client-Id` was supplied. `lastSeenAt` is the daemon-side `Date.now()` epoch (ms) the bridge stored. + +Errors: + +- `400` — `{ code: 'invalid_client_id' }` when the header is malformed (header-shape rule) or when it carries a `clientId` that isn't registered for this session (the bridge throws `InvalidClientIdError` before bumping any timestamp). +- `404` — unknown session. + +Capability gating: pre-flight `caps.features.client_heartbeat`. Older daemons return `404` for this path. + ### `POST /session/:id/model` Switch the active model **within** the session's currently bound model service. Serialized through the per-session model-change queue. @@ -231,6 +1099,149 @@ Response: On success, publishes `model_switched` to the SSE stream. On failure, publishes `model_switch_failed` (so passive subscribers see the failure, not just the caller). Races against the agent channel exit so a wedged child can't block the HTTP handler. +### Mutation: approval, tools, init, MCP restart + +Issue [#4175](https://github.com/QwenLM/qwen-code/issues/4175) Wave 4 PR 17 adds four mutation control routes that let remote clients change runtime posture without touching the daemon host's CLI. All four: + +- Are gated by the **strict** mutation gate from PR 15. A daemon configured without a bearer token rejects them with `401 {code: 'token_required'}`. Configure `--token` (or `QWEN_SERVER_TOKEN`) before opting in. +- Accept and stamp the `X-Qwen-Client-Id` header (PR 7 audit chain). When the header carries a trusted id, the daemon emits `originatorClientId` on the corresponding SSE event so cross-client UIs can suppress echoes of their own mutations. +- Pre-flight each per-tag capability before exposing the affordance. Older daemons return `404` for the route. + +Three of the four routes (`tools/:name/enable`, `init`, `mcp/:server/restart`) emit **workspace-scoped** events: every active session SSE bus receives the event, regardless of which session was attached when the mutation was triggered. `approval-mode` emits a **session-scoped** event because the change is local to one session's `Config`. + +#### `POST /session/:id/approval-mode` + +Capability tag: `session_approval_mode_control`. Bridge → ACP extMethod `qwen/control/session/approval_mode`. + +Change the approval mode of a live session. The new mode lands inside the ACP child's per-session `Config` immediately. Settings are NOT written to disk by default — pass `persist: true` to also write `tools.approvalMode` to workspace settings. + +Request: + +```json +{ "mode": "auto-edit", "persist": false } +``` + +`mode` must be one of `'plan' | 'default' | 'auto-edit' | 'auto' | 'yolo'` (mirror of core's `ApprovalMode` enum; the SDK exports `DAEMON_APPROVAL_MODES` for runtime validation). `persist` defaults to `false`. + +Response (200): + +```json +{ + "sessionId": "sess:42", + "mode": "auto-edit", + "previous": "default", + "persisted": false +} +``` + +Errors: + +- `400 {code: 'invalid_approval_mode', allowed: [...]}` — unknown mode literal. +- `400 {code: 'invalid_persist_flag'}` — `persist` is non-boolean. +- `403 {code: 'trust_gate', errorKind: 'auth_env_error'}` — the requested mode requires a trusted folder (privileged modes in untrusted workspaces are rejected by core's `Config.setApprovalMode`). +- `404` — session unknown. + +SSE event (session-scoped): `approval_mode_changed` with `{sessionId, previous, next, persisted, originatorClientId?}`. + +#### `POST /workspace/tools/:name/enable` + +Capability tag: `workspace_tool_toggle`. Pure file IO — no ACP roundtrip. + +Toggle a tool name in the workspace's `tools.disabled` settings list. Tools listed there are **not registered** at all (distinct from `permissions.deny`, which keeps the tool registered and rejects invocation). Both built-in tools and MCP-discovered tools flow through `ToolRegistry.registerTool`, which consults the disabled set. + +> ⚠️ **Names must match the registry's exposed identifier exactly.** No alias resolution happens — the route stores whatever string is in the path parameter into `tools.disabled`, and the next ACP child compares against `tool.name` at register time. Built-ins use their canonical registry name (snake_case verb form): `run_shell_command`, `read_file`, `write_file`, `list_directory`, `glob`, `search_file_content`, `ripgrep`, `web_fetch`, etc. — NOT the display labels (`Shell`, `Read`, `Write`) that the CLI surfaces. MCP-discovered tools use the qualified `mcp____` form (which is also the form `tool_toggled` events broadcast and what `GET /workspace/mcp` lists). Disabling `Bash` will NOT prevent `run_shell_command` from registering on the next session. + +Live ACP children retain already-registered tools — the toggle takes effect on the **next** ACP child spawn. Combine with `POST /workspace/mcp/:server/restart` (for MCP-sourced tools) or new-session creation to make the change effective in the current daemon. + +Unknown tool names are accepted: pre-disabling a not-yet-installed MCP tool is a legitimate use case. + +Request: + +```json +{ "enabled": false } +``` + +Response (200): + +```json +{ "toolName": "run_shell_command", "enabled": false } +``` + +Errors: + +- `400 {code: 'invalid_tool_name'}` — empty path parameter, or path parameter exceeds the 256-character cap. +- `400 {code: 'invalid_enabled_flag'}` — `enabled` missing or non-boolean. + +SSE event (workspace-scoped): `tool_toggled` with `{toolName, enabled, originatorClientId?}`. + +#### `POST /workspace/init` + +Capability tag: `workspace_init`. Pure file IO — no ACP roundtrip, **no LLM invocation**. + +Scaffold an empty `QWEN.md` (or whatever `getCurrentGeminiMdFilename()` returns under `--memory-file-name` overrides) at the daemon's bound workspace root. Mechanical only — for AI-driven content fill, follow up with `POST /session/:id/prompt`. + +Default refuses to overwrite when the target file exists with non-whitespace content. Whitespace-only files are treated as absent (matches the local `/init` slash command). + +Request: + +```json +{ "force": false } +``` + +Response (200): + +```json +{ "path": "/work/bound/QWEN.md", "action": "created" } +``` + +`action` is `'created'` for fresh creates, `'noop'` when an existing whitespace-only file was left untouched (no write performed), and `'overwrote'` when `force: true` replaced non-empty content. The `workspace_initialized` SSE event mirrors the response action — observers can filter for `action !== 'noop'` to react only to actual on-disk changes. + +Errors: + +- `400 {code: 'invalid_force_flag'}` — `force` is non-boolean. +- `409 {code: 'workspace_init_conflict', path, existingSize}` — file exists with non-whitespace content and `force` is omitted/false. Body carries the absolute path and size (bytes) so SDK clients can render an "overwrite N bytes?" prompt without re-stat'ing. + +SSE event (workspace-scoped): `workspace_initialized` with `{path, action, originatorClientId?}`. + +#### `POST /workspace/mcp/:server/restart` + +Capability tag: `workspace_mcp_restart`. Bridge → ACP extMethod `qwen/control/workspace/mcp/restart`. + +Restart a configured MCP server through the ACP child's `McpClientManager.discoverMcpToolsForServer` (disconnect + reconnect + rediscover). Pre-checks the live budget snapshot from PR 14 v1's accounting so a restart on a budget-saturated workspace returns a soft refusal rather than triggering a `BudgetExhaustedError` cascade. + +Request body is empty (`{}`). The path parameter is the URL-encoded server name as it appears in `mcpServers` config. + +Response (200) — discriminated union on `restarted`: + +```json +{ "serverName": "docs", "restarted": true, "durationMs": 1234 } +``` + +```json +{ + "serverName": "docs", + "restarted": false, + "skipped": true, + "reason": "budget_would_exceed" +} +``` + +Soft skip reasons (all return 200): + +| `reason` | Meaning | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `'in_flight'` | Another discovery / restart for this server is already in progress. The route returns immediately rather than awaiting the original promise. Caller should retry after a short delay. | +| `'disabled'` | Server is configured but listed in `excludedMcpServers`. Re-enable before restart. | +| `'budget_would_exceed'` | Daemon is `--mcp-budget-mode=enforce`, the target server is not currently in `reservedSlots`, and the live total has reached `clientBudget`. Caller should free a slot first. | + +Errors (non-2xx): + +- `400 {code: 'invalid_server_name'}` — empty path parameter. +- `404` — server name not in `mcpServers` config, or no live ACP channel exists (restart inherently requires a live `McpClientManager` instance). +- `500` — internal error (e.g. `ToolRegistry` not initialized). + +SSE events (workspace-scoped): `mcp_server_restarted` with `{serverName, durationMs, originatorClientId?}` on success; `mcp_server_restart_refused` with `{serverName, reason, originatorClientId?}` on soft skip. + ### `GET /session/:id/events` (SSE) Subscribe to the session's event stream. @@ -242,6 +1253,12 @@ Accept: text/event-stream Last-Event-ID: 42 ← optional, replays from after id 42 ``` +Query params: + +| Param | Required | Notes | +| ----------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `maxQueued` | no | Per-subscriber **live-backlog** cap. Range `[16, 2048]`, default 256. Replay frames force-pushed at subscribe time are exempt from the cap; what actually consumes it is live events that arrive while the subscriber is still draining a large `Last-Event-ID: 0` replay. Bump for cold reconnects so the live tail doesn't trip the slow-client warning / eviction before the consumer catches up. Out-of-range / non-decimal / present-but-empty values return `400 invalid_max_queued` before the SSE handshake opens. Pre-flight `caps.features.slow_client_warning` — old daemons silently ignore the param. | + Frame format. The `data:` line is the **full event envelope**, JSON-stringified on a single line — `{id?, v, type, data, originatorClientId?}`. The ACP-specific payload (`sessionUpdate`, `requestPermission` arguments, etc.) sits under the envelope's `data` field; the envelope's own `type` matches the SSE `event:` line. ``` @@ -261,28 +1278,30 @@ data: {"v":1,"type":"client_evicted","data":{"reason":"queue_overflow","droppedA The SSE-level `id:` / `event:` lines duplicate `envelope.id` / `envelope.type` for EventSource compatibility. Raw-`fetch` consumers (the SDK's `parseSseStream`) read everything off the JSON envelope and ignore the SSE preamble lines. -| Event type | Trigger | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `session_update` | Any ACP `sessionUpdate` notification (LLM chunks, tool calls, usage) | -| `permission_request` | Agent asked for tool approval | -| `permission_resolved` | Some client voted on a permission via `POST /permission/:requestId` | -| `model_switched` | `POST /session/:id/model` succeeded | -| `model_switch_failed` | `POST /session/:id/model` rejected | -| `session_died` | Agent child crashed unexpectedly. **Terminal: SSE stream closes after this frame; the session is gone from `byId`.** Subscribers should reconnect via `POST /session` to spawn a fresh one. | -| `client_evicted` | Subscriber-local: queue overflow. **Terminal: SSE stream closes after this frame** (no `id` — synthetic). Other subscribers on the same session continue. | -| `stream_error` | Daemon-side error during fan-out. **Terminal: SSE stream closes after this frame** (no `id` — synthetic). | +| Event type | Trigger | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `session_update` | Any ACP `sessionUpdate` notification (LLM chunks, tool calls, usage) | +| `permission_request` | Agent asked for tool approval | +| `permission_resolved` | Some client voted on a permission via `POST /permission/:requestId` | +| `model_switched` | `POST /session/:id/model` succeeded | +| `model_switch_failed` | `POST /session/:id/model` rejected | +| `session_died` | Agent child crashed unexpectedly. **Terminal: SSE stream closes after this frame; the session is gone from `byId`.** Subscribers should reconnect via `POST /session` to spawn a fresh one. | +| `slow_client_warning` | Subscriber-local: queue ≥ 75% full. **Non-terminal** — the stream continues; the warning is a heads-up before eviction. Carries `{queueSize, maxQueued, lastEventId}`. Fires ONCE per overflow episode; re-arms after the queue drains below 37.5%. No `id` (synthetic). Pre-flight `caps.features.slow_client_warning`. | +| `client_evicted` | Subscriber-local: queue overflow. **Terminal: SSE stream closes after this frame** (no `id` — synthetic). Other subscribers on the same session continue. | +| `stream_error` | Daemon-side error during fan-out. **Terminal: SSE stream closes after this frame** (no `id` — synthetic). | Reconnect semantics: -- Send `Last-Event-ID: ` to replay events with `id > n` from the per-session ring (default depth 4000) +- Send `Last-Event-ID: ` to replay events with `id > n` from the per-session ring (default depth **8000**, tunable via `qwen serve --event-ring-size `) - **Gap detection (client-side):** if `` predates the oldest event still in the ring (e.g. you reconnect with `Last-Event-ID: 50` but the ring now holds 200–1199), the daemon replays from the oldest available event without raising. Compare the first replayed event's `id` against `n + 1`; any difference is the size of the lost window. Stage 2 will inject an explicit `stream_gap` synthetic frame on the daemon side; in Stage 1 detection is the client's responsibility. - IDs are monotonic per session, starting at 1 -- Synthetic terminal frames (`client_evicted`, `stream_error`) intentionally omit `id` so they don't burn a sequence slot for other subscribers +- Synthetic frames (`client_evicted`, `slow_client_warning`, `stream_error`) intentionally omit `id` so they don't burn a sequence slot for other subscribers Backpressure: -- Per-subscriber queue defaults to `maxQueued: 256` live items (replay frames during reconnect bypass the cap) -- On overflow the bus emits the `client_evicted` terminal frame and closes the subscription +- Per-subscriber queue defaults to `maxQueued: 256` live items (replay frames during reconnect bypass the cap). Override via `?maxQueued=N` (range `[16, 2048]`) on the SSE request. +- When a subscriber's queue crosses 75% full the bus force-pushes a `slow_client_warning` synthetic frame to that subscriber (once per overflow episode; re-armed after drain below 37.5%). The stream stays open — the warning is a heads-up so the client can drain faster or detach + reconnect cleanly. +- If the queue actually overflows the warning, the bus emits the `client_evicted` terminal frame and closes the subscription. ### `POST /permission/:requestId` @@ -324,6 +1343,97 @@ Response: After a successful vote, every connected client sees `permission_resolved` with the same `requestId` and the chosen `outcome`. +### Auth device-flow routes (issue #4175 PR 21) + +The daemon brokers an OAuth 2.0 Device Authorization Grant (RFC 8628) so a remote SDK client can trigger a login whose tokens land on the **daemon** filesystem — not on the client. The daemon polls the IdP itself; the client's only job is to display the verification URL + user code and (optionally) subscribe to SSE for completion events. + +Capability tag: `auth_device_flow` (always advertised). Supported providers in v1: `qwen-oauth`. + +**Runtime locality.** The daemon never spawns a browser — even if it can. The client decides whether to call `open(verificationUri)` locally; on a headless pod (the canonical Mode B deployment) the user opens the URL on whatever device they have a browser on. See `docs/users/qwen-serve.md` for the recommended UX. + +**No token leakage in events.** `auth_device_flow_started` carries `{deviceFlowId, providerId, expiresAt}` only. The user code and verification URL come back point-to-point in the POST 201 body and via `GET /workspace/auth/device-flow/:id`; they are never broadcast on SSE. + +**Per-provider singleton.** A second `POST` for the same provider while a flow is pending is an idempotent take-over — it returns the existing entry with `attached: true` rather than starting a fresh IdP request. + +#### `POST /workspace/auth/device-flow` + +Strict mutation gate: requires a bearer token even on token-less loopback defaults (`401 token_required`). + +Request: + +```json +{ "providerId": "qwen-oauth" } +``` + +Response (`201` fresh start, `200` idempotent take-over): + +```json +{ + "deviceFlowId": "fa07c61b-…", + "providerId": "qwen-oauth", + "status": "pending", + "userCode": "USER-1", + "verificationUri": "https://chat.qwen.ai/api/v1/oauth2/device", + "verificationUriComplete": "https://chat.qwen.ai/api/v1/oauth2/device?user_code=USER-1", + "expiresAt": 1700000600000, + "intervalMs": 5000, + "attached": false +} +``` + +Errors: + +- `400 unsupported_provider` — unknown `providerId` (response includes `supportedProviders`) +- `409 too_many_active_flows` — workspace cap (4) reached; cancel one with `DELETE` +- `401 token_required` — strict gate denied a token-less request +- `502 upstream_error` — IdP returned an unexpected error + +#### `GET /workspace/auth/device-flow/:id` + +Read the current state. Pending entries echo `userCode/verificationUri/expiresAt/intervalMs`; terminal entries (5-min grace) drop them and surface `status` + optional `errorKind/hint`. + +Returns `404 device_flow_not_found` for unknown ids and post-grace evicted entries. + +#### `DELETE /workspace/auth/device-flow/:id` + +Idempotent cancel: + +- pending entry → `204` + emit `auth_device_flow_cancelled` +- terminal entry → `204` no-op (no event re-emit) +- unknown id → `404` + +#### `GET /workspace/auth/status` + +Snapshot of pending flows + supported providers: + +```json +{ + "v": 1, + "workspaceCwd": "/work/bound", + "providers": [], + "pendingDeviceFlows": [ + { + "deviceFlowId": "fa07c61b-…", + "providerId": "qwen-oauth", + "expiresAt": 1700000600000 + } + ], + "supportedDeviceFlowProviders": ["qwen-oauth"] +} +``` + +#### Device-flow SSE events + +Five typed events (workspace-scoped, fanned out to every active session bus): + +- `auth_device_flow_started` `{deviceFlowId, providerId, expiresAt}` — POST succeeded; SDK should subscribe (no userCode here, fetch via GET if needed) +- `auth_device_flow_throttled` `{deviceFlowId, intervalMs}` — daemon honored upstream `slow_down`; clients polling GET should bump their interval to match +- `auth_device_flow_authorized` `{deviceFlowId, providerId, expiresAt?, accountAlias?}` — credentials persisted; `accountAlias` is a non-PII label (never email/phone) +- `auth_device_flow_failed` `{deviceFlowId, errorKind, hint?}` — terminal; `errorKind` is one of `expired_token | access_denied | invalid_grant | upstream_error | persist_failed`. `persist_failed` is daemon-internal: the IdP exchange succeeded but the daemon couldn't durably store credentials (EACCES / EROFS / ENOSPC). The user should retry once the underlying disk condition is fixed. +- `auth_device_flow_cancelled` `{deviceFlowId}` — DELETE succeeded against a pending entry + +> **Not MCP-compatible.** The MCP authorization spec (2025-06-18) mandates OAuth 2.1 + PKCE auth-code with a redirect callback, which doesn't work for headless-pod daemons. Mode B's device-flow surface is daemon-private — clients targeting MCP-compliant servers should use a different auth path. + ## Streaming wire format Events are emitted as standard EventSource frames. The daemon writes one `data:` line per frame (the JSON has no embedded newlines after `JSON.stringify`); the SDK parser at `packages/sdk-typescript/src/daemon/sse.ts` handles both that and the spec-allowed multi-`data:` form on the receive side. @@ -348,15 +1458,17 @@ The connection then closes. ## Source layout -| Path | Purpose | -| ---------------------------------------------------- | ------------------------------------------------------------------ | -| `packages/cli/src/commands/serve.ts` | yargs command + flag schema | -| `packages/cli/src/serve/runQwenServe.ts` | listener lifecycle + signal handling | -| `packages/cli/src/serve/server.ts` | Express routes + middleware | -| `packages/cli/src/serve/auth.ts` | bearer + Host allowlist + CORS deny | -| `packages/cli/src/serve/httpAcpBridge.ts` | spawn-or-attach + per-session FIFO + permission registry | -| `packages/cli/src/serve/eventBus.ts` | bounded async queue + replay ring | -| `packages/sdk-typescript/src/daemon/DaemonClient.ts` | TS client | -| `packages/sdk-typescript/src/daemon/sse.ts` | EventSource frame parser | -| `integration-tests/cli/qwen-serve-routes.test.ts` | 18 cases, no LLM | -| `integration-tests/cli/qwen-serve-streaming.test.ts` | 3 cases, real `qwen --acp` child (skipped when `SKIP_LLM_TESTS=1`) | +| Path | Purpose | +| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `packages/cli/src/commands/serve.ts` | yargs command + flag schema | +| `packages/cli/src/serve/runQwenServe.ts` | listener lifecycle + signal handling | +| `packages/cli/src/serve/server.ts` | Express routes + middleware | +| `packages/cli/src/serve/auth.ts` | bearer + Host allowlist + CORS deny | +| `packages/cli/src/serve/httpAcpBridge.ts` | spawn-or-attach + per-session FIFO + permission registry | +| `packages/cli/src/serve/status.ts` | read-only daemon status wire types + `ServeErrorKind` + `BridgeTimeoutError` + `mapDomainErrorToErrorKind` | +| `packages/cli/src/serve/envSnapshot.ts` | pure helper that builds `/workspace/env` payloads from `process.*` state, including credential redaction | +| `packages/cli/src/serve/eventBus.ts` | bounded async queue + replay ring | +| `packages/sdk-typescript/src/daemon/DaemonClient.ts` | TS client | +| `packages/sdk-typescript/src/daemon/sse.ts` | EventSource frame parser | +| `integration-tests/cli/qwen-serve-routes.test.ts` | 18 cases, no LLM | +| `integration-tests/cli/qwen-serve-streaming.test.ts` | 3 cases, real `qwen --acp` child (skipped when `SKIP_LLM_TESTS=1`) | diff --git a/docs/developers/sdk-java.md b/docs/developers/sdk-java.md index d3eab983c57..8e2a8dbfec2 100644 --- a/docs/developers/sdk-java.md +++ b/docs/developers/sdk-java.md @@ -144,7 +144,7 @@ The SDK supports different permission modes for controlling tool execution: - **`default`**: Write tools are denied unless approved via `canUseTool` callback or in `allowedTools`. Read-only tools execute without confirmation. - **`plan`**: Blocks all write tools, instructing AI to present a plan first. -- **`auto-edit`**: Auto-approve edit tools (edit, write_file) while other tools require confirmation. +- **`auto-edit`**: Auto-approve edit tools (`edit`, `write_file`, `notebook_edit`) while other tools require confirmation. - **`yolo`**: All tools execute automatically without confirmation. ### Session Event Consumers and Assistant Content Consumers diff --git a/docs/developers/sdk-typescript.md b/docs/developers/sdk-typescript.md index 8ba590997d6..a3de0f2e57e 100644 --- a/docs/developers/sdk-typescript.md +++ b/docs/developers/sdk-typescript.md @@ -158,7 +158,7 @@ The SDK supports different permission modes for controlling tool execution: - **`default`**: Write tools are denied unless approved via `canUseTool` callback or in `allowedTools`. Read-only tools execute without confirmation. - **`plan`**: Blocks all write tools, instructing AI to present a plan first. -- **`auto-edit`**: Auto-approve edit tools (edit, write_file) while other tools require confirmation. +- **`auto-edit`**: Auto-approve edit tools (`edit`, `write_file`, `notebook_edit`) while other tools require confirmation. - **`yolo`**: All tools execute automatically without confirmation. ### Permission Priority Chain diff --git a/docs/developers/tools/_meta.ts b/docs/developers/tools/_meta.ts index 2662563769f..9f68d150ce9 100644 --- a/docs/developers/tools/_meta.ts +++ b/docs/developers/tools/_meta.ts @@ -3,6 +3,7 @@ export default { 'file-system': 'File System', 'multi-file': 'Multi-File Read', shell: 'Shell', + monitor: 'Monitor', 'todo-write': 'Todo Write', task: 'Task', 'exit-plan-mode': 'Exit Plan Mode', diff --git a/docs/developers/tools/file-system.md b/docs/developers/tools/file-system.md index 118f5e0b6d8..d07fd805c6b 100644 --- a/docs/developers/tools/file-system.md +++ b/docs/developers/tools/file-system.md @@ -44,7 +44,72 @@ Qwen Code provides a comprehensive suite of tools for interacting with the local - For other binary files: A message like `Cannot display content of binary file: /path/to/data.bin`. - **Confirmation:** No. -## 3. `write_file` (WriteFile) +### Jupyter notebook reads + +For Jupyter notebooks (`.ipynb`), `read_file` parses the notebook JSON and returns a structured, model-readable notebook view instead of raw JSON. The rendered output includes the notebook language, ordered cells, cell IDs, source, and summarized outputs. + +Notebook cells can then be edited with `notebook_edit`. The model should use the cell IDs shown by `read_file` when targeting a cell. + +`offset` and `limit` are not supported for `.ipynb` files. Notebook reads are treated as structured full-file reads; if the rendered notebook output is internally truncated because it is too large, `notebook_edit` will reject cell-level edits and ask you to reduce outputs or split the notebook before editing. + +## 3. `notebook_edit` (NotebookEdit) + +`notebook_edit` edits Jupyter notebook (`.ipynb`) files safely at the cell level. Use it instead of `edit` or `write_file` when changing notebook cells. + +- **Tool name:** `notebook_edit` +- **Display name:** NotebookEdit +- **File:** `notebook-edit.ts` +- **Parameters:** + - `notebook_path` (string, required): The absolute path to the `.ipynb` file. + - `cell_id` (string, optional): The target cell ID shown by `read_file`. Required for `replace` and `delete`. For `insert`, the new cell is inserted after this cell; if omitted, the new cell is inserted at the beginning. + - `new_source` (string, optional): The new cell source for `replace` and `insert`. Not required for `delete`. + - `cell_type` (`code` or `markdown`, optional): The cell type for inserted cells, or the target type when replacing a cell. + - `edit_mode` (`replace`, `insert`, or `delete`, optional): The edit operation. Defaults to `replace`. +- **Behavior:** + - Requires the notebook to have been read first with `read_file` in the current session. + - Targets cells using the IDs rendered by `read_file`, including real notebook cell IDs and displayed `cell-N` fallback IDs. + - Rejects ambiguous rendered cell IDs instead of guessing. + - For code cells, clears stale outputs and resets `execution_count` when source changes. + - Preserves notebook JSON formatting, line endings, encoding, and BOM where possible. + - Invalidates the prior-read state after structural edits when displayed fallback IDs can shift, so the next notebook edit requires a fresh `read_file`. +- **Output (`llmContent`):** A success message describing the edited notebook cell and, for non-delete operations, the updated source. +- **Confirmation:** Yes. Shows a notebook JSON diff and asks for user approval before writing, unless the current permission mode or rules auto-approve edit tools. + +### `notebook_edit` examples + +Replace a code cell: + +``` +notebook_edit( + notebook_path="/path/to/analysis.ipynb", + cell_id="load-data", + new_source="result = 41 + 1\nprint(result)" +) +``` + +Insert a markdown cell after an existing cell: + +``` +notebook_edit( + notebook_path="/path/to/analysis.ipynb", + edit_mode="insert", + cell_id="summary", + cell_type="markdown", + new_source="## Findings\n\nThe cleaned data is ready for modeling." +) +``` + +Delete a cell: + +``` +notebook_edit( + notebook_path="/path/to/analysis.ipynb", + edit_mode="delete", + cell_id="old-experiment" +) +``` + +## 4. `write_file` (WriteFile) `write_file` writes content to a specified file. If the file exists, it will be overwritten. If the file doesn't exist, it (and any necessary parent directories) will be created. @@ -56,11 +121,12 @@ Qwen Code provides a comprehensive suite of tools for interacting with the local - `content` (string, required): The content to write into the file. - **Behavior:** - Writes the provided `content` to the `file_path`. + - Does not write raw Jupyter notebook JSON. Use `notebook_edit` for `.ipynb` cell edits. - Creates parent directories if they don't exist. - **Output (`llmContent`):** A success message, e.g., `Successfully overwrote file: /path/to/your/file.txt` or `Successfully created and wrote to new file: /path/to/new/file.txt`. - **Confirmation:** Yes. Shows a diff of changes and asks for user approval before writing. -## 4. `glob` (Glob) +## 5. `glob` (Glob) `glob` finds files matching specific glob patterns (e.g., `src/**/*.ts`, `*.md`), returning absolute paths sorted by modification time (newest first). @@ -78,7 +144,7 @@ Qwen Code provides a comprehensive suite of tools for interacting with the local - **Output (`llmContent`):** A message like: `Found 5 file(s) matching "*.ts" within /path/to/search/dir, sorted by modification time (newest first):\n---\n/path/to/file1.ts\n/path/to/subdir/file2.ts\n---\n[95 files truncated] ...` - **Confirmation:** No. -## 5. `grep_search` (Grep) +## 6. `grep_search` (Grep) `grep_search` searches for a regular expression pattern within the content of files in a specified directory. Can filter files by a glob pattern. Returns the lines containing matches, along with their file paths and line numbers. @@ -131,7 +197,7 @@ Search for a pattern with file filtering and custom result limiting: grep_search(pattern="function", glob="*.js", limit=10) ``` -## 6. `edit` (Edit) +## 7. `edit` (Edit) `edit` replaces text within a file. By default it requires `old_string` to match a single unique location; set `replace_all` to `true` when you intentionally want to change every occurrence. This tool is designed for precise, targeted changes and requires significant context around the `old_string` to ensure it modifies the correct location. @@ -148,6 +214,7 @@ grep_search(pattern="function", glob="*.js", limit=10) - `replace_all` (boolean, optional): Replace all occurrences of `old_string`. Defaults to `false`. - **Behavior:** + - Does not edit raw Jupyter notebook JSON. Use `notebook_edit` for `.ipynb` cell edits. - If `old_string` is empty and `file_path` does not exist, creates a new file with `new_string` as content. - If `old_string` is provided, it reads the `file_path` and attempts to find exactly one occurrence unless `replace_all` is true. - If the match is unique (or `replace_all` is true), it replaces the text with `new_string`. diff --git a/docs/developers/tools/introduction.md b/docs/developers/tools/introduction.md index 1dafb14c885..2a6b3e2faeb 100644 --- a/docs/developers/tools/introduction.md +++ b/docs/developers/tools/introduction.md @@ -45,6 +45,7 @@ Qwen Code's built-in tools can be broadly categorized as follows: - **[File System Tools](./file-system.md):** For interacting with files and directories (reading, writing, listing, searching, etc.). - **[Shell Tool](./shell.md) (`run_shell_command`):** For executing shell commands. +- **[Monitor Tool](./monitor.md) (`monitor`):** For running long-lived shell commands that stream output back as background task notifications. - **[Web Fetch Tool](./web-fetch.md) (`web_fetch`):** For retrieving content from URLs. - **[Multi-File Read Tool](./multi-file.md) (`read_many_files`):** A specialized tool for reading content from multiple files or directories, often used by the `@` command. - **[Memory Tool](./memory.md) (`save_memory`):** For saving and recalling information across sessions. diff --git a/docs/developers/tools/monitor.md b/docs/developers/tools/monitor.md new file mode 100644 index 00000000000..c004552aa14 --- /dev/null +++ b/docs/developers/tools/monitor.md @@ -0,0 +1,154 @@ +# Monitor Tool (`monitor`) + +This document describes the `monitor` tool for Qwen Code. + +## Description + +Use `monitor` to start a long-running shell command that streams stdout and +stderr lines back to the agent as background task notifications. It is intended +for watch-style commands where new output matters over time, such as tailing +logs, watching build output, polling a health endpoint, or observing file +changes. + +The monitor runs in the background, so the agent can continue working while +events arrive. Each non-empty output line becomes a notification event, subject +to throttling. + +### Arguments + +`monitor` takes the following arguments: + +- `command` (string, required): The shell command to run and monitor. +- `description` (string, optional): A brief description of what the monitor is + watching. The display text is truncated to 80 characters. +- `max_events` (number, optional): Stop after this many notification events. + Must be a positive integer. Defaults to `1000`; maximum `10000` (values + outside this range are rejected, not silently clamped). +- `idle_timeout_ms` (number, optional): Stop if the command produces no output + for this many milliseconds. Must be a positive integer. Defaults to `300000` + (5 minutes); maximum `600000` (10 minutes), and values outside this range are + rejected. +- `directory` (string, optional): An absolute path to run the command in. Must + resolve (after symlink canonicalization) inside one of the registered + workspace directories, and must not be inside the user-skills directory. If + omitted, Qwen Code uses the project root. + +## How to use `monitor` with Qwen Code + +The model chooses the `monitor` tool when it needs to observe a process over +time instead of collecting a single command result. A successful invocation +returns a monitor ID, the command, the event limit, and the idle timeout. + +Usage: + +``` +monitor(command="tail -f logs/app.log", description="app log stream") +``` + +Monitor output is visible in the conversation as task notifications. You can +also inspect running and completed monitors with `/tasks` or the interactive +Background tasks dialog. + +To stop a running monitor, use the `task_stop` tool with the monitor ID: + +``` +task_stop(task_id="mon_abc123def4567890") +``` + +## `monitor` examples + +Watch an application log: + +``` +monitor( + command="tail -f logs/app.log", + description="application log stream", + max_events=200 +) +``` + +Monitor a dev server or build watcher: + +``` +monitor( + command="npm run build -- --watch", + description="watch build output", + idle_timeout_ms=600000 +) +``` + +Poll a local health endpoint: + +``` +monitor( + command="while true; do curl -s http://localhost:8080/health; sleep 5; done", + description="local health check", + max_events=120 +) +``` + +Run from a specific workspace directory: + +``` +monitor( + command="npm run dev", + description="frontend dev server", + directory="/absolute/path/to/workspace/packages/web" +) +``` + +## Monitor vs. background shell commands + +Use `monitor` when the agent needs to react to streaming output while the +command keeps running. Use `run_shell_command` instead when you need a one-shot +result or the complete command output. + +| Need | Use | +| :----------------------------------------------------- | :--------------------------------------- | +| Watch logs, build output, or periodic status updates | `monitor` | +| Run a one-time command and read the full output | `run_shell_command(is_background=false)` | +| Start a daemon that does not produce meaningful output | `run_shell_command(is_background=true)` | + +Do not add `&` to monitor commands. A trailing `&`, such as +`tail -f log &`, is stripped because the monitor manages backgrounding itself. +A non-final `&`, such as `cmd1 & cmd2`, is rejected outright; restructure such +commands without backgrounding instead. + +## Important notes + +- **Auto-stop behavior:** Monitors stop automatically when they reach + `max_events`, when `idle_timeout_ms` elapses without output, or when the + underlying command exits on its own. A monitor's status reflects the + command's outcome, not a tool error: a clean exit (`code 0`) becomes + `completed`, a non-zero exit code becomes `failed` with message + `Exit code N`, and termination by signal becomes `failed` with message + `Killed by signal SIG`. Commands cannot be interactive because stdin is + closed. When a monitor stops, Qwen Code sends `SIGTERM` to the command's + process group and escalates to `SIGKILL` after about 200 ms. On Windows, it + uses `taskkill /f /t`. If the Qwen Code process itself is hard-killed, + crashes, or runs out of memory, the detached process group is not cleaned up + automatically; recover by stopping the monitor with `task_stop` before exit + or by terminating the process group manually. +- **Concurrency limit:** Qwen Code allows up to 16 running monitors per CLI + session as a single shared pool. Monitors started by subagents count against + the same cap as monitors started by the main agent. Stop an existing monitor + before starting another if the limit is reached. +- **Output handling:** Stdout and stderr are merged into a single notification + stream with no stream prefix. Empty lines are ignored, ANSI color and control + characters are stripped, and individual lines longer than 2000 characters are + truncated. High-volume output is rate-limited with a burst of 5 events and + about 1 event per second after that; lines beyond the rate limit are dropped, + not buffered. Monitor output flows into the agent context as + `` content. Structural notification tags are defanged, but + the model still reads each line's text, so avoid monitoring streams that + external parties can write to unless you trust the model to ignore embedded + instructions. +- **Permissions:** `monitor` has its own permission boundary and permission + rules, such as `Monitor(git status)`. Read-only commands are automatically + allowed; commands that modify state require user approval; commands containing + command substitution (`$(...)`, backticks, `<(...)`, or `>(...)`) are rejected + outright. The `tools.core` and `tools.exclude` settings for + `run_shell_command` do not apply to `monitor`. +- **Workspace restriction:** The optional `directory` must be an absolute path + that resolves inside a registered workspace directory and outside the + user-skills directory. Symlinks that point outside the workspace are rejected. diff --git a/docs/developers/tools/swarm.md b/docs/developers/tools/swarm.md new file mode 100644 index 00000000000..280c185e5a0 --- /dev/null +++ b/docs/developers/tools/swarm.md @@ -0,0 +1,102 @@ +# Swarm Tool (`swarm`) + +Use `swarm` to run many independent, simple tasks through ephemeral worker +agents and return a structured aggregate result to the parent agent. + +Swarm is intended for map-reduce style work: + +- analyzing many files independently +- processing chunks of a large data file +- running independent searches where the first successful result is enough +- collecting per-item summaries, counts, or validation results + +For a few complex role-based tasks, use the [`task`](./task.md) tool instead. +For model comparison on the same task, use Agent Arena. + +## Arguments + +- `description` (string, required): Short description of the overall swarm job. +- `tasks` (array, required): Independent tasks. Each task becomes one worker. + - `id` (string, optional): Stable identifier returned in results. + - `description` (string, required): Short per-worker description. + - `prompt` (string, required): Complete instructions for the worker. +- `mode` (`wait_all` or `first_success`, optional): Defaults to `wait_all`. +- `max_concurrency` (number, optional): Maximum workers to run at once. +- `max_turns` (number, optional): Maximum model/tool turns per worker. + Defaults to `8`. +- `timeout_seconds` (number, optional): Per-worker wall-clock timeout. +- `worker_system_prompt` (string, optional): Shared worker system prompt. +- `allowed_tools` (string array, optional): Tool allowlist for workers. +- `disallowed_tools` (string array, optional): Tools removed from workers. + +If `max_concurrency` is omitted, Qwen Code uses +`QWEN_CODE_MAX_SWARM_CONCURRENCY`, then `QWEN_CODE_MAX_TOOL_CONCURRENCY`, then +`10`. + +## Result + +The tool returns JSON to the parent agent with: + +- `summary.total` +- `summary.completed` +- `summary.failed` +- `summary.cancelled` +- `summary.notStarted` +- `results[]` with one entry per task, including `taskId`, `status`, `output` + or `error`, duration, and execution stats when available + +Individual worker failures do not abort the whole swarm. The parent agent is +responsible for reading the aggregate result and presenting the final answer. + +## Examples + +Analyze files in parallel: + +```text +swarm( + description="Extract function names", + tasks=[ + { + id="src/a.ts", + description="Analyze src/a.ts", + prompt="Read /repo/src/a.ts and return the exported function names." + }, + { + id="src/b.ts", + description="Analyze src/b.ts", + prompt="Read /repo/src/b.ts and return the exported function names." + } + ], + max_concurrency=10 +) +``` + +Use first successful result: + +```text +swarm( + description="Find API route definition", + mode="first_success", + tasks=[ + { + description="Search routes directory", + prompt="Search /repo/src/routes for the user creation route." + }, + { + description="Search controllers directory", + prompt="Search /repo/src/controllers for the user creation route." + } + ] +) +``` + +## Notes + +Workers are lightweight and ephemeral: they are spawned, execute one task, +return a result, and are cleaned up. Workers cannot spawn further subagents or +cron jobs. + +Swarm workers run concurrently, so interactive permission prompts are avoided. +Permission hooks can still approve actions, and permissive approval modes still +apply where configured. Prefer read-only or disjoint file scopes for swarm +tasks. diff --git a/docs/e2e-tests/2026-05-18-qwen-memory-benchmark-report.md b/docs/e2e-tests/2026-05-18-qwen-memory-benchmark-report.md new file mode 100644 index 00000000000..1a7aaf32533 --- /dev/null +++ b/docs/e2e-tests/2026-05-18-qwen-memory-benchmark-report.md @@ -0,0 +1,286 @@ +# Qwen Code Runtime Memory Benchmark Report + +Date: 2026-05-18 + +## Summary + +This report records local memory benchmarks for Qwen Code runtime behavior. It +compares Qwen Code across models and compares Qwen Code with Claude Code on the +same task shapes where equivalent model endpoints were available. + +The headline result is consistent across the latest matrix (single run per cell, +not statistically repeated): + +- Qwen Code process-tree RSS peak: about `852-1062 MiB` (`0.83-1.04 GiB`). +- Claude Code process-tree RSS peak: about `279-366 MiB` (`0.27-0.36 GiB`). +- Qwen Code was about `2.3x-3.6x` higher in the tested + non-interactive CLI task benchmarks. + +Note: process-tree RSS includes MCP child processes (~350 MiB overhead on the +Qwen side). This inflates the absolute numbers but the relative comparison +remains informative since both CLIs were measured the same way. + +The difference reproduced in small PR review, code navigation, and synthetic +diff workloads. It is therefore unlikely to be explained only by one large PR +or by one model provider. + +This report is intended to make the current performance investigation visible: +what has been measured, what conclusion is already supported, what remains +unknown, and what diagnostics should be added next. + +## Test Environment + +| Item | Value | +| --------------------------------------------- | ------------------------------------------ | +| Date | 2026-05-18 | +| Platform | macOS local development machine | +| Qwen Code version | `0.15.11` | +| Qwen Code binary | PATH-resolved `qwen` binary | +| Claude Code version used in the latest matrix | `2.1.129` | +| Claude Code binary used in the latest matrix | PATH-resolved `claude` binary | +| Node.js version | v22.x (default system install) | +| Sampling method | External `ps` RSS sampling once per second | +| Headline metric | Process-tree RSS peak | + +Process-tree RSS is used as the headline metric because Qwen Code launches a +root wrapper and a child Node/Qwen worker. Looking only at the root process can +understate the memory footprint seen by users. + +Temporary CLI config directories were used for matrix runs so the benchmarks +did not depend on global CLI state. + +## Benchmark Artifacts + +Five local reports were produced before this consolidated report: + +1. Qwen Code PR review memory run. +2. Qwen Code model comparison run. +3. Strict Qwen Code vs Claude Code comparison with `pai/glm-5`. +4. Qwen Code vs Claude Code, two CLIs by two models. +5. Qwen Code vs Claude Code, five-case matrix. + +This consolidated report covers the conclusions and headline metrics from all +five reports. It does not embed every raw sample row, terminal transcript, or +temporary runner artifact. Those raw artifacts stayed in local `tmp/` +directories because they are experiment outputs rather than stable repository +fixtures. + +The latest matrix is the strongest evidence because it covers multiple task +shapes rather than only one PR review workload. + +## Preliminary Conclusion + +The current data is strong enough to say that Qwen Code has a higher runtime +memory footprint than Claude Code in these local non-interactive CLI task +benchmarks. It is not strong enough to name one final root cause yet. + +The leading explanation is a Qwen Code runtime/path difference rather than a +model provider difference: + +- the gap reproduces with both `pai/glm-5` and `qwen3.6-plus`; +- the gap reproduces in small PR and code-navigation tasks, not only in large + diff tasks; +- Qwen Code repeatedly sends or accounts for more tokens than Claude Code for + similar work; +- Qwen Code's largest observed component is the child Node/Qwen worker process, + which points toward task-time process footprint, module loading, context + assembly, live history, tool-result retention, or subagent/saved-output + paths. + +The most useful next measurement is therefore not another external RSS-only +run. The next measurement should split RSS into V8 heap, native memory, +session/history size, retained tool-result size, and subagent/process-tree +activity. + +## Initial Cause Analysis + +The benchmark does not yet prove one root cause, but it does narrow the likely +problem area. + +| Signal | What it suggests | What it does not prove | +| -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------- | +| Qwen remains near `1 GiB` in small PR and code-navigation cases | A high non-interactive task-time runtime cost is likely involved | It does not identify whether the footprint is V8 heap, native memory, module loading, or retained state | +| Diff size from 100 KiB to 5 MiB does not scale linearly with RSS | Raw diff bytes alone are probably not the primary driver | Large outputs can still amplify memory in real PR review flows | +| Qwen uses more tokens than Claude in every matrix cell | Qwen likely constructs or retains larger prompt/context/tool-result state for similar work | Token count is not the same as process memory and may be an effect rather than the cause | +| Tool call counts are similar, and Claude sometimes uses more turns/tool calls with lower RSS | A longer tool-call chain is unlikely to be the main explanation by itself | Tool output size and retention still need to be measured | +| Earlier large PR runs showed saved-output recovery and subagent amplification | Tool-output truncation and saved-output paths are likely heavy-workload amplifiers | They do not explain the entire small-task execution footprint | + +The current best explanation is therefore: + +1. **Task-time runtime cost first**: Qwen Code likely initializes or retains + more runtime state during non-interactive CLI task execution than Claude + Code. This may include agent runtime, tool registry, provider adapters, + session services, or UI/history structures that are not strictly needed for + a short non-interactive task. +2. **Context/tool-result volume second**: Qwen Code appears to carry larger + model-facing or session-facing context for similar work. The token gap makes + context assembly, tool result normalization, and history retention important + suspects. +3. **Large-output amplification third**: Large PR review can trigger additional + saved-output and subagent paths. These are probably not the only cause, but + they can make memory and token pressure worse in realistic review tasks. + +The next diagnostic run should answer where the `~1 GiB` sits: + +- high immediately after startup: module/runtime startup cost; +- jumps after tool execution: tool-output retention or result normalization; +- jumps during request assembly: context construction or duplicated histories; +- grows after streaming/compression: response retention or compression state; +- mostly RSS outside V8 heap: native buffers, loaded modules, or external + memory. + +## Latest Matrix + +The latest benchmark ran: + +- 2 CLIs: Qwen Code and Claude Code. +- 2 model labels: `pai/glm-5` and `qwen3.6-plus`. +- 5 cases: + - small PR review: PR `#4268`, one-line change + - code navigation: `rg` plus `sed` on compression-related files + - synthetic local diff, about 100 KiB + - synthetic local diff, about 1 MiB + - synthetic local diff, about 5 MiB + +All 20 runs exited `0` with no timeout. + +## Matrix Results + +| Case | Model | Qwen tree peak | Claude tree peak | Qwen / Claude | +| ---------------- | -------------- | -------------: | ---------------: | ------------: | +| small PR `#4268` | `pai/glm-5` | 1032.7 MiB | 357.8 MiB | 2.89x | +| small PR `#4268` | `qwen3.6-plus` | 852.2 MiB | 365.5 MiB | 2.33x | +| code navigation | `pai/glm-5` | 993.1 MiB | 359.6 MiB | 2.76x | +| code navigation | `qwen3.6-plus` | 996.9 MiB | 349.0 MiB | 2.86x | +| diff 100 KiB | `pai/glm-5` | 1012.1 MiB | 350.8 MiB | 2.89x | +| diff 100 KiB | `qwen3.6-plus` | 1001.1 MiB | 336.2 MiB | 2.98x | +| diff 1 MiB | `pai/glm-5` | 1008.3 MiB | 278.8 MiB | 3.62x | +| diff 1 MiB | `qwen3.6-plus` | 1003.3 MiB | 340.5 MiB | 2.95x | +| diff 5 MiB | `pai/glm-5` | 858.8 MiB | 323.2 MiB | 2.66x | +| diff 5 MiB | `qwen3.6-plus` | 1062.0 MiB | 331.2 MiB | 3.21x | + +Average process-tree RSS peak by case: + +| Case | Avg Qwen tree peak | Avg Claude tree peak | +| ---------------- | -----------------: | -------------------: | +| small PR `#4268` | 942.5 MiB | 361.6 MiB | +| code navigation | 995.0 MiB | 354.3 MiB | +| diff 100 KiB | 1006.6 MiB | 343.5 MiB | +| diff 1 MiB | 1005.8 MiB | 309.6 MiB | +| diff 5 MiB | 960.4 MiB | 327.2 MiB | + +## Runtime And Token Signals + +The same matrix also showed Qwen Code using more model-side tokens in every +tested case. + +Selected examples: + +| Case | Model | CLI | Duration | Turns | Total tokens | Tool calls | +| --------------- | -------------- | ------ | -------: | ----: | -----------: | ---------: | +| small PR | `pai/glm-5` | Qwen | 25.2s | 2 | 32,567 | 3 | +| small PR | `pai/glm-5` | Claude | 21.1s | 4 | 7,899 | 3 | +| code navigation | `qwen3.6-plus` | Qwen | 25.2s | 2 | 38,151 | 3 | +| code navigation | `qwen3.6-plus` | Claude | 46.9s | 6 | 25,861 | 5 | +| diff 100 KiB | `qwen3.6-plus` | Qwen | 16.5s | 3 | 57,185 | 2 | +| diff 100 KiB | `qwen3.6-plus` | Claude | 17.2s | 3 | 6,377 | 2 | +| diff 5 MiB | `pai/glm-5` | Qwen | 23.2s | 2 | 38,574 | 2 | +| diff 5 MiB | `pai/glm-5` | Claude | 9.8s | 3 | 5,285 | 2 | + +This token gap does not prove that token volume is the memory root cause, but it +does suggest that context assembly, tool result retention, or response +normalization should be measured alongside RSS and V8 heap statistics. + +## Token Usage Analysis + +The token gap is one of the strongest clues, but it needs internal request +metrics before it can be treated as a root cause. + +What the data supports today: + +- Qwen Code used more total tokens than Claude Code in every matrix cell. +- The gap appears even when tool-call counts are similar. +- Claude sometimes used more turns or tool calls while still using less memory. + +What this suggests: + +- The token delta is unlikely to come only from a longer tool-call chain. +- Qwen may be carrying larger static prompt/context state, larger tool schemas, + larger serialized tool results, or more retained conversation/session content. +- Large-output flows may add another layer through truncation, saved-output + recovery, or subagent paths. + +What is still missing: + +- per-request input token breakdown; +- system prompt and tool schema token sizes; +- retained message and tool-result sizes before each model request; +- whether large outputs are retained in multiple places, such as model history, + UI history, session recording, or saved-output storage. + +Those missing metrics are why the next step should add internal diagnostics +rather than only repeat the external RSS benchmark. + +## Earlier Large PR Review Signal + +An earlier strict PR review benchmark used PR `#4186` and showed the same broad +shape: + +| Model | CLI | Process-tree RSS peak | +| -------------- | ----------- | --------------------: | +| `pai/glm-5` | Qwen Code | 1000.7 MiB | +| `pai/glm-5` | Claude Code | 349.0 MiB | +| `qwen3.6-plus` | Qwen Code | 1095.8 MiB | +| `qwen3.6-plus` | Claude Code | 341.1 MiB | + +That earlier run was not enough by itself because a large PR can trigger unusual +tool-output and saved-output paths. The latest five-case matrix makes the +finding stronger because small PR and code-navigation tasks also reproduce the +gap. + +## Working Hypothesis + +The current evidence supports these hypotheses, in priority order: + +1. Qwen Code has a higher non-interactive task-time process footprint than + Claude Code. The Qwen child Node worker was typically the largest process in + local sampling, often around `0.7-0.8 GiB`. +2. Model choice is not the main explanation. Both `pai/glm-5` and + `qwen3.6-plus` showed the same broad Qwen-vs-Claude gap. +3. Large diff size alone is not the main explanation. The synthetic diff size + did not scale linearly from 100 KiB to 5 MiB, likely because tool-output + truncation caps how much output reaches the model. +4. Context/tool-result handling is still a likely contributor. Qwen Code used + more tokens than Claude Code in every matrix cell, and earlier large-PR runs + showed saved tool-output recovery and subagent amplification paths. +5. The next diagnostic layer should separate V8 heap, native RSS, loaded + module/runtime startup cost, session history, UI history, tool-result + retention, and subagent activity. External RSS alone cannot distinguish + those causes. + +## Caveats + +- These are single runs per matrix cell, not repeated statistical samples. +- RSS is external process RSS. It cannot distinguish V8 heap, native buffers, + module loading, retained tool output, UI state, or session history. +- Claude Code and Qwen Code use different runtime implementations and protocol + adapters, even when the model labels are the same. +- The benchmark was run locally on macOS. Linux servers should be tested before + drawing deployment-specific conclusions. + +## Recommended Follow-Up Measurements + +The next local investigation branch should add or use diagnostics for: + +- `process.memoryUsage()` before and after startup, tool execution, streaming, + compression, and session finalization. +- V8 heap statistics and heap spaces. +- Active handles and requests. +- Session message count and approximate retained character/token volume. +- Tool result count, total retained tool-result size, largest tool-result size, + and whether large outputs are retained by UI history or model history. +- Subagent count and child process/process-tree RSS. +- Tool-output truncation and saved-output recovery events. + +These measurements should be collected with the same benchmark matrix so the +current RSS comparison can be connected to internal Qwen Code state. diff --git a/docs/e2e-tests/2026-05-19-oom-reproduction-report.md b/docs/e2e-tests/2026-05-19-oom-reproduction-report.md new file mode 100644 index 00000000000..8716e208f56 --- /dev/null +++ b/docs/e2e-tests/2026-05-19-oom-reproduction-report.md @@ -0,0 +1,437 @@ +# OOM 压力测试与长任务 Replay 报告 + +**日期**: 2026-05-19 +**分支**: `codex/memory-diagnostics-local-run` +**测试人**: yiliang114 +**结论**: 成功复现并定位根因。v0.15.7 (#3735) 引入的 auto-compaction 使 `structuredClone` +调用频率倍增,在高 heap 压力时形成正反馈死循环导致 OOM。真实 debug 日志完整佐证了该机制。 + +--- + +## 一、背景 + +多个 issue(#4309, #4276, #4185, #4315, #4322, #2868)报告 qwen-code 在长会话中出现 V8 heap OOM crash: + +``` +FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory +``` + +用户报告的崩溃特征: +| Issue | 崩溃时 Heap | 运行时长 | 平台 | +|-------|------------|---------|------| +| #4276 | 4014 MB | ~110 分钟 | Linux x64 | +| #4315 | 2027 MB | ~19.6 小时 | macOS (默认 2GB limit) | +| #4322 | 4023 MB | ~7 小时 | Windows | +| #2868 | 2035 MB | ~1.7 分钟 | Linux | +| #4309 | 7020 MB | 未知 | Windows (设了 8GB limit 仍崩) | + +--- + +## 二、方法论修正 + +本报告区分两类测试: + +1. **低 heap 压力测试**:通过降低 `--max-old-space-size` 放大问题,用于快速定位 + “history 很大时整段复制导致瞬时峰值”的代码路径。它是诊断工具,不等价于用户真实 + 4G/8G OOM 复现。 +2. **默认 heap 长任务 replay**:不设置 `NODE_OPTIONS`,使用真实 JSONL 历史恢复并 + 继续执行 review 任务,同时从进程外采样 process-tree RSS。这类结果才用于判断 + 用户侧实际内存量级。 + +因此,低 heap 结果不能单独作为“真实 OOM 已修复”的证明。它只能说明某条路径在 +history 足够大时会产生峰值放大,需要再用默认 heap 长任务验证。 + +## 三、低 heap 压力测试条件 + +| 参数 | 值 | +| ------------------------ | ------------------------------------------------------------ | +| CLI 版本 | 0.15.11 (从 `codex/memory-diagnostics-local-run` 分支 build) | +| Model | `qwen3.6-plus` (128K context window) | +| Heap limit | `--max-old-space-size=512` | +| Heap-pressure safety net | **禁用** (HEAP_PRESSURE_COMPRESSION_RATIO 设为 99.0) | +| 操作模式 | YOLO + 自动化多轮 Read 文件任务 | +| 工作目录 | qwen-code monorepo (3538 .ts files, 1.26M lines) | + +### 关键配置修改 + +`packages/core/src/core/geminiChat.ts` 中将 heap-pressure compaction 阈值从 0.7 改为 99.0(使其永远不触发),模拟 #4186 修复前的状态。 + +--- + +## 四、低 heap 压力测试结果 + +### 崩溃时间线 + +``` +[21:26:59] #1 RSS:193.6MB Ctx:0% → Read geminiChat.ts (1500 行) +[21:27:46] #2 RSS:270.4MB Ctx:4.2% → Read agent.ts +[21:28:32] #3 RSS:397.5MB Ctx:4.3% → grep + Read 3 个文件 +[21:29:18] #4 RSS:452.7MB Ctx:5.7% → Read slashCommandProcessor.ts +[21:30:04] #5 RSS:515.0MB Ctx:5.9% → Read chatCompressionService.ts +[21:30:50] #6 RSS:649.1MB Ctx:4.0% ← TOKEN COMPACTION 触发 (5.9%→4.0%) + RSS 反增 134MB (structuredClone 峰值) +[21:31:36] #7 RSS:666.7MB Ctx:3.2% ← 再次 compaction, RSS 继续涨 +[21:32:22] CRASH — FATAL ERROR: Ineffective mark-compacts near heap limit +``` + +**总耗时**: ~5.5 分钟,7 轮任务后崩溃。 + +这证明在受限 heap 下,长 history + compaction/history clone 可以触发 V8 heap OOM。 +但该结果不代表默认 heap 下的真实用户 OOM 已经被完整复现。 + +### 更大 heap 的 synthetic 复现 + +为避免只依赖 512 MiB 低 heap 结论,补充了更大 heap 的 synthetic runtime +pressure 测试。该测试不调用模型,而是构造类似长 review/subagent 任务的历史: + +- root review turns: 10 +- subagent calls: 30 +- subagent transcript records: 780 +- retained tool result bytes: 193,986,560 +- serialized history bytes: 195,620,061 +- pressure mode: retained `structuredClone(history)` copies + +| Heap limit | Clone pressure | 结果 | 关键 GC / stack | +| ---------- | -----------------: | ---------------------------------------- | ------------------------------------------------------------ | +| 2 GiB | 8 retained clones | 未崩溃,RSS 2.42 GiB,heap used 1.87 GiB | 接近 heap limit | +| 2 GiB | 10 retained clones | OOM | `Reached heap limit`, `ValueDeserializer`, `StructuredClone` | +| 4 GiB | 20 retained clones | OOM | `Reached heap limit`, `ValueDeserializer`, `StructuredClone` | + +2 GiB 复现的 GC 摘要: + +``` +Mark-Compact 2042.9 (2081.9) -> 2042.9 (2081.1) MB +Mark-Compact 2048.9 (2087.2) -> 2048.9 (2087.2) MB +FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory +... +node::worker::(anonymous namespace)::StructuredClone +``` + +4 GiB 复现的 GC 摘要: + +``` +Mark-Compact 4082.5 (4126.8) -> 4082.5 (4126.3) MB +Mark-Compact 4095.1 (4139.0) -> 4095.1 (4139.0) MB +FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory +... +node::worker::(anonymous namespace)::StructuredClone +``` + +这组结果比 512 MiB 压力测试更接近用户报告的 2 GiB / 4 GiB heap OOM: +只要 history 中保留足够多的大 tool result / subagent transcript,对整段 history +做 retained 或瞬时 clone 都可以在 2-4 GiB heap 下触发 V8 OOM。它仍然是 synthetic +复现,不等价于完整业务长任务 replay,但能直接证明问题不是“小 heap 人为制造”的。 + +### 崩溃时 GC 状态 + +``` +[41381:0x130008000] 342468 ms: Mark-Compact 508.6 (526.7) -> 507.0 (526.9) MB, + pooled: 1 MB, 86.42 / 0.00 ms (average mu = 0.175, current mu = 0.150) + task; scavenge might not succeed + +[41381:0x130008000] 342568 ms: Mark-Compact 509.1 (526.9) -> 507.1 (528.2) MB, + pooled: 0 MB, 93.79 / 0.12 ms (average mu = 0.121, current mu = 0.068) + allocation failure; scavenge might not succeed + +FATAL ERROR: Ineffective mark-compacts near heap limit +Allocation failed - JavaScript heap out of memory +``` + +Mark-Compact 只能回收 1-2 MB(几乎所有对象都是 reachable),证明内存确实被合法持有的对象占满。 + +--- + +## 五、默认 heap 长任务 replay + +为了避免低 heap 结论过度外推,补充了默认 heap 的真实 JSONL replay: + +- 不设置 `NODE_OPTIONS` +- 不启用内部 runtime profiler,避免采样器自身影响 heap +- 每个 CLI 从同一份 rewound JSONL 复制出 fresh session +- 使用临时 `QWEN_HOME`,禁用 MCP 和 hooks,避免本地全局配置污染 +- 只用进程外采样统计 process-tree RSS + +| CLI | 结果 | 时长 | Tree RSS 峰值 | Root RSS 峰值 | Worker RSS 峰值 | 备注 | +| -------------------- | ---- | -----: | ------------: | ------------: | --------------: | ----------------------------------------------------------- | +| installed `qwen` | 成功 | 167.3s | 838.0 MiB | 230.2 MiB | 566.3 MiB | 第一次 fresh run 遇到模型服务端错误,未纳入结论;retry 成功 | +| local rebuilt bundle | 成功 | 106.3s | 527.5 MiB | 182.1 MiB | 345.4 MiB | 包含本地 clone 热路径修复 | + +默认 heap replay 的结论: + +1. 当前这份 review JSONL 可以稳定跑出数百 MiB 到约 0.8 GiB 的 process-tree RSS, + 但没有复现 4G/8G OOM。 +2. 本地 rebuilt bundle 在同起点 replay 上的峰值低于 installed CLI,说明减少 + history clone 热路径有实际收益。 +3. 这还不能证明所有用户 OOM 都已解决。真实 4G/8G OOM 仍需要更长任务、更大 + tool-result 累积,或保留 MCP/tool schema 压力的 replay 继续验证。 + +## 六、根因分析 + +### OOM 的三层机制 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Layer 3: V8 Heap Limit (512MB/2GB/4GB) │ ← 用户最终撞到这里 +├─────────────────────────────────────────────────────────┤ +│ Layer 2: structuredClone() 峰值放大 (瞬时 ~2x) │ ← 直接诱因 +├─────────────────────────────────────────────────────────┤ +│ Layer 1: History 中 tool result 累积 (线性增长) │ ← 基础增长 +├─────────────────────────────────────────────────────────┤ +│ Layer 0: Token compaction 触发时机 │ ← 控制点 +└─────────────────────────────────────────────────────────┘ +``` + +### 精确崩溃路径 + +``` +sendMessage() + → tryCompress() + → heapPressureRatio < threshold (safety net disabled) + → ChatCompressionService.compress() + → chat.getHistory(true) + → structuredClone(this._history) ← 峰值分配! + → V8 需要额外 ~N MB 来容纳 clone + → 如果 existing heap + N > limit → OOM +``` + +### 关键证据 + +| 观察 | 含义 | +| --------------------------------------- | ---------------------------------------------- | +| Task #5→#6: Context 5.9%→4.0% (降了) | Token compaction **成功执行**了 | +| Task #5→#6: RSS 515→649 MB (涨了 134MB) | Compaction 过程的 `structuredClone` 制造了峰值 | +| GC 只能回收 1-2 MB | 所有对象都是 live(history + clone 都在) | +| #4309 设 8GB limit 仍崩 | history 足够大时,clone 峰值可超任何 limit | + +需要注意:以上证据来自低 heap 压力测试和 issue 现象的组合推断。默认 heap replay +目前支持”clone 热路径会显著影响峰值 RSS”,但尚未单独复现 4G/8G OOM。 + +### 为什么 128K context window 更容易触发 + +- 128K × 70% = ~90K tokens 触发 compaction +- 大 context window (1M) 的 70% = 700K tokens,几乎不会触发 +- **compaction 越频繁 → structuredClone 越频繁 → OOM 风险越高** +- DeepSeek 等未配置 contextWindowSize 的模型默认 128K,更易触发 + +--- + +## 六.5、真实运行日志佐证 + +以下日志提取自本地 crash session 的 debug 输出。为避免泄露本地路径和 session id, +报告只保留时间线和关键日志内容。 + +该 session 启动于 `2026-05-19T13:26:35Z` (本地 21:26:35),crash 于 +`2026-05-19T13:32:10Z` (本地 21:32:10)。 + +### Heap Pressure 与 Auto-Compaction 事件时间线 + +``` +13:29:43 [WARN] Heap pressure at 74.9%; attempting auto-compaction before token threshold. +13:30:06 [DEBUG] [FILE_READ_CACHE] clear after auto tryCompress ← compaction #1 执行成功 +13:30:13 [WARN] Heap pressure at 70.7%; attempting auto-compaction before token threshold. + ← 刚压完 heap 从 74.9% 仅降到 70.7%,仍超阈值,立即再次尝试 +13:30:52 [DEBUG] Heap pressure at 86.0%; skipping heap-pressure auto-compaction during cooldown. + ← 30s cooldown 期间拒绝执行 +13:30:56 [WARN] Heap pressure at 85.3%; attempting auto-compaction before token threshold. + ← cooldown 过期,heap 已升至 85.3% +13:31:21 [DEBUG] [FILE_READ_CACHE] clear after auto tryCompress ← compaction #2 执行成功 +13:31:37 [WARN] Heap pressure at 88.8%; attempting auto-compaction before token threshold. + ← 压完后 heap 反弹至 88.8% +13:32:09 [DEBUG] Heap pressure at 90.2%; skipping heap-pressure auto-compaction during cooldown. + ← heap 已达 90.2%,cooldown 中无法执行 +13:32:10 ← 日志终止(进程 OOM crash) +``` + +### 日志证据解读 + +| 日志观察 | 含义 | +| ------------------------------------------------------------------------------------- | --------------------------------------------------------- | +| 2.5 分钟内触发 **4 次** heap-pressure auto-compaction 尝试(另有 2 次 cooldown 拒绝) | #3735 引入的 `tryCompress` 在高压时频繁触发 | +| 每次 compaction 执行后 heap 占比仍 >70% | `structuredClone()` 制造的临时峰值抵消了压缩收益 | +| 74.9% → 70.7% → 86% → 85.3% → 88.8% → 90.2% → crash | 正反馈循环:压缩→clone 峰值→heap 更高→再压缩→更高 | +| 日志在 90.2% 后 1 秒内断裂 | 下一次 `getHistory(true)` 的 `structuredClone()` 瞬间超限 | +| `[FILE_READ_CACHE] clear after auto tryCompress` 出现 2 次 | 证实 compaction 确实走了完整的 compress → setHistory 路径 | + +### 正反馈死循环机制 + +``` +heap 占比高 (>70%) + → 触发 heap-pressure auto-compaction + → tryCompress() 内部调用 getHistory(true) + → structuredClone(this._history) ← 瞬时 heap 峰值 +30~40% + → compaction 成功,释放旧 history + → 但 clone 峰值已经把 heap 推高到更危险的水位 + → 下一轮 send 继续累积 + → heap 占比更高 → 更频繁触发 → crash +``` + +--- + +## 六.6、版本归因:为什么 0.15.7 ~ 0.15.11 期间 OOM 报告增多 + +### 关键 commit 时间线 + +| 版本 | PR | 改动 | 对 `structuredClone` 调用频率的影响 | +| ------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------------------------------- | +| **v0.15.6** | — | `getHistory(true)` 仅在 `sendMessage` 入口调用 1 次 | 基线:每次 send 1 次 clone | +| **v0.15.7** | **#3735** `auto-compact subagent context` | 将 `tryCompress()` 下沉到 `GeminiChat`,**每次 send 前**先执行一次 compaction 检查 | **+1 次**:send 前 compress 检查 | +| **v0.15.10** | **#3879** `reactive compression on context overflow` | 当 provider 返回 context overflow 时,再次触发 `tryCompress()` + `getHistory(true)` | **+1~2 次**:overflow retry 路径 | +| **v0.15.10** | **#3985** `harden reactive compression` | 强化 reactive compression 重试逻辑 | 同上 | + +### v0.15.6 vs v0.15.11 的 `getHistory(true)` 调用点对比 + +**v0.15.6** (2 处): + +``` +L367: const requestContents = this.getHistory(true); ← send 构造 request +L618: const recoveryContents = self.getHistory(true); ← MAX_TOKENS escalation (极少触发) +``` + +**v0.15.11** (5 处): + +``` +L467: ChatCompressionService.compress() 内部调用 ← #3735: 每次 send 前的 auto-compact +L574: requestContents = this.getHistory(true); ← send 构造 request +L724: reactive tryCompress() 内部调用 ← #3879: context overflow 后 retry +L739: requestContents = self.getHistory(true); ← #3879: retry 构造新 request +L943: const recoveryContents = self.getHistory(true); ← MAX_TOKENS escalation +``` + +### 最坏路径:一次 send 可触发 4 次 `structuredClone` + +``` +sendMessage() + → tryCompress() ← #3735: getHistory(true) [clone #1] + → getHistory(true) ← 构造 request [clone #2] + → API 返回 context overflow + → reactive tryCompress() ← #3879: getHistory(true) [clone #3] + → getHistory(true) ← retry request [clone #4] +``` + +### 结论 + +**#3735 (v0.15.7)** 是 OOM 频率显著上升的最可能触发因素(非唯一根因)——它使每次 +`sendMessage` 都会先跑一次 `tryCompress()`,而 `tryCompress` 内部通过 +`ChatCompressionService.compress()` → `chat.getHistory(true)` 做全量 `structuredClone`。 +在 history 较大时,这个 “先 clone 再判断是否需要压缩” 的设计让内存峰值从 ~1.3x 升至 ~2x+。 +注:issue history 显示 OOM 报告在 #3735 之前就已存在,但 #3735 大幅增加了 structuredClone +的调用频率,从而显著提高了 OOM 的触发概率。 + +**#3879 (v0.15.10)** 进一步恶化了问题——在已经处于 heap 边界时 (provider 返回 context overflow) +再触发一次全量 clone,使原本就危险的 session 更容易 crash。 + +--- + +## 七、#4186 修复效果验证(对比测试) + +启用 heap-pressure safety net (HEAP_PRESSURE_COMPRESSION_RATIO = 0.7) 后的对比测试: + +| 指标 | 禁用 safety net | 启用 safety net | +| --------------- | ------------------ | ------------------------- | +| OOM 发生 | 是(7 轮后 crash) | 否(持续运行 >10 分钟) | +| RSS 峰值 | 666 MB → crash | 555 MB → GC 回收到 280 MB | +| Compaction 触发 | 仅 token threshold | heap 70% 时提前触发 | +| Context 行为 | 5.9%→4.0%→crash | 22.7%→17.0%(安全回落) | + +**结论**: #4186 的 heap-pressure safety net 有效防止了 OOM,但它是一个**缓解**而非根治: + +- 如果 history 本身已经占了 heap 的 60%+,即使提前 compact,clone 的峰值仍然可能超限 +- 这解释了为什么 #4309 用户设了 8GB limit 后仍然 crash + +--- + +## 八、内存占用分布 + +基于测试中的 RSS 增长模式估算: + +| 内存位置 | 占比 | 增长特征 | +| -------------------------------- | ------ | --------------------------- | +| `this._history[]` (tool results) | 40-50% | 线性累积,每轮 +30-100MB | +| `structuredClone()` 临时拷贝 | 30-40% | 瞬时峰值,compaction 时出现 | +| V8 runtime (GC metadata, code) | ~15% | 基本恒定 | +| UI/logging/stream buffers | ~5% | 缓慢增长 | + +--- + +## 九、复现脚本与环境 + +### 自动化驱动脚本 + +```bash +#!/bin/bash +# /tmp/oom-simple-driver.sh +SESSION="$1" + +TASKS=( + "用 Read 工具完整读取 packages/core/src/core/geminiChat.ts" + "用 Read 工具完整读取 packages/core/src/tools/agent/agent.ts" + "用 grep -rn structuredClone packages/core/src 然后 Read 前 3 个文件" + "用 Read 完整读取 packages/cli/src/ui/hooks/slashCommandProcessor.ts" + "用 Read 完整读取 packages/core/src/services/chatCompressionService.ts" + "用 find packages/cli/src/ui/commands -name '*.ts' 然后逐一 Read" + "用 Read 完整读取 packages/core/src/core/turn.ts" + # ... 更多任务 +) + +i=0 +while true; do + TASK="${TASKS[$((i % ${#TASKS[@]}))]}" + i=$((i + 1)) + + QWEN_PID=$(ps aux | grep "dist/index.js" | grep -v grep | awk '{print $2}' | sort -rn | head -1) + RSS=$(ps -o rss= -p $QWEN_PID 2>/dev/null) + [ -z "$RSS" ] && { echo "CRASH after $((i-1)) tasks!"; exit 0; } + + RSS_MB=$(echo "scale=1; $RSS/1024" | bc) + CTX=$(tmux capture-pane -t "$SESSION:1" -p 2>/dev/null | grep -oE "[0-9]+\.[0-9]+% 已用" | tail -1) + echo "[$(date +%H:%M:%S)] #$i RSS:${RSS_MB}MB Ctx:$CTX | ${TASK:0:55}" + + tmux send-keys -t "$SESSION:1" C-u + sleep 0.2 + tmux send-keys -t "$SESSION:1" "$TASK" Enter + sleep 0.5 + tmux send-keys -t "$SESSION:1" Enter + sleep 45 +done +``` + +### 启动命令 + +```bash +# 1. 禁用 heap-pressure safety net +# geminiChat.ts: HEAP_PRESSURE_COMPRESSION_RATIO = 99.0 + +# 2. Build +npm run build --workspace=packages/core && npm run build --workspace=packages/cli + +# 3. 启动 qwen (128K context model, 512MB heap) +SESSION="oom-test" +tmux new-session -d -s "$SESSION" -c "$REPO_DIR" +tmux send-keys -t "$SESSION" \ + "NODE_OPTIONS='--max-old-space-size=512' node packages/cli/dist/index.js --model 'qwen3.6-plus'" Enter + +# 4. 等待启动后运行驱动 +sleep 10 +bash /tmp/oom-simple-driver.sh "$SESSION" +``` + +--- + +## 十、后续建议 + +### 短期缓解(已有) + +- [x] #4186: heap-pressure auto-compaction safety net (0.7 threshold) +- [x] #4188: fileReadCache / crawlCache 上限 + +### 中期修复(建议) + +- [ ] 减少 `structuredClone()` 调用 — `nextSpeakerChecker` 只需最后一条消息,不需 clone 全量 +- [ ] Compaction 使用 slice + 引用替代全量 deep clone +- [ ] 大 tool result (>100KB) 写入临时文件,history 中只保留摘要引用 + +### 长期方向 + +- [ ] Tool result offload 到磁盘 + lazy load (#4184) +- [ ] 基于 RSS 的分级压缩策略(不仅是 token count) +- [ ] History 分段存储,避免单次全量操作 diff --git a/docs/e2e-tests/2026-05-19-qwen-runtime-diagnostics-benchmark-report.md b/docs/e2e-tests/2026-05-19-qwen-runtime-diagnostics-benchmark-report.md new file mode 100644 index 00000000000..e482f0f94c3 --- /dev/null +++ b/docs/e2e-tests/2026-05-19-qwen-runtime-diagnostics-benchmark-report.md @@ -0,0 +1,904 @@ +# Qwen Code Runtime Diagnostics Benchmark Report + +Date: 2026-05-19 + +## Scope + +This run repeats the previous Qwen Code benchmark shapes with the new opt-in +runtime diagnostics enabled. It only tests Qwen Code, not Claude Code. + +Initial model matrix: + +- `pai/glm-5` +- `qwen3.6-plus` + +Additional PR-size follow-up: + +- `DeepSeek/deepseek-v4-pro` through Anthropic-compatible protocol + +Cases: + +- small GitHub PR review: PR `#4268` +- code navigation: compression / compaction related code search and reads +- synthetic local diff: about 94.6 KiB +- synthetic local diff: about 968.5 KiB +- synthetic local diff: about 4.84 MiB + +The run used the local bundled CLI from the diagnostics branch, with +`QWEN_CODE_PROFILE_RUNTIME=1` and a temporary CLI home. Global MCP servers and +hooks were not loaded for this benchmark. + +Important caveat: these absolute RSS numbers are lower than the previous +PATH-resolved `qwen` runs because this run used `node dist/cli.js` from the +local branch plus a stripped temporary config. Treat this report as an internal +diagnostics distribution run, not a direct replacement for the earlier installed +CLI RSS comparison. + +## Installed CLI vs Local Bundle Sanity Check + +A follow-up sanity check used the same minimal prompt, model, and non-interactive +mode across the installed CLI and the local diagnostics bundle. The only +intentional variable was whether Qwen Code loaded a stripped temporary CLI home +or the normal user config. + +| CLI | Config mode | Total tokens | Tree RSS peak | Root RSS peak | Process count peak | Runtime diagnostics | +| ------------------- | --------------- | -----------: | ------------: | ------------: | -----------------: | ------------------- | +| PATH `qwen` | stripped config | 33,965 | 542.4 MiB | 249.9 MiB | 3 | no | +| local `dist/cli.js` | stripped config | 47,281 | 455.2 MiB | 214.2 MiB | 4 | yes | +| PATH `qwen` | normal config | 97,615 | 1,099.9 MiB | 250.1 MiB | 6 | no | +| local `dist/cli.js` | normal config | 97,954 | 1,105.4 MiB | 212.7 MiB | 8 | yes | + +This check changes the attribution: the earlier 1 GiB user-visible peak is +reproducible with the normal config even on the local diagnostics bundle. It is +therefore not primarily explained by the local branch including PR `#4186`. + +At the normal-config peak, the local process-tree sample was dominated by +multiple Node/MCP processes rather than the Qwen root process alone: + +| Role | Command shape | RSS at tree peak | +| ----- | ------------------------- | ---------------: | +| child | Node process | 252.9 MiB | +| child | Chrome DevTools MCP | 219.7 MiB | +| child | Node process | 219.2 MiB | +| root | Qwen Node process | 215.1 MiB | +| child | Chrome DevTools MCP setup | 175.2 MiB | + +PR `#4186` is present in the local diagnostics branch, but it is a V8 heap +pressure auto-compaction safety net. It triggers at about 70% V8 heap pressure; +on this environment the Node heap limit is about 4.1 GiB, while the stripped +benchmark end heap was about 99-143 MiB. Based on these numbers, the lower +stripped-config RSS is not caused by `#4186` actively compressing context during +these benchmark runs. + +### Bare Mode Config Attribution Check + +A second follow-up used `qwen3.6-plus` with the same PR-review prompt shape on +both the installed CLI and the local bundle. This is not a normal end-to-end +business benchmark. It is a controlled attribution check for startup/config +memory only. + +`--bare` changes the runtime inputs: it skips normal global settings discovery, +MCP startup, hooks, implicit context, skills, and other startup integrations. It +can therefore fail or behave differently when a model provider is configured +only in global settings. For this run, model credentials were supplied only +through the child-process environment because bare mode intentionally does not +load the normal provider settings. Nothing was written back to the user's global +config. + +This run did not produce useful token/tool-call statistics: the model completed +in one turn and did not call the requested shell command. Do not use these rows +as normal task benchmark results, and do not compare their token/tool-call +behavior with the matrix above. They are only useful for estimating how much +process-tree RSS comes from normal config and configured child processes. + +| CLI | Mode | Wall | Turns | Tool uses | Tree RSS peak | Root RSS peak | Process count peak | +| ------------------- | -------- | ---: | ----: | --------: | ------------: | ------------: | -----------------: | +| PATH `qwen` | normal | 5.5s | 1 | 0 | 1,021.3 MiB | 251.5 MiB | 5 | +| PATH `qwen` | `--bare` | 2.4s | 1 | 0 | 525.7 MiB | 246.4 MiB | 2 | +| local `dist/cli.js` | normal | 4.9s | 1 | 0 | 1,046.2 MiB | 213.3 MiB | 5 | +| local `dist/cli.js` | `--bare` | 2.3s | 1 | 0 | 454.3 MiB | 216.5 MiB | 3 | + +The result confirms the process-tree hypothesis for startup/config attribution. +On this machine, normal config adds roughly 0.50-0.59 GiB of user-visible +process-tree RSS over `--bare`, while root RSS stays in the same 0.21-0.25 GiB +band. At the normal-config peak, the extra RSS again came from additional +Node/MCP child processes, including a Chrome DevTools MCP process and its setup +wrapper. `--bare` removes those startup/config children and brings +installed/local runs back into the 0.45-0.53 GiB tree-RSS range. + +### Temporary Settings MCP / Hooks Isolation + +Because `--bare` changes too many runtime inputs to be treated as a normal +benchmark, a follow-up used temporary `QWEN_HOME` directories with generated +settings files derived from the normal settings. The run stayed on the normal +settings-loading path, but toggled only two config dimensions: + +- MCP disabled: `mcpServers` cleared and MCP allow/exclude lists emptied. +- Hooks disabled: `disableAllHooks` set to true. + +No global settings were modified. The case used `qwen3.6-plus` and a minimal +startup prompt, so it measures startup/config process-tree cost, not task +reasoning quality. + +| CLI | Temporary config | MCP servers | Tools | Tree RSS peak | Root RSS peak | Process count peak | +| ------------------- | -------------------- | ----------: | ----: | ------------: | ------------: | -----------------: | +| PATH `qwen` | full | 4 | 46 | 1,017.4 MiB | 249.8 MiB | 5 | +| PATH `qwen` | MCP disabled | 0 | 17 | 548.7 MiB | 252.4 MiB | 2 | +| PATH `qwen` | hooks disabled | 4 | 46 | 1,003.8 MiB | 246.4 MiB | 5 | +| PATH `qwen` | MCP + hooks disabled | 0 | 17 | 542.5 MiB | 248.0 MiB | 2 | +| local `dist/cli.js` | full | 4 | 48 | 865.9 MiB | 220.4 MiB | 6 | +| local `dist/cli.js` | MCP disabled | 0 | 19 | 442.9 MiB | 209.6 MiB | 2 | +| local `dist/cli.js` | hooks disabled | 4 | 48 | 848.3 MiB | 212.6 MiB | 5 | +| local `dist/cli.js` | MCP + hooks disabled | 0 | 19 | 447.2 MiB | 217.8 MiB | 2 | + +Interpretation: + +1. Disabling MCP is the dominant change. It removes 4 MCP servers, reduces the + advertised tool count by about 29 tools, and lowers process-tree RSS by about + 0.42-0.47 GiB in this startup/config case. +2. Disabling hooks alone barely changes RSS in this case. That is expected + because the prompt did not produce tool calls, so `PreToolUse` / + `PostToolUse` hooks were not executed. +3. The root process stays around 0.21-0.25 GiB across all rows. The large + difference is again process-tree composition, not root Qwen RSS. + +Two attempted code-navigation follow-ups with `qwen3.6-plus` and `pai/glm-5` +also reproduced the same MCP-vs-no-MCP memory split, but neither model produced +tool calls in those runs. Those rows are therefore not used as hooks execution +evidence. A valid hooks benchmark still needs a task/model combination that +reliably emits tool calls. + +### Per-MCP Isolation + +The previous row showed MCP as a group is the dominant startup/config memory +factor. A follow-up isolated each configured MCP server while keeping hooks +disabled for all rows. This keeps the test on the normal settings-loading path +but changes only the MCP server subset. + +Configured MCP server names: + +- `approval-bridge` +- `env-center` +- `chrome-devtools` +- `code` + +Single-pass isolation: + +| Variant | Enabled MCPs | Tools | MCP servers | Tree RSS peak | Root RSS peak | Interpretation | +| ------------------------- | -------------------------------------------------- | ----: | ----------: | ------------: | ------------: | ------------------------------------ | +| none | none | 19 | 0 | 444.4 MiB | 211.7 MiB | baseline without MCP | +| full | all 4 | 48 | 4 | 857.3 MiB | 215.9 MiB | full MCP startup shape | +| only `approval-bridge` | `approval-bridge` | 19 | 1 | 455.5 MiB | 214.0 MiB | near baseline | +| only `env-center` | `env-center` | 19 | 1 | 452.3 MiB | 214.4 MiB | near baseline | +| only `chrome-devtools` | `chrome-devtools` | 48 | 1 | 824.4 MiB | 209.5 MiB | large RSS increase and tool increase | +| only `code` | `code` | 19 | 1 | 452.1 MiB | 216.6 MiB | near baseline | +| without `approval-bridge` | `env-center`, `chrome-devtools`, `code` | 48 | 3 | 997.1 MiB | 215.4 MiB | still high; run showed variance | +| without `env-center` | `approval-bridge`, `chrome-devtools`, `code` | 48 | 3 | 863.8 MiB | 220.9 MiB | still high | +| without `chrome-devtools` | `approval-bridge`, `env-center`, `code` | 19 | 3 | 463.4 MiB | 221.6 MiB | returns near baseline | +| without `code` | `approval-bridge`, `env-center`, `chrome-devtools` | 48 | 3 | 858.1 MiB | 219.5 MiB | still high | + +Because startup RSS has some variance, the key variants were repeated twice: + +| Variant | Samples | Tree RSS range | Avg tree RSS | Result | +| ------------------------- | ------: | ------------------- | -----------: | ------------------------------ | +| none | 2 | 443.3-451.9 MiB | 447.6 MiB | stable no-MCP baseline | +| full | 2 | 856.1-922.8 MiB | 889.5 MiB | stable high-MCP range | +| only `chrome-devtools` | 2 | 1,007.1-1,021.2 MiB | 1,014.2 MiB | enough alone to reproduce high | +| without `chrome-devtools` | 2 | 461.1-461.6 MiB | 461.4 MiB | removes the high RSS | +| only `approval-bridge` | 2 | 449.1-449.9 MiB | 449.5 MiB | near baseline | +| only `env-center` | 2 | 438.7-449.5 MiB | 444.1 MiB | near baseline | +| only `code` | 2 | 450.6-451.3 MiB | 451.0 MiB | near baseline | + +Interpretation: + +1. `chrome-devtools` is the dominant MCP contributor in this environment. It is + sufficient by itself to reproduce the high process-tree RSS. +2. Removing `chrome-devtools` from the full MCP set returns RSS to the no-MCP + band. Removing other MCPs while keeping `chrome-devtools` does not. +3. The advertised tool count follows the same pattern: baseline is 19 tools, + while `chrome-devtools` raises the tool count to 48. That means this MCP is + also likely to increase request tool schema size and token pressure, not just + process-tree RSS. +4. `approval-bridge`, `env-center`, and `code` individually stay near the + no-MCP baseline in these startup/config runs. They emitted startup warnings + in this environment, so this result should be interpreted as "no persistent + startup RSS owner observed" rather than proof that they have zero cost in all + workflows. + +## Runtime Summary + +| Case | Model | Wall | Turns | Total tokens | Tree RSS peak | Root RSS peak | End heap | End RSS | +| ---------------- | -------------- | ----: | ----: | -----------: | ------------: | ------------: | --------: | --------: | +| small PR `#4268` | `pai/glm-5` | 20.1s | 7 | 173,216 | 362.1 MiB | 359.8 MiB | 103.1 MiB | 216.5 MiB | +| code navigation | `pai/glm-5` | 18.4s | 2 | 49,127 | 378.0 MiB | 376.0 MiB | 102.4 MiB | 313.4 MiB | +| diff 94.6 KiB | `pai/glm-5` | 16.6s | 6 | 135,716 | 367.9 MiB | 366.0 MiB | 99.1 MiB | 295.0 MiB | +| diff 968.5 KiB | `pai/glm-5` | 11.4s | 2 | 42,590 | 373.2 MiB | 362.5 MiB | 106.4 MiB | 345.6 MiB | +| diff 4.84 MiB | `pai/glm-5` | 12.0s | 4 | 95,119 | 414.2 MiB | 412.0 MiB | 123.6 MiB | 410.7 MiB | +| small PR `#4268` | `qwen3.6-plus` | 35.0s | 6 | 156,556 | 358.9 MiB | 356.9 MiB | 102.6 MiB | 293.1 MiB | +| code navigation | `qwen3.6-plus` | 28.9s | 4 | 99,800 | 370.3 MiB | 368.3 MiB | 105.8 MiB | 298.2 MiB | +| diff 94.6 KiB | `qwen3.6-plus` | 28.3s | 4 | 90,808 | 358.8 MiB | 356.9 MiB | 105.9 MiB | 307.0 MiB | +| diff 968.5 KiB | `qwen3.6-plus` | 30.9s | 6 | 151,782 | 366.1 MiB | 364.1 MiB | 101.0 MiB | 316.9 MiB | +| diff 4.84 MiB | `qwen3.6-plus` | 24.1s | 4 | 93,271 | 372.8 MiB | 366.0 MiB | 142.8 MiB | 366.0 MiB | + +Average by model: + +| Model | Avg tree RSS peak | Avg root RSS peak | Avg turns | Avg total tokens | Avg max wire body | Avg total tool result | +| -------------- | ----------------: | ----------------: | --------: | ---------------: | ----------------: | --------------------: | +| `pai/glm-5` | 379.1 MiB | 375.3 MiB | 4.2 | 99,154 | 111.8 KiB | 335.1 KiB | +| `qwen3.6-plus` | 365.4 MiB | 362.4 MiB | 4.8 | 118,443 | 119.3 KiB | 344.3 KiB | + +Overlapping small PR `#4268` model snapshot: + +| Model | Protocol | Wall | Turns | Total tokens | Tree RSS peak | Root RSS peak | Max wire body | +| -------------------------- | --------- | ----: | ----: | -----------: | ------------: | ------------: | ------------: | +| `pai/glm-5` | OpenAI | 20.1s | 7 | 173,216 | 362.1 MiB | 359.8 MiB | 113.8 KiB | +| `qwen3.6-plus` | OpenAI | 35.0s | 6 | 156,556 | 358.9 MiB | 356.9 MiB | 134.1 KiB | +| `DeepSeek/deepseek-v4-pro` | Anthropic | 39.7s | 2 | 43,362 | 346.9 MiB | 344.8 MiB | 103.0 KiB | + +## Request And Tool Diagnostics + +| Case | Model | Requests | Max wire body | Max system prompt | Max tool schema | Tool calls | Total tool result | Max tool result | Max function response in request | +| ---------------- | -------------- | -------: | ------------: | ----------------: | --------------: | ---------: | ----------------: | --------------: | -------------------------------: | +| small PR `#4268` | `pai/glm-5` | 7 | 113.8 KiB | 51.4 KiB | 40.2 KiB | 9 | 4.7 KiB | 3.9 KiB | 15.3 KiB | +| code navigation | `pai/glm-5` | 2 | 114.6 KiB | 51.5 KiB | 40.2 KiB | 3 | 17.5 KiB | 6.2 KiB | 18.4 KiB | +| diff 94.6 KiB | `pai/glm-5` | 6 | 111.2 KiB | 39.1 KiB | 37.2 KiB | 9 | 94.9 KiB | 92.6 KiB | 29.2 KiB | +| diff 968.5 KiB | `pai/glm-5` | 2 | 104.8 KiB | 39.1 KiB | 37.2 KiB | 2 | 772.1 KiB | 771.9 KiB | 25.6 KiB | +| diff 4.84 MiB | `pai/glm-5` | 4 | 114.7 KiB | 39.1 KiB | 37.2 KiB | 4 | 786.3 KiB | 783.2 KiB | 34.7 KiB | +| small PR `#4268` | `qwen3.6-plus` | 6 | 134.1 KiB | 51.4 KiB | 40.2 KiB | 5 | 34.6 KiB | 15.6 KiB | 36.6 KiB | +| code navigation | `qwen3.6-plus` | 4 | 114.9 KiB | 51.5 KiB | 40.2 KiB | 3 | 17.5 KiB | 6.2 KiB | 18.4 KiB | +| diff 94.6 KiB | `qwen3.6-plus` | 4 | 112.8 KiB | 39.1 KiB | 37.2 KiB | 3 | 92.9 KiB | 92.6 KiB | 33.0 KiB | +| diff 968.5 KiB | `qwen3.6-plus` | 6 | 113.1 KiB | 39.1 KiB | 37.2 KiB | 5 | 778.0 KiB | 771.9 KiB | 32.1 KiB | +| diff 4.84 MiB | `qwen3.6-plus` | 4 | 121.5 KiB | 39.1 KiB | 37.2 KiB | 4 | 798.5 KiB | 783.2 KiB | 41.3 KiB | + +## Observations + +1. Process-tree RSS is almost the same as root RSS in this local bundle run. + The root/tree gap is usually below 10 MiB. That means these runs did not + show a persistent child-process memory owner. The dominant process is the + main Node process. +2. The local bundle run peaks around 0.36-0.41 GiB, not the earlier + 0.83-1.04 GiB, because the matrix used a stripped temporary config. A + follow-up normal-config sanity check reproduced about 1.1 GiB tree RSS on + both PATH `qwen` and local `dist/cli.js`, with the extra memory coming from + child MCP/Node processes in the process tree. +3. V8 heap is much smaller than RSS. End heap is about 99-143 MiB while end RSS + is about 216-411 MiB. The remaining footprint is likely loaded modules, + native allocations, external buffers, or runtime overhead outside live JS + heap. +4. Static request overhead is large and repeated. The system prompt is about + 39-51 KiB per request, and tool schema is about 37-40 KiB per request. This + explains why even small tasks can produce high accumulated token counts when + the model takes several turns. +5. Large diff output is capped before it reaches the model request. The 968 KiB + and 4.84 MiB diff cases produced around 772-799 KiB of captured tool result, + but the largest model-facing function response in a request stayed around + 25-41 KiB, and max wire body stayed around 105-122 KiB. This points to + truncation / saved-output handling working on the model-facing path. +6. Memory still increases on large-output cases even though wire body remains + bounded. For example, the 4.84 MiB GLM run reached 414.2 MiB tree RSS and + 410.7 MiB end RSS, and the 4.84 MiB qwen3.6-plus run ended with 142.8 MiB + heap. That suggests large tool output can still affect local capture, + normalization, or retained runtime state even when the final request payload + is capped. +7. Model choice changed turns and token totals more than RSS in this run. + `qwen3.6-plus` averaged more tokens and turns than `pai/glm-5`, but its + average tree RSS peak was slightly lower. This supports the earlier + conclusion that model choice is not the main explanation for process memory. + +## Updated Working Inference + +The new diagnostics make the earlier hypothesis more precise: + +- The installed-CLI user-visible 1 GiB peak is now reproducible with the normal + config on the local diagnostics bundle. The stripped run should be used for + internal Qwen runtime attribution; the normal-config run should be used for + user-visible process-tree attribution. +- The largest observed difference between stripped and normal config is + process-tree shape: normal config starts additional MCP/Node child processes. + Those children explain most of the absolute jump from about 0.35-0.55 GiB to + about 1.1 GiB in the minimal prompt sanity check. +- The `--bare` follow-up confirms the same direction on `qwen3.6-plus`: normal + config costs about 0.50-0.59 GiB more process-tree RSS than bare mode for the + same prompt shape, while root RSS changes only slightly. +- The temporary-settings isolation is a better attribution test than `--bare`: + disabling MCP alone reduces process-tree RSS by about 0.42-0.47 GiB while + keeping the normal settings-loading path. Disabling hooks alone does not show + a meaningful RSS change in no-tool-call cases. +- Per-MCP isolation points to `chrome-devtools` as the dominant MCP contributor: + it is enough by itself to reproduce the high RSS band, and removing it returns + the run near the no-MCP baseline. +- Within the local Qwen runtime, the most suspicious areas are no longer "raw + diff bytes sent to the model". The model-facing request body is bounded. +- The stronger suspects are static per-request context cost, repeated request + rounds, tool schema size, and local retention/capture of large tool outputs + before or outside model-facing truncation. +- Because RSS remains much higher than V8 heap, the next profiling layer should + include module/startup accounting, external memory, and heap snapshots around + tool execution and final response emission. + +## RSS Attribution From Current Diagnostics + +The current counters do not identify an exact retained object or source file, +but they do narrow what is and is not driving RSS in these local runs: + +| Signal | Current evidence | RSS implication | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| Root RSS vs process-tree RSS | Root and tree peaks are usually within about 2-10 MiB; DeepSeek large PR is the widest gap at about 23.6 MiB | No persistent child process explains the RSS in this local bundle run; the main Node process dominates | +| Normal config process tree | Minimal-prompt normal-config runs reach about 1.1 GiB tree RSS while root RSS stays about 213-250 MiB | User-visible 1 GiB peaks can be dominated by MCP/Node child processes rather than Qwen root RSS alone | +| `--bare` comparison | `qwen3.6-plus` normal runs peak around 1.02-1.05 GiB tree RSS; bare runs peak around 0.45-0.53 GiB | Loading normal config adds about 0.50-0.59 GiB process-tree RSS in this environment | +| Temporary MCP isolation | Clearing MCP servers drops startup/config tree RSS from 865-1,017 MiB to 443-549 MiB | MCP startup and MCP child processes explain about 0.42-0.47 GiB of process-tree RSS in the controlled config check | +| Per-MCP isolation | `chrome-devtools` alone reaches about 1.0 GiB in repeated samples; without it the run stays around 461 MiB | `chrome-devtools` is the dominant MCP process-tree RSS contributor in this environment | +| Temporary hooks isolation | `disableAllHooks=true` with MCP still enabled changes tree RSS by only about 13-18 MiB in no-tool-call cases | Hook config alone is not a visible startup RSS driver here; hook execution still needs a tool-call benchmark | +| V8 heap vs RSS | End heap is about 99-143 MiB while end RSS is about 216-411 MiB | Live JS heap is not the whole footprint; loaded modules, native allocations, external buffers, or runtime overhead are likely significant | +| PR/diff size vs RSS | DeepSeek small/medium/large PRs scale from 1 to 4,750 changed lines, but tree RSS stays in a narrow 340.7-360.0 MiB band | Raw PR size is not linearly driving RSS once tool output is bounded | +| Tool output size | Large diff runs capture about 772-799 KiB tool results and show some higher end RSS / heap, but RSS does not scale linearly | Tool result capture/normalization contributes pressure, especially large-output cases, but is unlikely to be the only RSS driver | +| Request body size | Max model-facing body ranges from about 103-289 KiB while RSS stays near the same band | Request serialization size affects tokens and latency more clearly than RSS peak | +| Static per-request context | System prompt is about 39-51 KiB and tool schema about 37-48 KiB per request | Repeated rounds are a token/cost amplifier; this alone does not explain RSS but is a likely optimization target for token pressure | + +Working attribution: in the stripped local bundle benchmark, the RSS floor looks +mostly like task-time runtime/module/native footprint, with large tool output +adding incremental pressure. In the normal-config run, the user-visible 1 GiB +tree peak is mostly process-tree composition: Qwen root plus MCP/Node child +processes. The next targeted measurement should split Qwen root diagnostics +from configured MCP server diagnostics, then add startup/module/external-memory +checkpoints inside the Qwen root process. + +## Progress Snapshot + +Current confirmed signals: + +1. The user-visible 1 GiB startup/config peak is reproducible with both the + installed CLI and the local diagnostics bundle when the normal config is + loaded. It is not primarily explained by the diagnostics branch or PR `#4186`. +2. In this environment, that 1 GiB peak is mostly process-tree composition: + Qwen root process plus relaunch child process plus MCP child processes. +3. `chrome-devtools` is the dominant configured MCP contributor in the current + config. It is enough by itself to reproduce the high process-tree RSS band, + even when the prompt does not explicitly use that MCP. +4. The no-MCP normal relaunch shape still sits around 0.45 GiB process-tree RSS. + A single Qwen runtime process without the relaunch parent is closer to + 0.22-0.24 GiB in the startup attribution check. This means the 0.45 GiB + baseline is not a single-process root RSS number. +5. In stripped non-interactive task runs, model choice changes turns, token + totals, latency, and request sizes more clearly than RSS. RSS stayed in a + relatively narrow range across `pai/glm-5`, `qwen3.6-plus`, and + `DeepSeek/deepseek-v4-pro`. +6. Current short-task diagnostics show model-facing tool/function responses are + bounded, but local tool-result capture and runtime state can still increase + heap/RSS on large-output cases. This keeps large-output retention on the + investigation path. + +Current gaps: + +1. The short-task benchmark matrix is still short-lived. A later interactive + long-review run did reproduce a 41.9 min failure, but it is still one sample + and needs repeat runs plus heap/object attribution. +2. The current counters are enough to attribute process-tree RSS and request + size, but not enough to name the retained JS object graph during long + sessions. +3. Startup/config RSS and long-session OOM must remain separate tracks. MCP and + relaunch explain a large idle/startup RSS band; they do not by themselves + explain V8 heap OOM after long tasks. +4. Interactive TUI memory still needs a separate run from non-interactive mode, + because UI history and Ink static output are not exercised the same way. + +## Long-Task OOM Evidence From Issues And PRs + +Issue/PR evidence points to several different OOM shapes, not one single +failure mode: + +| Source | Evidence summary | Hypothesis to test | +| ---------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| [`#4309`](https://github.com/QwenLM/qwen-code/issues/4309) | User reports 5.84 GiB memory usage / 7.02 GiB warning with YOLO mode and DeepSeek backend; increasing Node memory to 8 GiB did not remove the symptom | Long autonomous tool loops can retain enough state that simply raising old-space limit is not a root fix | +| [`#4149`](https://github.com/QwenLM/qwen-code/issues/4149) | Multiple reports show `Ineffective mark-compacts near heap limit`, including 4 GiB and much larger heap-limit cases | A large fraction of heap is reachable application state, not immediately collectible garbage | +| [`#4116`](https://github.com/QwenLM/qwen-code/issues/4116) | OOM occurred while context display was around 9.5%; analysis points to `structuredClone`, UI history, Ink static tree, and large context windows | Token usage can be low while JS heap pressure is high; token threshold alone is not a reliable memory guard | +| [`#4167`](https://github.com/QwenLM/qwen-code/issues/4167) | User says the crash happened while compressing; analysis identifies compression peak memory as a distinct shape | Compression can itself create a peak when heap is already high, especially if history is cloned/stringified around the same time | +| [`#2128`](https://github.com/QwenLM/qwen-code/issues/2128) | Report identifies unbounded UI history, retained file diffs / terminal output, string-width caches, and checkpoint serialization | Interactive TUI long sessions may retain memory outside model history and outside non-interactive benchmarks | +| [`#2562`](https://github.com/QwenLM/qwen-code/issues/2562) | Report focuses on `GeminiChat.getHistory()` deep-cloning full history in long sessions | Full-history cloning can amplify memory peaks and should be measured separately from retained steady-state size | +| [`#4185`](https://github.com/QwenLM/qwen-code/issues/4185) | Tracks V8 heap pressure exceeding limit before token-based compaction runs | Heap-pressure guard is necessary, but it only mitigates symptoms if retained data remains large | +| [`#4184`](https://github.com/QwenLM/qwen-code/issues/4184) | Proposes diagnostics and offload/preview for large retained tool results | Large tool output may be bounded for model requests while still retained in local hot memory | +| [`#4186`](https://github.com/QwenLM/qwen-code/pull/4186) | Merged heap-pressure auto-compaction safety net and O(1) last-history access for `nextSpeakerChecker` | Covers part of heap-pressure and clone amplification, but does not claim to solve all OOM classes | +| [`#4127`](https://github.com/QwenLM/qwen-code/pull/4127), [`#4168`](https://github.com/QwenLM/qwen-code/pull/4168) | Open compaction-threshold PRs; one uses fixed heap thresholds, the other redesigns token thresholds and compression behavior | Useful related work, but long-task testing must verify whether heap, token, and compression signals line up in real runs | +| [`#3000`](https://github.com/QwenLM/qwen-code/issues/3000), [`#4183`](https://github.com/QwenLM/qwen-code/issues/4183) | Diagnostic roadmap calls out `/doctor memory`, heap snapshot, and bounded memory timeline | Snapshot/timeline support is needed to move from RSS attribution to retained-object attribution | + +Initial interpretation: + +- Unused configured MCP can consume memory because normal startup connects to + configured MCP servers and advertises their tools before the task needs them. + In the measured config, `chrome-devtools` starts extra Node/npm MCP processes + and also increases the tool schema count from 19 to 48. This explains a large + startup/config RSS band and can also increase repeated request overhead. +- The long-session OOM reports are a different layer. GC logs where + Mark-Compact frees very little memory suggest the heap is full of reachable + state. The strongest candidates are retained history/tool/UI objects, + full-history clones, compression intermediates, and streaming/logging + accumulators. +- PR `#4186` is a useful mitigation because it can compact based on heap + pressure before token thresholds trigger, and it removes one unnecessary + full-history clone. It should not be treated as proof that large tool-output + retention, UI history retention, or compression peak memory is already solved. + +## Long-Task Validation Plan + +The next benchmark should keep two tracks separate: + +1. Startup/config attribution: normal config vs MCP-disabled vs + `chrome-devtools`-only vs no-relaunch attribution. This explains what users + see before meaningful work begins. +2. Long-task runtime growth: repeated tool calls, large outputs, compression, + resume, and interactive UI history. This explains OOM after real work. + +Recommended long-task cases: + +| Case | Shape | Why it matters | +| ----------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| Long PR review loop | Repeat medium/large PR review prompts for 30, 60, and 120 minutes, with fixed model and fixed config | Closest to reported agent workflows; captures turns, tool calls, token growth, and RSS/heap trend | +| Large tool-output retention | Repeatedly produce bounded 1 MiB / 5 MiB / 20 MiB command outputs, then ask follow-up questions | Tests whether raw output is retained locally after model-facing truncation | +| Compression pressure | Use a lower controlled old-space limit and large-context prompts to trigger heap-pressure compaction | Verifies PR `#4186` triggers before OOM and whether compression itself creates a new peak | +| Interactive TUI history | Run the same long loop in tmux TUI mode and compare with non-interactive mode | Isolates UI history, Ink static output, rendered diffs, and terminal-output display retention | +| Resume stress | Resume a large saved session and immediately continue work | Targets `/resume` OOM reports and session reconstruction cost | +| Streaming/logging accumulator | Force long streamed responses with telemetry/logging enabled vs disabled | Tests the suspected `collected responses` / logging-retention path from issue analysis | +| MCP idle vs MCP active | Run no-MCP, `chrome-devtools` configured-but-unused, and `chrome-devtools` actively used variants | Separates idle MCP child RSS from actual MCP tool execution and tool schema/token overhead | + +Metrics that should be recorded per turn or per sampling interval: + +- Root RSS current/peak and process-tree RSS current/peak. +- Child process count and top child command shapes. +- V8 `heapUsed`, `heapTotal`, `heap_size_limit`, `external`, and + `arrayBuffers`. +- Turn count, request count, tool-call count, and tool-call rounds. +- Input/output/cache/total tokens by request and by whole task. +- Request body bytes, system prompt bytes, tool schema bytes, and function + response bytes. +- Tool-result count, total captured tool-result bytes, max tool-result bytes, + and retained tool-result bytes if available. +- Conversation history message count and approximate history byte size. +- Interactive-only UI history item count and approximate retained display size. +- Compression attempts, compression trigger reason, tokens before/after, heap + pressure before/after, and compression failure status. +- Heap snapshot or bounded memory timeline artifacts when heap pressure crosses + a configured threshold. + +Validation criteria: + +1. Repeat at least the key long-task cases twice. Startup RSS has visible + variance, so single-run conclusions should be avoided. +2. Report root RSS and process-tree RSS separately. User-facing memory pressure + can come from child processes, while V8 OOM comes from the Qwen root heap. +3. Treat a flat RSS line as important evidence. If tokens and tool calls grow + but heap/RSS stays flat, the issue is likely elsewhere. +4. When RSS or heap grows, correlate the growth with a specific signal: + tool-result bytes, history bytes, UI history count, compression event, + streaming accumulator size, or MCP process start. +5. If a heap snapshot is taken, write a structured diagnostics JSON first, then + the snapshot. Heap snapshots may be large and can contain sensitive strings, + so they should remain opt-in and local. + +## Interactive Long-Review Reproduction + +After the short non-interactive prompts kept finishing before the target window, +an interactive TUI benchmark was run with remote input. The CLI process stayed +alive in one session while a controller submitted one real PR-review turn at a +time. The next turn was only submitted after the assistant emitted that turn's +completion marker. This avoids treating a short one-shot prompt as a long-task +reproduction. + +Setup: + +- Installed Qwen Code `0.15.11`, model `qwen-latest-series-invite-beta-v28`. +- Temporary CLI home derived from the normal settings, with MCP and hook config + removed. No global config was modified. +- Interactive TUI mode with dual JSON event output and remote JSONL input. +- Static PR review only. The prompt disallowed dependency install, build, test, + Playwright, Docker, and other long external build commands. +- External RSS samplers recorded both process-tree RSS and the Qwen Node root + RSS every 5 seconds. + +Outcome: + +| Signal | Value | +| ----------------------------- | ----------: | +| Wall time before exit | 41.9 min | +| Exit status | 1 | +| Completed PR-review turns | 6 | +| Main chat records | 1,076 | +| API response telemetry | 335 | +| Tool-call telemetry | 607 | +| MCP tool-call telemetry | 0 | +| Main/root API responses | 36 | +| Subagent API responses | 299 | +| Root total tokens | 2.08M | +| Subagent total tokens | 17.24M | +| Total API telemetry tokens | 19.32M | +| Max root input tokens | 85,655 | +| Max subagent input tokens | 215,207 | +| `/usr/bin/time -l` max RSS | 1,072.4 MiB | +| Sampled Qwen root RSS peak | 1,028.2 MiB | +| Sampled process-tree RSS peak | 1,038.1 MiB | + +The process exited with: + +```text +libc++abi: terminating due to uncaught exception of type std::__1::system_error: thread constructor failed: Resource temporarily unavailable +``` + +This is a **thread exhaustion** error, not a V8 heap OOM. The failure mechanism +is distinct: the OS refused to create a new thread, likely due to per-process +resource limits (`RLIMIT_NPROC`) or memory fragmentation preventing stack +allocation. It is still relevant because it occurred in a disabled-MCP, +no-build/test, interactive long-session review where the Qwen Node process +itself crossed about 1 GiB RSS. +The failure happened during the final summary phase, after the controller had +already completed six review turns. + +Turn timeline and sampled Qwen root RSS: + +| Window | Turn state | Qwen root RSS max | Qwen root RSS at window end | +| ------------- | -------------------- | ----------------: | --------------------------: | +| 0.0-9.0 min | turn 1 completed | 701.2 MiB | 255.3 MiB | +| 9.0-15.1 min | turn 2 completed | 503.2 MiB | 494.4 MiB | +| 15.1-24.1 min | turn 3 completed | 468.7 MiB | 457.5 MiB | +| 24.1-31.9 min | turn 4 completed | 619.3 MiB | 602.3 MiB | +| 31.9-40.3 min | turn 5 completed | 955.5 MiB | 955.5 MiB | +| 40.3-40.4 min | turn 6 completed | 988.6 MiB | 988.6 MiB | +| 40.4-41.9 min | final summary / exit | 1,028.2 MiB | 1,028.2 MiB | + +Token and tool distribution: + +| Owner | API responses | Input tokens | Output tokens | Total tokens | Max input | +| ------------ | ------------: | -----------: | ------------: | -----------: | --------: | +| Root session | 36 | 2.06M | 22.2K | 2.08M | 85,655 | +| Subagents | 299 | 17.08M | 154.6K | 17.24M | 215,207 | + +Tool-call telemetry by function: + +| Tool | Calls | Captured content length | +| ------------------- | ----: | ----------------------: | +| `read_file` | 271 | 1.46 MB | +| `run_shell_command` | 181 | 164.4 KB | +| `web_fetch` | 80 | 846.3 KB | +| `grep_search` | 25 | 15.0 KB | +| `glob` | 15 | 27.8 KB | +| `todo_write` | 16 | 16.1 KB | +| `list_directory` | 8 | 6.2 KB | +| `agent` | 10 | 0 | +| `tool_search` | 1 | 2.1 KB | + +The top visible TUI token counter for a single agent reached about 3.83M +tokens. Telemetry also shows the heaviest subagent at about 4.05M total tokens +with a 215K-token max input request. That makes subagent amplification the +dominant signal in this reproduction. + +Interpretation: + +1. This run separates long-session growth from MCP startup/config memory. MCP + was disabled and there were no MCP tool calls, yet the Qwen root process + still reached about 1 GiB RSS. +2. The late memory peak aligns with subagent-heavy review turns and final + summary/merge-back, not with external build/test child processes. +3. The RSS curve is not a simple linear leak. It falls after early turns, then + rises sharply after later subagent turns and remains high near exit. +4. The failure mode is native resource exhaustion rather than a V8 heap-limit + stack, so the next run should add heap/external/arrayBuffer/thread-count + sampling. RSS alone cannot distinguish JS heap from native allocations or + thread-resource pressure. +5. The strongest code paths to inspect remain subagent transcript retention, + agent-result merge-back, full-history cloning, checkpoint/session recording, + and final summary/history assembly. + +## Deterministic Huge-Task Clone-Pressure Reproduction + +A deterministic stress harness was added as +`scripts/memory-pressure-repro.mjs`. It does not call a model. Instead, it +constructs a Qwen-like long-session object graph with root review turns, +subagent transcripts, large tool results, checkpoint JSON, and retained +`structuredClone()` copies. This gives a repeatable reproduction for the clone +and checkpoint peak suspected from the user-provided OOM stack. + +The harness has a lightweight script test: + +```bash +npx vitest run --config ./scripts/tests/vitest.config.ts \ + scripts/tests/memory-pressure-repro.test.js +``` + +Result: passed, 1 test. + +Controlled runs used `node --max-old-space-size=256` unless otherwise noted. + +| Case | History shape | Clone/checkpoint pressure | Result | Max RSS | +| ------------------------------------------------- | ----------------------------------------------------------------------- | -------------------------------------------------- | --------------------------------- | --------: | +| Small sanity | 2 turns, 2 KiB tool result, 1 subagent | 1 clone + 1 checkpoint | passed; 2.6 MiB history JSON | 89.7 MiB | +| Huge build only | 12 turns, 256 KiB tool result, 2 subagents x 12 subagent turns | no retained clone/checkpoint | passed; 76.2 MiB history JSON | 491.5 MiB | +| Huge + 1 clone | same as above | 1 retained `structuredClone()` | passed | 569.6 MiB | +| Huge + 2 clones | same as above | 2 retained `structuredClone()` copies | OOM, exit 134 | 496.5 MiB | +| Huge + 1 checkpoint | same as above | one checkpoint with original + cloned history JSON | passed; 152.5 MiB checkpoint JSON | 926.9 MiB | +| Huge + 2 checkpoints | same as above | two checkpoint copies | OOM, exit 134 | 920.1 MiB | +| Huge + 2 clones, no retained subagent transcripts | same generated subagent output, but parent history keeps only summaries | passed; parent history JSON drops to 3.8 MiB | 136.8 MiB | + +The failing huge-clone run produced: + +```text +FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory +``` + +The native stack included: + +- `v8::internal::ValueDeserializer::ReadObjectInternal` +- `v8::internal::ValueDeserializer::ReadDenseJSArray` +- `node::worker::Message::Deserialize` +- `node::worker::StructuredClone` + +This matches the same stack family as the user-provided OOM log. The controlled +reproduction also shows why 4 GiB / 8 GiB user reports are plausible: the +failure is not caused by a single large object, but by large retained +history/tool-result/subagent state plus one or more full-history clone or +checkpoint copies. Raising `--max-old-space-size` can delay the crash while +preserving the same amplification pattern. + +Important attribution from this deterministic run: + +1. Building a 76.2 MiB parent history JSON can succeed under the reduced heap. + The OOM appears when additional full-history clone/checkpoint copies are + retained. +2. A single checkpoint copy can push RSS close to 1 GiB even before OOM. +3. Removing retained subagent transcripts from the parent hot history changes + the same generated workload from OOM to a small 136.8 MiB RSS run. That is + the clearest mitigation signal so far. +4. This reproducer is synthetic and intentionally adversarial, but it exercises + the same object-graph shape as the long interactive review: parent session, + subagents, large tool outputs, transcript merge-back, and full-history clone + pressure. + +## DeepSeek PR-Size Follow-Up + +After the initial model matrix, an additional Qwen Code-only run tested +`DeepSeek/deepseek-v4-pro` across three real PR sizes. This model is configured +through the Anthropic-compatible protocol; OpenAI-compatible execution returned +404 in a smoke check, so the successful benchmark uses `--auth-type anthropic`. + +The diagnostics branch was extended to record Anthropic wire request summaries +with the same privacy rule as the OpenAI path: aggregate counts and byte sizes +only, no prompt text, diff content, tool arguments, headers, base URL, or API +key. + +PR sizes: + +| Size | PR | State | Files | Changed lines | Title | +| ------ | ------- | ------ | ----: | ------------: | ----------------------------------------------------------------------- | +| small | `#4268` | merged | 1 | 1 | fix(serve): add mcp_guardrails to E2E capabilities expectation | +| medium | `#4186` | merged | 6 | 494 | fix(core): add heap-pressure auto-compaction safety net | +| large | `#4168` | open | 25 | 4,750 | feat(core)!: redesign auto-compaction thresholds with three-tier ladder | + +Runtime: + +| Size | PR | Wall | Turns | Total tokens | Cache-read tokens | Tree RSS peak | Root RSS peak | End heap | End RSS | +| ------ | ------- | -----: | ----: | -----------: | ----------------: | ------------: | ------------: | --------: | --------: | +| small | `#4268` | 39.7s | 2 | 43,362 | 28,672 | 346.9 MiB | 344.8 MiB | 115.2 MiB | 304.3 MiB | +| medium | `#4186` | 142.6s | 4 | 135,120 | 115,840 | 340.7 MiB | 337.3 MiB | 103.5 MiB | 285.6 MiB | +| large | `#4168` | 191.1s | 8 | 386,891 | 332,928 | 360.0 MiB | 336.3 MiB | 119.3 MiB | 237.9 MiB | + +Request and tool diagnostics: + +| Size | PR | Requests | Anthropic wire requests | Max Anthropic body | Max system | Max tool schema | Tool calls | Total tool result | Max tool result | Max function response in request | +| ------ | ------- | -------: | ----------------------: | -----------------: | ---------: | --------------: | ---------: | ----------------: | --------------: | -------------------------------: | +| small | `#4268` | 2 | 2 | 103.0 KiB | 50.8 KiB | 47.6 KiB | 3 | 0.6 KiB | 0.5 KiB | 1.1 KiB | +| medium | `#4186` | 4 | 4 | 159.8 KiB | 50.8 KiB | 47.6 KiB | 5 | 30.2 KiB | 29.3 KiB | 56.7 KiB | +| large | `#4168` | 8 | 8 | 289.5 KiB | 50.8 KiB | 47.6 KiB | 11 | 235.0 KiB | 232.1 KiB | 182.4 KiB | + +DeepSeek observations: + +1. PR size scaled turns, tokens, Anthropic wire body size, and tool result size + clearly, but did not scale RSS proportionally. The small/medium/large tree + RSS peaks stayed in a narrow `340.7-360.0 MiB` band. +2. The large PR was expensive mostly in model rounds and token volume: + 8 requests and 386,891 total tokens. Its max Anthropic body was 289.5 KiB, + much larger than the OpenAI-compatible runs, but RSS still stayed near the + same local-bundle band. +3. The static Anthropic request cost is also visible: system prompt is about + 50.8 KiB and tool schema about 47.6 KiB per request. Repeated rounds are + therefore a major token amplifier. +4. The large PR produced 235.0 KiB of captured tool results and 182.4 KiB max + function response in a request. This is higher than the earlier small PR / + code-navigation cases and shows large PRs still put pressure on local + tool-result handling and request assembly, even when RSS does not spike. +5. The DeepSeek run reinforces the model-choice conclusion: provider/model + choice strongly changes turns, latency, token volume, and wire payload shape, + but the local bundle RSS peak remains dominated by Qwen Code runtime shape + rather than scaling linearly with PR size. + +## Long-Review JSONL Replay: History Clone Pressure + +A recent long PR-review chat record was analyzed as a post-mortem shape for +the reported OOM class. The raw JSONL is not included here because it contains +prompt and tool output text. The aggregate shape is: + +| Signal | Value | +| ----------------------- | ----------------------------- | +| Duration | 87.0 min | +| Qwen Code version | 0.15.10 | +| Model | qwen-latest-series beta model | +| API responses | 380 | +| Tool-call telemetry | 507 events | +| MCP tool-call telemetry | 4 events | +| Subagent API responses | 313 | +| Root API responses | 67 | +| Root prompt growth | 38,622 -> 168,555 tokens | +| Max prompt tokens | 168,555 | +| Total response tokens | 31.28M | + +This shape does not support MCP as the primary OOM cause for this case. Only +4 of 507 tool-call telemetry events were MCP, and all four recorded +`content_length=0`. The dominant shape is long-session/subagent amplification: +15 `agent` calls produced 313 subagent API responses and 403 subagent tool-call +events. + +The replay then rebuilt the chat `Content[]` message shape from the JSONL and +ran controlled clone/stringify pressure tests. The base retained message payload +is small, so it is not itself enough to OOM: + +| Replay scale | Retained clones | History JSON | Checkpoint JSON | End heap | End RSS | +| ------------ | --------------: | -----------: | --------------: | -------: | -------: | +| 1x | 8 | 0.54 MB | 1.08 MB | 18.0 MB | 88.8 MB | +| 30x | 8 | 14.46 MB | 28.92 MB | 260.0 MB | 577.8 MB | +| 60x | 8 | 28.86 MB | 57.71 MB | 510.3 MB | 960.8 MB | + +The scaled replay is not a user-data claim; it is a controlled amplification of +the observed JSONL shape to test whether full-history clone and checkpoint +serialization can create the same failure mode as the reports. + +A low-heap reproduction with `--max-old-space-size=256` confirms the mechanism: + +| Case | History JSON | Result | +| ------------------------- | -----------: | ----------------------------------------------------- | +| Build history only | 38.4 MB | Succeeded; heap 131.6 MB, RSS 378.2 MB | +| Build + one clone | 38.4 MB | Succeeded; heap 183.3 MB, RSS 463.4 MB | +| Build + repeated clones | 38.4 MB | OOM after several retained `structuredClone()` copies | +| Checkpoint double-history | 38.4 MB | OOM while holding history plus cloned client history | + +The repeated-clone OOM stack contains `ValueDeserializer::ReadObjectInternal`, +`ValueDeserializer::ReadDenseJSArray`, +`node::worker::Message::Deserialize`, and +`node::worker::StructuredClone`, matching the same stack family seen in the +user-provided OOM log. This proves that full-history `structuredClone()` can be +the immediate OOM trigger without any MCP server involvement. + +Current working hypothesis for this JSONL class: + +1. MCP can explain normal-config startup RSS in separate benchmarks, but it is + not the likely trigger for this long-review OOM shape. +2. Long task growth comes from retained chat history, large tool outputs, + subagent histories, observable agent messages, and UI/tool-result state. +3. The immediate OOM trigger can be a full-history clone or checkpoint-style + double serialization after the heap is already high. +4. Compression can mitigate retained history, but compression itself may create + a temporary peak if it first clones or serializes large history. + +### Local Mitigation Validation: Disabled-MCP PR Review Case + +Two targeted mitigations were applied locally and validated before rerunning a +disabled-MCP PR review case: + +1. `checkNextSpeaker()` now reads only the last curated message with + `getHistoryTail(1, true)` and sends only that message to the next-speaker + side query. The next-speaker prompt only asks about the immediately previous + model response, so sending full history was unnecessary clone and token + pressure. +2. `AgentToolInvocation` no longer retains full `responseParts` arrays inside + the live `task_execution.toolCalls` display. The real response parts still + flow through transcript/history paths, but the parent UI display now keeps + only a bounded text summary for nested tool-result streaming instead of + holding another full copy of large subagent tool outputs during long runs. +3. `GeminiChat.sendMessageStream()` now builds model request contents through + an internal curated-history view instead of calling public + `getHistory(true)`. Public `getHistory()` still returns a defensive + `structuredClone()` for external callers, but the request hot path no longer + deep-clones the whole retained chat history before every model call. + +TDD checks added for these mitigations: + +| Test | Expected protection | +| -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| `checkNextSpeaker > should send only the last curated model message to the side query` | Prevents full-history clone/send in next-speaker checks | +| `AgentTool > should not retain responseParts in live tool call display after TOOL_RESULT` | Prevents live subagent display from retaining large tool responses | +| `AgentTool > should keep only a bounded result summary in live tool call display` | Preserves nested result readability without retaining the full response body | +| `GeminiChat > sendMessageStream > does not deep-clone the full curated history when building request contents` | Prevents request setup from hitting the `ValueDeserializer` / `StructuredClone` OOM path | + +Additional reproduction and fix validation: + +| Step | Command shape | Result | +| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Pre-fix deterministic clone pressure | `node --max-old-space-size=256 scripts/memory-pressure-repro.mjs ... --clone-count=2 --mode=clone` | OOM, exit 134; stderr contained `Reached heap limit` and `ValueDeserializer` / `StructuredClone`; max RSS 528.1 MiB in the repeat run | +| Red test | targeted `GeminiChat` test with `structuredClone` forced to throw during request setup | failed at `GeminiChat.getHistory()` before the mitigation | +| Green test | same targeted `GeminiChat` test after the mitigation | passed | +| Built-code smoke | `node --max-old-space-size=256` against the built core package, with a 96-entry / about 48 MiB history and `structuredClone` forced to throw | passed; request had 97 contents; process RSS 161.4 MiB, `/usr/bin/time -l` max RSS 161.6 MiB | + +This narrows the earlier "same stack family" statement: the deterministic +synthetic OOM still proves retained full-history clones can fail in the same V8 +stack family as the user log, while the new `GeminiChat` red/green test proves +one real production request-setup path no longer reaches that clone point. +Checkpoint/resume and compression internals still need separate long-run +validation because they can legitimately need durable copied history. + +Verification commands: + +| Command | Result | +| ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `npx vitest run src/core/geminiChat.test.ts` | passed, 89 tests | +| `npx vitest run src/utils/nextSpeakerChecker.test.ts --coverage=false` | passed, 13 tests | +| `npx vitest run src/tools/agent/agent.test.ts --coverage=false` | passed, 77 tests | +| `npx vitest run --config ./scripts/tests/vitest.config.ts scripts/tests/memory-pressure-repro.test.js` | passed, 1 test | +| `npm run build --workspace=packages/core` | passed | +| `npm run build --workspace=packages/cli` | passed | +| `npm run typecheck --workspace=packages/core` | passed | +| `npm run typecheck --workspace=packages/cli` | passed | +| `npm run bundle` | passed | +| `npm run build` | failed in `packages/vscode-ide-companion` lint on existing internal-module import rules; core, CLI, bundle, and targeted tests above passed | + +The full root `npm run build` was not clean in this worktree because the +`vscode-ide-companion` package hit pre-existing `import/no-internal-modules` +lint errors. The core/CLI build and bundle needed for the local runtime test +completed successfully. + +The same PR review prompt was then run with a temporary config where MCP and +hooks were disabled. Both rows were interrupted after a bounded long-run window +instead of waiting for a full review to finish. **Caveat**: the two runs are +confounded by workload size (79K vs 390K tokens) and cannot be compared as a +controlled experiment. The comparison only shows directional evidence. + +| Variant | Runtime | MCP servers | Tools | Assistant messages | Tool use/result blocks | Parent tool ids | Total tokens | Max input tokens | Root max RSS | +| ----------------- | ------: | ----------: | ----: | -----------------: | ---------------------: | --------------: | -----------: | ---------------: | -----------: | +| before mitigation | 365.08s | 0 | 19 | 42 | 42 / 42 | 3 | 79,439 | 26,807 | 357.7 MiB | +| after mitigation | 404.52s | 0 | 19 | 58 | 52 / 42 | 2 | 390,339 | 54,000 | 310.5 MiB | + +This is not a deterministic apples-to-apples model benchmark: the patched run +did more work and consumed substantially more total tokens before the manual +cutoff. The useful signal is narrower: under a disabled-MCP review case with +more observed work, root max RSS did not increase and was about 47.2 MiB lower. +That supports the mitigation direction, but it does not prove the whole +long-task OOM class is fixed. + +Remaining high-risk clone/retention paths to inspect next: + +1. Compression still calls full `getHistory(true)` before summarization. If the + heap is already high, the compression attempt can create the peak that trips + OOM. +2. Checkpoint creation can hold original history, cloned client history, and a + serialized checkpoint payload at the same time. +3. Fork subagents still seed from parent history with `getHistory(true)`. +4. ACP/history export/summary/copy paths still call full `getHistory()` and + should be audited separately from the normal review loop. + +Version timing: + +| Issue | Created | Reported version | Signal | +| ----- | ---------- | ------------------------ | ---------------------------------------- | +| #2128 | 2026-03-05 | not specified | Long-session UI memory growth | +| #2562 | 2026-03-21 | not specified | `structuredClone` OOM in long sessions | +| #2868 | 2026-04-03 | 0.13.2 | Heap OOM | +| #2945 | 2026-04-07 | 0.14.0 | V8 heap OOM | +| #4116 | 2026-05-13 | 0.15.11 | OOM with structured-clone-style analysis | +| #4134 | 2026-05-14 | 0.15.11 | OOM | +| #4149 | 2026-05-14 | 0.15.10-nightly.20260513 | V8 heap OOM | +| #4167 | 2026-05-15 | 0.15.11 | Crash near compression | +| #4185 | 2026-05-15 | 0.15.11 | Heap pressure before token compaction | +| #4254 | 2026-05-17 | not specified | Memory keeps rising | +| #4276 | 2026-05-18 | 0.15.11 | V8 heap OOM | +| #4309 | 2026-05-19 | 0.15.11 | High memory warning around 7 GiB | + +The issue history does not prove that 0.15.10 introduced the OOM class; similar +reports existed in March and April. It does support a recent cluster beginning +around 2026-05-13, overlapping `v0.15.10`/`v0.15.11` releases. The relevant +diff between `v0.15.9` and `v0.15.10` touched subagent runtime, +non-interactive execution, `GeminiChat`, and compression code heavily, so this +range is a reasonable first bisect window. + +## Notes + +- The first code-navigation prompt allowed open-ended exploration and hit + `maxSessionTurns`; the successful rows above use a constrained command list. +- The first synthetic-diff attempt used a relative bundle path from inside the + temporary repositories; those failed immediately and are excluded from the + tables. The successful rows use the absolute local bundle path. +- Raw JSONL streams are not committed because they contain prompts, tool + commands, and tool output. The report only includes aggregate diagnostics. diff --git a/docs/e2e-tests/2026-05-21-qwen-0.15.11-default-heap-oom-stress-report.md b/docs/e2e-tests/2026-05-21-qwen-0.15.11-default-heap-oom-stress-report.md new file mode 100644 index 00000000000..e9579dee1b8 --- /dev/null +++ b/docs/e2e-tests/2026-05-21-qwen-0.15.11-default-heap-oom-stress-report.md @@ -0,0 +1,338 @@ +# Qwen Code 0.15.11 默认 Heap OOM 压测报告 + +日期:2026-05-21 + +## 测试范围 + +本报告记录了针对 Qwen Code `0.15.11` 最新本地构建的一轮默认 heap 压测。 +这轮测试的目标是验证:在不人为降低内存上限的情况下,当前代码是否还能复现 +issue 中提到的长会话 OOM,以及在更极端的大输出场景下还有没有新的风险。 + +本轮覆盖三个模型: + +- `pai/glm-5` +- `qwen3.6-plus` +- `DeepSeek/deepseek-v4-pro` + +测试分为两部分: + +1. 真实长任务、多 agent 并发 review 循环。 +2. amplified foreground stdout 压测,即用大规模前台 shell stdout 放大 + tool-output 路径压力。 + +## 测试环境 + +| 项目 | 值 | +| --------------------------- | --------------------------------------------- | +| 分支 | `codex/memory-investigation-draft-pr` | +| Commit | `c161e0aa4` | +| CLI | 本地 `dist/cli.js` | +| CLI 版本 | `0.15.11` | +| Node 默认 heap limit | `4144 MiB` | +| `NODE_OPTIONS` | 未设置 | +| 显式 `--max-old-space-size` | 未设置 | +| runner `ulimit` | runner 未设置 | +| 配置模式 | 临时复制 `~/.qwen`,并隔离 `QWEN_RUNTIME_DIR` | +| MCP / 正常配置 | 尽量按复制后的正常配置加载 | + +注意:这里的 CLI 版本显示为 `0.15.11`,是因为 package version 尚未 bump。 +实际测试对象是 commit `c161e0aa4` 下本地编译出的 `dist/cli.js`,不是 PATH +里的全局 `qwen` 可执行文件。 + +本轮没有修改全局 Qwen 配置。原始 runtime artifacts 在: + +- `.qwen/runtime-bench/2026-05-20T13-51-58-731Z-oom-stress` +- `.qwen/runtime-bench/2026-05-20T15-20-37-790Z-oom-amplified` + +注意:本轮里 `env-center` MCP server 启动失败,但其他内置工具和部分 +MCP/child process 仍然加载。因此这些结果代表当前本地环境,不是完全 stripped +的 `--bare` 环境。 + +## 核心结论 + +最新本地构建在 issue 最关心的“长会话 V8 heap OOM”路径上表现明显更好。 +基于这轮默认 heap、多模型、多 agent、长任务压测,可以认为本 PR 对此前遇到的 +long-session heap OOM 问题已经基本解决,至少在当前复现维度下已经不能再复现 +原始 heap OOM。 + +真实长任务、多 agent 并发测试一共执行了: + +- 23 个 worker turn +- 约 `719,094,118` reported total tokens +- 77 次 agent tool call +- 856 次总 tool call + +这部分没有复现任何传统 V8 heap OOM 特征: + +- `JavaScript heap out of memory` +- `Reached heap limit` +- `Ineffective mark-compacts near heap limit` +- `Allocation failed` + +真实长任务阶段最高 process-tree RSS 为 `874.7 MiB`,最高 root-process RSS 为 +`219.1 MiB`。这说明在默认 heap 下,当前代码没有轻易复现原 issue 中那种长任务 +跑挂的 heap OOM。 + +第二阶段 amplified stdout 压测更激进。它一共执行了 18 个 payload attempt, +覆盖三个模型和 `128 MiB` 到 `2048 MiB` 的 foreground stdout payload。 + +结果是: + +- 三个模型都成功跑过 `1536 MiB` payload。 +- 最高成功 process-tree RSS 是 `5964.7 MiB`,出现在 `qwen3.6-plus` + 的 `1536 MiB` payload。 +- 到 `2048 MiB` payload 时,出现了一个新的 extreme large-output failure。 + +`2048 MiB` 的结果: + +- `pai/glm-5`:`exit=1`,stdout 为空,没有标准 OOM 文本。 +- `qwen3.6-plus`:`exit=1`,stdout 为空,没有标准 OOM 文本。 +- `DeepSeek/deepseek-v4-pro`:出现 V8 fatal: + `Check failed: i::kMaxInt >= len`,栈在 + `v8::String::NewFromOneByte` / `node::StringBytes::Encode` / + `DecodeUTF8`。 + +这个新问题不是原 issue 中的传统 long-session heap OOM。它更像是 +multi-GiB foreground stdout 被解码/构造成 JS string 时触发的 V8 字符串长度 +限制或大输出处理问题。建议作为 large-output follow-up 跟踪,而不是把它当作 +当前长会话 heap-pressure 修复失败。 + +## Phase 1:真实长任务、多 Agent 并发压测 + +### 测试形态 + +每个模型 worker 都复用同一个 session,不断 `--resume`。每一轮要求 Qwen Code: + +- 进行只读代码审查和代码搜索; +- 在同一轮中并发启动至少 4 个 `agent` tool call; +- 重点检查 chat history、compaction、subagent runtime、non-interactive + streaming、provider adapters 等 memory 相关区域; +- 保留足够详细的最终回答,让 session history 自然增长。 + +runner 每秒采样 process-tree RSS,没有设置任何额外 heap cap。 + +这部分在观察到内存比较稳定后用 `SIGTERM` 主动停止,以便切换到第二阶段的 +amplified stdout 压测。因此表里的 `SIGTERM` 不是 OOM。 + +### 汇总结果 + +| Model | Worker turns | Total tokens | Agent calls | Tool calls | Peak tree RSS | Peak root RSS | Last exit | OOM | +| -------------------------- | -----------: | --------------: | ----------: | ---------: | ------------: | ------------: | --------- | ------ | +| `pai/glm-5` | 9 | 444,614,704 | 36 | 362 | 874.7 MiB | 217.4 MiB | `SIGTERM` | no | +| `qwen3.6-plus` | 7 | 101,425,927 | 17 | 346 | 862.7 MiB | 219.1 MiB | `SIGTERM` | no | +| `DeepSeek/deepseek-v4-pro` | 7 | 173,053,487 | 24 | 148 | 864.5 MiB | 213.8 MiB | `SIGTERM` | no | +| **Total / max** | **23** | **719,094,118** | **77** | **856** | **874.7 MiB** | **219.1 MiB** | - | **no** | + +### 分轮结果 + +| Model | Turn | Exit | Timed out | OOM | Peak tree RSS | Peak root RSS | Total tokens | Agent calls | Tool calls | +| -------------------------- | ---: | --------- | --------- | --- | ------------: | ------------: | -----------: | ----------: | ---------: | +| `DeepSeek/deepseek-v4-pro` | 1 | `0` | no | no | 709.1 MiB | 167.3 MiB | 5,565,147 | 4 | 37 | +| `DeepSeek/deepseek-v4-pro` | 2 | `0` | no | no | 674.5 MiB | 118.8 MiB | 13,989,721 | 4 | 29 | +| `DeepSeek/deepseek-v4-pro` | 3 | `0` | no | no | 734.1 MiB | 148.0 MiB | 22,621,542 | 4 | 24 | +| `DeepSeek/deepseek-v4-pro` | 4 | `0` | no | no | 771.1 MiB | 107.5 MiB | 33,470,249 | 4 | 22 | +| `DeepSeek/deepseek-v4-pro` | 5 | `0` | no | no | 864.5 MiB | 212.9 MiB | 43,540,313 | 4 | 19 | +| `DeepSeek/deepseek-v4-pro` | 6 | `0` | no | no | 807.6 MiB | 167.9 MiB | 53,866,515 | 4 | 17 | +| `DeepSeek/deepseek-v4-pro` | 7 | `SIGTERM` | no | no | 785.1 MiB | 213.8 MiB | n/a | n/a | n/a | +| `pai/glm-5` | 1 | `SIGTERM` | yes | no | 742.8 MiB | 170.5 MiB | 17,071,519 | 4 | 142 | +| `pai/glm-5` | 2 | `0` | no | no | 874.7 MiB | 217.4 MiB | 27,438,727 | 4 | 60 | +| `pai/glm-5` | 3 | `0` | no | no | 699.7 MiB | 102.1 MiB | 35,627,222 | 4 | 38 | +| `pai/glm-5` | 4 | `0` | no | no | 796.0 MiB | 194.0 MiB | 44,130,101 | 4 | 23 | +| `pai/glm-5` | 5 | `0` | no | no | 743.4 MiB | 152.1 MiB | 50,465,979 | 4 | 26 | +| `pai/glm-5` | 6 | `0` | no | no | 714.9 MiB | 125.2 MiB | 56,357,372 | 4 | 18 | +| `pai/glm-5` | 7 | `0` | no | no | 694.5 MiB | 96.6 MiB | 64,047,037 | 4 | 20 | +| `pai/glm-5` | 8 | `0` | no | no | 756.0 MiB | 136.8 MiB | 71,891,505 | 4 | 15 | +| `pai/glm-5` | 9 | `SIGTERM` | no | no | 755.7 MiB | 157.3 MiB | 77,585,242 | 4 | 20 | +| `qwen3.6-plus` | 1 | `0` | no | no | 735.1 MiB | 153.1 MiB | 3,890,508 | 4 | 83 | +| `qwen3.6-plus` | 2 | `0` | no | no | 702.4 MiB | 142.5 MiB | 4,300,186 | 1 | 9 | +| `qwen3.6-plus` | 3 | `0` | no | no | 862.7 MiB | 219.1 MiB | 8,635,953 | 4 | 88 | +| `qwen3.6-plus` | 4 | `SIGTERM` | yes | no | 685.8 MiB | 106.5 MiB | n/a | n/a | n/a | +| `qwen3.6-plus` | 5 | `0` | no | no | 610.5 MiB | 93.1 MiB | 40,191,337 | 4 | 87 | +| `qwen3.6-plus` | 6 | `0` | no | no | 723.6 MiB | 121.9 MiB | 44,407,943 | 4 | 79 | +| `qwen3.6-plus` | 7 | `SIGTERM` | no | no | 810.4 MiB | 116.0 MiB | n/a | n/a | n/a | + +### Phase 1 解读 + +这是本轮里最能说明原始 long-session OOM 已明显改善的数据。 + +这组测试比 5 月 18 日的小 PR review / code navigation 更重:它包含更多 +`--resume`、更多 subagent activity、更大的 reported token 量和更多 tool call。 +但 process-tree RSS 始终低于 `0.9 GiB`,也没有出现传统 V8 heap OOM。 + +这不能证明所有用户 OOM 都不可能再发生,但至少说明当前构建在默认 heap 下, +已经无法轻易复现 issue 中那类长会话 heap-pressure OOM。 + +## Phase 2:Amplified Foreground Stdout 压测 + +### 测试形态 + +第二阶段故意放大 shell-output 路径压力。每个模型、每个 payload size 都要求 +parent session 和并发 agents 运行前台 shell 命令,输出大量 `x` 到 stdout: + +```bash +node -e "const chunk='x'.repeat(1024*1024); for (let i=0; i= len. +... +v8::String::NewFromOneByte +node::StringBytes::Encode +node::encoding_binding::BindingData::DecodeUTF8 +``` + +触发条件: + +- Model:`DeepSeek/deepseek-v4-pro` +- Payload:`2048 MiB` +- Peak tree RSS:`4660.4 MiB` +- Largest process RSS:`4527.6 MiB` +- runner 记录 exit:`SIGTERM`,因为 fatal 输出已经捕获后,剩余子进程仍在高 CPU + 空转,被手动终止。 + +`pai/glm-5` 和 `qwen3.6-plus` 在 `2048 MiB` 也失败,表现为 stdout 为空、 +exit code `1`,但 stderr 没有捕获到 V8 fatal stack。 + +### 严重程度 + +这是一个真实的 robustness 问题,但触发条件是 multi-GiB foreground stdout, +不是正常代码审查任务。它也不能证明当前 long-session heap-pressure 修复失败。 + +### 是否是本 PR 引入? + +本轮没有证据表明 `2048 MiB` stdout failure 是当前 memory PR 引入的回归。 + +原因: + +- 失败路径是 foreground shell stdout decode / string construction。 +- 原 issue 路径是 long-session history、compaction、clone pressure。 +- 本轮没有做同 payload 的 pre-PR baseline,因此不能归因成 regression。 +- 该 failure 只在刻意极端的 `2048 MiB` payload 出现;`128 MiB` 到 + `1536 MiB` 都能完成。 + +建议把它作为 dedicated large-output follow-up:更早 stream / spool / hard-cap +foreground shell output,避免在内存里构造 multi-GiB JS string。除非当前 PR 的目标 +明确包含“任意 multi-GiB 前台 stdout 都必须可处理”,否则不建议把它作为当前 PR 的 +blocker。 + +## 结论 + +1. 最新本地 `0.15.11` 构建在 issue 报告的 long-session heap OOM 方向上明显更好。 + 基于当前默认 heap 压测结果,可以认为本 PR 已经基本解决此前遇到的 + long-session heap OOM 复现路径。 + +2. 在默认 Node heap 下,真实长任务 + 多 agent review loop 没有在 + `pai/glm-5`、`qwen3.6-plus`、`DeepSeek/deepseek-v4-pro` 三个模型上复现传统 + V8 heap OOM。 + +3. synthetic foreground stdout 压测仍能把 process-tree RSS 推得很高。当前构建在 + 三模型上都撑过了 `1536 MiB` payload,最高成功 tree RSS 是 `5964.7 MiB`。 + +4. 仍然存在一个独立的极端 large-output 问题:`2048 MiB` stdout 附近,Qwen Code + 可能在输出 JSON 结果前失败;DeepSeek case 捕获到了 V8 string-length fatal。 + +5. 这个新发现重要,但更像是后续 large-output robustness 问题,不应直接作为 + long-session heap-pressure mitigation 的 blocker。 + +## 建议发到 PR 的评论摘要 + +建议 PR 评论里只放精简摘要,完整数据放本文档: + +```markdown +I reran default-heap stress tests on the latest local build with +`pai/glm-5`, `qwen3.6-plus`, and `DeepSeek/deepseek-v4-pro`. + +No `NODE_OPTIONS`, `--max-old-space-size`, or runner `ulimit` was used. The +local Node heap limit was about 4144 MiB. + +Results: + +- Realistic long-session + multi-agent review loop: 23 worker turns, + ~719M reported total tokens, 77 agent calls, 856 total tool calls. + No traditional V8 heap OOM was reproduced. Peak process-tree RSS was + 874.7 MiB; peak root RSS was 219.1 MiB. +- Amplified stdout stress: 18 payload attempts across 128 MiB -> 2048 MiB. + All three models completed through 1536 MiB payloads without traditional + heap OOM. Highest successful process-tree RSS was 5964.7 MiB. +- At 2048 MiB foreground stdout, an extreme large-output failure remains. + DeepSeek captured a V8 fatal `Check failed: i::kMaxInt >= len` stack in + `String::NewFromOneByte` / `StringBytes::Encode` / `DecodeUTF8`. + +Conclusion: this PR appears to have effectively addressed the previously +observed long-session heap OOM reproduction path under default heap. The +2048 MiB stdout failure is a separate large-output/string-limit robustness issue +and should be tracked as a follow-up rather than treated as the same +long-session heap OOM regression. +``` diff --git a/docs/e2e-tests/worktree-phase-c.md b/docs/e2e-tests/worktree-phase-c.md new file mode 100644 index 00000000000..8f7ae8d6c80 --- /dev/null +++ b/docs/e2e-tests/worktree-phase-c.md @@ -0,0 +1,594 @@ +# Worktree Phase C E2E Test Plan + +## Scope + +End-to-end verification of Phase C features against the local build at +`/Users/mochi/code/qwen-code/.claude/worktrees/romantic-burnell-b6e48c/dist/cli.js`. + +Phase C delivers: + +- **Task 1, 3, 4** — `WorktreeSession` sidecar JSON file at + `~/.qwen/tmp//chats/.worktree.json` +- **Task 2** — `core.hooksPath` configured inside new worktrees +- **Task 5–6** — `useWorktreeSession` hook, `UIState.activeWorktree`, Footer + worktree indicator, `StatusLineCommandInput.worktree` field +- **Task 7** — `--resume` injects an INFO history item when active worktree + still exists; cleans up stale sidecar otherwise +- **Task 8** — `WorktreeExitDialog` with dirty-state inspection, intercepts + second Ctrl+C in active worktree + +## Binaries + +- **Local build**: `node /Users/mochi/code/qwen-code/.claude/worktrees/romantic-burnell-b6e48c/dist/cli.js` +- **Baseline (for pre-impl comparison if needed)**: globally installed `qwen` + +## Test environment template + +Each group runs in its own temp git repo and tmux session: + +```bash +TEST_DIR=$(mktemp -d -t qwen-wt-phc-XXXXXX) +TEST_DIR=$(cd "$TEST_DIR" && pwd -P) # resolve symlinks (macOS /var → /private/var) +cd "$TEST_DIR" +git init -q -b main +git config user.email t@e.com +git config user.name t +git config commit.gpgsign false +echo "hello" > README.md +git add README.md +git commit -q -m "initial" --no-verify +``` + +`QWEN=/Users/mochi/code/qwen-code/.claude/worktrees/romantic-burnell-b6e48c/dist/cli.js` + +--- + +## Group A: WorktreeSession sidecar (headless) + +**Mode:** headless, `--approval-mode yolo`, `--output-format json` + +### A1: enter_worktree writes sidecar with all fields + +**Steps:** + +```bash +SESSION=$(node $QWEN "use the enter_worktree tool with name='a1-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null \ + | jq -r '.[] | select(.type=="system") | .session_id' | head -1) + +PROJECT_ID=$(node -e "console.log(process.argv[1].replace(/[^a-zA-Z0-9]/g,'-'))" "$TEST_DIR") +SIDECAR=~/.qwen/projects/$PROJECT_ID/chats/$SESSION.worktree.json + +# Verify all fields present +cat "$SIDECAR" | jq '.slug, .worktreePath, .worktreeBranch, .originalCwd, .originalBranch, .originalHeadCommit' +``` + +**Expected:** + +- `slug` = "a1-test" +- `worktreePath` ends with `.qwen/worktrees/a1-test` +- `worktreeBranch` = "worktree-a1-test" +- `originalCwd` = `$TEST_DIR` (resolved) +- `originalBranch` = "main" +- `originalHeadCommit` matches `[0-9a-f]{40}` + +### A2: exit_worktree (keep) clears sidecar + +**Steps:** + +```bash +SESSION=$(node $QWEN "create a worktree named 'a2-test' using enter_worktree, then immediately exit it with action='keep' using exit_worktree" \ + --approval-mode yolo --output-format json 2>/dev/null \ + | jq -r '.[] | select(.type=="system") | .session_id' | head -1) + +SIDECAR=~/.qwen/projects/$PROJECT_ID/chats/$SESSION.worktree.json +test ! -f "$SIDECAR" && echo "PASS: sidecar removed" || echo "FAIL: sidecar still exists" +``` + +**Expected:** sidecar file does not exist after the exit_worktree call. + +### A3: exit_worktree (remove) clears sidecar + +**Steps:** + +```bash +SESSION=$(node $QWEN "create a worktree named 'a3-test' using enter_worktree, then immediately exit it with action='remove' and discard_changes=true using exit_worktree" \ + --approval-mode yolo --output-format json 2>/dev/null \ + | jq -r '.[] | select(.type=="system") | .session_id' | head -1) + +SIDECAR=~/.qwen/projects/$PROJECT_ID/chats/$SESSION.worktree.json +test ! -f "$SIDECAR" && echo "PASS: sidecar removed" || echo "FAIL: sidecar still exists" +# Also verify the worktree dir is gone +test ! -d "$TEST_DIR/.qwen/worktrees/a3-test" && echo "PASS: worktree dir removed" +``` + +**Expected:** both the sidecar AND the worktree directory are gone. + +--- + +## Group B: hooksPath configuration (headless) + +### B1: Without `.husky/`, hooksPath = `/.git/hooks` + +**Steps:** + +```bash +node $QWEN "use enter_worktree with name='b1-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null > /dev/null + +HOOKS_PATH=$(git -C "$TEST_DIR/.qwen/worktrees/b1-test" config --local core.hooksPath) +echo "Got hooksPath: $HOOKS_PATH" +test "$HOOKS_PATH" = "$TEST_DIR/.git/hooks" && echo "PASS" || echo "FAIL" +``` + +**Expected:** `$TEST_DIR/.git/hooks` + +### B2: With `.husky/`, hooksPath = `/.husky` + +**Steps:** + +```bash +mkdir -p "$TEST_DIR/.husky" +echo '#!/bin/sh' > "$TEST_DIR/.husky/pre-commit" +chmod +x "$TEST_DIR/.husky/pre-commit" + +node $QWEN "use enter_worktree with name='b2-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null > /dev/null + +HOOKS_PATH=$(git -C "$TEST_DIR/.qwen/worktrees/b2-test" config --local core.hooksPath) +test "$HOOKS_PATH" = "$TEST_DIR/.husky" && echo "PASS" || echo "FAIL got=$HOOKS_PATH" +``` + +**Expected:** `$TEST_DIR/.husky` + +### B3: Hooks in main repo actually fire from inside worktree + +**Steps:** + +```bash +# Set up a hook that writes a marker file +mkdir -p "$TEST_DIR/.git/hooks" +cat > "$TEST_DIR/.git/hooks/pre-commit" <<'EOF' +#!/bin/sh +echo "hook-fired" > /tmp/qwen-wt-hook-marker +EOF +chmod +x "$TEST_DIR/.git/hooks/pre-commit" + +node $QWEN "use enter_worktree with name='b3-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null > /dev/null + +# Commit something inside the worktree +WT="$TEST_DIR/.qwen/worktrees/b3-test" +echo "x" > "$WT/file.txt" +git -C "$WT" add file.txt +rm -f /tmp/qwen-wt-hook-marker +git -C "$WT" commit -m "trigger hook" 2>&1 +test -f /tmp/qwen-wt-hook-marker && echo "PASS: hook fired" || echo "FAIL: hook did not fire" +rm -f /tmp/qwen-wt-hook-marker +``` + +**Expected:** `/tmp/qwen-wt-hook-marker` exists after the commit. + +--- + +## Group C: --resume worktree restoration (headless) + +### C1: --resume injects worktree context when sidecar present and dir alive + +**Steps:** + +```bash +# Create initial session with worktree +INIT_OUT=$(node $QWEN "use enter_worktree with name='c1-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null) +SESSION=$(echo "$INIT_OUT" | jq -r '.[] | select(.type=="system") | .session_id' | head -1) + +# Resume the session and ask "what's my context?" +RESUMED=$(node $QWEN --resume "$SESSION" "say SIDECAR-CONFIRM" \ + --approval-mode yolo --output-format json 2>/dev/null) + +# Look for the injected INFO message text in the conversation +echo "$RESUMED" | grep -q "Resumed.*Active worktree.*c1-test" && echo "PASS" || echo "FAIL: no context injection" +``` + +**Expected:** the JSON stream contains an INFO message referencing `c1-test`. + +### C2: --resume cleans up stale sidecar when worktree dir is gone + +**Steps:** + +```bash +INIT_OUT=$(node $QWEN "use enter_worktree with name='c2-test' to create a worktree" \ + --approval-mode yolo --output-format json 2>/dev/null) +SESSION=$(echo "$INIT_OUT" | jq -r '.[] | select(.type=="system") | .session_id' | head -1) +SIDECAR=~/.qwen/projects/$PROJECT_ID/chats/$SESSION.worktree.json + +# Delete the worktree directory out-of-band +rm -rf "$TEST_DIR/.qwen/worktrees/c2-test" +test -f "$SIDECAR" || { echo "SKIP: sidecar was already gone"; exit 0; } + +# Resume — should clean up the stale sidecar +node $QWEN --resume "$SESSION" "hello" --approval-mode yolo --output-format json 2>/dev/null > /dev/null +test ! -f "$SIDECAR" && echo "PASS: stale sidecar cleaned" || echo "FAIL: stale sidecar still present" +``` + +**Expected:** sidecar file is removed. + +--- + +## Group D: Footer worktree indicator (interactive tmux) + +### D1: Footer shows worktree indicator after enter_worktree + +**Steps:** + +```bash +tmux new-session -d -s wt-d1 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-d1 "use enter_worktree with name='d1-test'" +sleep 0.5 +tmux send-keys -t wt-d1 Enter + +for i in $(seq 1 30); do + sleep 2 + tmux capture-pane -t wt-d1 -p | grep -q "Type your message" && break +done + +# Capture and look for the worktree indicator line in Footer area +tmux capture-pane -t wt-d1 -p -S -100 > /tmp/wt-d1.out +grep -E "⎇.*worktree-d1-test.*\(d1-test\)" /tmp/wt-d1.out && echo "PASS" || \ + { echo "FAIL — captured output:"; cat /tmp/wt-d1.out; } +tmux kill-session -t wt-d1 +``` + +**Expected:** Footer contains a line like `⎇ worktree-d1-test (d1-test)`. + +### D2: Footer indicator disappears after exit_worktree (keep) + +**Steps:** + +```bash +tmux new-session -d -s wt-d2 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-d2 "use enter_worktree with name='d2-test'" +sleep 0.5 +tmux send-keys -t wt-d2 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-d2 -p | grep -q "Type your message" && break; done + +# Verify indicator showed +tmux capture-pane -t wt-d2 -p -S -100 | grep -q "⎇.*d2-test" || { echo "FAIL: indicator missing before exit"; tmux kill-session -t wt-d2; exit 1; } + +# Exit the worktree (keep) +tmux send-keys -t wt-d2 "use exit_worktree with name='d2-test' action='keep'" +sleep 0.5 +tmux send-keys -t wt-d2 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-d2 -p | grep -q "Kept worktree" && break; done + +sleep 2 # give Footer a tick to refresh after sidecar removal +tmux capture-pane -t wt-d2 -p -S -100 > /tmp/wt-d2-after.out +# After exit, the indicator should be gone from the bottom panel area +tail -5 /tmp/wt-d2-after.out | grep -q "⎇.*d2-test" && \ + echo "FAIL: indicator still showing" || echo "PASS" +tmux kill-session -t wt-d2 +``` + +**Expected:** worktree indicator disappears from Footer within ~2s of `exit_worktree`. + +--- + +## Group E: WorktreeExitDialog (interactive tmux) + +### E1: Second Ctrl+C in worktree shows dialog instead of quitting + +**Steps:** + +```bash +tmux new-session -d -s wt-e1 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-e1 "use enter_worktree with name='e1-test'" +sleep 0.5 +tmux send-keys -t wt-e1 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-e1 -p | grep -q "Type your message" && break; done + +# First Ctrl+C (cleanup; should show "Press Ctrl+C again to exit") +tmux send-keys -t wt-e1 C-c +sleep 0.3 +tmux capture-pane -t wt-e1 -p | grep -q "Press Ctrl+C again" || \ + { echo "FAIL: first Ctrl+C didn't show warning"; tmux kill-session -t wt-e1; exit 1; } + +# Second Ctrl+C — should show the WorktreeExitDialog, NOT quit +tmux send-keys -t wt-e1 C-c +sleep 2 + +# Verify the dialog rendered +tmux capture-pane -t wt-e1 -p -S -50 > /tmp/wt-e1.out +grep -q "Active worktree.*e1-test" /tmp/wt-e1.out && \ + grep -q "Keep worktree" /tmp/wt-e1.out && \ + grep -q "Remove worktree" /tmp/wt-e1.out && \ + echo "PASS" || { echo "FAIL — captured:"; cat /tmp/wt-e1.out; } +tmux kill-session -t wt-e1 +``` + +**Expected:** dialog shows three options (Keep / Remove / Cancel) and process is still alive. + +### E2: Dialog shows dirty-state counts (commits + files) + +**Steps:** + +```bash +tmux new-session -d -s wt-e2 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-e2 "use enter_worktree with name='e2-test'" +sleep 0.5 +tmux send-keys -t wt-e2 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-e2 -p | grep -q "Type your message" && break; done + +# Make the worktree dirty: 1 new commit + 1 uncommitted file +WT="$TEST_DIR/.qwen/worktrees/e2-test" +echo "new" > "$WT/new.txt" +git -C "$WT" add new.txt +git -C "$WT" commit -q -m "test commit" --no-verify +echo "dirty" > "$WT/uncommitted.txt" + +# Trigger exit dialog via Ctrl+C double-press +tmux send-keys -t wt-e2 C-c +sleep 0.3 +tmux send-keys -t wt-e2 C-c +sleep 3 # allow time for git status / rev-list + +tmux capture-pane -t wt-e2 -p -S -50 > /tmp/wt-e2.out +grep -qE "new commit|uncommitted file" /tmp/wt-e2.out && echo "PASS" || \ + { echo "FAIL — captured:"; cat /tmp/wt-e2.out; } +tmux kill-session -t wt-e2 +``` + +**Expected:** dialog body contains both "X new commit(s)" and "Y uncommitted file(s)". + +### E3: Cancel option dismisses dialog without exiting + +**Steps:** + +```bash +tmux new-session -d -s wt-e3 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-e3 "use enter_worktree with name='e3-test'" +sleep 0.5 +tmux send-keys -t wt-e3 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-e3 -p | grep -q "Type your message" && break; done + +# Trigger dialog +tmux send-keys -t wt-e3 C-c +sleep 0.3 +tmux send-keys -t wt-e3 C-c +sleep 3 + +# Navigate to Cancel (DOWN DOWN) and press Enter +tmux send-keys -t wt-e3 Down +sleep 0.2 +tmux send-keys -t wt-e3 Down +sleep 0.2 +tmux send-keys -t wt-e3 Enter +sleep 2 + +# Dialog should be gone; input prompt should be back +tmux capture-pane -t wt-e3 -p | grep -q "Type your message" && echo "PASS" || \ + { echo "FAIL — captured:"; tmux capture-pane -t wt-e3 -p; } + +# Verify the worktree was NOT removed +test -d "$TEST_DIR/.qwen/worktrees/e3-test" && echo "worktree intact" || echo "FAIL: worktree gone" +tmux kill-session -t wt-e3 +``` + +**Expected:** dialog closes, input prompt returns, worktree directory still exists. + +### E4: Keep option exits session but preserves worktree + +**Steps:** + +```bash +tmux new-session -d -s wt-e4 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-e4 "use enter_worktree with name='e4-test'" +sleep 0.5 +tmux send-keys -t wt-e4 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-e4 -p | grep -q "Type your message" && break; done + +# Trigger dialog and pick Keep (first option, already selected) +tmux send-keys -t wt-e4 C-c +sleep 0.3 +tmux send-keys -t wt-e4 C-c +sleep 3 +tmux send-keys -t wt-e4 Enter + +# Wait for process to exit +for i in $(seq 1 20); do + sleep 1 + tmux has-session -t wt-e4 2>/dev/null || break + tmux capture-pane -t wt-e4 -p | grep -q "\$ " && break # shell prompt back +done + +# Worktree directory should still exist +test -d "$TEST_DIR/.qwen/worktrees/e4-test" && echo "PASS: worktree preserved" || \ + echo "FAIL: worktree was removed" +tmux kill-session -t wt-e4 2>/dev/null || true +``` + +**Expected:** process exits, worktree directory remains on disk. + +### E5: Remove option exits session and deletes worktree + +**Steps:** + +```bash +tmux new-session -d -s wt-e5 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +tmux send-keys -t wt-e5 "use enter_worktree with name='e5-test'" +sleep 0.5 +tmux send-keys -t wt-e5 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-e5 -p | grep -q "Type your message" && break; done + +# Trigger dialog and pick Remove (DOWN, Enter) +tmux send-keys -t wt-e5 C-c +sleep 0.3 +tmux send-keys -t wt-e5 C-c +sleep 3 +tmux send-keys -t wt-e5 Down +sleep 0.2 +tmux send-keys -t wt-e5 Enter + +# Wait for exit +for i in $(seq 1 20); do + sleep 1 + tmux has-session -t wt-e5 2>/dev/null || break + tmux capture-pane -t wt-e5 -p | grep -q "\$ " && break +done + +# Worktree directory should be GONE +test ! -d "$TEST_DIR/.qwen/worktrees/e5-test" && echo "PASS: worktree removed" || \ + echo "FAIL: worktree still on disk" +# Branch should also be deleted +git -C "$TEST_DIR" branch --list | grep -q "worktree-e5-test" && \ + echo "FAIL: branch still present" || echo "PASS: branch removed" +tmux kill-session -t wt-e5 2>/dev/null || true +``` + +**Expected:** process exits, worktree directory deleted, branch `worktree-e5-test` deleted. + +--- + +## Group F: Real-user workflow simulation (interactive tmux) + +### F1: Full enter → edit → commit → resume → exit (keep) flow + +**Steps:** + +```bash +tmux new-session -d -s wt-f1 -x 200 -y 50 \ + "cd $TEST_DIR && node $QWEN --approval-mode yolo" +sleep 3 + +# Step 1: enter worktree +tmux send-keys -t wt-f1 "use enter_worktree with name='f1-feature' to create a worktree" +sleep 0.5 +tmux send-keys -t wt-f1 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-f1 -p | grep -q "Type your message" && break; done + +# Step 2: read the absolute worktree path so the model knows where to write +WT="$TEST_DIR/.qwen/worktrees/f1-feature" +tmux send-keys -t wt-f1 "write the file $WT/hello.txt with content 'hi from worktree'" +sleep 0.5 +tmux send-keys -t wt-f1 Enter +for i in $(seq 1 60); do sleep 2; tmux capture-pane -t wt-f1 -p | grep -q "Type your message" && break; done + +# Verify the file was actually written INSIDE the worktree +test -f "$WT/hello.txt" && grep -q "hi from worktree" "$WT/hello.txt" && \ + echo "PASS: file written inside worktree" || echo "FAIL: file not in worktree" + +# Step 3: Exit with keep via the tool +tmux send-keys -t wt-f1 "use exit_worktree with name='f1-feature' action='keep'" +sleep 0.5 +tmux send-keys -t wt-f1 Enter +for i in $(seq 1 30); do sleep 2; tmux capture-pane -t wt-f1 -p | grep -q "Kept worktree" && break; done + +# Step 4: Verify worktree still on disk after exit +test -d "$WT" && echo "PASS: worktree kept" || echo "FAIL: worktree removed" +test -f "$WT/hello.txt" && echo "PASS: file persists" || echo "FAIL" + +tmux kill-session -t wt-f1 +``` + +**Expected:** + +- File written to worktree directory (not main repo) +- After exit `keep`, both the worktree directory and the file remain + +### F2: Custom statusline receives `worktree` payload + +**Steps:** + +```bash +# Create a statusline script that prints the JSON it receives via stdin +SETTINGS_DIR=~/.qwen +SETTINGS_FILE=$SETTINGS_DIR/settings.json +cp -f "$SETTINGS_FILE" /tmp/qwen-settings-backup.json 2>/dev/null || true +mkdir -p "$SETTINGS_DIR" +SL_SCRIPT=/tmp/qwen-wt-statusline.sh +cat > $SL_SCRIPT <<'EOF' +#!/bin/sh +INPUT=$(cat) +echo "$INPUT" > /tmp/qwen-wt-statusline-input.json +WT_NAME=$(echo "$INPUT" | jq -r '.worktree.name // "no-worktree"') +echo "WT=$WT_NAME" +EOF +chmod +x $SL_SCRIPT + +cat > "$SETTINGS_FILE" <