feat(cli): compress native update artifacts and decompress them while staging

.changeset/zstd-native-artifacts.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": patch
+---
+
+Download compressed native update artifacts and decompress them while staging.

apps/kimi-code/scripts/native/produce-manifest.mjs

@@ -1,20 +1,36 @@
 /**
- * Aggregate per-platform zip archive `.sha256` files into a single
- * `manifest.json` written into the same input directory.
+ * Build the per-release native artifact set and `manifest.json` from the
+ * matrix runners' zip archives.
  *
  * Usage:
  *   node produce-manifest.mjs <input-dir> <release-tag>
  *
- * Input dir must contain files matching: kimi-code-<target>.zip.sha256
- * (produced by package.mjs across the 6 native-build matrix runners).
- *
- * Output:
- *   <input-dir>/manifest.json   ← consumed by install.sh / install.ps1
+ * Input dir must contain files matching: kimi-code-<target>.zip(.sha256)
+ * (produced by package.mjs across the 6 native-build matrix runners). The
+ * zip is the only form in which binaries leave the matrix runners, so this
+ * script extracts each bare executable and emits next to it:
+ *   kimi-code-<target>.zst       zstd -19, consumed by the staged updater
+ *   kimi-code-<target>.tar.gz    consumed by install.sh / install.ps1
+ *   <artifact>.sha256            sidecars in `<hex>  <name>` format
+ *   manifest.json                platform entries pair the bare binary with
+ *                                its compressed variant (`checksum` is always
+ *                                the hash of what `compressed` inflates to)
  *
+ * Requires `unzip`, `zstd`, and `tar` on PATH (preinstalled on
+ * GitHub-hosted runners).
  */
 
-import { readFile, readdir, writeFile } from 'node:fs/promises';
-import { basename, resolve } from 'node:path';
+import { execFile } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { basename, join, resolve } from 'node:path';
+import { promisify } from 'node:util';
+
+import { fail, run } from './exec.mjs';
+
+const execFileAsync = promisify(execFile);
 
 const [, , inputDir, tag] = process.argv;
 if (!inputDir || !tag) {
@@ -25,26 +41,61 @@ if (!inputDir || !tag) {
 // Tag 格式 `@moonshot-ai/kimi-code@x.y.z` 或 `vx.y.z` 或 `x.y.z`，都归一化到 x.y.z
 const version = tag.replace(/^@moonshot-ai\/kimi-code@/, '').replace(/^v/, '');
 
+for (const tool of ['unzip', 'zstd', 'tar']) {
+  try {
+    await execFileAsync('sh', ['-c', `command -v ${tool}`]);
+  } catch {
+    fail(`produce-manifest.mjs requires \`${tool}\` on PATH (preinstalled on GitHub-hosted runners).`);
+  }
+}
+
+async function sha256File(path) {
+  return await new Promise((resolveHash, reject) => {
+    const hash = createHash('sha256');
+    const stream = createReadStream(path);
+    stream.on('error', reject);
+    stream.on('data', (chunk) => hash.update(chunk));
+    stream.on('end', () => resolveHash(hash.digest('hex')));
+  });
+}
+
 const entries = await readdir(inputDir);
 const sumFiles = entries.filter((f) => /^kimi-code-[a-z0-9-]+\.zip\.sha256$/.test(f));
-
 if (sumFiles.length === 0) {
-  console.error(`No kimi-code-<target>.zip.sha256 files found in ${inputDir}`);
-  process.exit(1);
+  fail(`No kimi-code-<target>.zip.sha256 files found in ${inputDir}`);
 }
 
 const platforms = {};
 for (const sumFile of sumFiles.sort()) {
-  const text = await readFile(resolve(inputDir, sumFile), 'utf-8');
-  const [checksum] = text.trim().split(/\s+/, 1);
-  if (!checksum || !/^[a-f0-9]{64}$/.test(checksum)) {
-    console.error(`Invalid checksum in ${sumFile}: ${checksum}`);
-    process.exit(1);
+  // kimi-code-darwin-arm64.zip.sha256 → darwin-arm64
+  const target = basename(sumFile, '.sha256').replace(/^kimi-code-/, '').replace(/\.zip$/, '');
+  const zipName = `kimi-code-${target}.zip`;
+  const exeName = target.startsWith('win32') ? 'kimi.exe' : 'kimi';
+  const binaryName = `kimi-code-${target}`;
+  const zstName = `${binaryName}.zst`;
+  const tarballName = `${binaryName}.tar.gz`;
+
+  const workDir = await mkdtemp(join(tmpdir(), `native-manifest-${target}-`));
+  try {
+    await run('unzip', ['-o', resolve(inputDir, zipName), '-d', workDir]);
+    const exePath = join(workDir, exeName);
+    const binaryChecksum = await sha256File(exePath);
+    await run('zstd', ['-T0', '-19', '-q', '-f', '-o', resolve(inputDir, zstName), exePath]);
+    await run('tar', ['-C', workDir, '-czf', resolve(inputDir, tarballName), exeName]);
+
+    const zstChecksum = await sha256File(resolve(inputDir, zstName));
+    const tarballChecksum = await sha256File(resolve(inputDir, tarballName));
+    await writeFile(resolve(inputDir, `${zstName}.sha256`), `${zstChecksum}  ${zstName}\n`);
+    await writeFile(resolve(inputDir, `${tarballName}.sha256`), `${tarballChecksum}  ${tarballName}\n`);
+
+    platforms[target] = {
+      filename: binaryName,
+      checksum: binaryChecksum,
+      compressed: { filename: zstName, checksum: zstChecksum },
+    };
+  } finally {
+    await rm(workDir, { recursive: true, force: true });
   }
-  const filename = basename(sumFile, '.sha256');
-  // kimi-code-darwin-arm64.zip → darwin-arm64
-  const target = filename.replace(/^kimi-code-/, '').replace(/\.zip$/, '');
-  platforms[target] = { filename, checksum };
 }
 
 const manifest = { version, tag, platforms };

apps/kimi-code/src/cli/update/native-manifest.ts

@@ -4,7 +4,8 @@
  * Published alongside the release and consumed by the install scripts; the
  * staged updater reuses the same file so checksums and file names have a
  * single source of truth. Entries point at the bare platform binary
- * (`kimi-code-<target>[.exe]`), not an archive.
+ * (`kimi-code-<target>[.exe]`), not an archive; an entry may additionally
+ * carry `compressed`, pointing at the zstd-compressed variant of that binary.
  */
 
 import { valid } from 'semver';
@@ -17,6 +18,12 @@ const MANIFEST_FETCH_TIMEOUT_MS = 10_000;
 const PlatformEntrySchema = z.object({
   filename: z.string().min(1),
   checksum: z.string().regex(/^[a-f0-9]{64}$/, { error: 'invalid sha256' }),
+  compressed: z
+    .object({
+      filename: z.string().min(1),
+      checksum: z.string().regex(/^[a-f0-9]{64}$/, { error: 'invalid sha256' }),
+    })
+    .optional(),
 });
 
 /**

apps/kimi-code/src/cli/update/native-stage.ts

@@ -12,6 +12,7 @@ import { createHash } from 'node:crypto';
 import { createReadStream } from 'node:fs';
 import { chmod, mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink } from 'node:fs/promises';
 import { basename, join } from 'node:path';
+import { createZstdDecompress } from 'node:zlib';
 
 import { valid } from 'semver';
 import { z } from 'zod';
@@ -186,14 +187,16 @@ export async function hashFileSha256(filePath: string): Promise<string | null> {
 /**
  * Whether a `.staging/` entry is an updater-owned artifact: a staged
  * executable (`kimi-<version>[.<pid>.<epoch-ms>.<n>][.exe]`) or a download
- * intermediate (the same plus `.part`). Ownership derives from the
- * semver/file-name contract (prerelease and build metadata included), so
- * foreign files in the directory are never matched.
+ * intermediate (the same plus `.part`, optionally with a `.zst` infix).
+ * Ownership derives from the semver/file-name contract (prerelease and
+ * build metadata included), so foreign files in the directory are never
+ * matched.
  */
 function isUpdaterOwnedStagingFile(entry: string): boolean {
   if (!entry.startsWith('kimi-')) return false;
   let name = entry.slice('kimi-'.length);
   if (name.endsWith('.part')) name = name.slice(0, -'.part'.length);
+  if (name.endsWith('.zst')) name = name.slice(0, -'.zst'.length);
   if (name.endsWith('.exe')) name = name.slice(0, -'.exe'.length);
   // Published artifacts may carry a unique per-worker infix after the
   // version (.<pid>.<epoch-ms>.<n>, or the older .<pid>.<n>) — try with and
@@ -365,6 +368,45 @@ async function downloadAndHash(
   return size;
 }
 
+/**
+ * Inflate a downloaded `.zst` artifact into `destPath`, hashing the plain
+ * bytes as they stream through; **throws** when the result does not match
+ * the manifest's bare-binary checksum. Returns the decompressed size.
+ */
+async function decompressAndHash(
+  zstPath: string,
+  destPath: string,
+  expectedSha256: string,
+): Promise<number> {
+  const hash = createHash('sha256');
+  let size = 0;
+  const file = await open(destPath, 'w');
+  try {
+    for await (const chunk of createReadStream(zstPath).pipe(createZstdDecompress())) {
+      hash.update(chunk as Buffer);
+      size += (chunk as Buffer).length;
+      // Same short-write loop as downloadAndHash: FileHandle.write may
+      // persist fewer bytes than requested, so loop until the chunk is
+      // fully on disk.
+      let offset = 0;
+      while (offset < (chunk as Buffer).length) {
+        const { bytesWritten } = await file.write(chunk as Buffer, offset);
+        if (bytesWritten === 0) {
+          throw new Error('failed to write the native binary to disk (disk full?)');
+        }
+        offset += bytesWritten;
+      }
+    }
+  } finally {
+    await file.close();
+  }
+  const digest = hash.digest('hex');
+  if (digest !== expectedSha256) {
+    throw new Error(`sha256 mismatch: expected ${expectedSha256}, got ${digest}`);
+  }
+  return size;
+}
+
 /**
  * Download + verify `version` next to the running executable.
  *
@@ -446,7 +488,32 @@ export async function stageNativeUpdate(
   try {
     const manifest = await fetchNativeReleaseManifest(options.version, fetchImpl);
     const entry = selectPlatformEntry(manifest, platform, arch);
-    const size = await downloadAndHash(
+    // Prefer the zstd-compressed artifact when the manifest carries one and
+    // the runtime can inflate it (~4x smaller than the bare binary). Any
+    // failure in the compressed path falls back to the bare download below.
+    let size: number | undefined;
+    if (entry.compressed !== undefined && typeof createZstdDecompress === 'function') {
+      const zstPartPath = join(stagingDir, `${exeFileName}.zst.part`);
+      try {
+        await downloadAndHash(
+          nativeBinaryUrl(options.version, entry.compressed.filename),
+          zstPartPath,
+          entry.compressed.checksum,
+          fetchImpl,
+          options.onProgress,
+          options.idleTimeoutMs,
+        );
+        size = await decompressAndHash(zstPartPath, partPath, entry.checksum);
+        await rm(zstPartPath, { force: true });
+      } catch (error) {
+        console.warn(
+          `[update] compressed artifact unavailable, falling back to uncompressed download: ${error instanceof Error ? error.message : String(error)}`,
+        );
+        await rm(zstPartPath, { force: true }).catch(() => {});
+        await rm(partPath, { force: true }).catch(() => {});
+      }
+    }
+    size ??= await downloadAndHash(
       nativeBinaryUrl(options.version, entry.filename),
       partPath,
       entry.checksum,

apps/kimi-code/test/cli/update/native-manifest.test.ts

@@ -59,6 +59,24 @@ describe('fetchNativeReleaseManifest', () => {
     expect(manifest.version).toBe(VERSION);
   });
 
+  it('parses a platform entry carrying a compressed artifact pointer', async () => {
+    const body = JSON.stringify({
+      version: VERSION,
+      platforms: {
+        'linux-x64': {
+          filename: 'kimi-code-linux-x64',
+          checksum: 'a'.repeat(64),
+          compressed: { filename: 'kimi-code-linux-x64.zst', checksum: 'b'.repeat(64) },
+        },
+      },
+    });
+    const manifest = await fetchNativeReleaseManifest(VERSION, mockFetch({ ok: true, status: 200, body }));
+    expect(manifest.platforms['linux-x64']?.compressed).toEqual({
+      filename: 'kimi-code-linux-x64.zst',
+      checksum: 'b'.repeat(64),
+    });
+  });
+
   it('rejects a non-semver version argument before hitting the network', async () => {
     const f = mockFetch({ ok: true, status: 200, body: MANIFEST_BODY });
     await expect(fetchNativeReleaseManifest('nope', f)).rejects.toThrow(/invalid semver/);