From 2217de4e768ef192103a8a9da63c0a89dad9282f Mon Sep 17 00:00:00 2001 From: Satya Patel Date: Mon, 4 May 2026 13:42:26 -0700 Subject: [PATCH] fix(cli): retry + cache prebuild downloads in build-dist; add docker test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cli-v0.2.4 darwin-arm64 release build failed with `curl: (56) ... 502` fetching the better-sqlite3 prebuild from GitHub Releases. The curl in build-dist.ts had no retry, so a single transient 502 brick the entire matrix run. Two changes: 1. Wrap all curl invocations in `curlDownload()` with `--retry 6 --retry-delay 2 --retry-all-errors` (GitHub Releases 5xx counts as a retryable error only with this flag), tight connect/max timeouts, and a `.partial → rename` atomic write so a half-finished download can't masquerade as a cache hit on the next run. Also cache the better-sqlite3 prebuild on disk like the Node tarball already is, keyed by version + ABI + target — so re-running the build after a partial failure is free. 2. Add `packages/cli/scripts/build-dist-linux-docker.sh` to reproduce the GH Actions Linux build flow (matches `.github/workflows/build-cli.yml`) inside `oven/bun:<.bun-version>`, with both linux-x64 and linux-arm64 targets. Runs the same install + npm rebuild + build-dist + smoke-test sequence CI runs, so we can validate workflow changes locally instead of burning release tag pushes. Verified locally on darwin-arm64 — full build + smoke test (`require('better-sqlite3')` etc.) passes; cache hit confirmed on second run. --- .../cli/scripts/build-dist-linux-docker.sh | 76 +++++++++++++++++++ packages/cli/scripts/build-dist.ts | 51 +++++++++++-- 2 files changed, 121 insertions(+), 6 deletions(-) create mode 100755 packages/cli/scripts/build-dist-linux-docker.sh diff --git a/packages/cli/scripts/build-dist-linux-docker.sh b/packages/cli/scripts/build-dist-linux-docker.sh new file mode 100755 index 00000000000..2adcf75d109 --- /dev/null +++ b/packages/cli/scripts/build-dist-linux-docker.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Reproduce the GitHub Actions Linux CLI build inside a Docker container. +# Mirrors `.github/workflows/build-cli.yml` so we can validate the full +# install + build + smoke-test flow without cutting a release. +# +# Usage: +# packages/cli/scripts/build-dist-linux-docker.sh [linux-x64|linux-arm64] +# +# Outputs the tarball at packages/cli/dist/superset-.tar.gz inside +# the container's copy of the repo and runs the same require() smoke test +# the CI workflow runs. +set -euo pipefail + +TARGET="${1:-linux-x64}" +case "$TARGET" in + linux-x64) PLATFORM="linux/amd64"; NODE_ARCH="x64" ;; + linux-arm64) PLATFORM="linux/arm64"; NODE_ARCH="arm64" ;; + *) echo "Usage: $0 [linux-x64|linux-arm64]" >&2; exit 1 ;; +esac + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +BUN_VERSION="$(cat "$REPO_ROOT/.bun-version")" +NODE_VERSION="22.22.2" + +echo "[docker-build] target=$TARGET platform=$PLATFORM bun=$BUN_VERSION node=$NODE_VERSION" +echo "[docker-build] repo: $REPO_ROOT" + +# Mount the repo read-only and copy it into a writable workdir inside the +# container so the host's darwin-arm64 node_modules don't bleed in. The +# container does its own `bun install` against the lockfile. +docker run --rm --platform "$PLATFORM" \ + -v "$REPO_ROOT:/host:ro" \ + -e TARGET="$TARGET" \ + -e NODE_ARCH="$NODE_ARCH" \ + -e NODE_VERSION="$NODE_VERSION" \ + -e RELAY_URL="${RELAY_URL:-https://relay.superset.sh}" \ + -e SUPERSET_API_URL="${SUPERSET_API_URL:-https://api.superset.sh}" \ + -e SUPERSET_WEB_URL="${SUPERSET_WEB_URL:-https://app.superset.sh}" \ + "oven/bun:${BUN_VERSION}" bash -euxc ' + apt-get update -qq + apt-get install -y --no-install-recommends \ + curl python3 make g++ ca-certificates xz-utils rsync >/dev/null + + curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 + node --version + bun --version + + rsync -a --exclude=node_modules --exclude=dist --exclude=.next /host/ /work/ + cd /work + + # Mirrors `.github/workflows/build-cli.yml` Linux install step. + # Bun occasionally hits transient integrity-check failures on cold caches + # in Docker, retry once before giving up. + bun install --frozen --ignore-scripts || \ + (rm -rf ~/.bun/install/cache && bun install --frozen --ignore-scripts) + PTY_DIR=$(ls -d node_modules/.bun/node-pty@*/node_modules/node-pty) + (cd "$PTY_DIR" && npx --yes node-gyp rebuild) + npm rebuild @parcel/watcher + + cd packages/cli + bun run build:dist --target="$TARGET" + + DIST="./dist/superset-${TARGET}" + "$DIST/bin/superset" --version + "$DIST/bin/superset" --help | head -5 + "$DIST/lib/node" --version + NODE_PATH="$DIST/lib/node_modules" "$DIST/lib/node" -e " + for (const m of [\"better-sqlite3\", \"node-pty\", \"@parcel/watcher\", \"libsql\"]) { + require(m); + console.log(m, \"OK\"); + } + " + echo "[docker-build] tarball: $(ls -la dist/superset-${TARGET}.tar.gz)" + ' diff --git a/packages/cli/scripts/build-dist.ts b/packages/cli/scripts/build-dist.ts index d001f82ddf5..a454411e7e8 100644 --- a/packages/cli/scripts/build-dist.ts +++ b/packages/cli/scripts/build-dist.ts @@ -27,6 +27,7 @@ import { readdirSync, readFileSync, realpathSync, + renameSync, rmSync, writeFileSync, } from "node:fs"; @@ -137,6 +138,35 @@ async function exec(cmd: string, args: string[], cwd?: string): Promise { }); } +/** + * curl wrapper that retries on network/HTTP flake (GitHub Releases 5xx, + * connection resets, etc). Writes atomically: download to a .partial + * sibling first, then rename — so a previous half-written file can't be + * mistaken for a cache hit on the next run. `--retry-all-errors` covers + * 5xx as well as transport errors; without it curl only retries a small + * subset by default. + */ +async function curlDownload(url: string, destPath: string): Promise { + const partial = `${destPath}.partial`; + rmSync(partial, { force: true }); + await exec("curl", [ + "-fsSL", + "--retry", + "6", + "--retry-delay", + "2", + "--retry-all-errors", + "--connect-timeout", + "15", + "--max-time", + "180", + "-o", + partial, + url, + ]); + renameSync(partial, destPath); +} + async function downloadAndExtractNode( target: Target, destDir: string, @@ -150,7 +180,7 @@ async function downloadAndExtractNode( if (!existsSync(archivePath)) { console.log(`[build-dist] downloading ${nodeDownloadUrl(target)}`); - await exec("curl", ["-fsSL", "-o", archivePath, nodeDownloadUrl(target)]); + await curlDownload(nodeDownloadUrl(target), archivePath); } if (!existsSync(extractedPath)) { @@ -282,13 +312,22 @@ async function fixNativeBinariesForNode( const bsqUrl = `https://github.com/WiseLibs/better-sqlite3/releases/download/` + `v${bsqVersion}/better-sqlite3-v${bsqVersion}-node-v${NODE_ABI}-${target}.tar.gz`; - console.log(`[build-dist] fetching Node-ABI better-sqlite3: ${bsqUrl}`); - const tmp = join(homedir(), ".superset-build-cache", `bsq-${target}`); + const cacheDir = join(homedir(), ".superset-build-cache"); + if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true }); + const cachedTarball = join( + cacheDir, + `better-sqlite3-v${bsqVersion}-node-v${NODE_ABI}-${target}.tar.gz`, + ); + if (!existsSync(cachedTarball)) { + console.log(`[build-dist] fetching Node-ABI better-sqlite3: ${bsqUrl}`); + await curlDownload(bsqUrl, cachedTarball); + } else { + console.log(`[build-dist] using cached better-sqlite3: ${cachedTarball}`); + } + const tmp = join(cacheDir, `bsq-${target}`); rmSync(tmp, { recursive: true, force: true }); mkdirSync(tmp, { recursive: true }); - const tarball = join(tmp, "bsq.tar.gz"); - await exec("curl", ["-fsSL", "-o", tarball, bsqUrl]); - await exec("tar", ["-xzf", tarball, "-C", tmp]); + await exec("tar", ["-xzf", cachedTarball, "-C", tmp]); rmSync(bsqDest, { recursive: true, force: true }); mkdirSync(bsqDest, { recursive: true }); cpSync(