diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 17926becd..6654830ee 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -57,12 +57,19 @@ jobs: - name: Prepare for publishing uses: ./.github/actions/publish-prepare - name: Publish packages + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Build common flags ARGS="" if [ "${{ inputs.dry-run }}" = "true" ]; then ARGS="$ARGS --dry-run" fi + # Enable github-release by default for non-workflow_dispatch triggers + # or when explicitly set to true in workflow_dispatch + if [ "${{ github.event_name }}" != "workflow_dispatch" ] || [ "${{ inputs.github-release }}" = "true" ]; then + ARGS="$ARGS --github-release" + fi pnpm code-infra publish-canary $ARGS diff --git a/packages/code-infra/src/cli/cmdPublishCanary.mjs b/packages/code-infra/src/cli/cmdPublishCanary.mjs index efdefdd8d..24e0bdf72 100644 --- a/packages/code-infra/src/cli/cmdPublishCanary.mjs +++ b/packages/code-infra/src/cli/cmdPublishCanary.mjs @@ -8,26 +8,390 @@ * @typedef {import('../utils/pnpm.mjs').PublishOptions} PublishOptions */ +import path from 'node:path'; +import { createActionAuth } from '@octokit/auth-action'; +import { Octokit } from '@octokit/rest'; import { $ } from 'execa'; +import gitUrlParse from 'git-url-parse'; import * as semver from 'semver'; -/** - * @typedef {Object} Args - * @property {boolean} [dryRun] - Whether to run in dry-run mode - */ - import { - getWorkspacePackages, + getCurrentGitSha, getPackageVersionInfo, + getWorkspacePackages, publishPackages, readPackageJson, - writePackageJson, - getCurrentGitSha, semverMax, + writePackageJson, } from '../utils/pnpm.mjs'; +/** + * @typedef {Object} Args + * @property {boolean} [dryRun] - Whether to run in dry-run mode + * @property {boolean} [githubRelease] - Whether to create GitHub releases for canary packages + */ + const CANARY_TAG = 'canary'; +/** + * Get Octokit instance with authentication + * @returns {Octokit} Authenticated Octokit instance + */ +function getOctokit() { + return new Octokit({ authStrategy: createActionAuth }); +} + +/** + * Get current repository info from git remote + * @returns {Promise<{owner: string, repo: string}>} Repository owner and name + */ +async function getRepositoryInfo() { + try { + const result = await $`git remote get-url origin`; + const url = result.stdout.trim(); + + const parsed = gitUrlParse(url); + if (parsed.source !== 'github.com') { + throw new Error('Repository is not hosted on GitHub'); + } + + return { + owner: parsed.owner, + repo: parsed.name, + }; + } catch (/** @type {any} */ error) { + throw new Error(`Failed to get repository info: ${error.message}`); + } +} + +/** + * @typedef {Object} Commit + * @property {string} sha - Commit SHA + * @property {string} message - Commit message + * @property {string} author - Commit author + */ + +/** + * @param {Object} param0 + * @param {string} param0.packagePath + * @returns {Promise} Commits between the tag and current HEAD for the package + */ +async function fetchCommitsForPackage({ packagePath }) { + /** + * @type {Commit[]} + */ + const results = []; + const fieldSeparator = '\u001f'; // ASCII unit separator is extremely unlikely to appear in git metadata + const formatArg = '--format=%H%x1f%s%x1f%an%x1f%ae'; // SHA, subject, author name, author email separated by unit separator + const res = $`git log --oneline --no-decorate ${ + // to avoid escaping by execa + [formatArg] + } ${CANARY_TAG}..HEAD -- ${packagePath}`; + for await (const line of res) { + const commitLine = line.trimEnd(); + if (!commitLine) { + continue; + } + const parts = commitLine.split(fieldSeparator); + if (parts.length < 3) { + console.error(`Failed to parse commit log line: ${commitLine}`); + continue; + } + const [sha, message, commitAuthor, commitEmail] = parts; + let author = commitAuthor; + // try to get github username from email + if (commitEmail) { + const emailUsername = commitEmail.split('@')[0]; + if (emailUsername) { + const [, githubUserName] = emailUsername.split('+'); + if (githubUserName) { + author = `@${githubUserName}`; + } + } + } + results.push({ sha, message, author }); + } + return results; +} + +const AUTHOR_EXCLUDE_LIST = ['renovate[bot]', 'dependabot[bot]']; + +/** + * @param {string} message + * @returns {string} + */ +function cleanupCommitMessage(message) { + // AI generated: clean up commit message by removing leading bracketed tokens except [breaking] + let msg = message || ''; + + // Extract and remove leading bracketed tokens like "[foo][bar] message" + const tokens = []; + const bracketRe = /^\s*\[([^\]]+)\]\s*/; + let match = msg.match(bracketRe); + while (match) { + tokens.push(match[1]); + msg = msg.slice(match[0].length); + match = msg.match(bracketRe); + } + msg = msg.trim(); + + // If any of the leading tokens is "breaking" keep that token (preserve original casing) + const breakingToken = tokens.find((t) => t.toLowerCase() === 'breaking'); + const prefix = breakingToken ? `[${breakingToken}]${msg ? ' ' : ''}` : ''; + + return `${prefix}${msg}`.trim(); +} + +async function getPackageToDependencyMap() { + /** + * @type {(PublicPackage & { dependencies: Record; private: boolean; })[]} + */ + const packagesWithDeps = JSON.parse( + (await $`pnpm ls -r --json --exclude-peers --only-projects --prod`).stdout, + ); + /** @type {Record} */ + const directPkgDependencies = packagesWithDeps + .filter((pkg) => !pkg.private) + .reduce((acc, pkg) => { + if (!pkg.name) { + return acc; + } + const deps = Object.keys(pkg.dependencies || {}); + if (!deps.length) { + return acc; + } + acc[pkg.name] = deps; + return acc; + }, /** @type {Record} */ ({})); + return directPkgDependencies; +} + +/** + * @param {Record} pkgGraph + */ +function resolveTransitiveDependencies(pkgGraph = {}) { + // Compute transitive (nested) dependencies limited to workspace packages and avoid cycles. + const workspacePkgNames = new Set(Object.keys(pkgGraph)); + const nestedMap = /** @type {Record} */ ({}); + + /** + * + * @param {string} pkgName + * @returns {string[]} + */ + const getTransitiveDeps = (pkgName) => { + /** + * @type {Set} + */ + const seen = new Set(); + const stack = (pkgGraph[pkgName] || []).slice(); + + while (stack.length) { + const dep = stack.pop(); + if (!dep || seen.has(dep)) { + continue; + } + // Only consider workspace packages for transitive expansion + if (!workspacePkgNames.has(dep)) { + // still record external deps as direct deps but don't traverse into them + seen.add(dep); + continue; + } + seen.add(dep); + const children = pkgGraph[dep] || []; + for (const c of children) { + if (!seen.has(c)) { + stack.push(c); + } + } + } + + return Array.from(seen); + }; + + for (const name of Object.keys(pkgGraph)) { + nestedMap[name] = getTransitiveDeps(name); + } + + return nestedMap; +} + +/** + * Prepare changelog data for packages using GitHub API + * @param {PublicPackage[]} packagesToPublish - Packages that will be published + * @param {PublicPackage[]} allPackages - All packages in the repository + * @param {Map} canaryVersions - Map of package names to their canary versions + * @returns {Promise>} Map of package names to their changelogs + */ +async function prepareChangelogsFromGitCli(packagesToPublish, allPackages, canaryVersions) { + /** + * @type {Map} + */ + const changelogs = new Map(); + + await Promise.all( + packagesToPublish.map(async (pkg) => { + const commits = await fetchCommitsForPackage({ packagePath: pkg.path }); + if (commits.length > 0) { + console.log(`Found ${commits.length} commits for package ${pkg.name}`); + } + const changeLogStrs = commits + // Exclude commits authored by bots + .filter( + // We want to allow commits from copilot or other AI tools, so only filter known bots + (commit) => !AUTHOR_EXCLUDE_LIST.includes(commit.author), + ) + .map((commit) => `- ${cleanupCommitMessage(commit.message)} by ${commit.author}`); + + if (changeLogStrs.length > 0) { + changelogs.set(pkg.name, changeLogStrs); + } + }), + ); + // Second pass: check for dependency updates in other packages not part of git history + const pkgDependencies = await getPackageToDependencyMap(); + const transitiveDependencies = resolveTransitiveDependencies(pkgDependencies); + + for (let i = 0; i < allPackages.length; i += 1) { + const pkg = allPackages[i]; + const depsToPublish = (transitiveDependencies[pkg.name] ?? []).filter((dep) => + packagesToPublish.some((p) => p.name === dep), + ); + if (depsToPublish.length === 0) { + continue; + } + const changelog = changelogs.get(pkg.name) ?? []; + changelog.push('- Updated dependencies:'); + depsToPublish.forEach((dep) => { + const depVersion = canaryVersions.get(dep); + if (depVersion) { + changelog.push(` - Bumped \`${dep}@${depVersion}\``); + } + }); + } + return changelogs; +} + +/** + * Prepare changelog data for packages + * @param {PublicPackage[]} packagesToPublish - Packages that will be published + * @param {PublicPackage[]} allPackages - All packages in the repository + * @param {Map} canaryVersions - Map of package names to their canary versions + * @returns {Promise>} Map of package names to their changelogs + */ +async function prepareChangelogsForPackages(packagesToPublish, allPackages, canaryVersions) { + console.log('\n๐Ÿ“ Preparing changelogs for packages...'); + + const repoInfo = await getRepositoryInfo(); + console.log(`๐Ÿ“‚ Repository: ${repoInfo.owner}/${repoInfo.repo}`); + + /** + * @type {Map} + */ + const changelogs = await prepareChangelogsFromGitCli( + packagesToPublish, + allPackages, + canaryVersions, + ); + + // Log changelog content for each package + for (const pkg of packagesToPublish) { + const version = canaryVersions.get(pkg.name); + if (!version) { + continue; + } + + const changelog = changelogs.get(pkg.name) || []; + console.log(`\n๐Ÿ“ฆ ${pkg.name}@${version}`); + if (changelog.length > 0) { + console.log( + ` Changelog:\n${changelog.map((/** @type {string} */ line) => ` ${line}`).join('\n')}`, + ); + } + } + + console.log('\nโœ… Changelogs prepared successfully'); + return changelogs; +} + +/** + * Create GitHub releases and tags for published packages + * @param {PublicPackage[]} publishedPackages - Packages that were published + * @param {Map} canaryVersions - Map of package names to their canary versions + * @param {Map} changelogs - Map of package names to their changelogs + * @param {{dryRun?: boolean}} options - Publishing options + * @returns {Promise} + */ +async function createGitHubReleasesForPackages( + publishedPackages, + canaryVersions, + changelogs, + options, +) { + console.log('\n๐Ÿš€ Creating GitHub releases and tags for published packages...'); + + const repoInfo = await getRepositoryInfo(); + const gitSha = await getCurrentGitSha(); + const octokit = getOctokit(); + + for (const pkg of publishedPackages) { + const version = canaryVersions.get(pkg.name); + if (!version) { + console.log(`โš ๏ธ No version found for ${pkg.name}, skipping...`); + continue; + } + + const changelog = changelogs.get(pkg.name); + if (!changelog) { + console.log(`โš ๏ธ No changelog found for ${pkg.name}, skipping release creation...`); + continue; + } + const tagName = `${pkg.name}@${version}`; + const releaseName = tagName; + + console.log(`\n๐Ÿ“ฆ Processing ${pkg.name}@${version}...`); + + // Create git tag + if (options.dryRun) { + console.log(`๐Ÿท๏ธ Would create and push git tag: ${tagName} (dry-run)`); + console.log(`๐Ÿ“ Would publish a Github release:`); + console.log(` - Name: ${releaseName}`); + console.log(` - Tag: ${tagName}`); + console.log(` - Body:\n${changelog.join('\n')}`); + } else { + // eslint-disable-next-line no-await-in-loop + await $({ + env: { + ...process.env, + GIT_COMMITTER_NAME: 'Code infra', + GIT_COMMITTER_EMAIL: 'code-infra@mui.com', + }, + })`git tag -a ${tagName} -m ${`Canary release ${pkg.name}@${version}`}`; + + // eslint-disable-next-line no-await-in-loop + await $`git push origin ${tagName}`; + console.log(`โœ… Created and pushed git tag: ${tagName}`); + + // Create GitHub release + // eslint-disable-next-line no-await-in-loop + const res = await octokit.repos.createRelease({ + owner: repoInfo.owner, + repo: repoInfo.repo, + tag_name: tagName, + target_commitish: gitSha, + name: releaseName, + body: changelog.join('\n'), + draft: false, + prerelease: true, // Mark as prerelease since these are canary versions + }); + + console.log(`โœ… Created GitHub release: ${releaseName} at ${res.data.html_url}`); + } + } + + console.log('\nโœ… Finished creating GitHub releases'); +} + /** * Check if the canary git tag exists * @returns {Promise} Canary tag name if exists, null otherwise @@ -66,11 +430,13 @@ async function createCanaryTag(dryRun = false) { } /** - * Publish canary versions with updated dependencies + * Publish canary versions with updated dependencies. A big assumption here is that + * all packages are already built before calling this function. + * * @param {PublicPackage[]} packagesToPublish - Packages that need canary publishing * @param {PublicPackage[]} allPackages - All workspace packages * @param {Map} packageVersionInfo - Version info map - * @param {PublishOptions} [options={}] - Publishing options + * @param {PublishOptions & {githubRelease?: boolean}} [options={}] - Publishing options * @returns {Promise} */ async function publishCanaryVersions( @@ -90,7 +456,6 @@ async function publishCanaryVersions( const gitSha = await getCurrentGitSha(); const canaryVersions = new Map(); - const originalPackageJsons = new Map(); // First pass: determine canary version numbers for all packages const changedPackageNames = new Set(packagesToPublish.map((pkg) => pkg.name)); @@ -117,41 +482,44 @@ async function publishCanaryVersions( } // Second pass: read and update ALL package.json files in parallel + // Packages are already built at this point. const packageUpdatePromises = allPackages.map(async (pkg) => { - const originalPackageJson = await readPackageJson(pkg.path); + let basePkgJson = await readPackageJson(pkg.path); + let pkgJsonDirectory = pkg.path; + if (basePkgJson.publishConfig?.directory) { + pkgJsonDirectory = path.join(pkg.path, basePkgJson.publishConfig.directory); + basePkgJson = await readPackageJson(pkgJsonDirectory); + } const canaryVersion = canaryVersions.get(pkg.name); if (canaryVersion) { const updatedPackageJson = { - ...originalPackageJson, + ...basePkgJson, version: canaryVersion, gitSha, }; - - await writePackageJson(pkg.path, updatedPackageJson); + await writePackageJson(pkgJsonDirectory, updatedPackageJson); console.log(`๐Ÿ“ Updated ${pkg.name} package.json to ${canaryVersion}`); } - - return { pkg, originalPackageJson }; + return { pkg, basePkgJson, pkgJsonDirectory }; }); const updateResults = await Promise.all(packageUpdatePromises); - // Build the original package.json map - for (const { pkg, originalPackageJson } of updateResults) { - originalPackageJsons.set(pkg.name, originalPackageJson); + // Prepare changelogs before building and publishing (so it can error out early if there are issues) + /** + * @type {Map} + */ + let changelogs = new Map(); + if (options.githubRelease) { + changelogs = await prepareChangelogsForPackages(packagesToPublish, allPackages, canaryVersions); } - // Run release build after updating package.json files - console.log('\n๐Ÿ”จ Running release build...'); - await $({ stdio: 'inherit' })`pnpm release:build`; - console.log('โœ… Release build completed successfully'); - // Third pass: publish only the changed packages using recursive publish let publishSuccess = false; try { console.log(`๐Ÿ“ค Publishing ${packagesToPublish.length} canary versions...`); - await publishPackages(packagesToPublish, { ...options, noGitChecks: true, tag: 'canary' }); + await publishPackages(packagesToPublish, { ...options, noGitChecks: true, tag: CANARY_TAG }); packagesToPublish.forEach((pkg) => { const canaryVersion = canaryVersions.get(pkg.name); @@ -161,9 +529,11 @@ async function publishCanaryVersions( } finally { // Always restore original package.json files in parallel console.log('\n๐Ÿ”„ Restoring original package.json files...'); - const restorePromises = allPackages.map(async (pkg) => { - const originalPackageJson = originalPackageJsons.get(pkg.name); - await writePackageJson(pkg.path, originalPackageJson); + const restorePromises = updateResults.map(async ({ pkg, basePkgJson, pkgJsonDirectory }) => { + // no need to restore package.json files in build directories + if (pkgJsonDirectory === pkg.path) { + await writePackageJson(pkg.path, basePkgJson); + } }); await Promise.all(restorePromises); @@ -172,6 +542,12 @@ async function publishCanaryVersions( if (publishSuccess) { // Create/update the canary tag after successful publish await createCanaryTag(options.dryRun); + + // Create GitHub releases if requested + if (options.githubRelease) { + await createGitHubReleasesForPackages(packagesToPublish, canaryVersions, changelogs, options); + } + console.log('\n๐ŸŽ‰ All canary versions published successfully!'); } } @@ -180,21 +556,31 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ command: 'publish-canary', describe: 'Publish canary packages to npm', builder: (yargs) => { - return yargs.option('dry-run', { - type: 'boolean', - default: false, - description: 'Run in dry-run mode without publishing', - }); + return yargs + .option('dry-run', { + type: 'boolean', + default: false, + description: 'Run in dry-run mode without publishing', + }) + .option('github-release', { + type: 'boolean', + default: false, + description: 'Create GitHub releases for published packages', + }); }, handler: async (argv) => { - const { dryRun = false } = argv; + const { dryRun = false, githubRelease = false } = argv; - const options = { dryRun }; + const options = { dryRun, githubRelease }; if (dryRun) { console.log('๐Ÿงช Running in DRY RUN mode - no actual publishing will occur\n'); } + if (githubRelease) { + console.log('๐Ÿ“ GitHub releases will be created for published packages\n'); + } + // Always get all packages first console.log('๐Ÿ” Discovering all workspace packages...'); const allPackages = await getWorkspacePackages({ publicOnly: true }); @@ -207,16 +593,12 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ // Check for canary tag to determine selective publishing const canaryTag = await getLastCanaryTag(); - console.log( - canaryTag - ? '๐Ÿ” Checking for packages changed since canary tag...' - : '๐Ÿ” No canary tag found, will publish all packages', - ); + console.log('๐Ÿ” Checking for packages changed since canary tag...'); const packages = canaryTag ? await getWorkspacePackages({ sinceRef: canaryTag, publicOnly: true }) : allPackages; - console.log(`๐Ÿ“‹ Found ${packages.length} packages that need canary publishing:`); + console.log(`๐Ÿ“‹ Found ${packages.length} packages(s) for canary publishing:`); packages.forEach((pkg) => { console.log(` โ€ข ${pkg.name}@${pkg.version}`); }); diff --git a/packages/code-infra/src/cli/packageJson.d.ts b/packages/code-infra/src/cli/packageJson.d.ts index 70cb30700..77fbe9268 100644 --- a/packages/code-infra/src/cli/packageJson.d.ts +++ b/packages/code-infra/src/cli/packageJson.d.ts @@ -708,6 +708,12 @@ declare namespace PackageJson { Default: `'latest'` */ tag?: string; + + /** + Specifies the directory to publish. + Default: The package root directory. + */ + directory?: string; }; } diff --git a/packages/code-infra/src/utils/changelog.mjs b/packages/code-infra/src/utils/changelog.mjs index 3da5f8a33..ba7622817 100644 --- a/packages/code-infra/src/utils/changelog.mjs +++ b/packages/code-infra/src/utils/changelog.mjs @@ -17,6 +17,7 @@ import { persistentAuthStrategy } from './github.mjs'; * @property {string} message * @property {string[]} labels * @property {number} prNumber + * @property {string} html_url * @property {{login: string; association: AuthorAssociation} | null} author */ @@ -122,6 +123,7 @@ async function fetchCommitsRest({ octokit, repo, lastRelease, release, org = 'mu message: commit.commit.message, labels, prNumber, + html_url: pr.data.html_url, author: pr.data.user?.login ? { login: pr.data.user.login, diff --git a/packages/code-infra/src/utils/pnpm.mjs b/packages/code-infra/src/utils/pnpm.mjs index 181f9ac26..d3b99e552 100644 --- a/packages/code-infra/src/utils/pnpm.mjs +++ b/packages/code-infra/src/utils/pnpm.mjs @@ -175,7 +175,7 @@ export async function publishPackages(packages, options = {}) { /** * Read package.json from a directory * @param {string} packagePath - Path to package directory - * @returns {Promise} Parsed package.json content + * @returns {Promise} Parsed package.json content */ export async function readPackageJson(packagePath) { const content = await fs.readFile(path.join(packagePath, 'package.json'), 'utf8');