From 4da087280c40dff6b3eb407883682f126127b9e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Caleb=20=E3=83=84=20Everett?= Date: Mon, 15 Nov 2021 13:35:01 -0800 Subject: [PATCH] feat: new npm copy command --- lib/commands/copy.js | 134 +++++++++++ lib/utils/cmd-list.js | 2 + .../tap-snapshots/test/index.js.test.cjs | 3 +- .../test/lib/commands/completion.js.test.cjs | 2 + .../test/lib/load-all-commands.js.test.cjs | 16 ++ .../test/lib/utils/cmd-list.js.test.cjs | 5 + .../test/lib/utils/npm-usage.js.test.cjs | 26 ++- test/lib/commands/copy.js | 221 ++++++++++++++++++ 8 files changed, 404 insertions(+), 5 deletions(-) create mode 100644 lib/commands/copy.js create mode 100644 test/lib/commands/copy.js diff --git a/lib/commands/copy.js b/lib/commands/copy.js new file mode 100644 index 0000000000000..a0ce04061f4ce --- /dev/null +++ b/lib/commands/copy.js @@ -0,0 +1,134 @@ +const Arborist = require('@npmcli/arborist') +const { join, relative, dirname } = require('path') +const packlist = require('npm-packlist') +const fs = require('@npmcli/fs') + +const BaseCommand = require('../base-command.js') + +class Copy extends BaseCommand { + static description = 'Copy package to new location' + + static name = 'copy' + + static params = [ + 'omit', + 'workspace', + 'workspaces', + 'include-workspace-root', + ] + + static ignoreImplicitWorkspace = false + + static usage = [''] + + async exec (args) { + await this.copyTo(args, true, new Set([])) + } + + // called when --workspace or --workspaces is passed. + async execWorkspaces (args, filters) { + await this.setWorkspaces(filters) + + await this.copyTo( + args, + this.includeWorkspaceRoot, + new Set(this.workspacePaths)) + } + + async copyTo (args, includeWorkspaceRoot, workspaces) { + if (args.length !== 1) { + throw this.usageError('Missing required destination argument') + } + const opts = { + ...this.npm.flatOptions, + path: this.npm.localPrefix, + log: this.npm.log, + } + const destination = args[0] + const omit = new Set(this.npm.flatOptions.omit) + + const tree = await new Arborist(opts).loadActual() + + // map of node to location in destination. + const destinations = new Map() + + // calculate the root set of packages. + if (includeWorkspaceRoot) { + const to = join(destination, tree.location) + destinations.set(tree, to) + } + for (const edge of tree.edgesOut.values()) { + if (edge.workspace && workspaces.has(edge.to.realpath)) { + const to = join(destination, edge.to.location) + destinations.set(edge.to, to) + } + } + + // copy the root set of packages and their dependencies. + for (const [node, dest] of destinations) { + if (node.isLink && node.target) { + const targetPath = destinations.get(node.target) + if (targetPath == null) { + // This is the first time the link target was seen, it will be the + // only copy in dest, other links to the same target will link to + // this copy. + destinations.set(node.target, dest) + } else { + // The link target is already in the destination + await relativeSymlink(targetPath, dest) + } + } else { + if (node.isWorkspace || node.isRoot) { + // workspace and root packages have not been published so they may + // have files that should be excluded. + await copyPacklist(node.target.realpath, dest) + } else { + // copy the modules files but not dependencies. + const nm = join(node.realpath, 'node_modules') + await fs.cp(node.realpath, dest, { + recursive: true, + errorOnExist: false, + filter: src => src !== nm, + }) + } + + // add dependency edges to the queue. + for (const edge of node.edgesOut.values()) { + if (!omit.has(edge.type) && edge.to != null) { + destinations.set( + edge.to, + join( + destinations.get(edge.to.parent) || destination, + relative(edge.to.parent.location, edge.to.location))) + } + } + } + } + } +} +module.exports = Copy + +async function copyPacklist (from, to) { + for (const file of await packlist({ path: from })) { + // packlist will include bundled node_modules. ignore it because we're + // already handling copying dependencies. + if (file.startsWith('node_modules/')) { + continue + } + + // using recursive copy because packlist doesn't list directories. + // TODO what is npm's preferred recursive copy? + await fs.cp( + join(from, file), + join(to, file), + { recursive: true, errorOnExist: false }) + } +} + +async function relativeSymlink (target, path) { + await fs.mkdir(dirname(path), { recursive: true }) + await fs.symlink( + './' + relative(dirname(path), target), + path // link to create + ) +} diff --git a/lib/utils/cmd-list.js b/lib/utils/cmd-list.js index c1d20186a82a6..a6f95f10b5ec3 100644 --- a/lib/utils/cmd-list.js +++ b/lib/utils/cmd-list.js @@ -40,6 +40,7 @@ const aliases = { x: 'exec', why: 'explain', la: 'll', + cp: 'copy', verison: 'version', ic: 'ci', @@ -137,6 +138,7 @@ const cmdList = [ 'version', 'view', 'whoami', + 'copy', ] const plumbing = ['birthday', 'help-search'] diff --git a/smoke-tests/tap-snapshots/test/index.js.test.cjs b/smoke-tests/tap-snapshots/test/index.js.test.cjs index 5a13b4e9a5c20..b60826ea4f928 100644 --- a/smoke-tests/tap-snapshots/test/index.js.test.cjs +++ b/smoke-tests/tap-snapshots/test/index.js.test.cjs @@ -29,7 +29,8 @@ All commands: pkg, prefix, profile, prune, publish, rebuild, repo, restart, root, run-script, search, set, set-script, shrinkwrap, star, stars, start, stop, team, test, token, - uninstall, unpublish, unstar, update, version, view, whoami + uninstall, unpublish, unstar, update, version, view, whoami, + copy Specify configs in the ini-formatted file: {CWD}/smoke-tests/test/tap-testdir-index/.npmrc diff --git a/tap-snapshots/test/lib/commands/completion.js.test.cjs b/tap-snapshots/test/lib/commands/completion.js.test.cjs index 232cfec669778..c89a4daa6dc38 100644 --- a/tap-snapshots/test/lib/commands/completion.js.test.cjs +++ b/tap-snapshots/test/lib/commands/completion.js.test.cjs @@ -110,6 +110,7 @@ Array [ version view whoami + copy login author home @@ -144,6 +145,7 @@ Array [ x why la + cp verison ic innit diff --git a/tap-snapshots/test/lib/load-all-commands.js.test.cjs b/tap-snapshots/test/lib/load-all-commands.js.test.cjs index cd8b0592c36e8..5de7445fca9af 100644 --- a/tap-snapshots/test/lib/load-all-commands.js.test.cjs +++ b/tap-snapshots/test/lib/load-all-commands.js.test.cjs @@ -153,6 +153,22 @@ alias: c Run "npm help config" for more info ` +exports[`test/lib/load-all-commands.js TAP load each command copy > must match snapshot 1`] = ` +Copy package to new location + +Usage: +npm copy + +Options: +[--omit [--omit ...]] +[-w|--workspace [-w|--workspace ...]] +[-ws|--workspaces] [--include-workspace-root] + +alias: cp + +Run "npm help copy" for more info +` + exports[`test/lib/load-all-commands.js TAP load each command dedupe > must match snapshot 1`] = ` Reduce duplication in the package tree diff --git a/tap-snapshots/test/lib/utils/cmd-list.js.test.cjs b/tap-snapshots/test/lib/utils/cmd-list.js.test.cjs index 9413f8e9a6d52..6b0a1c9068220 100644 --- a/tap-snapshots/test/lib/utils/cmd-list.js.test.cjs +++ b/tap-snapshots/test/lib/utils/cmd-list.js.test.cjs @@ -60,6 +60,9 @@ Object { "conf": "config", "confi": "config", "config": "config", + "cop": "copy", + "copy": "copy", + "cp": "cp", "cr": "create", "cre": "create", "crea": "create", @@ -358,6 +361,7 @@ Object { "cit": "install-ci-test", "clean-install": "ci", "clean-install-test": "cit", + "cp": "copy", "create": "init", "ddp": "dedupe", "dist-tags": "dist-tag", @@ -476,6 +480,7 @@ Object { "version", "view", "whoami", + "copy", ], "plumbing": Array [ "birthday", diff --git a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs index fef4cc65edc65..29025791be40e 100644 --- a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs +++ b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs @@ -29,7 +29,8 @@ All commands: pkg, prefix, profile, prune, publish, rebuild, repo, restart, root, run-script, search, set, set-script, shrinkwrap, star, stars, start, stop, team, test, token, - uninstall, unpublish, unstar, update, version, view, whoami + uninstall, unpublish, unstar, update, version, view, whoami, + copy Specify configs in the ini-formatted file: /some/config/file/.npmrc @@ -65,7 +66,8 @@ All commands: pkg, prefix, profile, prune, publish, rebuild, repo, restart, root, run-script, search, set, set-script, shrinkwrap, star, stars, start, stop, team, test, token, - uninstall, unpublish, unstar, update, version, view, whoami + uninstall, unpublish, unstar, update, version, view, whoami, + copy Specify configs in the ini-formatted file: /some/config/file/.npmrc @@ -101,7 +103,8 @@ All commands: pkg, prefix, profile, prune, publish, rebuild, repo, restart, root, run-script, search, set, set-script, shrinkwrap, star, stars, start, stop, team, test, token, - uninstall, unpublish, unstar, update, version, view, whoami + uninstall, unpublish, unstar, update, version, view, whoami, + copy Specify configs in the ini-formatted file: /some/config/file/.npmrc @@ -137,7 +140,8 @@ All commands: pkg, prefix, profile, prune, publish, rebuild, repo, restart, root, run-script, search, set, set-script, shrinkwrap, star, stars, start, stop, team, test, token, - uninstall, unpublish, unstar, update, version, view, whoami + uninstall, unpublish, unstar, update, version, view, whoami, + copy Specify configs in the ini-formatted file: /some/config/file/.npmrc @@ -286,6 +290,20 @@ All commands: Run "npm help config" for more info + copy Copy package to new location + + Usage: + npm copy + + Options: + [--omit [--omit ...]] + [-w|--workspace [-w|--workspace ...]] + [-ws|--workspaces] [--include-workspace-root] + + alias: cp + + Run "npm help copy" for more info + dedupe Reduce duplication in the package tree Usage: diff --git a/test/lib/commands/copy.js b/test/lib/commands/copy.js new file mode 100644 index 0000000000000..234b851da6581 --- /dev/null +++ b/test/lib/commands/copy.js @@ -0,0 +1,221 @@ +const t = require('tap') +const { load } = require('../../fixtures/mock-npm') +const path = require('path') +const fs = require('fs') + +const cwd = process.cwd() +t.afterEach(t => process.chdir(cwd)) + +t.test('should copy module files to destination', async t => { + const { npm, outputs, logs } = await load(t, { + prefixDir: { + 'package.json': JSON.stringify({ + name: 'test-package', + version: '1.0.0', + files: ['lib'], + }), + 'README.md': 'file', + lib: { + 'index.js': '// empty', + }, + src: { + 'index.js': '// empty', + }, + }, + }) + process.chdir(npm.prefix) + await npm.exec('copy', ['build']) + t.strictSame(outputs, []) + t.strictSame(logs.notice, []) + + // lib is included in files, while package.json and README.md are always included. + assertExists(path.join('build', 'package.json')) + assertExists(path.join('build', 'README.md')) + assertExists(path.join('build', 'lib', 'index.js')) + + // src should not be copied because it's excluded by files. + assertMissing(path.join('build', 'src')) +}) + +t.test('should copy dependencies', async t => { + const { npm, outputs, logs } = await load(t, { + prefixDir: { + 'package.json': JSON.stringify({ + name: 'test-package', + version: '1.0.0', + dependencies: { + foo: '^1.0.0', + bar: '^1.0.0', + }, + }), + node_modules: { + foo: { }, + bar: { }, + baz: { }, + }, + }, + }) + process.chdir(npm.prefix) + await npm.exec('copy', ['build']) + t.strictSame(outputs, []) + t.strictSame(logs.notice, []) + + assertExists(path.join('build', 'node_modules', 'foo')) + assertExists(path.join('build', 'node_modules', 'bar')) + // baz is missing because it is an extraneous dep. + assertMissing(path.join('build', 'node_modules', 'baz')) +}) + +t.test('should not copy bundled dependencies if they are omitted', async t => { + const { npm, outputs, logs } = await load(t, { + prefixDir: { + 'package.json': JSON.stringify({ + name: 'test-package', + version: '1.0.0', + bundledDependencies: true, + optionalDependencies: { + foo: '^1.0.0', + }, + }), + node_modules: { + foo: { + 'package.json': JSON.stringify({ + name: 'foo', + version: '1.0.0', + }), + }, + }, + }, + }) + process.chdir(npm.prefix) + npm.config.set('omit', ['optional']) + await npm.exec('copy', ['build']) + t.strictSame(outputs, []) + t.strictSame(logs.notice, []) + + assertMissing(path.join('build', 'node_modules', 'foo')) +}) + +t.test('workspaces', async t => { + const fixture = { + prefixDir: { + 'package.json': JSON.stringify({ + name: 'workspace-root', + workspaces: [ + 'pkgs/a', + 'pkgs/b', + ], + }), + pkgs: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + version: '1.0.0', + }), + 'README.md': 'a', + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + version: '1.0.0', + dependencies: { c: '^1.0.0' }, + }), + 'README.md': 'b', + }, + }, + node_modules: { + a: t.fixture('symlink', '../pkgs/a'), + b: t.fixture('symlink', '../pkgs/b'), + c: t.fixture('symlink', '../pkgs/a'), + }, + }, + } + + t.test('should only copy included workspaces', async t => { + const { npm } = await load(t, fixture) + process.chdir(npm.prefix) + npm.config.set('workspace', ['a']) + await npm.exec('copy', ['build']) + + assertExists(path.join('build', 'node_modules', 'a')) + assertMissing(path.join('build', 'node_modules', 'b')) + assertMissing(path.join('build', 'package.json')) + }) + + t.test('should copy all included workspaces', async t => { + const { npm } = await load(t, fixture) + process.chdir(npm.prefix) + npm.config.set('workspace', ['a', 'b']) + await npm.exec('copy', ['build']) + + assertExists(path.join('build', 'node_modules', 'a')) + assertExists(path.join('build', 'node_modules', 'b')) + assertMissing(path.join('build', 'package.json')) + }) + + t.test('should copy workspace root', async t => { + const { npm } = await load(t, fixture) + process.chdir(npm.prefix) + npm.config.set('workspaces', true) + npm.config.set('include-workspace-root', true) + await npm.exec('copy', ['build']) + + assertExists(path.join('build', 'node_modules', 'a')) + assertExists(path.join('build', 'node_modules', 'b')) + assertExists(path.join('build', 'package.json')) + }) + + t.test('should copy symlinks once', async t => { + const { npm } = await load(t, fixture) + process.chdir(npm.prefix) + npm.config.set('workspaces', true) + await npm.exec('copy', ['build']) + const canonPath = path.join('build', 'node_modules', 'a') + const linkPath = path.join('build', 'node_modules', 'c') + + // if we stat the linkPath windows errors with EPERM, so I want to stat the + // link target, which means resolving relative links. + const linkDest = path.resolve( + path.dirname(linkPath), + fs.readlinkSync(linkPath)) + + t.strictSame( + fs.statSync(canonPath), + fs.statSync(linkDest), + `${linkPath} should be a link to ${canonPath}`) + }) +}) + +t.test('requires destination argument', async t => { + const { npm, outputs } = await load(t) + process.chdir(npm.prefix) + await t.rejects( + npm.exec('copy', []), + /Missing required destination argument/ + ) + t.strictSame(outputs, []) +}) + +function assertExists (path) { + try { + fs.statSync(path) + t.pass(`${path} exists`) + } catch (err) { + if (err.code === 'ENOENT') { + return t.fail(`${path} should exist but does not`) + } + throw err + } +} + +function assertMissing (path) { + try { + fs.statSync(path) + t.fail(`${path} should not exist but does`) + } catch (err) { + if (err.code === 'ENOENT') { + return t.pass(`${path} does not exist`) + } + throw err + } +}