Adds support for multiple concurrent runtimes (#630)

* Adds support for multiple concurrent runtimes

* More fixes

* Adds documentation

* More fixes

* Updates snapshots

* Fixes tests for Windows

packages/acceptance-tests/pkg-tests-specs/sources/pnp.test.js

@@ -600,6 +600,41 @@ describe(`Plug'n'Play`, () => {
     ),
   );
 
+  test(
+    `it should be able to require files from a different dependency tree`,
+    makeTemporaryEnv({
+      dependencies: {
+        [`no-deps`]: `1.0.0`,
+      },
+    }, async ({path, run, source}) => {
+      await run(`install`);
+
+      const tmp = await createTemporaryFolder();
+
+      await xfs.writeJsonPromise(ppath.join(tmp, `package.json`), {
+        dependencies: {
+          [`no-deps`]: `2.0.0`,
+        },
+      });
+
+      await xfs.writeFilePromise(ppath.join(tmp, `index.js`), `
+        module.exports = require('no-deps');
+      `);
+
+      await run(`install`, {cwd: tmp});
+
+      await expect(source(`require('no-deps')`)).resolves.toEqual({
+        name: `no-deps`,
+        version: `1.0.0`,
+      });
+
+      await expect(source(`require(${JSON.stringify(tmp)})`)).resolves.toEqual({
+        name: `no-deps`,
+        version: `2.0.0`,
+      });
+    }),
+  );
+
   testIf(
     () => satisfies(process.versions.node, `>=8.9.0`),
     `it should throw when using require.resolve with unsupported options`,
@@ -641,7 +676,7 @@ describe(`Plug'n'Play`, () => {
         await writeFile(`${tmp}/index.js`, `require(process.argv[2])`);
         await writeFile(`${path}/index.js`, `require('no-deps')`);
 
-        await run(`node`, `${npath.fromPortablePath(tmp)}/index.js`, `${path}/index.js`);
+        await run(`node`, `${npath.fromPortablePath(tmp)}/index.js`, `${npath.fromPortablePath(path)}/index.js`);
       },
     ),
   );

packages/acceptance-tests/pkg-tests-specs/sources/pnpapi.test.js

@@ -285,11 +285,12 @@ describe(`Plug'n'Play API`, () => {
           const virtualPath = await source(`require.resolve('peer-deps')`);
 
           // Sanity check: to ensure that the test actually tests something :)
-          expect(xfs.existsSync(virtualPath)).toEqual(false);
+          expect(virtualPath).toMatch(`${npath.sep}$$virtual${npath.sep}`);
 
           const physicalPath = await source(`require('pnpapi').resolveVirtual(require.resolve('peer-deps'))`);
 
           expect(typeof physicalPath).toEqual(`string`);
+          expect(physicalPath).not.toEqual(virtualPath);
           expect(xfs.existsSync(physicalPath)).toEqual(true);
         }),
       );

packages/gatsby/src/pages/configuration/manifest.js

@@ -133,7 +133,7 @@ const PackageJsonDoc = () => <>
       <JsonObjectProperty
         name={`peerDependencies`}
         description={<>
-          Peer dependencies are inherited dependencies - the consumer of your package will be tasked to provide them. This is typically what you want when writing plugins, for example. Be careful: listing peer dependencies will have side effects on the way your package will be executed by your consumers. Check the documentation for more information.
+          Peer dependencies are inherited dependencies - the consumer of your package will be tasked to provide them. This is typically what you want when writing plugins, for example. Note that peer dependencies can also be listed as regular dependencies; in this case, Yarn will use the package provided by the ancestors if possible, but will fallback to the regular dependencies otherwise.
         </>}
       >
         <JsonScalarProperty

packages/gatsby/src/pages/configuration/yarnrc.js

@@ -382,9 +382,9 @@ const YarnrcDoc = () => <>
       </SymlArrayProperty>
       <SymlScalarProperty
         name={`virtualFolder`}
-        placeholder={`./.yarn/virtual`}
+        placeholder={`./.yarn/$$virtual`}
         description={<>
-          Due to a particularity in how Yarn installs packages which list peer dependencies, some packages will be mapped to multiple virtual directories that don't actually exist on the filesystem. This settings tells Yarn where to put them.
+          Due to a particularity in how Yarn installs packages which list peer dependencies, some packages will be mapped to multiple virtual directories that don't actually exist on the filesystem. This settings tells Yarn where to put them. Note that the folder name *must* be <code>$$virtual</code>.
         </>}
       />
       <SymlScalarProperty

packages/plugin-dlx/sources/commands/dlx.ts

@@ -83,7 +83,7 @@ export default class DlxCommand extends BaseCommand {
 
 function createTemporaryDirectory(name?: Filename) {
   return new Promise<PortablePath>((resolve, reject) => {
-    tmp.dir({unsafeCleanup: true}, (error, dirPath) => {
+    tmp.dir({unsafeCleanup: false}, (error, dirPath) => {
       if (error) {
         reject(error);
       } else {

packages/plugin-pnp/sources/PnpLinker.ts

@@ -2,7 +2,7 @@ import {Installer, Linker, LinkOptions, MinimalLinkOptions, Manifest, LinkType,
 import {FetchResult, Descriptor, Ident, Locator, Package, BuildDirective, BuildType}                         from '@yarnpkg/core';
 import {miscUtils, structUtils}                                                                              from '@yarnpkg/core';
 import {CwdFS, FakeFS, PortablePath, npath, ppath, toFilename, xfs}                                          from '@yarnpkg/fslib';
-import {PackageRegistry, generateInlinedScript, generateSplitScript}                                         from '@yarnpkg/pnp';
+import {PackageRegistry, generateInlinedScript, generateSplitScript, PnpSettings}                            from '@yarnpkg/pnp';
 import {UsageError}                                                                                          from 'clipanion';
 
 import {getPnpPath}                                                                                          from './index';
@@ -165,7 +165,6 @@ class PnpInstaller implements Installer {
     const ignorePattern = this.opts.project.configuration.get(`pnpIgnorePattern`);
     const packageRegistry = this.packageRegistry;
     const shebang = this.opts.project.configuration.get(`pnpShebang`);
-    const virtualRoots = [this.normalizeDirectoryPath(this.opts.project.configuration.get(`virtualFolder`))];
 
     if (pnpFallbackMode === `dependencies-only`)
       for (const pkg of this.opts.project.storedPackages.values())
@@ -175,15 +174,14 @@ class PnpInstaller implements Installer {
     const pnpPath = getPnpPath(this.opts.project);
     const pnpDataPath = this.opts.project.configuration.get(`pnpDataPath`);
 
-    const pnpSettings = {
+    const pnpSettings: PnpSettings = {
       blacklistedLocations,
       dependencyTreeRoots,
       enableTopLevelFallback,
       fallbackExclusionList,
       ignorePattern,
       packageRegistry,
       shebang,
-      virtualRoots,
     };
 
     if (this.opts.project.configuration.get(`pnpEnableInlining`)) {

packages/yarnpkg-builder/sources/commands/build/bundle.ts

@@ -84,7 +84,7 @@ export default class BuildBundleCommand extends Command {
       plugins: [
         new webpack.BannerPlugin({
           entryOnly: true,
-          banner: `#!/usr/bin/env node`,
+          banner: `#!/usr/bin/env node\n/* eslint-disable */`,
           raw: true,
         }),
         new webpack.DefinePlugin({

packages/yarnpkg-builder/sources/commands/build/plugin.ts

@@ -65,14 +65,14 @@ export default class BuildPluginCommand extends Command {
               for (const chunk of chunks) {
                 for (const file of chunk.files) {
                   compilation.assets[file] = new RawSource(reindent(`
-                    module.exports = {};
-
-                    module.exports.factory = function (require) {
-${reindent(compilation.assets[file].source().replace(/^ +/, ``), 11)}
-                      return plugin;
+                    /* eslint-disable*/
+                    module.exports = {
+                      name: ${JSON.stringify(name)},
+                      factory: function (require) {
+                        ${reindent(compilation.assets[file].source().replace(/^ +/, ``), 11)}
+                        return plugin;
+                      },
                     };
-
-                    module.exports.name = ${JSON.stringify(name)};
                   `));
                 }
               }

packages/yarnpkg-core/sources/Configuration.ts

@@ -160,9 +160,9 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     default: `./.yarn/cache`,
   },
   virtualFolder: {
-    description: `Folder where the virtual packages (cf doc) will be mapped on the disk`,
+    description: `Folder where the virtual packages (cf doc) will be mapped on the disk (must be named $$virtual)`,
     type: SettingsType.ABSOLUTE_PATH,
-    default: `./.yarn/virtual`,
+    default: `./.yarn/$$virtual`,
   },
   bstatePath: {
     description: `Path of the file where the current state of the built packages must be stored`,

packages/yarnpkg-fslib/sources/VirtualFS.ts

@@ -3,8 +3,15 @@ import {NodeFS}                        from './NodeFS';
 import {ProxiedFS}                     from './ProxiedFS';
 import {Filename, PortablePath, ppath} from './path';
 
-// https://github.com/benjamingr/RegExp.escape/blob/master/polyfill.js
-const escapeRegexp = (s: string) => s.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&');
+const NUMBER_REGEXP = /^[0-9]+$/;
+
+// $0: full path
+// $1: virtual folder
+// $2: virtual segment
+// $3: hash
+// $4: depth
+// $5: subpath
+const VIRTUAL_REGEXP = /^(\/(?:[^\/]+\/)*?\$\$virtual)((?:\/([^\/]+)(?:\/([^\/]+))?)?((?:\/.*)?))$/;
 
 export type VirtualFSOptions = {
   baseFs?: FakeFS<PortablePath>,
@@ -14,12 +21,10 @@ export type VirtualFSOptions = {
 export class VirtualFS extends ProxiedFS<PortablePath, PortablePath> {
   protected readonly baseFs: FakeFS<PortablePath>;
 
-  private readonly target: PortablePath;
-  private readonly virtual: PortablePath;
-
-  private readonly mapToBaseRegExp: RegExp;
-
   static makeVirtualPath(base: PortablePath, component: Filename, to: PortablePath) {
+    if (ppath.basename(base) !== `$$virtual`)
+      throw new Error(`Assertion failed: Virtual folders must be named "$$virtual"`);
+
     // Obtains the relative distance between the virtual path and its actual target
     const target = ppath.relative(ppath.dirname(base), to);
     const segments = target.split(`/`);
@@ -35,54 +40,62 @@ export class VirtualFS extends ProxiedFS<PortablePath, PortablePath> {
     return fullVirtualPath;
   }
 
-  constructor(virtual: PortablePath, {baseFs = new NodeFS()}: VirtualFSOptions = {}) {
-    super(ppath);
+  static resolveVirtual(p: PortablePath): PortablePath {
+    const match = p.match(VIRTUAL_REGEXP);
+    if (!match)
+      return p;
 
-    this.baseFs = baseFs;
+    const target = ppath.dirname(match[1] as PortablePath);
+    if (!match[3] || !match[4])
+      return target;
+
+    const isnum = NUMBER_REGEXP.test(match[4]);
+    if (!isnum)
+      return p;
 
-    this.target = ppath.dirname(virtual);
-    this.virtual = virtual;
+    const depth = Number(match[4]);
+    const backstep = `../`.repeat(depth) as PortablePath;
+    const subpath = (match[5] || `.`) as PortablePath;
 
-    this.mapToBaseRegExp = new RegExp(`^(${escapeRegexp(this.virtual)})((?:/([^\/]+)(?:/([^/]+))?)?((?:/.*)?))$`);
+    return VirtualFS.resolveVirtual(ppath.join(target, backstep, subpath));
+  }
+
+  constructor({baseFs = new NodeFS()}: VirtualFSOptions = {}) {
+    super(ppath);
+
+    this.baseFs = baseFs;
   }
 
   getRealPath() {
-    return this.pathUtils.resolve(this.baseFs.getRealPath(), this.target);
+    return this.baseFs.getRealPath();
   }
 
   realpathSync(p: PortablePath) {
-    const match = p.match(this.mapToBaseRegExp);
+    const match = p.match(VIRTUAL_REGEXP);
     if (!match)
       return this.baseFs.realpathSync(p);
 
     if (!match[5])
       return p;
 
     const realpath = this.baseFs.realpathSync(this.mapToBase(p));
-    return VirtualFS.makeVirtualPath(this.virtual, match[3] as Filename, realpath);
+    return VirtualFS.makeVirtualPath(match[1] as PortablePath, match[3] as Filename, realpath);
   }
 
   async realpathPromise(p: PortablePath) {
-    const match = p.match(this.mapToBaseRegExp);
+    const match = p.match(VIRTUAL_REGEXP);
     if (!match)
       return await this.baseFs.realpathPromise(p);
 
     if (!match[5])
       return p;
 
     const realpath = await this.baseFs.realpathPromise(this.mapToBase(p));
-    return VirtualFS.makeVirtualPath(this.virtual, match[3] as Filename, realpath);
+    return VirtualFS.makeVirtualPath(match[1] as PortablePath, match[3] as Filename, realpath);
   }
 
   mapToBase(p: PortablePath): PortablePath {
-    const match = p.match(this.mapToBaseRegExp);
-    if (!match)
-      return p;
-
-    if (match[3])
-      return this.mapToBase(ppath.join(this.target, `../`.repeat(Number(match[4])) as PortablePath, match[5] as PortablePath));
-
-    return this.target;
+    return VirtualFS.resolveVirtual(p);
   }
 
   mapFromBase(p: PortablePath) {

packages/yarnpkg-fslib/sources/ZipOpenFS.ts

@@ -12,6 +12,7 @@ const ZIP_FD = 0x80000000;
 export type ZipOpenFSOptions = {
   baseFs?: FakeFS<PortablePath>,
   filter?: RegExp | null,
+  maxOpenFiles?: number,
   readOnlyArchives?: boolean,
   useCache?: boolean,
 };
@@ -42,20 +43,22 @@ export class ZipOpenFS extends BasePortableFakeFS {
   private readonly fdMap: Map<number, [ZipFS, number]> = new Map();
   private nextFd = 3;
 
-  private readonly filter?: RegExp | null;
-  private readonly readOnlyArchives?: boolean;
+  private readonly filter: RegExp | null;
+  private readonly maxOpenFiles: number;
+  private readonly readOnlyArchives: boolean;
 
   private isZip: Set<string> = new Set();
   private notZip: Set<string> = new Set();
 
-  constructor({baseFs = new NodeFS(), filter = null, readOnlyArchives = false, useCache = true}: ZipOpenFSOptions = {}) {
+  constructor({baseFs = new NodeFS(), filter = null, maxOpenFiles = Infinity, readOnlyArchives = false, useCache = true}: ZipOpenFSOptions = {}) {
     super();
 
     this.baseFs = baseFs;
 
     this.zipInstances = useCache ? new Map() : null;
 
     this.filter = filter;
+    this.maxOpenFiles = maxOpenFiles;
     this.readOnlyArchives = readOnlyArchives;
 
     this.isZip = new Set();
@@ -724,6 +727,23 @@ export class ZipOpenFS extends BasePortableFakeFS {
     return null;
   }
 
+  private limitOpenFiles(max: number) {
+    if (this.zipInstances === null)
+      return;
+
+    let closeCount = this.zipInstances.size - max;
+
+    for (const [path, zipFs] of this.zipInstances.entries()) {
+      if (closeCount <= 0)
+        break;
+
+      zipFs.saveAndClose();
+      this.zipInstances.delete(path);
+
+      closeCount -= 1;
+    }
+  }
+
   private async getZipPromise<T>(p: PortablePath, accept: (zipFs: ZipFS) => Promise<T>) {
     const getZipOptions = async () => ({
       baseFs: this.baseFs,
@@ -735,7 +755,14 @@ export class ZipOpenFS extends BasePortableFakeFS {
       let zipFs = this.zipInstances.get(p);
 
       if (!zipFs)
-        this.zipInstances.set(p, zipFs = new ZipFS(p, await getZipOptions()));
+        zipFs = new ZipFS(p, await getZipOptions());
+
+      // Removing then re-adding the field allows us to easily implement
+      // a basic LRU garbage collection strategy
+      this.zipInstances.delete(p);
+      this.zipInstances.set(p, zipFs);
+
+      this.limitOpenFiles(this.maxOpenFiles);
 
       return await accept(zipFs);
     } else {