[Replace #2507] Support VSTS and other custom registries with unusual feeds (#3231)

* Fix issue 2505

* Fix flow check

* Fix test errors

* Load custom package host suffix from .yarnrc

* Update tests with new customHostPrefix parameter

* Use customHostSuffix parameter to determine if request is to registry

Author: lumaxis

__tests__/fixtures/install/install-from-authed-private-registry-no-slash/.npmrc

@@ -1,2 +1,3 @@
 //registry.yarnpkg.com/:_authToken=abc123
+//registry.yarnpkg.com/:always-auth=true
 @types:registry=https://registry.yarnpkg.com

__tests__/fixtures/install/install-from-authed-private-registry/.npmrc

@@ -1,2 +1,3 @@
 //registry.yarnpkg.com/:_authToken=abc123
+//registry.yarnpkg.com/:always-auth=true
 @types:registry=https://registry.yarnpkg.com/

__tests__/registries/is-request-to-registry.js

@@ -3,49 +3,70 @@
 
 import isRequestToRegistry from '../../src/registries/is-request-to-registry.js';
 
-test('isRequestToRegistry functional test', () => {
-  expect(isRequestToRegistry(
-    'http://foo.bar:80/foo/bar/baz',
-    'http://foo.bar/foo/',
-  )).toBe(true);
-
-  expect(isRequestToRegistry(
-    'http://foo.bar/foo/bar/baz',
-    'http://foo.bar/foo/',
-  )).toBe(true);
-
-  expect(isRequestToRegistry(
-    'https://foo.bar:443/foo/bar/baz',
-    'https://foo.bar/foo/',
-  )).toBe(true);
-
-  expect(isRequestToRegistry(
-    'https://foo.bar/foo/bar/baz',
-    'https://foo.bar:443/foo/',
-  )).toBe(true);
-
-  expect(isRequestToRegistry(
-    'http://foo.bar:80/foo/bar/baz',
-    'https://foo.bar/foo/',
-  )).toBe(true);
-
-  expect(isRequestToRegistry(
-    'http://foo.bar/blah/whatever/something',
-    'http://foo.bar/foo/',
-  )).toBe(false);
-
-  expect(isRequestToRegistry(
-    'https://wrong.thing/foo/bar/baz',
-    'https://foo.bar/foo/',
-  )).toBe(false);
-
-  expect(isRequestToRegistry(
-    'https://foo.bar:1337/foo/bar/baz',
-    'https://foo.bar/foo/',
-  )).toBe(false);
-
-  expect(isRequestToRegistry(
-    'http://foo.bar/foo/bar/baz',
-    'https://foo.bar/foo/bar/baz',
-  )).toBe(true);
+describe('isRequestToRegistry functional test', () => {
+  test('request to registry url matching', () => {
+    expect(isRequestToRegistry(
+      'http://foo.bar:80/foo/bar/baz',
+      'http://foo.bar/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'http://foo.bar/foo/bar/baz',
+      'http://foo.bar/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'http://foo.bar/foo/00000000-1111-4444-8888-000000000000/baz',
+      'http://foo.bar/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'https://foo.bar:443/foo/bar/baz',
+      'https://foo.bar/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+        'https://foo.bar/foo/bar/baz',
+        'https://foo.bar:443/foo/',
+      )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'https://foo.bar/foo/bar/baz',
+      'https://foo.bar:443/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'http://foo.bar:80/foo/bar/baz',
+      'https://foo.bar/foo/',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'https://wrong.thing/foo/bar/baz',
+      'https://foo.bar/foo/',
+    )).toBe(false);
+
+    expect(isRequestToRegistry(
+      'https://foo.bar:1337/foo/bar/baz',
+      'https://foo.bar/foo/',
+    )).toBe(false);
+  });
+
+  test('isRequestToRegistry with custom host prefix', () => {
+    expect(isRequestToRegistry(
+      'http://pkgs.host.com:80/foo/bar/baz',
+      'http://pkgs.host.com/bar/baz',
+      'some.host.org',
+    )).toBe(false);
+
+    expect(isRequestToRegistry(
+      'http://foo.bar/foo/bar/baz',
+      'https://foo.bar/foo/bar/baz',
+    )).toBe(true);
+
+    expect(isRequestToRegistry(
+      'http://pkgs.host.com:80/foo/bar/baz',
+      'http://pkgs.host.com/bar/baz',
+      'pkgs.host.com',
+    )).toBe(true);
+  });
 });

src/fetchers/base-fetcher.js

@@ -13,6 +13,7 @@ const path = require('path');
 export default class BaseFetcher {
   constructor(dest: string, remote: PackageRemote, config: Config) {
     this.reporter = config.reporter;
+    this.packageName = remote.packageName;
     this.reference = remote.reference;
     this.registry = remote.registry;
     this.hash = remote.hash;
@@ -24,6 +25,7 @@ export default class BaseFetcher {
   reporter: Reporter;
   remote: PackageRemote;
   registry: RegistryNames;
+  packageName: ?string;
   reference: string;
   config: Config;
   hash: ?string;

src/fetchers/tarball-fetcher.js

@@ -178,7 +178,7 @@ export default class TarballFetcher extends BaseFetcher {
           .pipe(extractorStream)
           .on('error', reject);
       },
-    });
+    }, this.packageName);
   }
 
   async _fetch(): Promise<FetchedOverride> {

src/registries/is-request-to-registry.js

@@ -2,17 +2,22 @@
 
 import url from 'url';
 
-export default function isRequestToRegistry(requestUrl: string, registry: string): boolean {
+export default function isRequestToRegistry(requestUrl: string, registry: string, customHostSuffix: ? any): boolean {
   const requestParsed = url.parse(requestUrl);
   const registryParsed = url.parse(registry);
+  const requestHost = requestParsed.hostname || '';
+  const registryHost = registryParsed.hostname || '';
   const requestPort = getPortOrDefaultPort(requestParsed.port, requestParsed.protocol);
   const registryPort = getPortOrDefaultPort(registryParsed.port, registryParsed.protocol);
   const requestPath = requestParsed.path || '';
   const registryPath = registryParsed.path || '';
 
-  return (requestParsed.hostname === registryParsed.hostname) &&
-        (requestPort === registryPort) &&
-        requestPath.startsWith(registryPath);
+  return (requestHost === registryHost) &&
+    (requestPort === registryPort) &&
+    (requestPath.startsWith(registryPath) ||
+      // For some registries, the package path does not prefix with the registry path
+      (!!customHostSuffix && customHostSuffix.length > 0 && requestHost.endsWith(customHostSuffix))
+    );
 }
 
 function getPortOrDefaultPort(port: ?string, protocol: ?string): ?string {

src/registries/npm-registry.js

@@ -9,7 +9,7 @@ import * as fs from '../util/fs.js';
 import NpmResolver from '../resolvers/registries/npm-resolver.js';
 import envReplace from '../util/env-replace.js';
 import Registry from './base-registry.js';
-import {addSuffix, removePrefix} from '../util/misc';
+import {addSuffix} from '../util/misc';
 import isRequestToRegistry from './is-request-to-registry.js';
 
 const userHome = require('../util/user-home-dir').default;
@@ -18,6 +18,8 @@ const url = require('url');
 const ini = require('ini');
 
 const DEFAULT_REGISTRY = 'https://registry.npmjs.org/';
+const REGEX_REGISTRY_PREFIX = /^https?:/;
+const REGEX_REGISTRY_SUFFIX = /registry\/?$/;
 
 function getGlobalPrefix(): string {
   if (process.env.PREFIX) {
@@ -51,18 +53,17 @@ export default class NpmRegistry extends Registry {
     return name.replace('/', '%2f');
   }
 
-  request(pathname: string, opts?: RegistryRequestOptions = {}): Promise<*> {
-    const registry = addSuffix(this.getRegistry(pathname), '/');
+  request(pathname: string, opts?: RegistryRequestOptions = {}, packageName: ?string): Promise<*> {
+    const registry = this.getRegistry(packageName || pathname);
     const requestUrl = url.resolve(registry, pathname);
-    const alwaysAuth = this.getScopedOption(registry.replace(/^https?:/, ''), 'always-auth')
-      || this.getOption('always-auth')
-      || removePrefix(requestUrl, registry)[0] === '@';
+    const alwaysAuth = this.getRegistryOrGlobalOption(registry, 'always-auth');
+    const customHostSuffix = this.getRegistryOrGlobalOption(registry, 'custom-host-suffix');
 
     const headers = Object.assign({
       'Accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*',
     }, opts.headers);
-    if (this.token || (alwaysAuth && isRequestToRegistry(requestUrl, registry))) {
-      const authorization = this.getAuth(pathname);
+    if (this.token || (alwaysAuth && isRequestToRegistry(requestUrl, registry, customHostSuffix))) {
+      const authorization = this.getAuth(packageName || pathname);
       if (authorization) {
         headers.authorization = authorization;
       }
@@ -160,15 +161,15 @@ export default class NpmRegistry extends Registry {
       const availableRegistries = this.getAvailableRegistries();
       const registry = availableRegistries.find((registry) => packageName.startsWith(registry));
       if (registry) {
-        return registry;
+        return addSuffix(registry, '/');
       }
     }
 
     for (const scope of [this.getScope(packageName), '']) {
       const registry = this.getScopedOption(scope, 'registry')
                     || this.registries.yarn.getScopedOption(scope, 'registry');
       if (registry) {
-        return String(registry);
+        return addSuffix(String(registry), '/');
       }
     }
 
@@ -180,28 +181,26 @@ export default class NpmRegistry extends Registry {
       return this.token;
     }
 
-    for (let registry of [this.getRegistry(packageName), '', DEFAULT_REGISTRY]) {
-      registry = registry.replace(/^https?:/, '');
+    const registry = this.getRegistry(packageName);
 
-      // Check for bearer token.
-      let auth = this.getScopedOption(registry.replace(/\/?$/, '/'), '_authToken');
-      if (auth) {
-        return `Bearer ${String(auth)}`;
-      }
+    // Check for bearer token.
+    const authToken = this.getRegistryOrGlobalOption(registry, '_authToken');
+    if (authToken) {
+      return `Bearer ${String(authToken)}`;
+    }
 
-      // Check for basic auth token.
-      auth = this.getScopedOption(registry, '_auth');
-      if (auth) {
-        return `Basic ${String(auth)}`;
-      }
+    // Check for basic auth token.
+    const auth = this.getRegistryOrGlobalOption(registry, '_auth');
+    if (auth) {
+      return `Basic ${String(auth)}`;
+    }
 
-      // Check for basic username/password auth.
-      const username = this.getScopedOption(registry, 'username');
-      const password = this.getScopedOption(registry, '_password');
-      if (username && password) {
-        const pw = new Buffer(String(password), 'base64').toString();
-        return 'Basic ' + new Buffer(String(username) + ':' + pw).toString('base64');
-      }
+    // Check for basic username/password auth.
+    const username = this.getRegistryOrGlobalOption(registry, 'username');
+    const password = this.getRegistryOrGlobalOption(registry, '_password');
+    if (username && password) {
+      const pw = new Buffer(String(password), 'base64').toString();
+      return 'Basic ' + new Buffer(String(username) + ':' + pw).toString('base64');
     }
 
     return '';
@@ -210,4 +209,23 @@ export default class NpmRegistry extends Registry {
   getScopedOption(scope: string, option: string): mixed {
     return this.getOption(scope + (scope ? ':' : '') + option);
   }
+
+  getRegistryOption(registry: string, option: string): mixed {
+    const pre = REGEX_REGISTRY_PREFIX;
+    const suf = REGEX_REGISTRY_SUFFIX;
+
+    // When registry is used config scope, the trailing '/' is required
+    const reg = addSuffix(registry, '/');
+
+    // 1st attempt, try to get option for the given registry URL
+    // 2nd attempt, remove the 'https?:' prefix of the registry URL
+    // 3nd attempt, remove the 'registry/?' suffix of the registry URL
+    return this.getScopedOption(reg, option)
+      || reg.match(pre) && this.getRegistryOption(reg.replace(pre, ''), option)
+      || reg.match(suf) && this.getRegistryOption(reg.replace(suf, ''), option);
+  }
+
+  getRegistryOrGlobalOption(registry: string, option: string): mixed {
+    return this.getRegistryOption(registry, option) || this.getOption(option);
+  }
 }

src/resolvers/registries/npm-resolver.js

@@ -191,6 +191,7 @@ export default class NpmResolver extends RegistryResolver {
         reference: this.cleanRegistry(dist.tarball),
         hash: dist.shasum,
         registry: 'npm',
+        packageName: info.name,
       };
     }
 

src/types.js

@@ -42,6 +42,7 @@ export type PackageRemote = {
   reference: string,
   resolved?: ?string,
   hash: ?string,
+  packageName?: string,
 };
 
 // `dependencies` field in package info