Resolve #2075 Allow PyPI mirrors

docs/advanced.rst

@@ -38,6 +38,17 @@ If you'd like a specific package to be installed with a specific package index,
 
 Very fancy.
 
+☤ Using a PyPI Mirror
+----------------------------
+
+If you'd like to override the default PyPI index urls with the url for a PyPI mirror, you can use the following::
+
+    $ pipenv install --pypi-mirror <mirror_url>
+
+    $ pipenv update --pypi-mirror <mirror_url>
+
+Alternatively, you can set the ``PIPENV_PYPI_MIRROR`` environment variable.
+
 ☤ Injecting credentials into Pipfiles via environment variables
 -----------------------------------------------------------------
 

pipenv/cli.py

@@ -24,6 +24,7 @@
 
 from . import environments
 from .environments import *
+from .utils import is_valid_url
 
 # Enable shell completion.
 init_completion()
@@ -79,7 +80,10 @@ def validate_python_path(ctx, param, value):
             raise BadParameter('Expected Python at path %s does not exist' % value)
     return value
 
-
+def validate_pypi_mirror(ctx, param, value):
+    if value and not is_valid_url(value):
+        raise BadParameter('Invalid PyPI mirror URL: %s' % value)
+    return value
 @group(
     cls=PipenvGroup,
     invoke_without_command=True,
@@ -302,6 +306,13 @@ def cli(
     callback=validate_python_path,
     help="Specify which version of Python virtualenv should use.",
 )
+@option(
+    '--pypi-mirror',
+    default=PIPENV_PYPI_MIRROR,
+    nargs=1,
+    callback=validate_pypi_mirror,
+    help="Specify a PyPI mirror.",
+)
 @option(
     '--system', is_flag=True, default=False, help="System pip management."
 )
@@ -368,6 +379,7 @@ def install(
     dev=False,
     three=False,
     python=False,
+    pypi_mirror=False,
     system=False,
     lock=True,
     ignore_pipfile=False,
@@ -389,6 +401,7 @@ def install(
         dev=dev,
         three=three,
         python=python,
+        pypi_mirror=pypi_mirror,
         system=system,
         lock=lock,
         ignore_pipfile=ignore_pipfile,
@@ -494,6 +507,13 @@ def uninstall(
     callback=validate_python_path,
     help="Specify which version of Python virtualenv should use.",
 )
+@option(
+    '--pypi-mirror',
+    default=PIPENV_PYPI_MIRROR,
+    nargs=1,
+    callback=validate_pypi_mirror,
+    help="Specify a PyPI mirror.",
+)
 @option(
     '--verbose',
     '-v',
@@ -531,6 +551,7 @@ def uninstall(
 def lock(
     three=None,
     python=False,
+    pypi_mirror=False,
     verbose=False,
     requirements=False,
     dev=False,
@@ -543,7 +564,7 @@ def lock(
     # Ensure that virtualenv is available.
     ensure_project(three=three, python=python)
     if requirements:
-        do_init(dev=dev, requirements=requirements)
+        do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror)
     do_lock(
         verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated
     )
@@ -695,6 +716,13 @@ def check(
     callback=validate_python_path,
     help="Specify which version of Python virtualenv should use.",
 )
+@option(
+    '--pypi-mirror',
+    default=PIPENV_PYPI_MIRROR,
+    nargs=1,
+    callback=validate_pypi_mirror,
+    help="Specify a PyPI mirror.",
+)
 @option(
     '--verbose',
     '-v',
@@ -747,6 +775,7 @@ def update(
     ctx,
     three=None,
     python=False,
+    pypi_mirror=False,
     system=False,
     verbose=False,
     clear=False,
@@ -822,6 +851,7 @@ def update(
             dev=dev,
             three=three,
             python=python,
+            pypi_mirror=pypi_mirror,
             system=system,
             lock=True,
             ignore_pipfile=False,

pipenv/core.py

@@ -40,6 +40,8 @@
     prepare_pip_source_args,
     temp_environ,
     is_valid_url,
+    is_pypi_url,
+    create_mirror_source,
     download_file,
     is_pinned,
     is_star,
@@ -717,6 +719,7 @@ def do_install_dependencies(
     verbose=False,
     concurrent=True,
     requirements_dir=None,
+    pypi_mirror = False,
 ):
     """"Executes the install functionality.
 
@@ -819,6 +822,7 @@ def cleanup_procs(procs, concurrent):
                 index=index,
                 requirements_dir=requirements_dir,
                 extra_indexes=extra_indexes,
+                pypi_mirror=pypi_mirror,
             )
             c.dep = dep
             c.ignore_hash = ignore_hash
@@ -1280,6 +1284,7 @@ def do_init(
     pre=False,
     keep_outdated=False,
     requirements_dir=None,
+    pypi_mirror=False,
 ):
     """Executes the init functionality."""
     if not system:
@@ -1372,6 +1377,7 @@ def do_init(
         verbose=verbose,
         concurrent=concurrent,
         requirements_dir=requirements_dir.name,
+        pypi_mirror=pypi_mirror,
     )
     requirements_dir.cleanup()
     # Activate virtualenv instructions.
@@ -1392,6 +1398,7 @@ def pip_install(
     selective_upgrade=False,
     requirements_dir=None,
     extra_indexes=None,
+    pypi_mirror = None,
 ):
     from notpip._internal import logger as piplogger
     from notpip._vendor.pyparsing import ParseException
@@ -1463,6 +1470,8 @@ def pip_install(
                     sources.append({'url': idx['url']})
     else:
         sources = project.pipfile_sources
+    if pypi_mirror:
+        sources = [create_mirror_source(pypi_mirror) if is_pypi_url(source['url']) else source for source in sources]
     if package_name.startswith('-e '):
         install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1])
     elif r:
@@ -1757,6 +1766,7 @@ def do_install(
     dev=False,
     three=False,
     python=False,
+    pypi_mirror=False,
     system=False,
     lock=True,
     ignore_pipfile=False,
@@ -1985,6 +1995,7 @@ def do_install(
                 requirements_dir=requirements_directory.name,
                 index=index,
                 extra_indexes=extra_indexes,
+                pypi_mirror=pypi_mirror,
             )
             # Warn if --editable wasn't passed.
             try:

pipenv/environments.py

@@ -76,3 +76,5 @@
 SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno()))
 PIPENV_SHELL = os.environ.get('SHELL') or os.environ.get('PYENV_SHELL')
 PIPENV_CACHE_DIR = os.environ.get('PIPENV_CACHE_DIR', user_cache_dir('pipenv'))
+# Tells pipenv to override PyPI index urls with a mirror.
+PIPENV_PYPI_MIRROR = os.environ.get('PIPENV_PYPI_MIRROR')

pipenv/utils.py

@@ -442,6 +442,7 @@ def resolve_deps(
             collected_hashes = []
             if any('python.org' in source['url'] or 'pypi.org' in source['url']
                    for source in sources):
+                #FIXME Add support for mirrors.
                 pkg_url = 'https://pypi.org/pypi/{0}/json'.format(name)
                 session = _get_requests_session()
                 try:
@@ -924,6 +925,14 @@ def is_valid_url(url):
     return all([pieces.scheme, pieces.netloc])
 
 
+def is_pypi_url(url):
+    return bool(re.match(r'^http[s]?:\/\/pypi(?:\.python)?\.org\/simple[\/]?$', url))
+
+
+def create_mirror_source(url):
+    return {'url': url, 'verify_ssl': url.startswith('https://'), 'name': urlparse(url).hostname}
+
+
 def download_file(url, filename):
     """Downloads file from url to a path with filename"""
     r = _get_requests_session().get(url, stream=True)

tests/integration/test_install_basic.py

@@ -40,6 +40,35 @@ def test_basic_install(PipenvInstance, pypi):
         assert 'certifi' in p.lockfile['default']
 
 
+@pytest.mark.install
+@pytest.mark.needs_internet
+@flaky
+def test_mirror_install(PipenvInstance, pypi):
+    with PipenvInstance(chdir=True) as p:
+        with open(p.pipfile_path, 'w') as f:
+            f.write("""
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+            """.strip())
+        # This should sufficiently demonstrate the mirror functionality
+        # since pypi.org is the default.
+        c = p.pipenv('install requests --pypi-mirror https://pypi.python.org/simple')
+        assert c.return_code == 0
+        # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources
+        assert len(p.pipfile['source']) == 1
+        assert len(p.lockfile["_meta"]["sources"]) == 1
+        assert 'https://pypi.org/simple' == p.pipfile['source'][0]['url']
+        assert 'https://pypi.org/simple' == p.lockfile['_meta']['sources'][0]['url']
+
+        assert 'requests' in p.pipfile['packages']
+        assert 'requests' in p.lockfile['default']
+        assert 'chardet' in p.lockfile['default']
+        assert 'idna' in p.lockfile['default']
+        assert 'urllib3' in p.lockfile['default']
+        assert 'certifi' in p.lockfile['default']
+
 @pytest.mark.complex
 @pytest.mark.lock
 @pytest.mark.skip(reason='Does not work unless you can explicitly install into py2')