From 9154f886478c4beab8481aa9a68c6c0fcc245436 Mon Sep 17 00:00:00 2001 From: Jacob Henner Date: Mon, 4 Jun 2018 23:47:09 -0400 Subject: [PATCH] Add support for PyPI mirrors Adds support for the --pypi-mirror command line parameter and the PIPENV_PYPI_MIRROR environment variable for most pipenv operations. This permits pipenv to function without pypi.org, which is necessary for users: 1. behind restrictive networks 2. facing strict artifact sourcing policies 3. experiencing poor performance connecting to pypi.org 4. who've configured a local cache for performance reasons When specified, the value of this parameter replaces all instances of pypi.org and pypi.python.org within pipenv operations without modifying or requring the modification of Pipfiles. - Resolves #2075 --- docs/advanced.rst | 17 +++++++ pipenv/cli.py | 61 ++++++++++++++++++++++--- pipenv/core.py | 40 ++++++++++++---- pipenv/environments.py | 2 + pipenv/resolver.py | 3 +- pipenv/utils.py | 16 ++++++- tests/integration/test_install_basic.py | 29 ++++++++++++ 7 files changed, 150 insertions(+), 18 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index de16e2e2db..a782014601 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -38,6 +38,23 @@ If you'd like a specific package to be installed with a specific package index, Very fancy. +☤ Using a PyPI Mirror +---------------------------- + +If you'd like to override the default PyPI index urls with the url for a PyPI mirror, you can use the following:: + + $ pipenv install --pypi-mirror + + $ pipenv update --pypi-mirror + + $ pipenv sync --pypi-mirror + + $ pipenv lock --pypi-mirror + + $ pipenv uninstall --pypi-mirror + +Alternatively, you can set the ``PIPENV_PYPI_MIRROR`` environment variable. + ☤ Injecting credentials into Pipfiles via environment variables ----------------------------------------------------------------- diff --git a/pipenv/cli.py b/pipenv/cli.py index 3a23b81d00..790837f0ed 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -24,6 +24,7 @@ from . import environments from .environments import * +from .utils import is_valid_url # Enable shell completion. init_completion() @@ -79,7 +80,10 @@ def validate_python_path(ctx, param, value): raise BadParameter('Expected Python at path %s does not exist' % value) return value - +def validate_pypi_mirror(ctx, param, value): + if value and not is_valid_url(value): + raise BadParameter('Invalid PyPI mirror URL: %s' % value) + return value @group( cls=PipenvGroup, invoke_without_command=True, @@ -302,6 +306,13 @@ def cli( callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) +@option( + '--pypi-mirror', + default=PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @option( '--system', is_flag=True, default=False, help="System pip management." ) @@ -368,6 +379,7 @@ def install( dev=False, three=False, python=False, + pypi_mirror=None, system=False, lock=True, ignore_pipfile=False, @@ -389,6 +401,7 @@ def install( dev=dev, three=three, python=python, + pypi_mirror=pypi_mirror, system=system, lock=lock, ignore_pipfile=ignore_pipfile, @@ -452,6 +465,13 @@ def install( default=False, help=u"Keep out–dated dependencies from being updated in Pipfile.lock.", ) +@option( + '--pypi-mirror', + default=PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) def uninstall( package_name=False, more_packages=False, @@ -463,6 +483,7 @@ def uninstall( all=False, verbose=False, keep_outdated=False, + pypi_mirror=None, ): from .core import do_uninstall @@ -477,6 +498,7 @@ def uninstall( all=all, verbose=verbose, keep_outdated=keep_outdated, + pypi_mirror=pypi_mirror, ) @@ -494,6 +516,13 @@ def uninstall( callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) +@option( + '--pypi-mirror', + default=PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @option( '--verbose', '-v', @@ -531,6 +560,7 @@ def uninstall( def lock( three=None, python=False, + pypi_mirror=None, verbose=False, requirements=False, dev=False, @@ -543,9 +573,9 @@ def lock( # Ensure that virtualenv is available. ensure_project(three=three, python=python) if requirements: - do_init(dev=dev, requirements=requirements) + do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) do_lock( - verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated + verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror ) @@ -695,6 +725,13 @@ def check( callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) +@option( + '--pypi-mirror', + default=PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @option( '--verbose', '-v', @@ -747,6 +784,7 @@ def update( ctx, three=None, python=False, + pypi_mirror=None, system=False, verbose=False, clear=False, @@ -774,7 +812,7 @@ def update( if not outdated: outdated = bool(dry_run) if outdated: - do_outdated() + do_outdated(pypi_mirror=pypi_mirror) if not package: echo( '{0} {1} {2} {3}{4}'.format( @@ -786,7 +824,7 @@ def update( ) ) do_lock( - verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated + verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror ) do_sync( ctx=ctx, @@ -801,6 +839,7 @@ def update( clear=clear, unused=False, sequential=sequential, + pypi_mirror=pypi_mirror, ) else: for package in ([package] + list(more_packages) or []): @@ -814,7 +853,7 @@ def update( err=True, ) sys.exit(1) - ensure_lockfile(keep_outdated=project.lockfile_exists) + ensure_lockfile(keep_outdated=project.lockfile_exists, pypi_mirror=pypi_mirror) # Install the dependencies. do_install( package_name=package, @@ -822,6 +861,7 @@ def update( dev=dev, three=three, python=python, + pypi_mirror=pypi_mirror, system=system, lock=True, ignore_pipfile=False, @@ -922,6 +962,13 @@ def run_open(module, three=None, python=None): callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) +@option( + '--pypi-mirror', + default=PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @option('--bare', is_flag=True, default=False, help="Minimal output.") @option( '--clear', is_flag=True, default=False, help="Clear the dependency cache." @@ -946,6 +993,7 @@ def sync( unused=False, package_name=None, sequential=False, + pypi_mirror=None, ): from .core import do_sync @@ -962,6 +1010,7 @@ def sync( clear=clear, unused=unused, sequential=sequential, + pypi_mirror=pypi_mirror, ) diff --git a/pipenv/core.py b/pipenv/core.py index 504ce47e0d..3dfc6847b0 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -40,6 +40,8 @@ prepare_pip_source_args, temp_environ, is_valid_url, + is_pypi_url, + create_mirror_source, download_file, is_pinned, is_star, @@ -717,6 +719,7 @@ def do_install_dependencies( verbose=False, concurrent=True, requirements_dir=None, + pypi_mirror = False, ): """"Executes the install functionality. @@ -819,6 +822,7 @@ def cleanup_procs(procs, concurrent): index=index, requirements_dir=requirements_dir, extra_indexes=extra_indexes, + pypi_mirror=pypi_mirror, ) c.dep = dep c.ignore_hash = ignore_hash @@ -1001,6 +1005,7 @@ def do_lock( pre=False, keep_outdated=False, write=True, + pypi_mirror = None, ): """Executes the freeze functionality.""" from .utils import get_vcs_deps @@ -1055,6 +1060,7 @@ def do_lock( clear=clear, pre=pre, allow_global=system, + pypi_mirror=pypi_mirror ) # Add develop dependencies to lockfile. for dep in results: @@ -1105,6 +1111,7 @@ def do_lock( clear=False, pre=pre, allow_global=system, + pypi_mirror=pypi_mirror, ) # Add default dependencies to lockfile. for dep in results: @@ -1280,6 +1287,7 @@ def do_init( pre=False, keep_outdated=False, requirements_dir=None, + pypi_mirror=None, ): """Executes the init functionality.""" if not system: @@ -1337,7 +1345,7 @@ def do_init( ), err=True, ) - do_lock(system=system, pre=pre, keep_outdated=keep_outdated) + do_lock(system=system, pre=pre, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) # Write out the lockfile if it doesn't exist. if not project.lockfile_exists and not skip_lock: # Unless we're in a virtualenv not managed by pipenv, abort if we're @@ -1363,6 +1371,7 @@ def do_init( pre=pre, keep_outdated=keep_outdated, verbose=verbose, + pypi_mirror=pypi_mirror, ) do_install_dependencies( dev=dev, @@ -1372,6 +1381,7 @@ def do_init( verbose=verbose, concurrent=concurrent, requirements_dir=requirements_dir.name, + pypi_mirror=pypi_mirror, ) requirements_dir.cleanup() # Activate virtualenv instructions. @@ -1392,6 +1402,7 @@ def pip_install( selective_upgrade=False, requirements_dir=None, extra_indexes=None, + pypi_mirror = None, ): from notpip._internal import logger as piplogger from notpip._vendor.pyparsing import ParseException @@ -1463,6 +1474,8 @@ def pip_install( sources.append({'url': idx['url']}) else: sources = project.pipfile_sources + if pypi_mirror: + sources = [create_mirror_source(pypi_mirror) if is_pypi_url(source['url']) else source for source in sources] if package_name.startswith('-e '): install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1]) elif r: @@ -1684,7 +1697,7 @@ def warn_in_virtualenv(): ) -def ensure_lockfile(keep_outdated=False): +def ensure_lockfile(keep_outdated=False, pypi_mirror=None): """Ensures that the lockfile is up–to–date.""" if not keep_outdated: keep_outdated = project.settings.get('keep_outdated') @@ -1702,9 +1715,9 @@ def ensure_lockfile(keep_outdated=False): ), err=True, ) - do_lock(keep_outdated=keep_outdated) + do_lock(keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) else: - do_lock(keep_outdated=keep_outdated) + do_lock(keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) def do_py(system=False): @@ -1714,7 +1727,7 @@ def do_py(system=False): click.echo(crayons.red('No project found!')) -def do_outdated(): +def do_outdated(pypi_mirror=None): packages = {} results = delegator.run('{0} freeze'.format(which('pip'))).out.strip( ).split( @@ -1725,7 +1738,7 @@ def do_outdated(): dep = Requirement.from_line(result) packages.update(dep.as_pipfile()) updated_packages = {} - lockfile = do_lock(write=False) + lockfile = do_lock(write=False, pypi_mirror=pypi_mirror) for section in ('develop', 'default'): for package in lockfile[section]: try: @@ -1757,6 +1770,7 @@ def do_install( dev=False, three=False, python=False, + pypi_mirror=None, system=False, lock=True, ignore_pipfile=False, @@ -1908,7 +1922,7 @@ def do_install( # Capture . argument and assign it to nothing if package_name == '.': package_name = False - # Install editable local packages before locking - this givves us acceess to dist-info + # Install editable local packages before locking - this gives us access to dist-info if project.pipfile_exists and ( not project.lockfile_exists or not project.virtualenv_exists ): @@ -1940,6 +1954,7 @@ def do_install( deploy=deploy, pre=pre, requirements_dir=requirements_directory, + pypi_mirror=pypi_mirror, ) requirements_directory.cleanup() sys.exit(0) @@ -1985,6 +2000,7 @@ def do_install( requirements_dir=requirements_directory.name, index=index, extra_indexes=extra_indexes, + pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. try: @@ -2059,6 +2075,7 @@ def do_install( keep_outdated=keep_outdated, requirements_dir=requirements_directory, deploy=deploy, + pypi_mirror=pypi_mirror, ) requirements_directory.cleanup() @@ -2074,6 +2091,7 @@ def do_uninstall( all=False, verbose=False, keep_outdated=False, + pypi_mirror=None, ): # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: @@ -2146,7 +2164,7 @@ def do_uninstall( project.remove_package_from_pipfile(package_name, dev=True) project.remove_package_from_pipfile(package_name, dev=False) if lock: - do_lock(system=system, keep_outdated=keep_outdated) + do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) def do_shell(three=None, python=False, fancy=False, shell_args=None): @@ -2545,6 +2563,7 @@ def do_sync( clear=False, unused=False, sequential=False, + pypi_mirror=None, ): # The lock file needs to exist because sync won't write to it. if not project.lockfile_exists: @@ -2570,17 +2589,18 @@ def do_sync( concurrent=(not sequential), requirements_dir=requirements_dir, ignore_pipfile=True, # Don't check if Pipfile and lock match. + pypi_mirror=pypi_mirror, ) requirements_dir.cleanup() click.echo(crayons.green('All dependencies are now up-to-date!')) def do_clean( - ctx, three=None, python=None, dry_run=False, bare=False, verbose=False + ctx, three=None, python=None, dry_run=False, bare=False, verbose=False, pypi_mirror=None ): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False) - ensure_lockfile() + ensure_lockfile(pypi_mirror=pypi_mirror) installed_package_names = [] pip_freeze_command = delegator.run('{0} freeze'.format(which_pip())) diff --git a/pipenv/environments.py b/pipenv/environments.py index 3837363df2..de2f9ae8d2 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -76,3 +76,5 @@ SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) PIPENV_SHELL = os.environ.get('SHELL') or os.environ.get('PYENV_SHELL') PIPENV_CACHE_DIR = os.environ.get('PIPENV_CACHE_DIR', user_cache_dir('pipenv')) +# Tells pipenv to override PyPI index urls with a mirror. +PIPENV_PYPI_MIRROR = os.environ.get('PIPENV_PYPI_MIRROR') \ No newline at end of file diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 5bce576835..f39aebfe04 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -48,6 +48,7 @@ def main(): for i, package in enumerate(packages): if package.startswith('--'): del packages[i] + pypi_mirror_source = pipenv.utils.create_mirror_source(os.environ['PIPENV_PYPI_MIRROR']) if 'PIPENV_PYPI_MIRROR' in os.environ else None project = pipenv.core.project def resolve(packages, pre, sources, verbose, clear, system): @@ -66,7 +67,7 @@ def resolve(packages, pre, sources, verbose, clear, system): results = resolve( packages, pre=do_pre, - sources=project.pipfile_sources, + sources = pipenv.utils.replace_pypi_sources(project.pipfile_sources, pypi_mirror_source) if pypi_mirror_source else project.pipfile_sources, verbose=is_verbose, clear=do_clear, system=system, diff --git a/pipenv/utils.py b/pipenv/utils.py index 7aff3d3d09..2f5af540dc 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -277,6 +277,7 @@ class PipCommand(basecommand.Command): pypi = PyPIRepository( pip_options=pip_options, use_json=True, session=session ) + print(pip_options) if verbose: logging.log.verbose = True piptools_logging.log.verbose = True @@ -325,7 +326,7 @@ class PipCommand(basecommand.Command): def venv_resolve_deps( - deps, which, project, pre=False, verbose=False, clear=False, allow_global=False + deps, which, project, pre=False, verbose=False, clear=False, allow_global=False, pypi_mirror=None ): from .vendor import delegator from . import resolver @@ -343,6 +344,8 @@ def venv_resolve_deps( ) with temp_environ(): os.environ['PIPENV_PACKAGES'] = '\n'.join(deps) + if pypi_mirror: + os.environ['PIPENV_PYPI_MIRROR'] = str(pypi_mirror) c = delegator.run(cmd, block=True) try: assert c.return_code == 0 @@ -442,6 +445,7 @@ def resolve_deps( collected_hashes = [] if any('python.org' in source['url'] or 'pypi.org' in source['url'] for source in sources): + #FIXME Add support for mirrors. pkg_url = 'https://pypi.org/pypi/{0}/json'.format(name) session = _get_requests_session() try: @@ -922,6 +926,16 @@ def is_valid_url(url): return all([pieces.scheme, pieces.netloc]) +def is_pypi_url(url): + return bool(re.match(r'^http[s]?:\/\/pypi(?:\.python)?\.org\/simple[\/]?$', url)) + +def replace_pypi_sources(sources, pypi_replacement_source): + return [pypi_replacement_source] + [source for source in sources if not is_pypi_url(source['url'])] + +def create_mirror_source(url): + return {'url': url, 'verify_ssl': url.startswith('https://'), 'name': urlparse(url).hostname} + + def download_file(url, filename): """Downloads file from url to a path with filename""" r = _get_requests_session().get(url, stream=True) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f995ef91ba..5c69897f05 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -40,6 +40,35 @@ def test_basic_install(PipenvInstance, pypi): assert 'certifi' in p.lockfile['default'] +@pytest.mark.install +@pytest.mark.needs_internet +@flaky +def test_mirror_install(PipenvInstance, pypi): + with PipenvInstance(chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + """.strip()) + # This should sufficiently demonstrate the mirror functionality + # since pypi.org is the default. + c = p.pipenv('install requests --pypi-mirror https://pypi.python.org/simple') + assert c.return_code == 0 + # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources + assert len(p.pipfile['source']) == 1 + assert len(p.lockfile["_meta"]["sources"]) == 1 + assert 'https://pypi.org/simple' == p.pipfile['source'][0]['url'] + assert 'https://pypi.org/simple' == p.lockfile['_meta']['sources'][0]['url'] + + assert 'requests' in p.pipfile['packages'] + assert 'requests' in p.lockfile['default'] + assert 'chardet' in p.lockfile['default'] + assert 'idna' in p.lockfile['default'] + assert 'urllib3' in p.lockfile['default'] + assert 'certifi' in p.lockfile['default'] + @pytest.mark.complex @pytest.mark.lock @pytest.mark.skip(reason='Does not work unless you can explicitly install into py2')