distlib v0.2.6 breaks --relocatable for long shebangs

[lithammer]

pip v10 ships with distlib v0.2.6, which introduced a clever way to work around the shebang limit on certain systems. The problem is that this shebang isn't recognized at all by the --relocatable flag.

        else:
            result = b'#!/bin/sh\n'
            result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n'
            result += b"' '''"
        return result

https://github.com/pypa/pip/blob/87d2735487fe2e8296d777c43ecb9920a2201bd6/src/pip/_vendor/distlib/scripts.py#L164-L168

Here's an example of what it looks like:

#!/bin/sh
'''exec' /very/long/shebang/[...]/env/bin/python "$0" "$@"
' '''
# -*- coding: utf-8 -*-
import re
import sys

from foobar import main

if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
    sys.exit(main())

[lobsterdore]

I've been hit by this issue as well, virtualenv throws this error when encountering such scripts:

Script {LONG PATH} cannot be made relative (it's not a normal script that starts with #!{LONG PATH}/bin/python)

[lobsterdore]

I've put together a little script to get around this issue, I've extracted the relevant parts from virtualenv.py and added in support for the new style:

#!/usr/bin/env python
"""Relocatable functions extracted from virtualenv package
ensures that we can relocate scripts that start with this format:

#!/bin/sh
'''exec' [VENV_PATH]/bin/python "$0" "$@"
' '''

See https://github.com/pypa/virtualenv/issues/1169
"""

import argparse
import logging
import os
import sys
from os.path import join

PY_VERSION = "python{}.{}".format(sys.version_info[0], sys.version_info[1])
ABI_FLAGS = getattr(sys, "abiflags", "")


# Relocating the environment:
def make_environment_relocatable(home_dir):
    """
    Makes the already-existing environment use relative paths, and takes out
    the #!-based environment selection in scripts.
    """
    home_dir, _, _, bin_dir = path_locations(home_dir)
    activate_this = os.path.join(bin_dir, "activate_this.py")
    if not os.path.exists(activate_this):
        logging.fatal(
            "The environment doesn't have a file %s -- please re-run "
            "virtualenv on this environment to update it",
            activate_this,
        )
    fixup_scripts(home_dir, bin_dir)
    fixup_pth_and_egg_link(home_dir)


OK_ABS_SCRIPTS = [
    "python",
    "python{}".format(sys.version[:3]),
    "activate",
    "activate.bat",
    "activate_this.py",
    "activate.fish",
    "activate.csh",
    "activate.xsh",
]


def fixup_scripts(_, bin_dir):
    new_shebang_args = ("/usr/bin/env", sys.version[:3], "")

    # This is what we expect at the top of scripts:
    shebangs = [
        "#!/bin/sh\n'''exec' {} \"$0\" \"$@\"\n' '''".format(
            os.path.normcase(
                os.path.join(
                    os.path.abspath(bin_dir),
                    "python{}".format(new_shebang_args[2])
                )
            )
        ),
        "#!{}".format(
            os.path.normcase(
                os.path.join(
                    os.path.abspath(bin_dir),
                    "python{}".format(new_shebang_args[2])
                )
            )
        )
    ]

    # This is what we'll put:
    new_shebang = "#!{} python{}{}".format(*new_shebang_args)

    for filename in os.listdir(bin_dir):
        filename = os.path.join(bin_dir, filename)
        if not os.path.isfile(filename):
            # ignore child directories, e.g. .svn ones.
            continue
        with open(filename, "rb") as f:
            try:
                lines = f.read().decode("utf-8").splitlines()
            except UnicodeDecodeError:
                # This is probably a binary program instead
                # of a script, so just ignore it.
                continue
        if not lines:
            logging.warn("Script %s is an empty file", filename)
            continue

        already_relative = False
        made_relative = False

        for shebang in shebangs:
            shebang_length = len(shebang.split('\n'))
            old_shebang = '\n'.join(lines[0:shebang_length]).strip()
            old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:])

            if old_shebang.startswith(shebang):
                made_relative = True
                logging.info("Making script %s relative", filename)
                script = relative_script(
                    [new_shebang] + lines[shebang_length:]
                )
                with open(filename, "wb") as f:
                    f.write("\n".join(script).encode("utf-8"))
                break
            elif lines[0].strip() == new_shebang:
                already_relative = True
                break

        if not made_relative:
            if os.path.basename(filename) in OK_ABS_SCRIPTS:
                logging.debug("Cannot make script %s relative", filename)
            elif already_relative:
                logging.info(
                    "Script %s has already been made relative", filename
                )
            else:
                logging.warn(
                    "Script %s cannot be made relative "
                    "(it's not a normal script)",
                    filename
                )
            continue


def relative_script(lines):
    """Return a script that'll work in a relocatable environment."""
    activate = (
        "import os; "
        "activate_this=os.path.join(os.path.dirname("
        "os.path.realpath(__file__)), 'activate_this.py'); "
        "exec(compile(open(activate_this).read(), activate_this, 'exec'), "
        "{ '__file__': activate_this}); "
        "del os, activate_this"
    )
    # Find the last future statement in the script. If we insert the activation
    # line before a future statement, Python will raise a SyntaxError.
    activate_at = None
    for idx, line in reversed(list(enumerate(lines))):
        if line.split()[:3] == ["from", "__future__", "import"]:
            activate_at = idx + 1
            break
    if activate_at is None:
        # Activate after the shebang.
        activate_at = 1
    return lines[:activate_at] + ["", activate, ""] + lines[activate_at:]


def fixup_pth_and_egg_link(home_dir, sys_path=None):
    """Makes .pth and .egg-link files use relative paths"""
    home_dir = os.path.normcase(os.path.abspath(home_dir))
    if sys_path is None:
        sys_path = sys.path
    for a_path in sys_path:
        if not a_path:
            a_path = "."
        if not os.path.isdir(a_path):
            continue
        a_path = os.path.normcase(os.path.abspath(a_path))
        if not a_path.startswith(home_dir):
            logging.debug(
                "Skipping system (non-environment) directory %s", a_path
            )
            continue
        for filename in os.listdir(a_path):
            filename = os.path.join(a_path, filename)
            if filename.endswith(".pth"):
                if not os.access(filename, os.W_OK):
                    logging.warn(
                        "Cannot write .pth file %s, skipping", filename
                    )
                else:
                    fixup_pth_file(filename)
            if filename.endswith(".egg-link"):
                if not os.access(filename, os.W_OK):
                    logging.warn(
                        "Cannot write .egg-link file %s, skipping", filename
                    )
                else:
                    fixup_egg_link(filename)


def fixup_pth_file(filename):
    lines = []
    with open(filename) as f:
        prev_lines = f.readlines()
    for line in prev_lines:
        line = line.strip()
        if not line or line.startswith("#") or line.startswith("import ") \
                or os.path.abspath(line) != line:
            lines.append(line)
        else:
            new_value = make_relative_path(filename, line)
            if line != new_value:
                logging.debug("Rewriting path {} as {} (in {})".format(
                    line, new_value, filename
                ))
            lines.append(new_value)
    if lines == prev_lines:
        logging.info("No changes to .pth file %s", filename)
        return
    logging.info("Making paths in .pth file %s relative", filename)
    with open(filename, "w") as f:
        f.write("\n".join(lines) + "\n")


def fixup_egg_link(filename):
    with open(filename) as f:
        link = f.readline().strip()
    if os.path.abspath(link) != link:
        logging.debug("Link in %s already relative", filename)
        return
    new_link = make_relative_path(filename, link)
    logging.info(
        "Rewriting link {} in {} as {}".format(link, filename, new_link)
    )
    with open(filename, "w") as f:
        f.write(new_link)


def make_relative_path(source, dest, dest_is_directory=True):
    """
    Make a filename relative, where the filename is dest, and it is
    being referred to from the filename source.

        >>> make_relative_path('/usr/share/something/a-file.pth',
        ...                    '/usr/share/another-place/src/Directory')
        '../another-place/src/Directory'
        >>> make_relative_path('/usr/share/something/a-file.pth',
        ...                    '/home/user/src/Directory')
        '../../../home/user/src/Directory'
        >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
        './'
    """
    source = os.path.dirname(source)
    if not dest_is_directory:
        dest_filename = os.path.basename(dest)
        dest = os.path.dirname(dest)
    else:
        dest_filename = None
    dest = os.path.normpath(os.path.abspath(dest))
    source = os.path.normpath(os.path.abspath(source))
    dest_parts = dest.strip(os.path.sep).split(os.path.sep)
    source_parts = source.strip(os.path.sep).split(os.path.sep)
    while dest_parts and source_parts and dest_parts[0] == source_parts[0]:
        dest_parts.pop(0)
        source_parts.pop(0)
    full_parts = [".."] * len(source_parts) + dest_parts
    if not dest_is_directory and dest_filename is not None:
        full_parts.append(dest_filename)
    if not full_parts:
        # Special case for the current directory (otherwise it'd be '')
        return "./"
    return os.path.sep.join(full_parts)


def path_locations(home_dir, dry_run=False):
    """Return the path locations for the environment (where libraries are,
    where scripts go, etc)"""
    home_dir = os.path.abspath(home_dir)
    lib_dir, inc_dir, bin_dir = None, None, None

    lib_dir = join(home_dir, "lib", PY_VERSION)
    inc_dir = join(home_dir, "include", PY_VERSION + ABI_FLAGS)
    bin_dir = join(home_dir, "bin")

    return home_dir, lib_dir, inc_dir, bin_dir


def main():
    parser = argparse.ArgumentParser(description='WIP')
    parser.add_argument(
        "home_dir",
        help="[REQUIRED] directory to relocate"
    )

    args = parser.parse_args()
    home_dir = args.home_dir
    make_environment_relocatable(home_dir)


if __name__ == "__main__":
    main()

This is specifically designed for Linux systems as I've removed all of the OS detection parts.

[gaborbernat]

Superseded by #1473