Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix sdist installation #3080

Merged
merged 15 commits into from
Jul 14, 2023
107 changes: 85 additions & 22 deletions python/packaging/frontend_sdist/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,42 +15,103 @@
# limitations under the License.
#

import os
import subprocess
import sys

from setuptools import setup
from setuptools.command.install import install
import subprocess as sp

tensorrt_module = "##TENSORRT_MODULE##"
tensorrt_version = "##TENSORRT_PYTHON_VERSION##"
tensorrt_submodules = [
"{}_libs=={}".format(tensorrt_module, tensorrt_version),
"{}_bindings=={}".format(tensorrt_module, tensorrt_version),
]
nvidia_pip_index_url = os.environ.get("NVIDIA_PIP_INDEX_URL", "https://pypi.nvidia.com")
disable_internal_pip = os.environ.get("NVIDIA_TENSORRT_DISABLE_INTERNAL_PIP", False)


def run_pip_command(args, call_func):
try:
return call_func([sys.executable, "-m", "pip"] + args)
except subprocess.CalledProcessError:
return call_func([os.path.join(sys.exec_prefix, "bin", "pip")] + args)


class InstallCommand(install):
def run(self):
def install_dep(package_name):
status = sp.run(
[
sys.executable,
"-m",
"pip",
"install",
"{:}==##TENSORRT_PYTHON_VERSION##".format(package_name),
"--index-url",
"https://pypi.nvidia.com",
]
)
status.check_returncode()

install_dep("{:}_libs".format(tensorrt_module))
install_dep("{:}_bindings".format(tensorrt_module))

install.run(self)
# pip-inside-pip hack ref #3080
run_pip_command(
[
"install",
"--extra-index-url",
nvidia_pip_index_url,
*tensorrt_submodules,
],
subprocess.check_call,
)

super().run()


def pip_config_list():
Copy link
Collaborator

@ttyio ttyio Jul 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we replace the function with below code, to solve some issue we see in internal CI/CD, thanks!

def run_pip_command(args, call_func):
    try:
        return call_func([sys.executable, "-m", "pip"] + args)
    except subprocess.CalledProcessError:
        return call_func([os.path.join(sys.exec_prefix, "bin", "pip")] + args)

def pip_config_list():
    """Get the current pip config (env vars, config file, etc)."""
    return run_pip_command(["config", "list"], subprocess.check_output).decode()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"""Get the current pip config (env vars, config file, etc)."""
return run_pip_command(["config", "list"], subprocess.check_output).decode()


def parent_command_line():
"""Get the command line of the parent PID."""
pid = os.getppid()
# try retrieval using psutil
try:
import psutil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know what the install base looks like for this Python module, but it does look like the best solution if available.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it has zero dependencies, but not all platform specific wheels are available and building it from source will fail e.g. on graviton processors ref giampaolo/psutil#2103


return " ".join(psutil.Process(pid).cmdline())
except ModuleNotFoundError:
pass
# fall back to shell
try:
return subprocess.check_output(
["ps", "-p", str(pid), "-o", "command", "--no-headers"]
).decode()
except subprocess.CalledProcessError:
return ""


# use pip-inside-pip hack only if the nvidia index is not set in the environment
if (
disable_internal_pip
or nvidia_pip_index_url in pip_config_list()
or nvidia_pip_index_url in parent_command_line()
):
install_requires = tensorrt_submodules
cmdclass = {}
else:
install_requires = []
cmdclass = {"install": InstallCommand}


setup(
name=tensorrt_module,
version="##TENSORRT_PYTHON_VERSION##",
version=tensorrt_version,
description="A high performance deep learning inference library",
long_description="A high performance deep learning inference library",
long_description="""A high performance deep learning inference library
To install, please execute the following:
```
pip install tensorrt --extra-index-url {}
```
Or add the index URL to the (space-separated) PIP_EXTRA_INDEX_URL environment variable:
```
export PIP_EXTRA_INDEX_URL='{}'
pip install tensorrt
```
When the extra index url does not contain `{}`, a nested `pip install` will run with the proper extra index url hard-coded.
""".format(
nvidia_pip_index_url, nvidia_pip_index_url, nvidia_pip_index_url
),
long_description_content_type="text/markdown",
author="NVIDIA Corporation",
license="Proprietary",
classifiers=[
Expand All @@ -59,12 +120,14 @@ def install_dep(package_name):
"Programming Language :: Python :: 3",
],
packages=[tensorrt_module],
install_requires=install_requires,
python_requires=">=3.6", # ref https://pypi.nvidia.com/tensorrt-bindings/
cmdclass=cmdclass,
extras_require={"numpy": "numpy"},
package_data={tensorrt_module: ["*.so*", "*.pyd", "*.pdb"]},
include_package_data=True,
zip_safe=True,
keywords="nvidia tensorrt deeplearning inference",
url="https://developer.nvidia.com/tensorrt",
download_url="https://github.com/nvidia/tensorrt/tags",
cmdclass={"install": InstallCommand},
)
Loading