Skip to content

Bit more complicated parsing of the requirements_to_exclude during install_depend_packages.py #21459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
scbedd merged 4 commits into from
Oct 28, 2021
Merged

Bit more complicated parsing of the requirements_to_exclude during install_depend_packages.py #21459

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2e5b071
we do a little bit more work to filter out packages now
scbedd Oct 27, 2021
30f1007
add docstring to check_req_against_exclusion
scbedd Oct 28, 2021
43b4ecd
small docstring update
scbedd Oct 28, 2021
38b0475
specifiers
scbedd Oct 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 60 additions & 30 deletions eng/tox/install_depend_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@
import logging
from packaging.specifiers import SpecifierSet
from pkg_resources import Requirement, parse_version
import re

from pypi_tools.pypi import PyPIClient

setup_parser_path = os.path.abspath(
os.path.join(os.path.abspath(__file__), "..", "..", "versioning")
)
setup_parser_path = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "versioning"))
sys.path.append(setup_parser_path)
from setup_parser import get_install_requires

Expand All @@ -30,19 +29,18 @@

# both min and max overrides are *inclusive* of the version targeted
MINIMUM_VERSION_SUPPORTED_OVERRIDE = {
'azure-common': '1.1.10',
'msrest': '0.6.10',
'typing-extensions': '3.6.5',
'opentelemetry-api': '1.3.0',
'opentelemetry-sdk': '1.3.0',
'azure-core': '1.11.0',
'requests': '2.19.0',
'six': '1.12.0'
"azure-common": "1.1.10",
"msrest": "0.6.10",
"typing-extensions": "3.6.5",
"opentelemetry-api": "1.3.0",
"opentelemetry-sdk": "1.3.0",
"azure-core": "1.11.0",
"requests": "2.19.0",
"six": "1.12.0",
}

MAXIMUM_VERSION_SUPPORTED_OVERRIDE = {
'cryptography': '3.4.8'
}
MAXIMUM_VERSION_SUPPORTED_OVERRIDE = {"cryptography": "3.4.8"}


def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir):
# This method identifies latest/ minimal version of dependent packages and installs them from pyPI
Expand All @@ -65,15 +63,15 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir):
pkgs_file_path = os.path.join(temp_dir, PKGS_TXT_FILE)
with open(pkgs_file_path, "w") as pkgs_file:
for package in released_packages:
pkgs_file.write(package + '\n')
pkgs_file.write(package + "\n")
logging.info("Created file %s to track azure packages found on PyPI", pkgs_file_path)


def find_released_packages(setup_py_path, dependency_type):
# this method returns list of required available package on PyPI in format <package-name>==<version>

# parse setup.py and find install requires
requires = [r for r in get_install_requires(setup_py_path) if '-nspkg' not in r]
requires = [r for r in get_install_requires(setup_py_path) if "-nspkg" not in r]

# Get available version on PyPI for each required package
avlble_packages = [x for x in map(lambda x: process_requirement(x, dependency_type), requires) if x]
Expand All @@ -99,11 +97,14 @@ def process_requirement(req, dependency_type):
logging.info("Versions available on PyPI for %s: %s", pkg_name, versions)

if pkg_name in MINIMUM_VERSION_SUPPORTED_OVERRIDE:
versions = [v for v in versions if parse_version(v) >= parse_version(MINIMUM_VERSION_SUPPORTED_OVERRIDE[pkg_name])]
versions = [
v for v in versions if parse_version(v) >= parse_version(MINIMUM_VERSION_SUPPORTED_OVERRIDE[pkg_name])
]

if pkg_name in MAXIMUM_VERSION_SUPPORTED_OVERRIDE:
versions = [v for v in versions if parse_version(v) <= parse_version(MAXIMUM_VERSION_SUPPORTED_OVERRIDE[pkg_name])]

versions = [
v for v in versions if parse_version(v) <= parse_version(MAXIMUM_VERSION_SUPPORTED_OVERRIDE[pkg_name])
]

# Search from lowest to latest in case of finding minimum dependency
# Search from latest to lowest in case of finding latest required version
Expand All @@ -114,13 +115,43 @@ def process_requirement(req, dependency_type):
# return first version that matches specifier in <package-name>==<version> format
for version in versions:
if version in spec:
logging.info("Found %s version %s that matches specifier %s", dependency_type, version, spec)
logging.info(
"Found %s version %s that matches specifier %s",
dependency_type,
version,
spec,
)
return pkg_name + "==" + version

logging.error("No version is found on PyPI for package %s that matches specifier %s", pkg_name, spec)
logging.error(
"No version is found on PyPI for package %s that matches specifier %s",
pkg_name,
spec,
)
return ""


def check_req_against_exclusion(req, req_to_exclude):
"""
This function evaluates a requirement from a dev_requirements file against a file name. Returns True
if the requirement is for the same package listed in "req_to_exclude". False otherwise.

:param req: An incoming "req" looks like a requirement that appears in a dev_requirements file. EG: [ "../../../tools/azure-devtools",
"https://docsupport.blob.core.windows.net/repackaged/cffi-1.14.6-cp310-cp310-win_amd64.whl; sys_platform=='win32' and python_version >= '3.10'",
"msrestazure>=0.4.11", "pytest" ]

:param req_to_exclude: A valid and complete python package name. No specifiers.
"""
req_id = ""
for c in req:
if re.match(r"[A-Za-z0-9_-]", c):
req_id += c
else:
break

return req_id == req_to_exclude


def filter_dev_requirements(setup_py_path, released_packages, temp_dir):
# This method returns list of requirements from dev_requirements by filtering out packages in given list
dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE)
Expand All @@ -130,17 +161,18 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir):

# filter out any package available on PyPI (released_packages)
# include packages without relative reference and packages not available on PyPI
released_packages = [p.split('==')[0] for p in released_packages]
released_packages = [p.split("==")[0] for p in released_packages]
# find prebuilt whl paths in dev requiremente
prebuilt_dev_reqs = [os.path.basename(req.replace('\n', '')) for req in requirements if os.path.sep in req]
prebuilt_dev_reqs = [os.path.basename(req.replace("\n", "")) for req in requirements if os.path.sep in req]
# filter any req if wheel is for a released package
req_to_exclude = [req for req in prebuilt_dev_reqs if req.split('-')[0].replace('_', '-') in released_packages]
req_to_exclude = [req for req in prebuilt_dev_reqs if req.split("-")[0].replace("_", "-") in released_packages]
req_to_exclude.extend(released_packages)

filtered_req = [
req
for req in requirements
if os.path.basename(req.replace('\n', '')) not in req_to_exclude and not any([req.startswith(i) for i in req_to_exclude])
if os.path.basename(req.replace("\n", "")) not in req_to_exclude
and not any([check_req_against_exclusion(req, i) for i in req_to_exclude])
]

logging.info("Filtered dev requirements: %s", filtered_req)
Expand Down Expand Up @@ -175,9 +207,7 @@ def install_packages(packages, req_file):


if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Install either latest or minimum version of dependent packages."
)
parser = argparse.ArgumentParser(description="Install either latest or minimum version of dependent packages.")

parser.add_argument(
"-t",
Expand All @@ -191,7 +221,7 @@ def install_packages(packages, req_file):
"-d",
"--dependency-type",
dest="dependency_type",
choices=['Latest', 'Minimum'],
choices=["Latest", "Minimum"],
help="Dependency type to install. Dependency type is either 'Latest' or 'Minimum'",
required=True,
)
Expand All @@ -207,7 +237,7 @@ def install_packages(packages, req_file):
args = parser.parse_args()
setup_path = os.path.join(os.path.abspath(args.target_package), "setup.py")

if not(os.path.exists(setup_path) and os.path.exists(args.work_dir)):
if not (os.path.exists(setup_path) and os.path.exists(args.work_dir)):
logging.error("Invalid arguments. Please make sure target directory and working directory are valid path")
sys.exit(1)

Expand Down