|
| 1 | +"""Update version specifiers of certain dependencies to comply with SPEC0-like rules. |
| 2 | +
|
| 3 | +SPEC0 (https://scientific-python.org/specs/spec-0000/) recommends that support for |
| 4 | +Python versions are dropped 3 years after initial release, and for core package |
| 5 | +dependencies 2 years after initial release. |
| 6 | +
|
| 7 | +MNE-Python follows a SPEC0-like policy that reduces maintenance burden whilst |
| 8 | +accommodating users in minimum version support similarly to when before this policy was |
| 9 | +adopted. |
| 10 | +
|
| 11 | +MNE-Python's policy differs from SPEC0 in the following ways: |
| 12 | +- Python versions are supported for at least 3 years after release, but possibly longer |
| 13 | + at the discretion of the MNE-Python maintainers based on, e.g., maintainability, |
| 14 | + features. |
| 15 | +- Not all core dependencies have minimum versions pinned, and some optional dependencies |
| 16 | + have minimum versions pinned. Only those dependencies whose older versions require |
| 17 | + considerable work to maintain compatibility with (e.g., due to API changes) have |
| 18 | + minimum versions pinned. |
| 19 | +- Micro/patch versions are not pinned as minimum versions (unless there is an issue with |
| 20 | + a specific patch), as these should not introduce breaking changes. |
| 21 | +- Minimum versions for dependencies are set to the latest minor release that was |
| 22 | + available 2 years prior. The rationale behind this is discussed here: |
| 23 | + https://github.com/mne-tools/mne-python/pull/13451#discussion_r2445337934 |
| 24 | +
|
| 25 | +For example, in October 2025: |
| 26 | +- The latest version of NumPy available 2 years prior was 1.26.1 (released October |
| 27 | + 2023), making the latest minor release 1.26, which would be pinned. Support for 1.26 |
| 28 | + would be dropped in June 2026 in favour of 2.0, which was released in June 2024. |
| 29 | +- The latest version of SciPy available 2 years prior was 1.11.3 (release September |
| 30 | + 2023), making the latest minor release 1.11, which would be pinned. Support for 1.11 |
| 31 | + would be dropped in January 2026 in favour of 1.12, which was released in January |
| 32 | + 2024. |
| 33 | +""" |
| 34 | + |
| 35 | +# Authors: The MNE-Python contributors. |
| 36 | +# License: BSD-3-Clause |
| 37 | +# Copyright the MNE-Python contributors. |
| 38 | + |
| 39 | +import collections |
| 40 | +import datetime |
| 41 | +import re |
| 42 | + |
| 43 | +import requests |
| 44 | +from packaging.requirements import Requirement |
| 45 | +from packaging.specifiers import SpecifierSet |
| 46 | +from packaging.version import InvalidVersion, Version |
| 47 | +from tomlkit import parse |
| 48 | +from tomlkit.items import Comment, Trivia |
| 49 | +from tomlkit.toml_file import TOMLFile |
| 50 | + |
| 51 | +SORT_PACKAGES = [ |
| 52 | + "matplotlib", |
| 53 | + "numpy", |
| 54 | + "pandas", |
| 55 | + "pyvista", |
| 56 | + "pyvistaqt", |
| 57 | + "scikit-learn", |
| 58 | + "scipy", |
| 59 | +] |
| 60 | +SUPPORT_TIME = datetime.timedelta(days=365 * 2) |
| 61 | +CURRENT_DATE = datetime.datetime.now() |
| 62 | + |
| 63 | + |
| 64 | +def get_release_and_drop_dates(package): |
| 65 | + """Get release and drop dates for a given package from pypi.org.""" |
| 66 | + releases = {} |
| 67 | + print(f"Querying pypi.org for {package} versions...", end="", flush=True) |
| 68 | + response = requests.get( |
| 69 | + f"https://pypi.org/simple/{package}", |
| 70 | + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, |
| 71 | + timeout=10, |
| 72 | + ).json() |
| 73 | + print("OK") |
| 74 | + file_date = collections.defaultdict(list) |
| 75 | + for f in response["files"]: |
| 76 | + if f["filename"].endswith(".tar.gz") or f["filename"].endswith(".zip"): |
| 77 | + continue |
| 78 | + if f["yanked"]: |
| 79 | + continue |
| 80 | + ver = f["filename"].split("-")[1] |
| 81 | + try: |
| 82 | + version = Version(ver) |
| 83 | + except InvalidVersion: |
| 84 | + continue |
| 85 | + if version.is_prerelease: |
| 86 | + continue |
| 87 | + release_date = datetime.datetime.fromisoformat(f["upload-time"]).replace( |
| 88 | + tzinfo=None |
| 89 | + ) |
| 90 | + if not release_date: |
| 91 | + continue |
| 92 | + file_date[version].append(release_date) |
| 93 | + release_date = {v: min(file_date[v]) for v in file_date} |
| 94 | + for ver, release_date in sorted(release_date.items()): |
| 95 | + cutoff_date = CURRENT_DATE - SUPPORT_TIME |
| 96 | + pre_cutoff = bool(release_date <= cutoff_date) # was available X time ago |
| 97 | + releases[ver] = {"release_date": release_date, "pre_cutoff": pre_cutoff} |
| 98 | + return releases |
| 99 | + |
| 100 | + |
| 101 | +def update_specifiers(dependencies, releases): |
| 102 | + """Update dependency version specifiers.""" |
| 103 | + for idx, dep in enumerate(dependencies): |
| 104 | + req = Requirement(dep) |
| 105 | + pkg_name = req.name |
| 106 | + pkg_spec = req.specifier |
| 107 | + if pkg_name in releases.keys(): # check if this is a package to update |
| 108 | + # Find package versions matching current specifiers |
| 109 | + package_vers = releases[pkg_name].keys() |
| 110 | + matches = list(pkg_spec.filter(package_vers)) # drop excluded versions |
| 111 | + pre_cutoff_matches = [ |
| 112 | + ver for ver in matches if releases[pkg_name][ver]["pre_cutoff"] |
| 113 | + ] |
| 114 | + if len(pre_cutoff_matches) == 0: |
| 115 | + raise RuntimeError( |
| 116 | + f"{pkg_name} had no versions available {SUPPORT_TIME.days / 365} " |
| 117 | + "years ago compliant with the following specifier(s): " |
| 118 | + f"{pkg_spec if pkg_spec else 'None'}", |
| 119 | + ) |
| 120 | + post_cutoff_matches = [ |
| 121 | + ver for ver in matches if not releases[pkg_name][ver]["pre_cutoff"] |
| 122 | + ] |
| 123 | + |
| 124 | + # Find latest pre-cutoff version to pin as the minimum version |
| 125 | + min_ver = max(pre_cutoff_matches) |
| 126 | + min_ver, min_ver_release = _find_version_to_pin_and_release( |
| 127 | + min_ver, pkg_spec, pre_cutoff_matches, releases[pkg_name] |
| 128 | + ) |
| 129 | + |
| 130 | + # Find earliest post-cutoff version to pin next |
| 131 | + next_ver = None |
| 132 | + next_ver_release = None |
| 133 | + for ver in post_cutoff_matches: |
| 134 | + if _as_minor_version(ver) > min_ver: # if a new minor version |
| 135 | + next_ver, next_ver_release = _find_version_to_pin_and_release( |
| 136 | + ver, pkg_spec, post_cutoff_matches, releases[pkg_name] |
| 137 | + ) |
| 138 | + break |
| 139 | + |
| 140 | + # Update specifiers with new minimum version |
| 141 | + min_ver_spec = SpecifierSet(f">={str(min_ver)}") |
| 142 | + new_spec = [str(min_ver_spec)] |
| 143 | + for spec in str(pkg_spec).split(","): |
| 144 | + spec = spec.strip() |
| 145 | + if spec.startswith(">"): |
| 146 | + continue # ignore old min ver |
| 147 | + if spec.startswith("!=") and not min_ver_spec.contains(spec[2:]): |
| 148 | + continue # ignore outdated exclusions |
| 149 | + new_spec.append(spec) # keep max vers and in-date exclusions |
| 150 | + req.specifier = SpecifierSet(",".join(new_spec)) |
| 151 | + |
| 152 | + dependencies._value[idx] = _add_date_comment( |
| 153 | + dependencies._value[idx], min_ver_release, next_ver, next_ver_release |
| 154 | + ) |
| 155 | + dependencies[idx] = _prettify_requirement(req) |
| 156 | + return dependencies |
| 157 | + |
| 158 | + |
| 159 | +def _as_minor_version(ver): |
| 160 | + """Convert a version to its major.minor form.""" |
| 161 | + return Version(f"{ver.major}.{ver.minor}") |
| 162 | + |
| 163 | + |
| 164 | +def _find_version_to_pin_and_release(version, specifier, all_versions, release_dates): |
| 165 | + """Find the version to pin based on specifiers and that version's release date. |
| 166 | +
|
| 167 | + Tries to reduce this to an unpatched major.minor form if possible (and use the |
| 168 | + unpatched version's release date). If the unpatched minor form is excluded by the |
| 169 | + specifier, finds the earliest patch version (and its release date) that is not |
| 170 | + excluded. |
| 171 | +
|
| 172 | + E.g., if version=1.2.3 but 1.2.0 is excluded by the specifier, find the earliest |
| 173 | + patched version (e.g., 1.2.1) and pin this. If 1.2.0 is not excluded, we can just |
| 174 | + pin 1.2. |
| 175 | +
|
| 176 | + If the unpatched version is not excluded by the specifier but it has been yanked, we |
| 177 | + don't need to pin the patched version, but we do have to rely on the release date of |
| 178 | + the earliest patched version. |
| 179 | + """ |
| 180 | + # Find earliest micro form of this minor version |
| 181 | + version = min( |
| 182 | + ver |
| 183 | + for ver in all_versions |
| 184 | + if _as_minor_version(ver) == _as_minor_version(version) |
| 185 | + ) |
| 186 | + # Check unpatched form of this version is not excluded by existing specifiers |
| 187 | + use_patch = not specifier.contains(_as_minor_version(version)) |
| 188 | + # Find release date of version to be pinned |
| 189 | + release = release_dates[version]["release_date"] |
| 190 | + # Discard patch info if not needed |
| 191 | + version = _as_minor_version(version) if not use_patch else version |
| 192 | + |
| 193 | + return version, release |
| 194 | + |
| 195 | + |
| 196 | +def _prettify_requirement(req): |
| 197 | + """Add spacing to make a requirement specifier prettier.""" |
| 198 | + specifiers = [] |
| 199 | + spec_order = _find_specifier_order(req.specifier) |
| 200 | + for spec in req.specifier: |
| 201 | + spec = str(spec) |
| 202 | + split = re.search(r"[<>=]\d", spec).span()[1] - 1 # find end of operator |
| 203 | + specifiers.append(f" {spec[:split]} {spec[split:]},") # pad operator w/ spaces |
| 204 | + specifiers = [specifiers[i] for i in spec_order] # order by ascending version |
| 205 | + specifiers = "".join(specifiers) |
| 206 | + specifiers = specifiers.rstrip(",") # remove trailing comma |
| 207 | + req.specifier = SpecifierSet() # remove ugly specifiers (from str repr) |
| 208 | + # Add pretty specifiers to name alongside trailing info (extras, markers, url) |
| 209 | + return req.name + specifiers + str(req)[len(req.name) :] |
| 210 | + |
| 211 | + |
| 212 | +def _add_date_comment(dependency, min_ver_release, next_ver, next_ver_release): |
| 213 | + """Add comment for when the min version was released and when it will be changed.""" |
| 214 | + comment = f"# released {min_ver_release.strftime('%Y-%m-%d')}" |
| 215 | + if next_ver is not None: |
| 216 | + comment += ( |
| 217 | + f", will become {str(next_ver)} on " |
| 218 | + f"{(next_ver_release + SUPPORT_TIME).strftime('%Y-%m-%d')}" |
| 219 | + ) |
| 220 | + else: |
| 221 | + comment += ", no newer version available" |
| 222 | + dependency.comment = Comment( |
| 223 | + Trivia(indent=" ", comment_ws="", comment=comment, trail="") |
| 224 | + ) |
| 225 | + return dependency |
| 226 | + |
| 227 | + |
| 228 | +def _find_specifier_order(specifiers): |
| 229 | + """Find ascending order of specifiers according to their version.""" |
| 230 | + versions = [] |
| 231 | + for spec in specifiers: |
| 232 | + versions.append(Version(re.sub(r"[<>=!~]+", "", str(spec)))) # extract version |
| 233 | + return sorted(range(len(versions)), key=lambda i: versions[i]) # sorted indices |
| 234 | + |
| 235 | + |
| 236 | +# Find release and drop dates for desired packages |
| 237 | +package_releases = { |
| 238 | + package: get_release_and_drop_dates(package) for package in SORT_PACKAGES |
| 239 | +} |
| 240 | + |
| 241 | +# Get dependencies from pyproject.toml |
| 242 | +pyproject = TOMLFile("pyproject.toml") |
| 243 | +pyproject_data = pyproject.read() |
| 244 | +project_info = pyproject_data.get("project") |
| 245 | +core_dependencies = project_info["dependencies"] |
| 246 | +opt_dependencies = project_info.get("optional-dependencies", {}) |
| 247 | + |
| 248 | +# Update version specifiers |
| 249 | +core_dependencies = update_specifiers(core_dependencies, package_releases) |
| 250 | +for key in opt_dependencies: |
| 251 | + opt_dependencies[key] = update_specifiers(opt_dependencies[key], package_releases) |
| 252 | +pyproject_data["project"]["dependencies"] = core_dependencies |
| 253 | +if opt_dependencies: |
| 254 | + pyproject_data["project"]["optional-dependencies"] = opt_dependencies |
| 255 | + |
| 256 | +# Save updated pyproject.toml (replace ugly \" with ' first) |
| 257 | +pyproject_data = parse(pyproject_data.as_string().replace('\\"', "'")) |
| 258 | +pyproject.write(pyproject_data) |
![autofix-ci[bot]](https://avatars.githubusercontent.com/in/243519?v=4&size=40){kind=avatar}
0 commit comments