From 25761092e93dec0897fd7e49073abf44ab3bcf13 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:19:47 +0100 Subject: [PATCH 1/8] docs: pin version in links to external docs --- src/lightning_utilities/docs/formatting.py | 78 +++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/src/lightning_utilities/docs/formatting.py b/src/lightning_utilities/docs/formatting.py index 3bd2f06d..5c0928a7 100644 --- a/src/lightning_utilities/docs/formatting.py +++ b/src/lightning_utilities/docs/formatting.py @@ -1,10 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # http://www.apache.org/licenses/LICENSE-2.0 # +import glob +import importlib import inspect +import logging import os +import re import sys -from typing import Tuple +from typing import Iterable, Optional, Tuple, Union def _transform_changelog(path_in: str, path_out: str) -> None: @@ -63,3 +67,75 @@ def find_source() -> Tuple[str, int, int]: branch = {"latest": "master", "stable": "master"}.get(branch, branch) filename = "/".join([branch] + filename.split("/")[1:]) return f"https://github.com/{github_user}/{github_repo}/blob/{filename}" + + +def _update_link_based_imported_package(link: str, pkg_ver: str, version_digits: Optional[int]) -> str: + """Adjust the linked external docs to be local. + + Args: + link: the source link to be replaced + pkg_ver: the target link to be replaced, if ``{package.version}`` is included it will be replaced accordingly + version_digits: for semantic versioning, how many digits to be considered + + """ + pkg_att = pkg_ver.split(".") + # load the package with all additional sub-modules + module = importlib.import_module(".".join(pkg_att[:-1])) + # load the attribute + ver = getattr(module, pkg_att[-1]) + # drop any additional context after `+` + ver = ver.split("+")[0] + # crop the version to the number of digits + ver = ".".join(ver.split(".")[:version_digits]) + # replace the version + return link.replace(f"{{{pkg_ver}}}", ver) + + +def _adjust_linked_external_docs( + source_link: str, + target_link: str, + browse_folder: Union[str, Iterable[str]], + file_extensions: Tuple[str] = (".rst", ".py"), + version_digits: int = 2, +) -> None: + r"""Adjust the linked external docs to be local. + + Args: + source_link: the link to be replaced + target_link: the link to be replaced, if ``{package.version}`` is included it will be replaced accordingly + browse_folder: the location of the browsable folder + file_extensions: what kind of files shall be scanned + version_digits: for semantic versioning, how many digits to be considered + + Examples: + >>> _adjust_linked_external_docs( + ... "https://numpy.org/doc/stable/", + ... "https://numpy.org/doc/{numpy.__version__}/", + ... "docs/source", + ... ) + + """ + list_files = [] + if isinstance(browse_folder, str): + browse_folder = [browse_folder] + for folder in browse_folder: + for ext in file_extensions: + list_files += glob.glob(os.path.join(folder, "**", f"*{ext}"), recursive=True) + if not list_files: + logging.warning(f'no files were listed in folder "{browse_folder}" and pattern "{file_extensions}"') + return + + # find the expression for package version in {} brackets if any, use re to find it + pkg_ver_all = re.findall(r"{([\w.]+)}", target_link) + for pkg_ver in pkg_ver_all: + target_link = _update_link_based_imported_package(target_link, pkg_ver, version_digits) + + # make it unique + list_files = set(list_files) + # replace the source link with target link + for fpath in list_files: + with open(fpath, encoding="UTF-8") as fopen: + body = fopen.read() + body = body.replace(source_link, target_link) + with open(fpath, "w", encoding="UTF-8") as fw: + fw.write(body) From a6afe93338f73b8668559e86328cbb1bbfea14fb Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:21:19 +0100 Subject: [PATCH 2/8] chlog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8c02721..98d9bd00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- +- docs: enable pin version in links to external docs ([#236](https://github.com/Lightning-AI/utilities/pull/236)) ### Changed From 21e510a2dcf79ddcdd1ccee01ca48434f1c5702b Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:24:21 +0100 Subject: [PATCH 3/8] mypy --- src/lightning_utilities/docs/formatting.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lightning_utilities/docs/formatting.py b/src/lightning_utilities/docs/formatting.py index 5c0928a7..288a51d4 100644 --- a/src/lightning_utilities/docs/formatting.py +++ b/src/lightning_utilities/docs/formatting.py @@ -95,7 +95,7 @@ def _adjust_linked_external_docs( source_link: str, target_link: str, browse_folder: Union[str, Iterable[str]], - file_extensions: Tuple[str] = (".rst", ".py"), + file_extensions: Iterable[str] = (".rst", ".py"), version_digits: int = 2, ) -> None: r"""Adjust the linked external docs to be local. @@ -130,10 +130,8 @@ def _adjust_linked_external_docs( for pkg_ver in pkg_ver_all: target_link = _update_link_based_imported_package(target_link, pkg_ver, version_digits) - # make it unique - list_files = set(list_files) # replace the source link with target link - for fpath in list_files: + for fpath in set(list_files): with open(fpath, encoding="UTF-8") as fopen: body = fopen.read() body = body.replace(source_link, target_link) From 9ac6c5efd44a41d2d18ff23a79b97a57beac6ad8 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:30:43 +0100 Subject: [PATCH 4/8] use in docs --- docs/source/conf.py | 6 ++++++ src/lightning_utilities/docs/formatting.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 91426a99..e66c2515 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,6 +14,7 @@ import lightning_utilities import pt_lightning_sphinx_theme from lightning_utilities.docs import fetch_external_assets +from lightning_utilities.docs.formatting import adjust_linked_external_docs # -- Path setup -------------------------------------------------------------- @@ -46,6 +47,11 @@ # -- Project documents ------------------------------------------------------- fetch_external_assets(docs_folder=_PATH_HERE) +adjust_linked_external_docs( + source_link="https://numpy.org/doc/stable/", + target_link="https://numpy.org/doc/{numpy.__version__}/", + browse_folder=_PATH_HERE, +) # export the READme diff --git a/src/lightning_utilities/docs/formatting.py b/src/lightning_utilities/docs/formatting.py index 288a51d4..e6158706 100644 --- a/src/lightning_utilities/docs/formatting.py +++ b/src/lightning_utilities/docs/formatting.py @@ -91,7 +91,7 @@ def _update_link_based_imported_package(link: str, pkg_ver: str, version_digits: return link.replace(f"{{{pkg_ver}}}", ver) -def _adjust_linked_external_docs( +def adjust_linked_external_docs( source_link: str, target_link: str, browse_folder: Union[str, Iterable[str]], @@ -108,7 +108,7 @@ def _adjust_linked_external_docs( version_digits: for semantic versioning, how many digits to be considered Examples: - >>> _adjust_linked_external_docs( + >>> adjust_linked_external_docs( ... "https://numpy.org/doc/stable/", ... "https://numpy.org/doc/{numpy.__version__}/", ... "docs/source", From 04fd71fd5d9b7e51bd6e00a89d63d3855befcc5f Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:33:37 +0100 Subject: [PATCH 5/8] numpy --- requirements/_docs.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/_docs.txt b/requirements/_docs.txt index 0ce9d9f8..2110c882 100644 --- a/requirements/_docs.txt +++ b/requirements/_docs.txt @@ -14,3 +14,5 @@ sphinx-autodoc-typehints >=1.0 sphinx-paramlinks >=0.5.1 sphinx-togglebutton >=0.2 sphinx-copybutton >=0.3 + +numpy # for validation of custom/local version adjustemnt From 2b84b4182639a9c1fce4d3e413a909b1c01555c8 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 13:58:13 +0100 Subject: [PATCH 6/8] logging --- src/lightning_utilities/docs/formatting.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lightning_utilities/docs/formatting.py b/src/lightning_utilities/docs/formatting.py index e6158706..32512ca4 100644 --- a/src/lightning_utilities/docs/formatting.py +++ b/src/lightning_utilities/docs/formatting.py @@ -122,7 +122,7 @@ def adjust_linked_external_docs( for ext in file_extensions: list_files += glob.glob(os.path.join(folder, "**", f"*{ext}"), recursive=True) if not list_files: - logging.warning(f'no files were listed in folder "{browse_folder}" and pattern "{file_extensions}"') + logging.warning(f'No files were listed in folder "{browse_folder}" and pattern "{file_extensions}"') return # find the expression for package version in {} brackets if any, use re to find it @@ -134,6 +134,9 @@ def adjust_linked_external_docs( for fpath in set(list_files): with open(fpath, encoding="UTF-8") as fopen: body = fopen.read() - body = body.replace(source_link, target_link) + body_ = body.replace(source_link, target_link) + if body == body_: + continue + logging.debug(f'links adjusting in {fpath}: "{source_link}" -> "{target_link}"') with open(fpath, "w", encoding="UTF-8") as fw: - fw.write(body) + fw.write(body_) From 0a17172cb40c3711ab916336c36f242a29fc3c01 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 20:29:15 +0100 Subject: [PATCH 7/8] add test --- src/lightning_utilities/docs/__init__.py | 5 +++- tests/unittests/docs/test_formatting.py | 32 ++++++++++++++++++++++++ tests/unittests/docs/test_retriever.py | 4 +-- 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/docs/test_formatting.py diff --git a/src/lightning_utilities/docs/__init__.py b/src/lightning_utilities/docs/__init__.py index 7b2e41cc..093cce5c 100644 --- a/src/lightning_utilities/docs/__init__.py +++ b/src/lightning_utilities/docs/__init__.py @@ -1,3 +1,6 @@ """General tools for Docs.""" -from lightning_utilities.docs.retriever import fetch_external_assets # noqa: F401 +from lightning_utilities.docs.formatting import adjust_linked_external_docs +from lightning_utilities.docs.retriever import fetch_external_assets + +__all__ = ["adjust_linked_external_docs", "fetch_external_assets"] diff --git a/tests/unittests/docs/test_formatting.py b/tests/unittests/docs/test_formatting.py new file mode 100644 index 00000000..1329d6b1 --- /dev/null +++ b/tests/unittests/docs/test_formatting.py @@ -0,0 +1,32 @@ +import os.path + +import pytest +from lightning_utilities.docs import adjust_linked_external_docs + + +@pytest.mark.online() +def test_adjust_linked_external_docs(temp_docs): + # take config as it includes API references with `stable` + path_conf = os.path.join(temp_docs, "conf.py") + + def _get_line_with_numpy(path_rst: str) -> str: + with open(path_rst, encoding="UTF-8") as fopen: + lines = fopen.readlines() + # find the first line with figure reference + return next(ln for ln in lines if ln.lstrip().startswith('"numpy":')) + + # validate the initial expectations + line = _get_line_with_numpy(path_conf) + assert "https://numpy.org/doc/stable/" in line + + adjust_linked_external_docs( + "https://numpy.org/doc/stable/", "https://numpy.org/doc/{numpy.__version__}/", temp_docs + ) + + import numpy as np + + np_version = np.__version__.split(".") + + # validate the final state of index page + line = _get_line_with_numpy(path_conf) + assert f"https://numpy.org/doc/{'.'.join(np_version[:2])}/" in line diff --git a/tests/unittests/docs/test_retriever.py b/tests/unittests/docs/test_retriever.py index 2b503e1f..2a5ba541 100644 --- a/tests/unittests/docs/test_retriever.py +++ b/tests/unittests/docs/test_retriever.py @@ -22,7 +22,7 @@ def _get_line_with_figure(path_rst: str) -> str: # validate the initial expectations line = _get_line_with_figure(path_index) - # that the image exists~ + # that the image exists assert "Lightning.gif" in line # and it is sourced in S3 assert ".s3." in line @@ -31,7 +31,7 @@ def _get_line_with_figure(path_rst: str) -> str: # validate the final state of index page line = _get_line_with_figure(path_index) - # that the image exists~ + # that the image exists assert os.path.join("fetched-s3-assets", "Lightning.gif") in line # but it is not sourced from S3 assert ".s3." not in line From 07555b4f7393a2b1ea9ad346d0e9e1f3bfb8e9ca Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 6 Mar 2024 20:36:04 +0100 Subject: [PATCH 8/8] import --- docs/source/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index e66c2515..a3007a04 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,8 +13,7 @@ import lightning_utilities import pt_lightning_sphinx_theme -from lightning_utilities.docs import fetch_external_assets -from lightning_utilities.docs.formatting import adjust_linked_external_docs +from lightning_utilities.docs import adjust_linked_external_docs, fetch_external_assets # -- Path setup --------------------------------------------------------------