ci/docs: fetch assets only for deployment, omit PR (#18609)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/ci-check-md-links.yml

@@ -16,4 +16,4 @@ jobs:
   check-md-links:
     uses: Lightning-AI/utilities/.github/workflows/[email protected]
     with:
-      config-file: ".github/markdown.links.config.json"
+      config-file: ".github/markdown-links-config.json"

.github/workflows/docs-build.yml

@@ -42,10 +42,13 @@ jobs:
       matrix:
         pkg-name: ["app", "fabric", "pytorch"]
         check: ["html", "doctest", "linkcheck"]
+    env:
+      FAST_DOCS_DEV: 1
     steps:
       - uses: actions/checkout@v3
         with:
-          submodules: true
+          # only Pytorch has/uses notebooks
+          submodules: ${{ matrix.pkg-name == 'pytorch' }}
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
@@ -74,28 +77,28 @@ jobs:
           pip install .[all] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
             -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL}
           pip list
+      - name: Install req. for Notebooks/tutorials
+        if: matrix.pkg-name == 'pytorch'
+        run: pip install -q -r _notebooks/.actions/requires.txt
 
       - name: Test Examples in Docs
         if: ${{ matrix.check == 'doctest' }}
         working-directory: ./docs/source-${{ matrix.pkg-name }}
-        env:
-          FAST_DOCS_DEV: 1
         run: |
           make doctest
           make coverage
 
       - name: Check External Links
         if: ${{ matrix.check == 'linkcheck' }}
         working-directory: ./docs/source-${{ matrix.pkg-name }}
-        env:
-          FAST_DOCS_DEV: 1
         run: make linkcheck SPHINXOPTS="--keep-going"
 
+      - name: Full build for deployment
+        if: github.event_name == 'push'
+        run: echo "FAST_DOCS_DEV=0" >> $GITHUB_ENV
       - name: Make Documentation
         if: ${{ matrix.check == 'html' }}
         working-directory: ./docs/source-${{ matrix.pkg-name }}
-        env:
-          FAST_DOCS_DEV: 0
         run: make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going"
 
       - name: Keep artifact

.readthedocs.yml

@@ -42,7 +42,7 @@ build:
     - pip install -U pip awscli --user
     - python -m awscli s3 sync --no-sign-request s3://sphinx-packages/ dist/ ; ls -lh dist/
     - >
-      pip install -e . -q -r requirements/pytorch/docs.txt \
+      pip install -e . -q -r requirements/pytorch/docs.txt -r _notebooks/.actions/requires.txt \
           -f 'https://download.pytorch.org/whl/cpu/torch_stable.html' -f dist/ ;
       pip list
     # this need to be split so `sphinx-build` is picked from previous installation

Makefile

@@ -48,20 +48,22 @@ test: clean
 
 docs: docs-pytorch
 
-pull-template:
+sphinx-theme:
 	pip install -q awscli
+	mkdir -p dist/
 	aws s3 sync --no-sign-request s3://sphinx-packages/ dist/
+	pip install lai-sphinx-theme -f dist/
 
-docs-app: clean pull-template
-	pip install -e .[all] --quiet -r requirements/app/docs.txt -f dist/
+docs-app: clean sphinx-theme
+	pip install -e .[all] --quiet -r requirements/app/docs.txt
 	cd docs/source-app && $(MAKE) html --jobs $(nproc)
 
-docs-fabric: clean pull-template
-	pip install -e .[all] --quiet -r requirements/fabric/docs.txt -f dist/
+docs-fabric: clean sphinx-theme
+	pip install -e .[all] --quiet -r requirements/fabric/docs.txt
 	cd docs/source-fabric && $(MAKE) html --jobs $(nproc)
 
-docs-pytorch: clean pull-template
-	pip install -e .[all] --quiet -r requirements/pytorch/docs.txt -f dist/
+docs-pytorch: clean sphinx-theme
+	pip install -e .[all] --quiet -r requirements/pytorch/docs.txt -r _notebooks/.actions/requires.txt
 	cd docs/source-pytorch && $(MAKE) html --jobs $(nproc)
 
 update:

docs/source-pytorch/conf.py

@@ -36,7 +36,8 @@
 _PATH_HERE = os.path.abspath(os.path.dirname(__file__))
 _PATH_ROOT = os.path.join(_PATH_HERE, "..", "..")
 _PATH_RAW_NB = os.path.join(_PATH_ROOT, "_notebooks")
-_SHOULD_COPY_NOTEBOOKS = True
+_PATH_RAW_NB_ACTIONS = os.path.join(_PATH_RAW_NB, ".actions")
+_COPY_NOTEBOOKS = int(os.getenv("DOCS_COPY_NOTEBOOKS", True))
 _FOLDER_GENERATED = "generated"
 
 
@@ -48,24 +49,27 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 
 
 assist_local = _load_py_module("assistant", os.path.join(_PATH_ROOT, ".actions", "assistant.py"))
-
-if os.path.isdir(os.path.join(_PATH_RAW_NB, ".actions")):
-    assist_nb = _load_py_module("assistant", os.path.join(_PATH_RAW_NB, ".actions", "assistant.py"))
+if os.path.isdir(_PATH_RAW_NB_ACTIONS):
+    assist_nb = _load_py_module("assistant", os.path.join(_PATH_RAW_NB_ACTIONS, "assistant.py"))
 else:
-    _SHOULD_COPY_NOTEBOOKS = False
-    warnings.warn("To build the code, please run: `git submodule update --init --recursive`", stacklevel=2)
+    _COPY_NOTEBOOKS = False
+    warnings.warn(
+        "To build full docs you need to include also tutorials/notebooks from submodule code."
+        " Please run: `git submodule update --init --recursive`",
+        stacklevel=2,
+    )
 
 
 # -- Project documents -------------------------------------------------------
 
-if _SHOULD_COPY_NOTEBOOKS:
+if _COPY_NOTEBOOKS:
     assist_nb.AssistantCLI.copy_notebooks(
         _PATH_RAW_NB,
         _PATH_HERE,
         "notebooks",
         patterns=[".", "course_UvA-DL", "lightning_examples"],
     )
-    # TODO: Complete converting the missing items and add them back
+    # TODO(@aniketmaurya): Complete converting the missing items and add them back
     ignore = [
         "course_UvA-DL/13-contrastive-learning",
         "lightning_examples/augmentation_kornia",
@@ -202,7 +206,8 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     "notebooks/sample-template*",
 ]
 
-if _FAST_DOCS_DEV:
+# todo: checking cross link will fail because `tutorials.rst` is referred in `common/notebooks.rst`
+if not _COPY_NOTEBOOKS:
     exclude_patterns.append("notebooks/*")
     exclude_patterns.append("tutorials.rst")
 

requirements/pytorch/docs.txt

@@ -1,7 +1,6 @@
 -r ../docs.txt
 
+nbformat  # used for generate empty notebook
 ipython[notebook] <8.7.0
 setuptools<58.0  # workaround for `error in ipython setup command: use_2to3 is invalid.`
 lai-sphinx-theme
-
--r ../../_notebooks/.actions/requires.txt