Merge remote-tracking branch 'origin/main' into publication

.actions/assistant.py

@@ -95,8 +95,8 @@
 
 def load_requirements(path_req: str = PATH_REQ_DEFAULT) -> list:
     """Load the requirements from a file."""
-    with open(path_req) as fp:
-        req = fp.readlines()
+    with open(path_req) as fopen:
+        req = fopen.readlines()
     req = [r[: r.index("#")] if "#" in r else r for r in req]
     req = [r.strip() for r in req]
     req = [r for r in req if r]
@@ -252,7 +252,7 @@ def _parse_requirements(folder: str) -> Tuple[str, str]:
             for k, v in meta.items()
             if k.startswith(AssistantCLI._META_PIP_KEY)
         }
-        pip_args = ["--extra-index-url https://download.pytorch.org/whl/" + _RUNTIME_VERSIONS.get("DEVICE")]
+        pip_args = ['--find-links="https://download.pytorch.org/whl/"' + _RUNTIME_VERSIONS.get("DEVICE")]
         for pip_key in meta_pip_args:
             if not isinstance(meta_pip_args[pip_key], (list, tuple, set)):
                 meta_pip_args[pip_key] = [meta_pip_args[pip_key]]
@@ -335,8 +335,8 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
         cmd.append(f"git add {pub_ipynb}")
         if not output_file:
             return os.linesep.join(cmd)
-        with open(output_file, "w") as fp:
-            fp.write(os.linesep.join(cmd))
+        with open(output_file, "w") as fopen:
+            fopen.write(os.linesep.join(cmd))
 
     @staticmethod
     def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]:
@@ -386,8 +386,8 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
         cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
         if not output_file:
             return os.linesep.join(cmd)
-        with open(output_file, "w") as fp:
-            fp.write(os.linesep.join(cmd))
+        with open(output_file, "w") as fopen:
+            fopen.write(os.linesep.join(cmd))
 
     @staticmethod
     def convert_ipynb(folder: str) -> None:
@@ -397,8 +397,8 @@ def convert_ipynb(folder: str) -> None:
             folder: folder with python script
         """
         fpath, _, _ = AssistantCLI._valid_folder(folder, ext=".py")
-        with open(fpath) as fp:
-            py_script = fp.readlines()
+        with open(fpath) as fopen:
+            py_script = fopen.readlines()
 
         meta = AssistantCLI._load_meta(folder, strict=True)
         meta.update(
@@ -414,8 +414,8 @@ def convert_ipynb(folder: str) -> None:
 
         py_script = AssistantCLI._replace_images(py_script, folder)
 
-        with open(fpath, "w") as fp:
-            fp.writelines(py_script)
+        with open(fpath, "w") as fopen:
+            fopen.writelines(py_script)
 
         os.system(f'python -m jupytext --set-formats "ipynb,py:percent" {fpath}')
 
@@ -442,8 +442,8 @@ def _replace_images(lines: list, local_dir: str) -> list:
             else:
                 url_path = "/".join([URL_PL_DOWNLOAD, local_dir, p_img])
                 p_local_img = os.path.join(local_dir, p_img)
-                with open(p_local_img, "rb") as fp:
-                    im = fp.read()
+                with open(p_local_img, "rb") as fopen:
+                    im = fopen.read()
             im_base64 = base64.b64encode(im).decode("utf-8")
             _, ext = os.path.splitext(p_img)
             md = md.replace(f'src="{p_img}"', f'src="{url_path}"')
@@ -489,8 +489,8 @@ def group_folders(
             $ python assistant.py group-folders ../target-diff.txt \
                 --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
         """
-        with open(fpath_gitdiff) as fp:
-            changed = [ln.strip() for ln in fp.readlines()]
+        with open(fpath_gitdiff) as fopen:
+            changed = [ln.strip() for ln in fopen.readlines()]
         dirs = [os.path.dirname(ln) for ln in changed]
         # not empty paths
         dirs = [ln for ln in dirs if ln]
@@ -520,22 +520,23 @@ def group_folders(
                 raise FileNotFoundError(f"{msg} nor sub-folder: \n {os.linesep.join(dirs_invalid)}")
 
         dirs_change = [d for d in dirs_exist if AssistantCLI._find_meta(d)]
-        with open(fpath_change_folders, "w") as fp:
-            fp.write(os.linesep.join(sorted(dirs_change)))
+        with open(fpath_change_folders, "w") as fopen:
+            fopen.write(os.linesep.join(sorted(dirs_change)))
 
         dirs_drop = [d for d in dirs if not os.path.isdir(d)]
-        with open(fpath_drop_folders, "w") as fp:
-            fp.write(os.linesep.join(sorted(dirs_drop)))
+        with open(fpath_drop_folders, "w") as fopen:
+            fopen.write(os.linesep.join(sorted(dirs_drop)))
 
     @staticmethod
-    def generate_matrix(fpath_change_folders: str) -> str:
+    def generate_matrix(fpath_change_folders: str, json_indent: Optional[int] = None) -> str:
         """Generate Azure matrix with leaf for each changed notebook.
 
         Args:
             fpath_change_folders: output of previous ``group_folders``
+            json_indent: makes the json more readable, recommendation is 4
         """
-        with open(fpath_change_folders) as fp:
-            folders = [ln.strip() for ln in fp.readlines()]
+        with open(fpath_change_folders) as fopen:
+            folders = [ln.strip() for ln in fopen.readlines()]
         # set default so the matrix has at least one runner
         if not folders:
             return ""
@@ -548,7 +549,7 @@ def generate_matrix(fpath_change_folders: str) -> str:
                 # TODO: allow defining some custom images with with python or PT
                 "docker-image": AssistantCLI._AZURE_DOCKER,
             }
-        return json.dumps(mtx)
+        return json.dumps(mtx, indent=json_indent)
 
     @staticmethod
     def _get_card_item_cell(path_ipynb: str, path_meta: str, path_thumb: Optional[str]) -> Dict[str, Any]:
@@ -652,7 +653,12 @@ def copy_notebooks(
 
     @staticmethod
     def _copy_notebook(
-        path_ipynb: str, path_root: str, docs_root: str, path_docs_ipynb: str, path_docs_images: str
+        path_ipynb: str,
+        path_root: str,
+        docs_root: str,
+        path_docs_ipynb: str,
+        path_docs_images: str,
+        json_indent: Optional[int] = None,
     ) -> str:
         """Copy particular notebook."""
         ipynb = path_ipynb.split(os.path.sep)
@@ -678,7 +684,7 @@ def _copy_notebook(
         ipynb["cells"].append(AssistantCLI._get_card_item_cell(path_ipynb, path_meta, path_thumb))
 
         with open(new_ipynb, "w") as fopen:
-            json.dump(ipynb, fopen, indent=4)
+            json.dump(ipynb, fopen, indent=json_indent)
         return path_ipynb_in_dir
 
     @staticmethod
@@ -691,8 +697,8 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
         """
         meta = AssistantCLI._load_meta(folder)
         # default is COU runtime
-        with open(PATH_REQ_DEFAULT) as fp:
-            req = fp.readlines()
+        with open(PATH_REQ_DEFAULT) as fopen:
+            req = fopen.readlines()
         req += meta.get("requirements", [])
         req = [r.strip() for r in req]
 

.azure/ipynb-publish.yml

@@ -42,10 +42,11 @@ jobs:
 
       - bash: |
           notebooks=$(python .actions/assistant.py generate-matrix changed-folders.txt)
-          printf "Changed notebooks: $notebooks\n"
           echo "##vso[task.setVariable variable=dirs;isOutput=true]$notebooks"
         name: mtrx
         displayName: "Changed matrix"
+      - bash: echo '$(mtrx.dirs)' | python -m json.tool
+        displayName: "Show matrix"
 
       - bash: |
           # second half with || [...] is needed for reading the last line
@@ -79,10 +80,10 @@ jobs:
     # - For 60 minutes on Microsoft-hosted agents with a private project or private repository
     timeoutInMinutes: "180"
 
-    pool: "$(agent-pool)"
+    pool: $(agent-pool)
     # this need to have installed docker in the base machine/image...
     container:
-      image: "$(docker-image)"
+      image: $(docker-image)
       options: "--gpus=all --shm-size=32g -v /usr/bin/docker:/tmp/docker:ro"
 
     variables:
@@ -131,12 +132,14 @@ jobs:
       - bash: |
           set -e
           pip --version
+          # todo: export requirements for notebooks to file and execute
+          # todo: adjust torch ecosystem versions
           pip install -r requirements.txt -r _requirements/data.txt
-          pip list
         displayName: "Install dependencies"
 
       - bash: |
           set -e
+          pip list
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu > 0, f'GPU: {mgpu}'"
           python -m papermill --version
         displayName: "Sanity check"

.azure/ipynb-tests.yml

@@ -29,10 +29,11 @@ jobs:
 
       - bash: |
           notebooks=$(python .actions/assistant.py generate-matrix changed-folders.txt)
-          printf "Changed notebooks: $notebooks\n"
           echo "##vso[task.setVariable variable=dirs;isOutput=true]$notebooks"
         name: mtrx
         displayName: "Changed matrix"
+      - bash: echo '$(mtrx.dirs)' | python -m json.tool
+        displayName: "Show matrix"
 
   - job: nbval
     dependsOn: check_diff
@@ -43,10 +44,10 @@ jobs:
     # how much time to give 'run always even if cancelled tasks' before stopping them
     cancelTimeoutInMinutes: "2"
 
-    pool: "$(agent-pool)"
+    pool: $(agent-pool)
     # this need to have installed docker in the base image...
     container:
-      image: "$(docker-image)"
+      image: $(docker-image)
       options: "--gpus=all --shm-size=32g -v /usr/bin/docker:/tmp/docker:ro"
 
     variables:

.github/workflows/ci_docs.yml

@@ -4,9 +4,7 @@ on: # Trigger the workflow on push or pull request
   #  push:
   #    branches: [main]
   pull_request: {}
-  schedule:
-    # At the end of every day
-    - cron: "0 0 * * *"
+  #workflow_dispatch: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}

.github/workflows/docker-build.yml

@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 env:
-  PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
+  PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }}
 
 jobs:
   build-cuda:

.gitignore

@@ -25,7 +25,6 @@ share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -55,16 +54,6 @@ coverage.xml
 *.mo
 *.pot
 
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
 # Scrapy stuff:
 .scrapy
 
@@ -84,13 +73,8 @@ ipython_config.py
 # pyenv
 .python-version
 
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
+# PyCharm
+.idea/
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
@@ -128,10 +112,10 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-.idea/
-
 # data artifacts
 logs/
 lightning_logs/
 cifar-10-batches-py
 *.tar.gz
+
+.ruff_cache/

_dockers/ubuntu-cuda/Dockerfile

@@ -43,6 +43,7 @@ RUN \
         openmpi-bin \
         cmake \
         git \
+        git-lfs \
         wget \
         curl \
         unzip \

course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py

@@ -227,7 +227,7 @@
 # if the hyperparameters are not well tuned.
 # We will rely on training tricks proposed in the paper
 # [Implicit Generation and Generalization in Energy-Based Models](https://arxiv.org/abs/1903.08689)
-# by Yilun Du and Igor Mordatch ([blog](https://openai.com/blog/energy-based-models/)).
+# by Yilun Du and Igor Mordatch ([blog](https://openai.com/research/energy-based-models)).
 # The important part of this notebook is however to see how the theory above can actually be used in a model.
 #
 # ### Dataset