Update documentation

recommenders-team · Nov 15, 2024 · 59f7c7d · 59f7c7d
1 parent d0be838
commit 59f7c7d
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 107 deletions.
diff --git a/build/lib/tests/ci/azureml_tests/aml_utils.py b/build/lib/tests/ci/azureml_tests/aml_utils.py
@@ -8,14 +8,15 @@
 * https://learn.microsoft.com/en-us/azure/machine-learning/reference-migrate-sdk-v1-mlflow-tracking?view=azureml-api-2&tabs=aml%2Ccli%2Cmlflow
 """
 import pathlib
-import tempfile
+import re
 
 from azure.ai.ml import MLClient, command
 from azure.ai.ml.entities import AmlCompute, BuildContext, Environment, Workspace
 from azure.ai.ml.exceptions import JobException
 from azure.core.exceptions import ResourceExistsError
 from azure.identity import DefaultAzureCredential
 
+
 def get_client(subscription_id, resource_group, workspace_name):
     """
     Get the client with specified AzureML workspace, or create one if not existing.
@@ -61,9 +62,8 @@ def get_or_create_environment(
     environment_name,
     use_gpu,
     use_spark,
-    conda_pkg_jdk,
+    conda_openjdk_version,
     python_version,
-    commit_sha,
 ):
     """
     AzureML requires the run environment to be setup prior to submission.
@@ -77,81 +77,39 @@ def get_or_create_environment(
             added to the conda environment, else False
         use_spark (bool): True if PySpark packages should be
             added to the conda environment, else False
-        conda_pkg_jdk (str): "openjdk=8" by default
-        python_version (str): python version, such as "3.9"
-        commit_sha (str): the commit that triggers the workflow
+        conda_openjdk_version (str): "21" by default
+        python_version (str): python version, such as "3.11"
     """
-    conda_env_name = "reco"
-    conda_env_yml = "environment.yml"
-    condafile = fr"""
-name: {conda_env_name}
-channels:
-  - conda-forge
-dependencies:
-  - python={python_version}
-  - {conda_pkg_jdk}
-  - pip
-  - pip:
-    - recommenders[dev{",gpu" if use_gpu else ""}{",spark" if use_spark else ""}]@git+https://github.com/recommenders-team/recommenders.git@{commit_sha}
-"""
-    # See https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04
-    image = "mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04"
-    # See https://github.com/Azure/AzureML-Containers/blob/master/base/gpu/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04
-    dockerfile = fr"""# syntax=docker/dockerfile:1
-FROM nvcr.io/nvidia/cuda:12.5.1-devel-ubuntu22.04
-SHELL ["/bin/bash", "-c"]
-USER root:root
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && \
-    apt-get install -y wget git-all && \
-    apt-get clean -y && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install Conda
-ENV CONDA_PREFIX /opt/miniconda
-RUN wget -qO /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_24.5.0-0-Linux-x86_64.sh && \
-    bash /tmp/miniconda.sh -bf -p ${{CONDA_PREFIX}} && \
-    ${{CONDA_PREFIX}}/bin/conda update --all -c conda-forge -y && \
-    ${{CONDA_PREFIX}}/bin/conda clean -ay && \
-    rm -rf ${{CONDA_PREFIX}}/pkgs && \
-    rm /tmp/miniconda.sh && \
-    find / -type d -name __pycache__ | xargs rm -rf
-
-# Create Conda environment
-COPY {conda_env_yml} /tmp/{conda_env_yml}
-RUN ${{CONDA_PREFIX}}/bin/conda env create -f /tmp/{conda_env_yml}
-
-# Activate Conda environment
-ENV CONDA_DEFAULT_ENV {conda_env_name}
-ENV CONDA_PREFIX ${{CONDA_PREFIX}}/envs/${{CONDA_DEFAULT_ENV}}
-ENV PATH="${{CONDA_PREFIX}}/bin:${{PATH}}"  LD_LIBRARY_PATH="${{CONDA_PREFIX}}/lib:$LD_LIBRARY_PATH"
-"""
-
-    with tempfile.TemporaryDirectory() as tmpdir:
-        tmpdir = pathlib.Path(tmpdir)
-        dockerfile_path = tmpdir / "Dockerfile"
-        condafile_path = tmpdir / conda_env_yml
-        build = BuildContext(path=tmpdir, dockerfile_path=dockerfile_path.name)
-
-        with open(dockerfile_path, "w") as file:
-            file.write(dockerfile)
-        with open(condafile_path, "w") as file:
-            file.write(condafile)
-
-        try:
-            client.environments.create_or_update(
-                Environment(
-                    name=environment_name,
-                    image=None if use_gpu else image,
-                    build=build if use_gpu else None,
-                    conda_file=None if use_gpu else condafile_path,
-                )
+    compute = "gpu" if use_gpu else "cpu"
+    extras = (
+        "[dev" + (",gpu" if use_gpu else "") + (",spark" if use_spark else "") + "]"
+    )
+    dockerfile = pathlib.Path("tools/docker/Dockerfile")
+
+    # Docker's --build-args is not supported by AzureML Python SDK v2 as shown
+    # in [the issue #33902](https://github.com/Azure/azure-sdk-for-python/issues/33902)
+    # so the build args are configured by regex substituion
+    text = dockerfile.read_text()
+    text = re.sub(r"(ARG\sCOMPUTE=).*", rf'\1"{compute}"', text)
+    text = re.sub(r"(ARG\sEXTRAS=).*", rf'\1"{extras}"', text)
+    text = re.sub(r"(ARG\sGIT_REF=).*", r'\1""', text)
+    text = re.sub(r"(ARG\sJDK_VERSION=).*", rf'\1"{conda_openjdk_version}"', text)
+    text = re.sub(r"(ARG\sPYTHON_VERSION=).*", rf'\1"{python_version}"', text)
+    dockerfile.write_text(text)
+
+    try:
+        client.environments.create_or_update(
+            Environment(
+                name=environment_name,
+                build=BuildContext(
+                    # Set path for Docker to access to Recommenders root
+                    path=".",
+                    dockerfile_path=dockerfile,
+                ),
             )
-        except ResourceExistsError:
-            pass
+        )
+    except ResourceExistsError:
+        pass
 
 
 def run_tests(

diff --git a/build/lib/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/build/lib/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py
@@ -108,15 +108,15 @@ def parse_args():
         help="Environment name on AzureML",
     )
     parser.add_argument(
-        "--conda_pkg_jdk",
+        "--conda-openjdk-version",
         action="store",
-        default="openjdk=8",
-        help="Conda package for JDK",
+        default="21",
+        help="Conda OpenJDK package version",
     )
     parser.add_argument(
         "--python-version",
         action="store",
-        default="3.8",
+        default="3.11",
         help="Python version",
     )
     parser.add_argument(
@@ -133,19 +133,16 @@ def parse_args():
     logger = logging.getLogger("submit_groupwise_azureml_pytest.py")
     args = parse_args()
 
-    logger.info(f"Setting up workspace {args.ws}")
+    logger.info("Setting up workspace %s", args.ws)
     client = get_client(
         subscription_id=args.subid,
         resource_group=args.rg,
         workspace_name=args.ws,
     )
 
-    logger.info(f"Setting up compute {args.cluster}")
+    logger.info("Setting up compute %s", args.cluster)
     create_or_start_compute(
-        client=client,
-        name=args.cluster,
-        size=args.vmsize,
-        max_instances=args.maxnodes
+        client=client, name=args.cluster, size=args.vmsize, max_instances=args.maxnodes
     )
 
     # TODO: Unlike Azure DevOps pipelines, GitHub Actions only has simple
@@ -159,19 +156,18 @@ def parse_args():
     #       * on AzureML
     #           recommenders-unit-group_cpu_001-python3_8-c8adeafabc011b549f875dc145313ffbe3fc53a8
     environment_name = correct_resource_name(args.envname)
-    logger.info(f"Setting up environment {environment_name}")
+    logger.info("Setting up environment %s", environment_name)
     get_or_create_environment(
         client=client,
         environment_name=environment_name,
-        use_gpu=True if "gpu" in args.testgroup else False,
-        use_spark=True if "spark" in args.testgroup else False,
-        conda_pkg_jdk=args.conda_pkg_jdk,
+        use_gpu="gpu" in args.testgroup,
+        use_spark="spark" in args.testgroup,
+        conda_openjdk_version=args.conda_openjdk_version,
         python_version=args.python_version,
-        commit_sha=args.sha,
     )
 
     experiment_name = correct_resource_name(args.expname)
-    logger.info(f"Running experiment {experiment_name}")
+    logger.info("Running experiment %s", experiment_name)
     run_tests(
         client=client,
         compute=args.cluster,

diff --git a/docs/_build/html/.doctrees/environment.pickle b/docs/_build/html/.doctrees/environment.pickle
diff --git a/recommenders.egg-info/PKG-INFO b/recommenders.egg-info/PKG-INFO
@@ -60,19 +60,19 @@ Requires-Dist: pytest>=7.2.1; extra == "dev"
 Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
 Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
 Provides-Extra: all
-Requires-Dist: pytest-mock>=3.10.0; extra == "all"
-Requires-Dist: spacy<=3.7.5; python_version <= "3.8" and extra == "all"
-Requires-Dist: tf-slim>=1.1.0; extra == "all"
-Requires-Dist: pyarrow>=10.0.1; extra == "all"
+Requires-Dist: black>=23.3.0; extra == "all"
 Requires-Dist: fastai<3,>=2.7.11; extra == "all"
+Requires-Dist: numpy<1.25.0; python_version <= "3.8" and extra == "all"
+Requires-Dist: spacy<=3.7.5; python_version <= "3.8" and extra == "all"
 Requires-Dist: pytest-cov>=4.1.0; extra == "all"
-Requires-Dist: black>=23.3.0; extra == "all"
+Requires-Dist: pytest>=7.2.1; extra == "all"
 Requires-Dist: torch<3,>=2.0.1; extra == "all"
+Requires-Dist: pytest-mock>=3.10.0; extra == "all"
 Requires-Dist: pyspark<=4,>=3.3.0; extra == "all"
-Requires-Dist: tensorflow!=2.10.0.*,!=2.9.0.*,!=2.9.1,!=2.9.2,<2.16,>=2.8.4; extra == "all"
 Requires-Dist: nvidia-ml-py>=11.525.84; extra == "all"
-Requires-Dist: numpy<1.25.0; python_version <= "3.8" and extra == "all"
-Requires-Dist: pytest>=7.2.1; extra == "all"
+Requires-Dist: tensorflow!=2.10.0.*,!=2.9.0.*,!=2.9.1,!=2.9.2,<2.16,>=2.8.4; extra == "all"
+Requires-Dist: tf-slim>=1.1.0; extra == "all"
+Requires-Dist: pyarrow>=10.0.1; extra == "all"
 Provides-Extra: experimental
 Requires-Dist: xlearn==0.40a1; extra == "experimental"
 Requires-Dist: vowpalwabbit<9,>=8.9.0; extra == "experimental"

diff --git a/recommenders.egg-info/requires.txt b/recommenders.egg-info/requires.txt
@@ -23,21 +23,21 @@ pandera[strategies]>=0.15.0
 statsmodels>=0.14.4
 
 [all]
-pytest-mock>=3.10.0
-tf-slim>=1.1.0
-pyarrow>=10.0.1
+black>=23.3.0
 fastai<3,>=2.7.11
 pytest-cov>=4.1.0
-black>=23.3.0
+pytest>=7.2.1
 torch<3,>=2.0.1
+pytest-mock>=3.10.0
 pyspark<=4,>=3.3.0
-tensorflow!=2.10.0.*,!=2.9.0.*,!=2.9.1,!=2.9.2,<2.16,>=2.8.4
 nvidia-ml-py>=11.525.84
-pytest>=7.2.1
+tensorflow!=2.10.0.*,!=2.9.0.*,!=2.9.1,!=2.9.2,<2.16,>=2.8.4
+tf-slim>=1.1.0
+pyarrow>=10.0.1
 
 [all:python_version <= "3.8"]
-spacy<=3.7.5
 numpy<1.25.0
+spacy<=3.7.5
 
 [dev]
 black>=23.3.0