Deprecate python 3.8 support.

.github/workflows/linux.yml

@@ -35,7 +35,6 @@ jobs:
           # Empty string is the floating most recent version of python
           # (useful to catch new compatibility issues in nightly builds)
           - ""
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"

Makefile

@@ -6,14 +6,16 @@ PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos[-]?//')
 ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml
 
 # Find the non-build python files we should consider as rule dependencies.
-PYTHON_FILES := $(shell find ./ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./(mlos_(core|bench|viz)/)?build/' -e '^./doc/source/')
-MLOS_CORE_PYTHON_FILES := $(shell find ./mlos_core/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_core/build/')
-MLOS_BENCH_PYTHON_FILES := $(shell find ./mlos_bench/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_bench/build/')
-MLOS_VIZ_PYTHON_FILES := $(shell find ./mlos_viz/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_viz/build/')
-NOTEBOOK_FILES := $(shell find ./ -type f -name '*.ipynb' 2>/dev/null | egrep -v -e '/build/')
-SCRIPT_FILES := $(shell find ./ -name '*.sh' -or -name '*.ps1' -or -name '*.cmd')
-SQL_FILES := $(shell find ./ -name '*.sql')
-MD_FILES := $(shell find ./ -name '*.md' | grep -v '^./doc/')
+# Do a single find and multiple filters for better performance.
+REPO_FILES := $(shell find . -type f 2>/dev/null | egrep -v -e '^./(mlos_(core|bench|viz)/)?build/' -e '^./doc/source/' -e '^./doc/build/')
+PYTHON_FILES := $(filter %.py, $(REPO_FILES))
+MLOS_CORE_PYTHON_FILES := $(filter ./mlos_core/%, $(PYTHON_FILES))
+MLOS_BENCH_PYTHON_FILES := $(filter ./mlos_bench/%, $(PYTHON_FILES))
+MLOS_VIZ_PYTHON_FILES := $(filter ./mlos_viz/%, $(PYTHON_FILES))
+NOTEBOOK_FILES := $(filter %.ipynb, $(REPO_FILES))
+SCRIPT_FILES := $(filter %.sh %.ps1 %.cmd, $(REPO_FILES))
+SQL_FILES := $(filter %.sql, $(REPO_FILES))
+MD_FILES := $(filter-out ./doc/%, $(filter %.md, $(REPO_FILES)))
 
 DOCKER := $(shell which docker)
 # Make sure the build directory exists.
@@ -70,6 +72,8 @@ ifneq (,$(filter format,$(MAKECMDGOALS)))
 endif
 
 build/format.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+# TODO: Disabled for now.  Will enable and format in a future PR.
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/isort.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/black.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/docformatter.${CONDA_ENV_NAME}.build-stamp
@@ -96,6 +100,35 @@ build/licenseheaders.${CONDA_ENV_NAME}.build-stamp:
 		-x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
 	touch $@
 
+.PHONY: pyupgrade
+pyupgrade: build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter pyupgrade,$(MAKECMDGOALS)))
+    FORMAT_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp:
+	touch $@
+
+PYUPGRADE_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+PYUPGRADE_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+PYUPGRADE_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+PYUPGRADE_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/pyupgrade.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/pyupgrade.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/pyupgrade.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/pyupgrade.%.${CONDA_ENV_NAME}.build-stamp: $(PYUPGRADE_COMMON_PREREQS)
+	# Reformat python file imports with pyupgrade.
+	conda run -n ${CONDA_ENV_NAME} pyupgrade --py39-plus --exit-zero-even-if-changed $(filter %.py,$+)
+	touch $@
+
 .PHONY: isort
 isort: build/isort.${CONDA_ENV_NAME}.build-stamp
 
@@ -118,6 +151,9 @@ ISORT_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 ISORT_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+ISORT_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ISORT_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
 ISORT_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
 
@@ -149,6 +185,9 @@ BLACK_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 BLACK_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ifneq (,$(filter format isort,$(MAKECMDGOALS)))
 BLACK_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
 endif
@@ -183,6 +222,9 @@ DOCFORMATTER_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 DOCFORMATTER_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+DOCFORMATTER_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ifneq (,$(filter format isort,$(MAKECMDGOALS)))
 DOCFORMATTER_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
 endif
@@ -757,6 +799,7 @@ endif
 .PHONY: nginx_port_env
 nginx_port_env:
 	@echo "Starting nginx docker container for serving docs."
+	mkdir -p doc/build
 	./doc/nginx-docker.sh restart
 	nginx_port=`docker port mlos-doc-nginx | grep 0.0.0.0:8080 | cut -d/ -f1` \
 		&& echo nginx_port=$${nginx_port} > doc/build/nginx_port.env

conda-envs/mlos.yml

@@ -6,8 +6,8 @@ dependencies:
   # Basic dev environment packages.
   # All other dependencies for the mlos modules come from pip.
   - pip
-  - pylint
   - black
+  - pyupgrade
   - pycodestyle
   - pydocstyle
   - flake8
@@ -29,6 +29,7 @@ dependencies:
     - isort
     - docformatter
     - licenseheaders
+    - pylint
     - mypy
     - pandas-stubs
     - types-beautifulsoup4

conftest.py

@@ -44,7 +44,8 @@ def pytest_configure(config: pytest.Config) -> None:
                         (
                             "DISPLAY environment variable is set, "
                             "which can cause problems in some setups (e.g. WSL). "
-                            f"Adjusting matplotlib backend to '{matplotlib.rcParams['backend']}' "
+                            "Adjusting matplotlib backend to "
+                            f"""'{matplotlib.rcParams["backend"]}' """
                             "to compensate."
                         )
                     )

doc/copy-source-tree-docs.sh

@@ -23,15 +23,27 @@ for readme_file_path in README.md mlos_core/README.md mlos_bench/README.md mlos_
 
     cp "$readme_file_path" "doc/source/source_tree_docs/$file_dir/index.md"
 
+    case "$OSTYPE" in
+        darwin*)
+            sed_args='-i ""'
+            ;;
+        *gnu*)
+            sed_args='-i'
+            ;;
+        default)
+            echo "Unsupported OS: $OSTYPE"
+            exit 1
+            ;;
+    esac
+
     # Tweak source source code links.
-    # FIXME: This sed expression doesn't work in MacOS.
-    sed -i -r -e "s|\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)|\]\(https://github.com/microsoft/MLOS/tree/main/$file_dir/\1\2\)|g" \
+    sed $sed_args -r -e "s|\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)|\]\(https://github.com/microsoft/MLOS/tree/main/$file_dir/\1\2\)|g" \
         "doc/source/source_tree_docs/$file_dir/index.md"
     # Tweak the lexers for local expansion by pygments instead of github's.
-    sed -i -r -e 's/```jsonc/```json/' \
+    sed $sed_args -r -e 's/```jsonc/```json/' \
         "doc/source/source_tree_docs/$file_dir/index.md"
 done
 
 # Do an explicit fixup for some static content.
-sed -i -r -e 's#="[^"]*(_static/[^"]+)"#="../\1"#g' \
+sed $sed_args -r -e 's#="[^"]*(_static/[^"]+)"#="../\1"#g' \
     "doc/source/source_tree_docs/index.md"

doc/source/conf.py

@@ -172,6 +172,7 @@ def is_on_github_actions():
     ("T_co", "class"): "data",
     ("CoroReturnType", "class"): "data",
     ("FutureReturnType", "class"): "data",
+    ("NullableT", "class"): "data",
 }
 
 
@@ -209,6 +210,7 @@ def setup(app: SphinxApp) -> None:
 # sphinx has a hard time finding typealiases and typevars instead of classes.
 # See Also: https://github.com/sphinx-doc/sphinx/issues/10974
 nitpick_ignore = [
+    ("py:class", "Ellipsis"),
     # Internal typevars and aliases:
     ("py:class", "EnvironType"),
     # External typevars and aliases:

mlos_bench/mlos_bench/config/environments/apps/redis/scripts/local/generate_redis_config.py

@@ -14,9 +14,10 @@
 
 
 def _main(fname_input: str, fname_output: str) -> None:
-    with open(fname_input, "rt", encoding="utf-8") as fh_tunables, open(
-        fname_output, "wt", encoding="utf-8", newline=""
-    ) as fh_config:
+    with (
+        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
+        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+    ):
         for key, val in json.load(fh_tunables).items():
             line = f"{key} {val}"
             fh_config.write(line + "\n")

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py

@@ -16,9 +16,10 @@
 
 
 def _write_config() -> None:
-    with open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json, open(
-        NEW_CFG, "w", encoding="UTF-8"
-    ) as fh_config:
+    with (
+        open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json,
+        open(NEW_CFG, "w", encoding="UTF-8") as fh_config,
+    ):
         for key, val in json.load(fh_json).items():
             fh_config.write(
                 'GRUB_CMDLINE_LINUX_DEFAULT="$' f'{{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"\n'

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/generate_grub_config.py

@@ -14,9 +14,10 @@
 
 
 def _main(fname_input: str, fname_output: str) -> None:
-    with open(fname_input, "rt", encoding="utf-8") as fh_tunables, open(
-        fname_output, "wt", encoding="utf-8", newline=""
-    ) as fh_config:
+    with (
+        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
+        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+    ):
         for key, val in json.load(fh_tunables).items():
             line = f'GRUB_CMDLINE_LINUX_DEFAULT="${{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"'
             fh_config.write(line + "\n")

mlos_bench/mlos_bench/event_loop_context.py

@@ -21,11 +21,8 @@
 CoroReturnType = TypeVar("CoroReturnType")  # pylint: disable=invalid-name
 """Type variable for the return type of an :external:py:mod:`asyncio` coroutine."""
 
-if sys.version_info >= (3, 9):
-    FutureReturnType: TypeAlias = Future[CoroReturnType]
-    """Type variable for the return type of a :py:class:`~concurrent.futures.Future`."""
-else:
-    FutureReturnType: TypeAlias = Future
+FutureReturnType: TypeAlias = Future[CoroReturnType]
+"""Type variable for the return type of a :py:class:`~concurrent.futures.Future`."""
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/os_environ.py

@@ -28,7 +28,7 @@
     # pylint: disable=protected-access,disable=unsubscriptable-object
     EnvironType: TypeAlias = os._Environ[str]
 else:
-    EnvironType: TypeAlias = os._Environ  # pylint: disable=protected-access
+    assert False, "Unsupported Python version."
 
 # Handle case sensitivity differences between platforms.
 # https://stackoverflow.com/a/19023293

mlos_bench/mlos_bench/tests/config/cli/test_load_cli_config_examples.py

@@ -146,8 +146,8 @@ def test_load_cli_config_examples_via_launcher(
         assert launcher.global_config["trial_id"] == config["trial_id"]
 
     expected_log_level = logging.getLevelName(config.get("log_level", "INFO"))
-    if isinstance(expected_log_level, int):
-        expected_log_level = logging.getLevelName(expected_log_level)
+    if isinstance(expected_log_level, int):  # type: ignore[unreachable]
+        expected_log_level = logging.getLevelName(expected_log_level)  # type: ignore[unreachable]
     current_log_level = logging.getLevelName(logging.root.getEffectiveLevel())
     assert current_log_level == expected_log_level
 

mlos_bench/mlos_bench/tests/services/config_persistence_test.py

@@ -5,20 +5,14 @@
 """Unit tests for configuration persistence service."""
 
 import os
-import sys
+from importlib.resources import files
 
 import pytest
 
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.util import path_join
 
-if sys.version_info < (3, 9):
-    from importlib_resources import files
-else:
-    from importlib.resources import files
-
-
 # pylint: disable=redefined-outer-name