Deprecate python 3.8 support. (#902)

# Pull Request

## Title

Deprecates Python 3.8 support

---

## Description

Per https://endoflife.date/python, 3.8 is no longer supported.

This PR drops Python 3.8 from the CI pipeline and changes a few small
places in the code where we have to check for the python version, but
otherwise leaves the rest of the pylint and syntax changes for a future
PR.

To manage that, we also prepare the use of `pyupgrade` as a formatter,
though may convert all formatting checks to use `pre-commit` hooks
instead in the future in order to reduce some of the `Makefile`
complexity.

- See Also: #749 

---

## Type of Change

- ✨ Dropped feature
- ⚠️ Breaking change
- 🔄 Refactor

---

## Testing

This is a small change for now, without rewrites, to keep testing
easier.
So usual CI should suffice.

---

## Additional Notes (optional)

- Python 3.9 will be supported for another 10 months. We could also
consider to drop support for it early as well and reduce another round
of large codebase rewrites. On the other hand, it might be simple to do
later with the introduction of pyupgrade anyways.

---

.github/workflows/linux.yml

@@ -35,7 +35,6 @@ jobs:
           # Empty string is the floating most recent version of python
           # (useful to catch new compatibility issues in nightly builds)
           - ""
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"

Makefile

@@ -72,6 +72,8 @@ ifneq (,$(filter format,$(MAKECMDGOALS)))
 endif
 
 build/format.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+# TODO: Disabled for now.  Will enable and format in a future PR.
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/isort.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/black.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp: build/docformatter.${CONDA_ENV_NAME}.build-stamp
@@ -98,6 +100,35 @@ build/licenseheaders.${CONDA_ENV_NAME}.build-stamp:
 		-x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
 	touch $@
 
+.PHONY: pyupgrade
+pyupgrade: build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter pyupgrade,$(MAKECMDGOALS)))
+    FORMAT_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp: build/pyupgrade.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/pyupgrade.${CONDA_ENV_NAME}.build-stamp:
+	touch $@
+
+PYUPGRADE_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+PYUPGRADE_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+PYUPGRADE_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+PYUPGRADE_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/pyupgrade.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/pyupgrade.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/pyupgrade.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/pyupgrade.%.${CONDA_ENV_NAME}.build-stamp: $(PYUPGRADE_COMMON_PREREQS)
+	# Reformat python file imports with pyupgrade.
+	conda run -n ${CONDA_ENV_NAME} pyupgrade --py39-plus --exit-zero-even-if-changed $(filter %.py,$+)
+	touch $@
+
 .PHONY: isort
 isort: build/isort.${CONDA_ENV_NAME}.build-stamp
 
@@ -120,6 +151,9 @@ ISORT_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 ISORT_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+ISORT_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ISORT_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
 ISORT_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
 
@@ -151,6 +185,9 @@ BLACK_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 BLACK_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ifneq (,$(filter format isort,$(MAKECMDGOALS)))
 BLACK_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
 endif
@@ -185,6 +222,9 @@ DOCFORMATTER_COMMON_PREREQS :=
 ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
 DOCFORMATTER_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
 endif
+ifneq (,$(filter format pyupgrade,$(MAKECMDGOALS)))
+DOCFORMATTER_COMMON_PREREQS += build/pyupgrade.${CONDA_ENV_NAME}.build-stamp
+endif
 ifneq (,$(filter format isort,$(MAKECMDGOALS)))
 DOCFORMATTER_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
 endif

conda-envs/mlos.yml

@@ -7,6 +7,7 @@ dependencies:
   # All other dependencies for the mlos modules come from pip.
   - pip
   - black
+  - pyupgrade
   - pycodestyle
   - pydocstyle
   - flake8

conftest.py

@@ -44,7 +44,8 @@ def pytest_configure(config: pytest.Config) -> None:
                         (
                             "DISPLAY environment variable is set, "
                             "which can cause problems in some setups (e.g. WSL). "
-                            f"Adjusting matplotlib backend to '{matplotlib.rcParams['backend']}' "
+                            "Adjusting matplotlib backend to "
+                            f"""'{matplotlib.rcParams["backend"]}' """
                             "to compensate."
                         )
                     )

mlos_bench/mlos_bench/config/environments/apps/redis/scripts/local/generate_redis_config.py

@@ -14,9 +14,10 @@
 
 
 def _main(fname_input: str, fname_output: str) -> None:
-    with open(fname_input, "rt", encoding="utf-8") as fh_tunables, open(
-        fname_output, "wt", encoding="utf-8", newline=""
-    ) as fh_config:
+    with (
+        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
+        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+    ):
         for key, val in json.load(fh_tunables).items():
             line = f"{key} {val}"
             fh_config.write(line + "\n")

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py

@@ -16,9 +16,10 @@
 
 
 def _write_config() -> None:
-    with open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json, open(
-        NEW_CFG, "w", encoding="UTF-8"
-    ) as fh_config:
+    with (
+        open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json,
+        open(NEW_CFG, "w", encoding="UTF-8") as fh_config,
+    ):
         for key, val in json.load(fh_json).items():
             fh_config.write(
                 'GRUB_CMDLINE_LINUX_DEFAULT="$' f'{{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"\n'

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/generate_grub_config.py

@@ -14,9 +14,10 @@
 
 
 def _main(fname_input: str, fname_output: str) -> None:
-    with open(fname_input, "rt", encoding="utf-8") as fh_tunables, open(
-        fname_output, "wt", encoding="utf-8", newline=""
-    ) as fh_config:
+    with (
+        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
+        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+    ):
         for key, val in json.load(fh_tunables).items():
             line = f'GRUB_CMDLINE_LINUX_DEFAULT="${{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"'
             fh_config.write(line + "\n")

mlos_bench/mlos_bench/event_loop_context.py

@@ -21,11 +21,8 @@
 CoroReturnType = TypeVar("CoroReturnType")  # pylint: disable=invalid-name
 """Type variable for the return type of an :external:py:mod:`asyncio` coroutine."""
 
-if sys.version_info >= (3, 9):
-    FutureReturnType: TypeAlias = Future[CoroReturnType]
-    """Type variable for the return type of a :py:class:`~concurrent.futures.Future`."""
-else:
-    FutureReturnType: TypeAlias = Future
+FutureReturnType: TypeAlias = Future[CoroReturnType]
+"""Type variable for the return type of a :py:class:`~concurrent.futures.Future`."""
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/os_environ.py

@@ -28,7 +28,7 @@
     # pylint: disable=protected-access,disable=unsubscriptable-object
     EnvironType: TypeAlias = os._Environ[str]
 else:
-    EnvironType: TypeAlias = os._Environ  # pylint: disable=protected-access
+    assert False, "Unsupported Python version."
 
 # Handle case sensitivity differences between platforms.
 # https://stackoverflow.com/a/19023293

mlos_bench/mlos_bench/tests/services/config_persistence_test.py

@@ -5,20 +5,14 @@
 """Unit tests for configuration persistence service."""
 
 import os
-import sys
+from importlib.resources import files
 
 import pytest
 
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.util import path_join
 
-if sys.version_info < (3, 9):
-    from importlib_resources import files
-else:
-    from importlib.resources import files
-
-
 # pylint: disable=redefined-outer-name
 
 

mlos_bench/mlos_bench/tests/services/remote/azure/azure_fileshare_test.py

@@ -28,11 +28,11 @@ def test_download_file(
     local_path = f"{local_folder}/{filename}"
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client, patch.object(
-        mock_share_client, "get_directory_client"
-    ) as mock_get_directory_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+        patch.object(mock_share_client, "get_directory_client") as mock_get_directory_client,
+    ):
 
         mock_get_directory_client.return_value = Mock(exists=Mock(return_value=False))
 
@@ -86,11 +86,11 @@ def test_download_folder_non_recursive(
     mock_share_client = azure_fileshare._share_client  # pylint: disable=protected-access
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_directory_client"
-    ) as mock_get_directory_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_directory_client") as mock_get_directory_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+    ):
 
         mock_get_directory_client.side_effect = lambda x: dir_client_returns[x]
 
@@ -120,11 +120,11 @@ def test_download_folder_recursive(
     dir_client_returns = make_dir_client_returns(remote_folder)
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_directory_client"
-    ) as mock_get_directory_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_directory_client") as mock_get_directory_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+    ):
         mock_get_directory_client.side_effect = lambda x: dir_client_returns[x]
         azure_fileshare.download(config, remote_folder, local_folder, recursive=True)
 
@@ -162,9 +162,10 @@ def test_upload_file(
     mock_isdir.return_value = False
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+    ):
         azure_fileshare.upload(config, local_path, remote_path)
 
     mock_get_file_client.assert_called_with(remote_path)
@@ -235,9 +236,10 @@ def test_upload_directory_non_recursive(
     mock_share_client = azure_fileshare._share_client  # pylint: disable=protected-access
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+    ):
         azure_fileshare.upload(config, local_folder, remote_folder, recursive=False)
 
     mock_get_file_client.assert_called_with(f"{remote_folder}/a_file_1.csv")
@@ -261,9 +263,10 @@ def test_upload_directory_recursive(
     mock_share_client = azure_fileshare._share_client  # pylint: disable=protected-access
 
     config: dict = {}
-    with patch.object(azure_fileshare, "_share_client") as mock_share_client, patch.object(
-        mock_share_client, "get_file_client"
-    ) as mock_get_file_client:
+    with (
+        patch.object(azure_fileshare, "_share_client") as mock_share_client,
+        patch.object(mock_share_client, "get_file_client") as mock_get_file_client,
+    ):
         azure_fileshare.upload(config, local_folder, remote_folder, recursive=True)
 
     mock_get_file_client.assert_has_calls(

mlos_bench/pyproject.toml

@@ -18,15 +18,14 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
 license = { "text" = "MIT" }
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
     { name = "Microsoft", email = "[email protected]" },
 ]

mlos_core/pyproject.toml

@@ -16,15 +16,14 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
 license = { "text" = "MIT" }
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
     { name = "Microsoft", email = "[email protected]" },
 ]

mlos_core/setup.py

@@ -95,11 +95,7 @@ def _get_long_desc_from_readme(base_url: str) -> dict:
         "scipy>=1.3.2",
         "numpy>=1.24",
         'pandas >= 2.2.0;python_version>="3.9"',
-        'pandas >= 1.0.3;python_version<"3.9"',
         "ConfigSpace>=1.0",
-        # pyparsing (required by matplotlib and needed for the notebook examples)
-        # 3.2 has incompatibilities with python 3.8 due to type hints
-        'pyparsing<3.2; python_version<"3.9"',
     ],
     extras_require=extra_requires,
     **_get_long_desc_from_readme("https://github.com/microsoft/MLOS/tree/main/mlos_core"),

mlos_viz/pyproject.toml

@@ -19,15 +19,14 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
 license = { "text" = "MIT" }
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
     { name = "Microsoft", email = "[email protected]" },
 ]