Switch from setuptools to hatchling

.ci/variables.json

@@ -1,8 +1,9 @@
 {
   "prerequisite_versions": {
-    "PIP": "21.2.4",
-    "SETUPTOOLS": "58.0.4",
-    "WHEEL": "0.37.0"
+    "HATCH": "1.3.1",
+    "HATCHLING": "1.6.0",
+    "PIP": "22.2",
+    "WHEEL": "0.37.1"
   },
   "python_versions": {
     "PY38": "3.8.13",

Makefile

@@ -24,8 +24,9 @@ PIP_WRAPPER = $(PY_BIN) -m pip
 export PY38 = $(shell jq -r '.python_versions.PY38' .ci/variables.json)
 export PY39 = $(shell jq -r '.python_versions.PY39' .ci/variables.json)
 export PY310 = $(shell jq -r '.python_versions.PY310' .ci/variables.json)
+export HATCH_VERSION = $(shell jq -r '.prerequisite_versions.HATCH' .ci/variables.json)
+export HATCHLING_VERSION = $(shell jq -r '.prerequisite_versions.HATCHLING' .ci/variables.json)
 export PIP_VERSION = $(shell jq -r '.prerequisite_versions.PIP' .ci/variables.json)
-export SETUPTOOLS_VERSION = $(shell jq -r '.prerequisite_versions.SETUPTOOLS' .ci/variables.json)
 export WHEEL_VERSION = $(shell jq -r '.prerequisite_versions.WHEEL' .ci/variables.json)
 VIRTUAL_ENV ?= .venv
 VENV_ACTIVATE_FILE = $(VIRTUAL_ENV)/bin/activate
@@ -60,7 +61,7 @@ check-venv:
 	fi
 
 install-user: venv-create
-	. $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install --upgrade pip==$(PIP_VERSION) setuptools==$(SETUPTOOLS_VERSION) wheel==$(WHEEL_VERSION)
+	. $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install --upgrade hatch==$(HATCH_VERSION) hatchling==$(HATCHLING_VERSION) pip==$(PIP_VERSION) wheel==$(WHEEL_VERSION)
 	. $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install -e .
 
 install: install-user
@@ -79,18 +80,8 @@ docs-clean:
 python-caches-clean:
 	-@find . -name "__pycache__" -prune -exec rm -rf -- \{\} \;
 
-# Force recreation of the virtual environment used by tox.
-#
-# See https://tox.readthedocs.io/en/latest/#system-overview:
-#
-# > Note pip will not update project dependencies (specified either in the install_requires or the extras
-# > section of the setup.py) if any version already exists in the virtual environment; therefore we recommend
-# > to recreate your environments whenever your project dependencies change.
-tox-env-clean:
-	rm -rf .tox
-
 lint: check-venv
-	@. $(VENV_ACTIVATE_FILE); find esrally benchmarks scripts tests it setup.py -name "*.py" -exec pylint -j0 -rn --rcfile=$(CURDIR)/.pylintrc \{\} +
+	@. $(VENV_ACTIVATE_FILE); find esrally benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --rcfile=$(CURDIR)/.pylintrc \{\} +
 	@. $(VENV_ACTIVATE_FILE); black --check --diff .
 	@. $(VENV_ACTIVATE_FILE); isort --check --diff .
 
@@ -110,7 +101,7 @@ test: check-venv
 precommit: lint
 
 # checks min and max python versions
-it: check-venv python-caches-clean tox-env-clean
+it: check-venv python-caches-clean
 	. $(VENV_ACTIVATE_FILE); tox -e py38
 	. $(VENV_ACTIVATE_FILE); tox -e py310
 

docker/Dockerfiles/Dockerfile-dev

@@ -11,10 +11,8 @@ RUN apt-get -y update && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /rally/esrally
-COPY setup.py /rally/
-COPY setup.cfg /rally/
+COPY pyproject.toml /rally/
 COPY README.md /rally/
-COPY MANIFEST.in /rally/
 COPY esrally/ /rally/esrally/
 
 RUN python3 -m venv /rally/venv
@@ -24,7 +22,7 @@ WORKDIR /rally
 # Wipe away any lingering caches, copied over from the local machine
 RUN find /rally -name "__pycache__" -exec rm -rf -- \{\} \; 2>/dev/null || true
 RUN find /rally -name ".pyc" -exec rm -rf -- \{\} \; 2>/dev/null || true
-RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 install --upgrade hatch hatchling pip wheel
 RUN pip3 install /rally
 
 ################################################################################

docker/Dockerfiles/Dockerfile-release

@@ -12,7 +12,7 @@ RUN apt-get -y update && \
 RUN groupadd --gid 1000 rally && \
     useradd -d /rally -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash rally
 
-RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 install --upgrade hatch hatchling pip wheel
 RUN pip3 install esrally==$RALLY_VERSION
 
 WORKDIR /rally

docs/create_scheduler_plot.py

@@ -17,7 +17,7 @@
 
 import sys
 
-# we don't include matplotlib in setup.py as it's just a development dependency
+# we don't add matplotlib as a dependency as it's infrequently used and heavy
 try:
     import matplotlib
     import matplotlib.pyplot as plt

pyproject.toml

@@ -1,6 +1,135 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "esrally/_version.py"
+
+
+[project]
+name = "esrally"
+dynamic = ["version"]
+authors = [
+  {name="Daniel Mitterdorfer", email="[email protected]"},
+]
+description = "Macrobenchmarking framework for Elasticsearch"
+readme = "README.md"
+license = {text = "Apache License 2.0"}
+requires-python = ">=3.8"
+classifiers=[
+    "Topic :: System :: Benchmark",
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: POSIX",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+]
+################################################################################################
+#
+# Adapt `create-notice.sh` whenever changing dependencies here.
+#
+# That script grabs all license files so we include them in the notice file.
+#
+################################################################################################
+dependencies = [
+    # License: Apache 2.0
+    # transitive dependencies:
+    #   urllib3: MIT
+    #   aiohttp: Apache 2.0
+    "elasticsearch[async]==7.14.0",
+    "urllib3==1.26.9",
+    # License: BSD
+    "psutil==5.8.0",
+    # License: MIT
+    "py-cpuinfo==7.0.0",
+    # License: MIT
+    "tabulate==0.8.9",
+    # License: MIT
+    "jsonschema==3.1.1",
+    # License: BSD
+    # transitive dependency Markupsafe: BSD
+    "Jinja2==2.11.3",
+    "markupsafe==2.0.1",
+    # License: MIT
+    "thespian==3.10.1",
+    # recommended library for thespian to identify actors more easily with `ps`
+    # "setproctitle==1.1.10",
+    # always use the latest version, these are certificate files...
+    # License: MPL 2.0
+    "certifi",
+    # License: MIT
+    "yappi==1.3.3",
+    # License: BSD
+    "ijson==2.6.1",
+    # License: Apache 2.0
+    # transitive dependencies:
+    #   google-crc32c: Apache 2.0
+    "google-resumable-media[requests]==1.1.0",
+    # License: Apache 2.0
+    "google-auth==1.22.1",
+]
+
+[project.optional-dependencies]
+s3 = [
+    # License: Apache 2.0
+    # transitive dependencies:
+    #   botocore: Apache 2.0
+    #   jmespath: MIT
+    #   s3transfer: Apache 2.0
+    "boto3==1.18.46",
+]
+# These packages are only required when developing Rally
+develop = [
+    # s3
+    "boto3==1.18.46",
+    # tests
+    "ujson",
+    "pytest==6.2.5",
+    "pytest-benchmark==3.2.2",
+    "pytest-asyncio==0.18.1",
+    "tox==3.14.0",
+    "sphinx==4.2.0",
+    "furo==2022.06.21",
+    "twine==1.15.0",
+    "wheel==0.37.1",
+    "github3.py==1.3.0",
+    "pylint==2.6.0",
+    "black==22.3.0",
+    "isort==5.8.0",
+    "trustme==0.9.0",
+    "pytest-httpserver==1.0.4",
+]
+
+[project.scripts]
+esrally = "esrally.rally:main"
+esrallyd = "esrally.rallyd:main"
+
+[project.urls]
+"Changelog" = "https://github.com/elastic/rally/blob/master/CHANGELOG.md"
+"Documentation" = "https://esrally.readthedocs.io/"
+"Code" = "https://github.com/elastic/rally"
+"Issue tracker" = "https://github.com/elastic/rally/issues"
+
+
 [tool.black]
 line-length = 140
 target-version = ['py38']
 
 [tool.isort]
 profile = 'black'
+
+[tool.pytest.ini_options]
+# set to true for more verbose output of tests
+log_cli = false
+log_level = "INFO"
+addopts = "--verbose --color=yes"
+testpaths = "tests"
+junit_family = "xunit2"
+junit_logging = "all"
+asyncio_mode = "strict"

release.sh

@@ -46,19 +46,17 @@ then
 fi
 
 echo "Updating changelog"
-# For exit on error to work we have to separate 
+# For exit on error to work we have to separate
 #  CHANGELOG.md generation into two steps.
 CHANGELOG="$(python3 changelog.py ${RELEASE_VERSION})"
 printf "$CHANGELOG\n\n$(cat CHANGELOG.md)" > CHANGELOG.md
 git commit -a -m "Update changelog for Rally release $RELEASE_VERSION"
 
-# * Update version in `setup.py` and `docs/conf.py`
 echo "Updating release version number"
 printf '__version__ = "%s"\n' $RELEASE_VERSION > esrally/_version.py
 git commit -a -m "Bump version to $RELEASE_VERSION"
 
-# --upgrade is required for virtualenv
-python3 setup.py develop --upgrade
+pip install --editable .
 
 # Check version
 if ! [[ $(esrally --version) =~ "esrally ${RELEASE_VERSION} (git revision" ]]
@@ -68,7 +66,7 @@ then
 fi
 
 # Build new version
-python3 setup.py bdist_wheel
+hatch build -t wheel
 # Upload to PyPI with retries in case of authentication failure
 printf "\033[0;31mUploading to PyPI. Please enter your credentials ...\033[0m\n"
 for i in 1 2 3 4 5; do twine upload dist/esrally-${RELEASE_VERSION}-*.whl && break || sleep 1; done

run.sh

@@ -28,18 +28,17 @@
 readonly BINARY_NAME="${__RALLY_INTERNAL_BINARY_NAME}"
 readonly HUMAN_NAME="${__RALLY_INTERNAL_HUMAN_NAME}"
 
-install_esrally_with_setuptools () {
+install_esrally() {
     # Check if optional parameter with Rally binary path, points to an existing executable file.
     if [[ $# -ge 1 && -n $1 ]]; then
         if [[ -f $1 && -x $1 ]]; then return; fi
     fi
 
     if [[ ${IN_VIRTUALENV} == 0 ]]; then
-        # https://setuptools.readthedocs.io/en/latest/setuptools.html suggests not invoking setup.py directly
-        # Also workaround system pip conflicts, https://github.com/pypa/pip/issues/5599
-        python3 -m pip install --quiet --user --upgrade --no-use-pep517 --editable .[develop]
+        # workaround system pip conflicts, https://github.com/pypa/pip/issues/5599
+        python3 -m pip install --quiet --user --upgrade --editable .[develop]
     else
-        python3 -m pip install --quiet --upgrade --no-use-pep517 --editable .[develop]
+        python3 -m pip install --quiet --upgrade --editable .[develop]
     fi
 }
 
@@ -99,7 +98,7 @@ then
       then
         echo "Auto-updating Rally from ${REMOTE}"
         git rebase ${REMOTE}/master --quiet
-        install_esrally_with_setuptools
+        install_esrally
       #else
       # offline - skipping update
       fi
@@ -125,14 +124,14 @@ if [[ $IN_VIRTUALENV == 0 ]]
 then
     RALLY_ROOT=$(python3 -c "import site; print(site.USER_BASE)")
     RALLY_BIN=${RALLY_ROOT}/bin/${BINARY_NAME}
-    install_esrally_with_setuptools "${RALLY_BIN}"
+    install_esrally "${RALLY_BIN}"
     if [[ -x $RALLY_BIN ]]; then
         ${RALLY_BIN} "$@"
     else
         echo "Cannot execute ${HUMAN_NAME} in ${RALLY_BIN}."
     fi
 else
-    install_esrally_with_setuptools "${BINARY_NAME}"
+    install_esrally "${BINARY_NAME}"
 
     ${BINARY_NAME} "$@"
 fi