From 95e6237f1d021da9b8678b6bb9bd90d15c7d0fba Mon Sep 17 00:00:00 2001 From: Julian Edwards Date: Mon, 22 Jul 2024 10:21:04 +1000 Subject: [PATCH] Use Hatch instead of Tox - Removes the old setup.[py,cfg] in favour of a modern pyproject.toml - Remove Tox in favour of Hatch - Add tests! --- .github/workflows/tests.yml | 8 +- .gitignore | 5 +- .vscode/settings.json | 19 ++-- ChangeLog | 6 ++ README.rst | 15 ++- RELEASING.txt | 17 ++-- dbtesttools/engines/postgres.py | 24 ++--- dbtesttools/fixtures.py | 15 +-- dbtesttools/tests/__init__.py | 13 +++ dbtesttools/tests/models.py | 11 +++ dbtesttools/tests/test_isolation.py | 140 ++++++++++++++++++++++++++++ pyproject.toml | 87 +++++++++++++++++ requirements.txt | 6 -- setup.cfg | 33 ------- setup.py | 11 --- test-requirements.txt | 9 -- tox.ini | 31 ------ 17 files changed, 308 insertions(+), 142 deletions(-) create mode 100644 dbtesttools/tests/__init__.py create mode 100644 dbtesttools/tests/models.py create mode 100644 dbtesttools/tests/test_isolation.py delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 test-requirements.txt delete mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1efdd53..e5e0f18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -22,9 +22,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install tox + pip install hatch - - name: Run Tox + - name: Run tests run: | - tox + hatch run ci diff --git a/.gitignore b/.gitignore index ea8e650..9e43151 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ -.tox/ +.hatch/ +.stestr/ db_testtools.egg-info/ __pycache__/ *.pyc +dbtesttools/_version.py +dist/ diff --git a/.vscode/settings.json b/.vscode/settings.json index cfd93cf..e38c4ff 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,23 +1,28 @@ { - "python.pythonPath": "${workspaceFolder}/.tox/py3/bin/python3", - "python.formatting.provider": "black", - "python.formatting.blackArgs": [ + "[python]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + } + }, + "black-formatter.args": [ "--line-length", "79" ], - "python.linting.flake8Enabled": true, - "python.linting.enabled": true, "python.testing.pytestEnabled": false, - "python.testing.nosetestsEnabled": false, "python.testing.unittestEnabled": true, "files.watcherExclude": { "**/.coverage/**": true, "**/.eggs": true, "**/.stestr/**": true, "**/.testrepository/**": true, - "**/.tox/**": true + "${workspaceFolder}/build/**": true, + "**/.hatch/**": true }, "git.allowForcePush": true, "editor.wordWrapColumn": 79, "vim.textwidth": 79, + "githubPullRequests.defaultMergeMethod": "rebase" } + diff --git a/ChangeLog b/ChangeLog index 9ae4d07..2be4842 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,12 @@ CHANGES ======= +2024.07.21 +---------- +* Use Hatch to test and build the project +* Replace Black with Ruff for formatting and checking +* Allow overriding of the PG server IP address + 2023.01.27 ---------- diff --git a/README.rst b/README.rst index 01666ed..846f2ef 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ after each test completes. Requirements ------------ -Python 3.7 and beyond should work. +Python 3.8 and beyond should work. Quickstart ---------- @@ -76,7 +76,8 @@ dropped and re-instated on every test. The PostgresContainerFixture starts its own Postgres instance in a local Docker container. Therefore you must have Docker installed before using -this fixture. +this fixture. The Postgres image used by default is 16.3-alpine, but this +fixture is known to work all the way back to v11. If you are already running inside Docker you will need to start the container with `--network-"host"` so that 127.0.0.1 routes to the started @@ -90,16 +91,12 @@ PG containers. You will need to do up to two extra things: can set the DBTESTTOOLS_PG_IP_ADDR environment variable. -Help needed! ------------- -This fixture suite is currently not tested itself and would benefit from -anyone willing to contribute some unit tests. However, it has been in -use daily on a large project at Cisco for a few years now, and is very -stable. +This code has been in use daily on a large project at Cisco for a few years +now, and is very stable. Copyright --------- -db-testtools is copyright (c) 2021-2023 Cisco Systems, Inc. and its affiliates +db-testtools is copyright (c) 2021-2024 Cisco Systems, Inc. and its affiliates All rights reserved. diff --git a/RELEASING.txt b/RELEASING.txt index 3740414..64621fd 100644 --- a/RELEASING.txt +++ b/RELEASING.txt @@ -3,14 +3,11 @@ To release a new version of db-testtools: 1. Tag the current revision with the required release number, e.g. `git tag 2021.09.29` 2. Build the package: - `tox -e build-python-package` - 3. This will generate a new ChangeLog file. It needs to be committed as - a new revision. - 4. After committing, move the tag: `git tag -f 2021.09.29` - 5. Build the package again because PBR. - 6. Upload the package to testpyi first to make sure it is ok: - `tox -e testpypi dist/*2021.9.29*` - 7. If that looks ok, upload to the real pypi: - `tox -e pypi dist/*2021.9.29*` - 8. Push the new tag to Github: + `hatch build` + 3. Upload the package to testpyi first to make sure it is ok: + `hatch run testpypi dist/*2021.9.29*` + 4. If that looks ok, upload to the real pypi: + `hatch run pypi dist/*2021.9.29*` + 5. Push the new tag to Github: `git push origin 2021.09.29` + 6. Make a Github release diff --git a/dbtesttools/engines/postgres.py b/dbtesttools/engines/postgres.py index b0454bc..44614f2 100644 --- a/dbtesttools/engines/postgres.py +++ b/dbtesttools/engines/postgres.py @@ -21,6 +21,8 @@ LC_CTYPE = 'en_US.utf8'; CREATE USER testing WITH ENCRYPTED PASSWORD 'testing'; GRANT ALL PRIVILEGES ON DATABASE testing TO testing; +GRANT ALL ON SCHEMA public TO testing; +ALTER DATABASE testing OWNER TO testing; """ NEXT_ID = count(1) @@ -53,10 +55,10 @@ def __init__( # Using the larger non-alpine image causes sort-order errors # because of locale collation differences. # image='postgres:11.4', - image='postgres:11.11-alpine', - name='testdb', + image="postgres:16.3-alpine", + name="testdb", init_sql=None, - pg_data='/tmp/pgdata', # noqa: S108 + pg_data="/tmp/pgdata", # noqa: S108 isolation=None, future=False, ip_address=None, @@ -71,7 +73,7 @@ def __init__( self.isolation = isolation self.future = future self.ip_address = ip_address or os.getenv( - 'DBTESTTOOLS_PG_IP_ADDR', '127.0.0.1' + "DBTESTTOOLS_PG_IP_ADDR", "127.0.0.1" ) def connect(self): @@ -95,7 +97,7 @@ def setUp(self): self.wait_for_pg_start() self.set_up_test_database() self.engine = sa.create_engine( - 'postgresql://testing:testing@{ip}:{port}/testing'.format( + "postgresql://testing:testing@{ip}:{port}/testing".format( ip=self.ip_address, port=self.local_port ), isolation_level=self.isolation, @@ -111,18 +113,18 @@ def pull_image(self): self.client.images.pull(self.image) def start_container(self): - env = dict(POSTGRES_PASSWORD='postgres', PGDATA=self.pg_data) # noqa: S106 - ports = {'5432': self.local_port} + env = dict(POSTGRES_PASSWORD="postgres", PGDATA=self.pg_data) # noqa: S106 + ports = {"5432": self.local_port} print("Starting Postgres container ...", file=sys.stderr) # Uniq-ify the name as threaded tests will create multiple containers. - name = '{}-{}.{}'.format(self.name, os.getpid(), next(NEXT_ID)) + name = "{}-{}.{}".format(self.name, os.getpid(), next(NEXT_ID)) self.container = self.client.containers.run( self.image, detach=True, auto_remove=True, environment=env, name=name, - network_mode='bridge', + network_mode="bridge", ports=ports, remove=True, ) @@ -133,7 +135,7 @@ def find_free_port(self): # real free port. We close the socket after determnining which port # that was. with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.bind(('localhost', 0)) + s.bind(("localhost", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.local_port = s.getsockname()[1] print("Using port {}".format(self.local_port), file=sys.stderr) @@ -148,7 +150,7 @@ def set_up_test_database(self): ) c.autocommit = True cur = c.cursor() - for stmt in self.init_sql.split(';'): + for stmt in self.init_sql.split(";"): if stmt.strip(): cur.execute(stmt) cur.close() diff --git a/dbtesttools/fixtures.py b/dbtesttools/fixtures.py index cd69959..d12b281 100644 --- a/dbtesttools/fixtures.py +++ b/dbtesttools/fixtures.py @@ -70,7 +70,7 @@ def __init__( ModelBase, models_module, # NOSONAR patch_query_property=False, - engine_fixture_name='SqliteMemoryFixture', + engine_fixture_name="SqliteMemoryFixture", sessionmaker_class=None, engine_fixture_kwargs=None, future=False, @@ -83,7 +83,7 @@ def __init__( self.sessionmaker_class = sessionmaker_class self.engine_fixture_kwargs = engine_fixture_kwargs or {} self.future = future - self.engine_fixture_kwargs['future'] = self.future + self.engine_fixture_kwargs["future"] = self.future def make(self, dep_resources): print("Creating new database resource...", file=sys.stderr) @@ -101,22 +101,21 @@ def has_savepoint(self): def clean(self, resource): print("Cleaning up database resource...", file=sys.stderr) - self.drop_tables(self.engine) self.db.cleanUp() def pick_engine_fixture(self): - env_name = os.environ.get('TEST_ENGINE_FIXTURE', None) + env_name = os.environ.get("TEST_ENGINE_FIXTURE", None) if env_name is not None: self.engine_fixture_name = env_name import dbtesttools.engines for _, module, _ in pkgutil.iter_modules(dbtesttools.engines.__path__): - mod = importlib.import_module(f'dbtesttools.engines.{module}') + mod = importlib.import_module(f"dbtesttools.engines.{module}") for name, obj in mod.__dict__.items(): if name == self.engine_fixture_name: return obj(**self.engine_fixture_kwargs) - raise AttributeError(f'{self.engine_fixture_name} not found') + raise AttributeError(f"{self.engine_fixture_name} not found") def initialize_engine(self): self.db = self.pick_engine_fixture() @@ -177,10 +176,6 @@ def create_tables(self, engine): metadata = self.ModelBase.metadata metadata.create_all(bind=engine) - def drop_tables(self, engine): - metadata = self.ModelBase.metadata - metadata.drop_all(bind=engine) - def rollback_transaction(self, txn): """Roll back an in-progress transaction. diff --git a/dbtesttools/tests/__init__.py b/dbtesttools/tests/__init__.py new file mode 100644 index 0000000..b7e4465 --- /dev/null +++ b/dbtesttools/tests/__init__.py @@ -0,0 +1,13 @@ +import os + +import testresources +from testscenarios import generate_scenarios + + +def load_tests(loader, tests, pattern): + this_dir = os.path.dirname(__file__) + mytests = loader.discover(start_dir=this_dir, pattern=pattern) + result = testresources.OptimisingTestSuite() + result.addTests(generate_scenarios(mytests)) + result.addTests(generate_scenarios(tests)) + return result diff --git a/dbtesttools/tests/models.py b/dbtesttools/tests/models.py new file mode 100644 index 0000000..5d54410 --- /dev/null +++ b/dbtesttools/tests/models.py @@ -0,0 +1,11 @@ +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base + +ModelBase = declarative_base() + + +class TestModel(ModelBase): + __tablename__ = "test_model" + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.String) + value = sa.Column(sa.Integer) diff --git a/dbtesttools/tests/test_isolation.py b/dbtesttools/tests/test_isolation.py new file mode 100644 index 0000000..19fad2d --- /dev/null +++ b/dbtesttools/tests/test_isolation.py @@ -0,0 +1,140 @@ +import testresources +import testscenarios +import testtools +from sqlalchemy import func + +from dbtesttools.fixtures import DatabaseResource, SessionFixture +from dbtesttools.tests.models import ModelBase, TestModel + + +class DBTestCaseSqlite(testresources.ResourcedTestCase, testtools.TestCase): + db_fixture = DatabaseResource( + ModelBase, + "dbtesttools.tests.models", + engine_fixture_name="SqliteMemoryFixture", + future=True, + ) + resources = [("database", db_fixture)] + + def setUp(self): + super().setUp() + self.session_fixture = SessionFixture(self.database, future=True) + self.useFixture(self.session_fixture) + self.session = self.session_fixture.session + + +class DBTestCasePostgres(testresources.ResourcedTestCase, testtools.TestCase): + db_fixture = DatabaseResource( + ModelBase, + "dbtesttools.tests.models", + engine_fixture_name="PostgresContainerFixture", + future=True, + ) + resources = [("database", db_fixture)] + + def setUp(self): + super().setUp() + self.session_fixture = SessionFixture(self.database, future=True) + self.useFixture(self.session_fixture) + self.session = self.session_fixture.session + + +# Use Scenarios to force a big list of tests to run that will re-use a single +# DB resource. This relies on a limited concurrency setting in +# pyproject.toml's scripts.py3 args for stestr. + + +class TestIsolationSqlite(testscenarios.TestWithScenarios, DBTestCaseSqlite): + scenarios = [ + ("1", dict()), + ("2", dict()), + ("3", dict()), + ("4", dict()), + ("5", dict()), + ("6", dict()), + ("7", dict()), + ("8", dict()), + ("9", dict()), + ("10", dict()), + ("11", dict()), + ("12", dict()), + ("13", dict()), + ("14", dict()), + ("15", dict()), + ("16", dict()), + ("17", dict()), + ("18", dict()), + ("19", dict()), + ("20", dict()), + ("21", dict()), + ("22", dict()), + ("23", dict()), + ("24", dict()), + ("25", dict()), + ("26", dict()), + ("27", dict()), + ("28", dict()), + ("29", dict()), + ("30", dict()), + ] + + def test_isolation_1(self): + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 0) + self.session.add(TestModel(name="test", value=1)) + self.session.commit() + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 1) + + def test_isolation_2(self): + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 0) + self.session.add(TestModel(name="test", value=1)) + self.session.commit() + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 1) + + +class TestIsolationPostrgres( + testscenarios.TestWithScenarios, DBTestCasePostgres +): + scenarios = [ + ("1", dict()), + ("2", dict()), + ("3", dict()), + ("4", dict()), + ("5", dict()), + ("6", dict()), + ("7", dict()), + ("8", dict()), + ("9", dict()), + ("10", dict()), + ("11", dict()), + ("12", dict()), + ("13", dict()), + ("14", dict()), + ("15", dict()), + ("16", dict()), + ("17", dict()), + ("18", dict()), + ("19", dict()), + ("20", dict()), + ("21", dict()), + ("22", dict()), + ("23", dict()), + ("24", dict()), + ("25", dict()), + ("26", dict()), + ("27", dict()), + ("28", dict()), + ("29", dict()), + ("30", dict()), + ] + + def test_isolation_1(self): + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 0) + self.session.add(TestModel(name="test", value=1)) + self.session.commit() + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 1) + + def test_isolation_2(self): + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 0) + self.session.add(TestModel(name="test", value=1)) + self.session.commit() + self.assertEqual(self.session.scalar(func.count(TestModel.id)), 1) diff --git a/pyproject.toml b/pyproject.toml index b5dfd97..2cf0ff5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,90 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "db-testtools" +dynamic = ["version"] +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.8" +authors = [ + { name = "Julian Edwards", email = "juledwar@cisco.com" }, +] +maintainers = [ + { name = "Julian Edwards", email = "juledwar@cisco.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Framework :: Hatch", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Utilities", +] +urls.Source = "https://github.com/juledwar/db-testtools" + +dependencies = [ + "docker>=5.0.2", + "fixtures>=3.0.0", + "psycopg2-binary>=2.9.1", + "retry>=0.9.2", + "sqlalchemy>=1.4.23", + "testresources>=2.0.1", +] +[project.optional-dependencies] +test = [ + "build>=0.7.0", + "coverage[toml]>=6.0.2", + "ipython>=7.28.0", + "pdbpp>=0.10.3", + "ruff>=0.5.3", + "sqlalchemy", + "stestr>=3.2.1", + "testresources", + "testscenarios>=0.5.0", + "testtools>=2.5.0", + "twine>=3.4.2", + +] + +[tool.hatch.version] +source = "vcs" +[tool.hatch.build.hooks.vcs] +version-file = "dbtesttools/_version.py" +[tool.hatch.build.targets.sdist] +include = ["/dbtesttools"] +exclude = ["tests"] + +[tool.hatch.envs.default] +path = ".hatch" +features = ["test"] + +scripts.debug = ["python -m testtools.run discover -v -s dbtesttools/tests -t {root} -p test*.py {args}"] +# Concurrency explicitly set to two because: +# 1. Want to make sure that 1 DB resource is brought up per thread +# 2. Want to make sure that tests will re-use the same DB resource +scripts.py3 = ["stestr run --concurrency 2 -t dbtesttools/tests --top-dir {root} {args}"] +scripts.formatcheck = [ + "ruff format --check dbtesttools", + "ruff check --select I --show-fixes dbtesttools" +] +scripts.format = [ + "ruff check --select I --fix-only --show-fixes dbtesttools", + "ruff format dbtesttools", +] +scripts.ci = ["py3", "formatcheck"] +scripts.testpypi = ["twine upload --repository testpypi {args}"] +scripts.pypi = ["twine upload --repository pypi {args}"] + [tool.coverage.run] omit = [ '*dbtesttools/tests*', diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index df27dd3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -docker>=5.0.2 -fixtures>=3.0.0 -psycopg2-binary>=2.9.1 -retry>=0.9.2 -sqlalchemy>=1.4.23 -testresources>=2.0.1 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4b39bbe..0000000 --- a/setup.cfg +++ /dev/null @@ -1,33 +0,0 @@ -[metadata] -name = db-testtools -url = https://github.com/juledwar/db-testtools -long_description = file: README.rst -long_description_content_type = text/x-rst -license = Apache2.0 -python_requires = >=3.7 -maintainer = Julian Edwards -maintainer_email = juledwar@cisco.com -classifier = - Development Status :: 5 - Production/Stable - Environment :: Console - Intended Audience :: Developers - Intended Audience :: Information Technology - Natural Language :: English - Operating System :: POSIX :: Linux - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Topic :: Utilities - -[files] -packages = dbtesttools -exclude = - dbtesttools.testing - dbtesttools.tests - -[bdist_wheel] -universal = 1 - -[pbr] -skip_git_sdist = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 87a862b..0000000 --- a/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -import setuptools - -setuptools_version = tuple(map(int, setuptools.__version__.split(".", 2)[:2])) -if setuptools_version < (34, 4): - raise RuntimeError("setuptools 34.4 or newer is required, detected ", - setuptools_version) - -if __name__ == "__main__": - setuptools.setup( - setup_requires=["pbr>=2.0.0"], - pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 309f346..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -build>=0.7.0 -coverage[toml]==6.0.2 -ipython==7.28.0 -pdbpp==0.10.3 -ruff>=0.5.3 -stestr==3.2.1 -testscenarios==0.5.0 -testtools==2.5.0 -twine>=3.4.2 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c34386d..0000000 --- a/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -envlist = pep8, formatcheck - -[testenv] -passenv = - BASEPYTHON - PYTHONBREAKPOINT -basepython = python3 -envdir ={toxworkdir}/py3 -usedevelop = True -setenv = - VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -commands = - repl: ipython {posargs} - pep8: ruff check dbtesttools {posargs} - build-python-package: python -m build {posargs} - pypi: python -m twine upload {posargs} - testpypi: python -m twine upload --repository testpypi {posargs} - -[testenv:formatcheck] -commands = - ruff check --select I --show-fixes dbtesttools - ruff format --check dbtesttools - -[testenv:format] -commands = - ruff check --select I --fix-only --show-fixes dbtesttools - ruff format dbtesttools